# Bazaar merge directive format 2 (Bazaar 0.90) # revision_id: someone1@gmail.com-20130520141919-bpr69objzzaokajt # target_branch: bzr+ssh://bazaar.launchpad.net/+branch/duplicity/ # testament_sha1: be3ee67f4ef4d58dd9ae3d40173e1c4b3c714c7a # timestamp: 2013-05-20 10:19:44 -0400 # base_revision_id: kenneth@loafman.com-20130427150127-\ # gdw2oo4m6g5d6xh4 # # Begin patch === modified file 'duplicity/backends/_boto_multi.py' --- duplicity/backends/_boto_multi.py 2013-01-25 13:35:33 +0000 +++ duplicity/backends/_boto_multi.py 2013-05-20 14:16:42 +0000 @@ -390,7 +390,9 @@ consumer = ConsumerThread(queue) consumer.start() - pool = multiprocessing.Pool(processes=chunks) + number_of_procs = min(chunks, globals.s3_multipart_max_procs) + log.Debug("Setting pool to %d processes" % number_of_procs) + pool = multiprocessing.Pool(processes=number_of_procs) for n in range(chunks): params = [self.scheme, self.parsed_url, self.bucket_name, mp.id, filename, n, chunk_size, globals.num_retries, === modified file 'duplicity/commandline.py' --- duplicity/commandline.py 2013-04-27 14:48:39 +0000 +++ duplicity/commandline.py 2013-05-20 14:19:19 +0000 @@ -481,6 +481,10 @@ parser.add_option("--s3-multipart-chunk-size", type="int", action="callback", metavar=_("number"), callback=lambda o, s, v, p: setattr(p.values, "s3_multipart_chunk_size", v*1024*1024)) + # Number of processes to set the Processor Pool to when uploading multipart + # uploads to S3. Use this to control the maximum simultaneous uploads to S3. + parser.add_option("--s3-multipart-max-procs", type="int", metavar=_("number")) + # Option to allow the s3/boto backend use the multiprocessing version. # By default it is off since it does not work for Python 2.4 or 2.5. if sys.version_info[:2] >= (2,6): === modified file 'duplicity/globals.py' --- duplicity/globals.py 2013-04-27 14:48:39 +0000 +++ duplicity/globals.py 2013-05-20 14:16:42 +0000 @@ -186,6 +186,9 @@ # Minimum chunk size accepted by S3 s3_multipart_minimum_chunk_size = 5*1024*1024 +# Maximum number of processes to use while doing a multipart upload to S3 +s3_multipart_max_procs = 8 + # Whether to use the full email address as the user name when # logging into an imap server. If false just the user name # part of the email address is used. # Begin bundle IyBCYXphYXIgcmV2aXNpb24gYnVuZGxlIHY0CiMKQlpoOTFBWSZTWaf36GcABEffgEQQWmf/93YD SgC////wYAd99bVoAADQpEAANASSKaZNTAJtTxMU9Go09Q9QYIaAHMABMAATAAAAAA5gAJgACYAA AAACU1MpNT9UfqaGo8iaeoAAADQADmAAmAAJgAAAAAJJATQAEBNJ6TTaTU2ptFPUYaD1Ry4QKdtq u0k1UiWWXp8Gcv8TToMcbdapCUDOWE4wuSRd2fExut5vqLS18JuiQqlmIIC2yi4PoFgISEmQm1eQ 2zBiL8biqHhQ4ikVOJA5lL73Qa79xzOXIqOfDRlYn5cXwCi/TbAXnZIY8b5Q+NsCeEv9/gsgLuwo 8DLTBp6odjHuhtzsKQ58vl4Dju4gxFqTtMbRwlU0GHvoQRXnvEmYWu0sNNXu581uFY6HMsqPIYNy NAjkd4hcwUlxHMLisCR2Emj/J4+txFFhrbF5gqMXDMZpq2TXhhmRXZW0pSpuqsZiQnArmBd5jspQ EkeI6LSBOMwW6pEGBDHEh4jAvlDUFUBRFor5ekD+XFROhEm0MROi96GfxUOh7S/F4YfocylbT0qm VGxghtNWycxAQnzvGReA0ARabtxMlh+S4mEAUlsrbyw1EpZFmR+uh8PcS4C4VxYW1DVDFoYCRsSI F5fw4SBYWmmeJYJOISBbCQYAjBhwJ4K4FIFpVAFlZXjyLB49TRHDdyK3kTKaDwBPqLzJ1x7rKy8t Ptq4dWCl8Xl4hV4mRYhZxgzGB0Eq4GWhmPOBCosgsVtovW/MrLDU5PeSwraUDbAfz+GADNgwlkwr SglCiKFKEzcMjKBG4hhuc+kwJGBc8u77NawtFUeS21TxKJhgUsgOvckuoYEITyIoyBFpQsg7ApUZ FlrhjErsHPeanGo8lMoQMbsnGsCuMpEolgXTckoxO/fPCZGrQ15ChsXJOKMO4y1MDsop1uw2ub1e yN9ZZG438rB1VDEuAfVh3CUjfiXxHEjWTOdo5lbdmOtrscSDNiLIai6vlWVjFxlpMtyJ5k6WESsg Xwoe5QSlVEHMPFm2QJom/Mlwik5zWD6tMTKUiNle4w1v0PuMT1OZuv+ZTEvdfrgbOwFneCzqHJUj I1NQjAoNMpJEyApDgGHjREwqIJ1DHqLijAvoyYZaAOcWp7NTs0nEOsiEixEUpGsSZhkMzc9j8iHR HYMFreznhIKDIHHlxMln8+gza5VASLqiAHPgCteG97x1Y4ccOcTeUP07dlp042vDa0k9Gpog7hI2 PEX5PpUHIIkke9Q4T8hfL00QiKkMSYLG9NeJccewQgfc3bgjxPqquAVFFaLuICiJYCY3XC2lNolF nMxZnOjPrKYhbNB1le9hKthlYCXIPM7bH2daLcv/UzN/IyH6HNfqTyK9Thsk3BXAPLSsqEcenJW/ kxiPqLewn2C2o8ezFhtwD6swxMOduV24TTkkYrwZst7hPghEiR39A0DrqTBLR2P8wtDVQq2PJmcd IZUFcs8BN9zcW0FhfMRiNBJ1SPNo41ERDC4uicWNzGDNvApJ2DeJgHCRhqWzTViQUO8YPIqOImHt RH7WCKCIHY3efU5lmJ2JOcd/LpoVFisXY8i4iK4ecjYX9B4B6BWje4PkHIevbBxYvsqkjz7Ff9iR ifsuiHdQTjD7DvUjmI2IB4ERLIxPwIYidRZiR58gUAwxM/HzQXCx4W0o2/8dxvKiMhDJYfMRkXm8 SkF/sJRFXmC+IuJUsxMoo3oOHASijqJHoCqAvEXjgII6sBeeAlxE7cRII+CPYP7kEaxgJAmLhC+q YmJHVHAXgjw02HUFaEqCRuMyTDQIIOAlIkJ8qpqjhz1rFzE0MJmIA4egYUE4UAE/eh4plkkaWiXo ClBCajgK/Db+HyFn1ApyrT+Al6i3CVDVQ57Cqky6AlWJDCLkjISOg/0XQ/0piJMX9xZpFAyJuFAu bxLu7iMJREtegwMB++gpC9JGp3pBQF3V4Q++AoB87dTwZYl2H7icfIS9jq82O9lkyYTnLgQQYncf CQlcdjQVMaIsFRhWEHFEpEpsLqWGkTc0A+kT4huw2nh1Jjz3s+OsgxdYMjQHwxEPCxWMDLe9TnBN NhEhS7SeQYcY1FmdQn4loJDvkMRCIHqB3lZwrgTKYmgEiSHJ0S2suCe3OaVQE7hK5BeJ4xWwGBcw lO/XpPzPBI960fOYlRF1zseJtULERUpzInxBMfdMagpiMSwNyYcDgXgJSuFsjYFUey2Hi/+LuSKc KEhT+/QzgA==