=== modified file 'duplicity-bin' --- duplicity-bin 2011-07-29 21:13:16 +0000 +++ duplicity-bin 2011-08-07 14:01:03 +0000 @@ -327,7 +327,8 @@ vol_num += 1 dest_filename = file_naming.get(backup_type, vol_num, encrypted=globals.encryption, - gzipped=not globals.encryption) + compressed=not globals.encryption, + compression=globals.compression) tdp = dup_temp.new_tempduppath(file_naming.parse(dest_filename)) # write volume @@ -335,7 +336,8 @@ at_end = gpg.GPGWriteFile(tarblock_iter, tdp.name, globals.gpg_profile, globals.volsize) else: - at_end = gpg.GzipWriteFile(tarblock_iter, tdp.name, globals.volsize) + at_end = gpg.CompressedWriteFile(tarblock_iter, tdp.name, + globals.volsize, compression = globals.compression) tdp.setdata() # Add volume information to manifest @@ -395,7 +397,9 @@ manifest=True) remote_man_filename = file_naming.get(backup_type, manifest=True, - encrypted=globals.encryption) + encrypted=globals.encryption, + compressed=not globals.encryption, + compression=globals.compression) fh = dup_temp.get_fileobj_duppath(globals.archive_dir, part_man_filename, @@ -420,12 +424,14 @@ assert sig_type in ["full-sig", "new-sig"] part_sig_filename = file_naming.get(sig_type, - gzipped=False, + compressed=False, partial=True) perm_sig_filename = file_naming.get(sig_type, - gzipped=True) - remote_sig_filename = file_naming.get(sig_type, encrypted=globals.encryption, - gzipped=not globals.encryption) + compressed=True, compression='gzip') + remote_sig_filename = file_naming.get(sig_type, + encrypted=globals.encryption, + compressed=not globals.encryption, + compression=globals.compression) fh = dup_temp.get_fileobj_duppath(globals.archive_dir, part_sig_filename, @@ -954,7 +960,8 @@ if pr.manifest: copy_raw(src_iter, tdp.name) else: - gpg.GzipWriteFile(src_iter, tdp.name, size=sys.maxint) + gpg.CompressedWriteFile(src_iter, tdp.name, size=sys.maxint, + compression=globals.compression) tdp.setdata() tdp.move(globals.archive_dir.append(loc_name)) === modified file 'duplicity.1' --- duplicity.1 2011-07-16 18:37:47 +0000 +++ duplicity.1 2011-08-07 14:01:48 +0000 @@ -286,6 +286,14 @@ enough storage space is required to store two volumes. .TP +.BI "--compression " +Specifies the the compression used when unencrypted volumes are created. +Only has an effect when +.BR --no-encryption +is specified, otherwise does nothing. Currently gzip and lzma compression +are supported. The default is gzip compression. + +.TP .BI "--dry-run " Calculate what would be done, but do not perform any backend actions === modified file 'duplicity/commandline.py' --- duplicity/commandline.py 2011-07-16 18:37:47 +0000 +++ duplicity/commandline.py 2011-08-06 09:44:09 +0000 @@ -365,6 +365,9 @@ # If set to false, then do not encrypt files on remote system parser.add_option("--no-encryption", action="store_false", dest="encryption") + # Set the compression type + parser.add_option("--compression", metavar=_("compression")) + # If set, print the statistics after every backup session parser.add_option("--no-print-statistics", action="store_false", dest="print_statistics") === modified file 'duplicity/dup_temp.py' --- duplicity/dup_temp.py 2011-06-17 11:42:33 +0000 +++ duplicity/dup_temp.py 2011-08-07 14:03:29 +0000 @@ -175,7 +175,8 @@ tgt = self.dirpath.append(self.remname) src_iter = SrcIter(src) if pr.compressed: - gpg.GzipWriteFile(src_iter, tgt.name, size = sys.maxint) + gpg.CompressedWriteFile(src_iter, tgt.name, size = sys.maxint, + compression = globals.compression) elif pr.encrypted: gpg.GPGWriteFile(src_iter, tgt.name, globals.gpg_profile, size = sys.maxint) else: @@ -195,7 +196,8 @@ src_iter = SrcIter(src) pr = file_naming.parse(self.permname) if pr.compressed: - gpg.GzipWriteFile(src_iter, tgt.name, size = sys.maxint) + gpg.CompressedWriteFile(src_iter, tgt.name, size = sys.maxint, + compression = globals.compression) os.unlink(src.name) else: os.rename(src.name, tgt.name) === modified file 'duplicity/file_naming.py' --- duplicity/file_naming.py 2010-07-22 19:15:11 +0000 +++ duplicity/file_naming.py 2011-08-07 14:05:18 +0000 @@ -140,29 +140,36 @@ return total -def get_suffix(encrypted, gzipped): +def get_suffix(encrypted, compressed, compression): """ Return appropriate suffix depending on status of encryption, compression, and short_filenames. """ - assert not (encrypted and gzipped) + assert not (encrypted and compressed) + if compressed: + assert compression in ['gzip', 'lzma'] if encrypted: if globals.short_filenames: suffix = '.g' else: suffix = ".gpg" - elif gzipped: + elif compressed and compression == 'gzip': if globals.short_filenames: suffix = ".z" else: suffix = '.gz' + elif compressed and compression == 'lzma': + if globals.short_filenames: + suffix = ".x" + else: + suffix = '.xz' else: suffix = "" return suffix def get(type, volume_number = None, manifest = False, - encrypted = False, gzipped = False, partial = False): + encrypted = False, compressed = False, compression = None, partial = False): """ Return duplicity filename of specified type @@ -171,8 +178,7 @@ filename is of a full or inc manifest file. """ assert dup_time.curtimestr - assert not (encrypted and gzipped) - suffix = get_suffix(encrypted, gzipped) + suffix = get_suffix(encrypted, compressed, compression) part_string = "" if globals.short_filenames: if partial: @@ -342,8 +348,14 @@ if (filename.endswith('.z') or not globals.short_filenames and filename.endswith('gz')): pr.compressed = 1 + pr.compression = 'gzip' + elif (filename.endswith('.x') or + not globals.short_filenames and filename.endswith('xz')): + pr.compressed = 1 + pr.compression = 'lzma' else: pr.compressed = None + pr.compression = None if (filename.endswith('.g') or not globals.short_filenames and filename.endswith('.gpg')): @@ -368,7 +380,7 @@ """ def __init__(self, type, manifest = None, volume_number = None, time = None, start_time = None, end_time = None, - encrypted = None, compressed = None, partial = False): + encrypted = None, compressed = None, compression = None, partial = False): assert type in ["full-sig", "new-sig", "inc", "full"] @@ -385,7 +397,8 @@ self.time = time self.start_time, self.end_time = start_time, end_time - self.compressed = compressed # true if gzip compressed + self.compressed = compressed # true if compressed + self.compression = compression # currently gzip or lzma self.encrypted = encrypted # true if gpg encrypted self.partial = partial === modified file 'duplicity/globals.py' --- duplicity/globals.py 2011-03-29 17:00:36 +0000 +++ duplicity/globals.py 2011-08-06 09:45:49 +0000 @@ -128,6 +128,9 @@ # If set to false, then do not encrypt files on remote system encryption = True +# Type of compression +compression = 'gzip' + # volume size. default 25M volsize = 25*1024*1024 === modified file 'duplicity/gpg.py' --- duplicity/gpg.py 2011-07-16 18:37:47 +0000 +++ duplicity/gpg.py 2011-08-07 13:19:21 +0000 @@ -23,7 +23,7 @@ duplicity's gpg interface, builds upon Frank Tobin's GnuPGInterface """ -import os, types, tempfile, re, gzip +import os, types, tempfile, re, gzip, lzma from duplicity import misc from duplicity import globals @@ -307,11 +307,11 @@ return at_end_of_blockiter -def GzipWriteFile(block_iter, filename, +def CompressedWriteFile(block_iter, filename, size = 200 * 1024 * 1024, - max_footer_size = 16 * 1024): + max_footer_size = 16 * 1024, compression = 'gzip'): """ - Write gzipped compressed file of given size + Write compressed file of given size This is like the earlier GPGWriteFile except it writes a gzipped file instead of a gpg'd file. This function is somewhat out of @@ -335,8 +335,29 @@ def close(self): return self.fileobj.close() + class LZMAFileWrite: + """ + Wrapper around lzma.LZMACompressor to allow writing to a fileobj + """ + def __init__(self, fileobj, level=6): + self.fileobj = fileobj + self.LZMA = lzma.LZMACompressor(options={'level':level}) + def write(self, buf): + dat = self.LZMA.compress(buf) + return self.fileobj.write(dat) + def close(self): + # could also instead flush at every write + dat = self.LZMA.flush() + self.fileobj.write(dat) + return self.fileobj.close() + file_counted = FileCounted(open(filename, "wb")) - gzip_file = gzip.GzipFile(None, "wb", 6, file_counted) + + assert compression in ['gzip', 'lzma'] + if compression == 'gzip': + compressed_file = gzip.GzipFile(None, "wb", 6, file_counted) + elif compression == 'lzma': + compressed_file = LZMAFileWrite(file_counted, 6) at_end_of_blockiter = 0 while True: bytes_to_go = size - file_counted.byte_count @@ -347,9 +368,9 @@ except StopIteration: at_end_of_blockiter = 1 break - gzip_file.write(new_block.data) + compressed_file.write(new_block.data) - assert not gzip_file.close() and not file_counted.close() + assert not compressed_file.close() and not file_counted.close() return at_end_of_blockiter === modified file 'testing/gpgtest.py' --- testing/gpgtest.py 2010-11-20 15:32:59 +0000 +++ testing/gpgtest.py 2011-08-07 12:58:23 +0000 @@ -124,18 +124,18 @@ profile, size = size) #print os.stat("testfiles/output/gpgwrite.gpg").st_size - def test_GzipWriteFile(self): - """Test GzipWriteFile""" + def test_CompressedWriteFile(self): + """Test CompressedWriteFile""" self.deltmp() size = 400 * 1000 gwfh = GPGWriteFile_Helper() for i in range(10): #@UnusedVariable - gpg.GzipWriteFile(gwfh, "testfiles/output/gzwrite.gz", + gpg.CompressedWriteFile(gwfh, "testfiles/output/gzwrite.gz", size = size) #print os.stat("testfiles/output/gzwrite.gz").st_size-size assert size - 64 * 1024 <= os.stat("testfiles/output/gzwrite.gz").st_size <= size + 64 * 1024 gwfh.set_at_end() - gpg.GzipWriteFile(gwfh, "testfiles/output/gzwrite.gpg", size = size) + gpg.CompressedWriteFile(gwfh, "testfiles/output/gzwrite.gpg", size = size) #print os.stat("testfiles/output/gzwrite.gz").st_size