From 86d6eecbbc9bad2f83b3250b09aef015eeaf186a Mon Sep 17 00:00:00 2001 From: Ji Tao Date: Thu, 7 May 2015 17:58:08 +0800 Subject: [PATCH] Add single file status with --file-changed argument in collection-status mode --- bin/duplicity | 6 +++++- bin/duplicity.1 | 13 +++++++++++-- duplicity/collections.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++- duplicity/commandline.py | 5 +++++ duplicity/diffdir.py | 2 +- duplicity/globals.py | 3 +++ duplicity/log.py | 4 ++++ duplicity/manifest.py | 22 ++++++++++++++++++++++ duplicity/statistics.py | 13 ++++++++++++- 9 files changed, 111 insertions(+), 6 deletions(-) diff --git a/bin/duplicity b/bin/duplicity index 6c4ffca..31b8c71 100755 --- a/bin/duplicity +++ b/bin/duplicity @@ -458,6 +458,7 @@ def write_multivol(backup_type, tarblock_iter, man_outfp, sig_outfp, backend): # Upload the collection summary. # bytes_written += write_manifest(mf, backup_type, backend) + mf.set_files_changed_info(diffdir.stats.get_delta_entries_file()) return bytes_written @@ -1450,7 +1451,10 @@ def do_backup(action): elif action == "list-current": list_current(col_stats) elif action == "collection-status": - log.PrintCollectionStatus(col_stats, True) + if not globals.file_changed: + log.PrintCollectionStatus(col_stats, True) + else: + log.PrintCollectionFileChangedStatus(col_stats, globals.file_changed, True) elif action == "cleanup": cleanup(col_stats) elif action == "remove-old": diff --git a/bin/duplicity.1 b/bin/duplicity.1 index 4c91077..4dbe0a7 100755 --- a/bin/duplicity.1 +++ b/bin/duplicity.1 @@ -21,7 +21,7 @@ source_directory target_url source_url target_directory .B duplicity collection-status -.I [options] +.I [options] [--file-changed ] target_url .B duplicity list-current-files @@ -180,7 +180,7 @@ The option enables data comparison (see below). .TP -.BI "collection-status " "" +.BI "collection-status " "[--file-changed ]" "" Summarize the status of the backup repository by printing the chains and sets found, and the number of volumes in each. @@ -423,6 +423,15 @@ See the argument for more information. .TP +.BI "--file-changed " path +This option may be given in collection-status mode, causing only +.I path +status to be collect instead of the entire contents of the backup archive. +.I path +should be given relative to the root of the directory backed up. + + +.TP .BI "--file-prefix, --file-prefix-manifest, --file-prefix-archive, --file-prefix-signature Adds a prefix to all files, manifest files, archive files, and/or signature files. diff --git a/duplicity/collections.py b/duplicity/collections.py index 1fbefd4..ede97b9 100644 --- a/duplicity/collections.py +++ b/duplicity/collections.py @@ -60,6 +60,7 @@ class BackupSet: self.end_time = None # will be set if inc self.partial = False # true if a partial backup self.encrypted = False # true if an encrypted backup + self.files_changed = [] def is_complete(self): """ @@ -123,6 +124,10 @@ class BackupSet: self.encrypted = bool(pr.encrypted) self.info_set = True + def set_files_changed(self): + mf = self.get_manifest() + self.files_changed = mf.get_files_changed() + def set_manifest(self, remote_filename): """ Add local and remote manifest filenames to backup set @@ -140,6 +145,8 @@ class BackupSet: and pr.end_time == self.end_time): self.local_manifest_path = \ globals.archive_dir.append(local_filename) + + self.set_files_changed() break def delete(self): @@ -272,13 +279,15 @@ class BackupSet: return self.end_time assert 0, "Neither self.time nor self.end_time set" + def get_files_changed(self): + return self.files_changed + def __len__(self): """ Return the number of volumes in the set """ return len(self.volume_name_dict.keys()) - class BackupChain: """ BackupChain - a number of linked BackupSets @@ -1153,3 +1162,41 @@ class CollectionsStatus: old_sets = filter(lambda s: s.get_time() < t, chain.get_all_sets()) result_sets.extend(old_sets) return self.sort_sets(result_sets) + + def get_file_changed_record(self, filepath): + """ + Returns time line of specified file changed + """ + if not self.matched_chain_pair: + return "" + + all_backup_set = self.matched_chain_pair[1].get_all_sets() + specified_backup_set = [] + for bs in all_backup_set: + if filepath in bs.get_files_changed(): + specified_backup_set.append(bs) + + return FileChangedStatus(filepath, specified_backup_set) + + +class FileChangedStatus: + def __init__(self, filepath, backup_set_list): + self.filepath = filepath + self.file_changed_backups = backup_set_list + + def __unicode__(self): + set_schema = "%20s %30s" + l = ["-------------------------", + _("File path: %s") % (self.filepath), + _("Total number of backup: %d") % (len(self.file_changed_backups)), + set_schema % (_("Type of backup set:"), _("Time:"))] + + for s in self.file_changed_backups: + if s.time: + type = _("Full") + else: + type = _("Incremental") + l.append(set_schema % (type, dup_time.timetopretty(s.get_time()))) + + l.append("-------------------------") + return "\n".join(l) diff --git a/duplicity/commandline.py b/duplicity/commandline.py index e1798ac..25a6603 100644 --- a/duplicity/commandline.py +++ b/duplicity/commandline.py @@ -614,6 +614,11 @@ def parse_cmdline_options(arglist): parser.add_option("--volsize", type="int", action="callback", metavar=_("number"), callback=lambda o, s, v, p: setattr(p.values, "volsize", v * 1024 * 1024)) + # If set, collect only the file status, not the whole root. + parser.add_option("--file-changed", action="callback", type="file", + metavar=_("path"), dest="file_changed", + callback=lambda o, s, v, p: setattr(p.values, "file_changed", v.rstrip('/'))) + # parse the options (options, args) = parser.parse_args() diff --git a/duplicity/diffdir.py b/duplicity/diffdir.py index 0ee8a8c..6aaabd3 100644 --- a/duplicity/diffdir.py +++ b/duplicity/diffdir.py @@ -209,7 +209,7 @@ def get_delta_iter(new_iter, sig_iter, sig_fileobj=None): ti = ROPath(sig_path.index).get_tarinfo() ti.name = "deleted/" + "/".join(sig_path.index) sigTarFile.addfile(ti) - stats.add_deleted_file() + stats.add_deleted_file(sig_path) yield ROPath(sig_path.index) elif not sig_path or new_path != sig_path: # Must calculate new signature and create delta diff --git a/duplicity/globals.py b/duplicity/globals.py index 4f6dcdc..5e83e43 100644 --- a/duplicity/globals.py +++ b/duplicity/globals.py @@ -284,3 +284,6 @@ par2_options = "" # Whether to enable gio backend use_gio = False + +# If set, collect only the file status, not the whole root. +file_changed = None diff --git a/duplicity/log.py b/duplicity/log.py index 797e163..1596a48 100644 --- a/duplicity/log.py +++ b/duplicity/log.py @@ -219,6 +219,10 @@ def PrintCollectionStatus(col_stats, force_print=False): Log(unicode(col_stats), 8, InfoCode.collection_status, '\n' + '\n'.join(col_stats.to_log_info()), force_print) +def PrintCollectionFileChangedStatus(col_stats, filepath, force_print=False): + """Prints a collection status to the log""" + Log(unicode(col_stats.get_file_changed_record(filepath)), 8, InfoCode.collection_status, None, force_print) + def Notice(s): """Shortcut used for notice messages (verbosity 3, the default).""" diff --git a/duplicity/manifest.py b/duplicity/manifest.py index 22a6640..6d01635 100644 --- a/duplicity/manifest.py +++ b/duplicity/manifest.py @@ -55,6 +55,7 @@ class Manifest: self.local_dirname = None self.volume_info_dict = {} # dictionary vol numbers -> vol infos self.fh = fh + self.files_changed = [] def set_dirinfo(self): """ @@ -110,6 +111,13 @@ class Manifest: "--allow-source-mismatch switch to avoid seeing this " "message"), code, code_extra) + def set_files_changed_info(self, files_changed): + self.files_changed = files_changed + if self.fh: + self.fh.write("Filelist %d\n" % len(self.files_changed)) + for filepath in self.files_changed: + self.fh.write(" %s\n" % Quote(filepath)) + def add_volume_info(self, vi): """ Add volume info vi to manifest and write to manifest @@ -150,6 +158,10 @@ class Manifest: result += "Hostname %s\n" % self.hostname if self.local_dirname: result += "Localdir %s\n" % Quote(self.local_dirname) + if self.files_changed: + result += "Filelist %d\n" % len(self.files_changed) + for filepath in self.files_changed: + result += " %s\n" % Quote(filepath) vol_num_list = self.volume_info_dict.keys() vol_num_list.sort() @@ -178,6 +190,13 @@ class Manifest: self.hostname = get_field("hostname") self.local_dirname = get_field("localdir") + #Get file changed list + filelist_regexp = re.compile("(^|\\n)filelist\\s([0-9]+)\\n(.*?)(\\nvolume\\s|$)", re.I | re.S) + match = filelist_regexp.search(s) + filecount = int(match.group(2)) + self.files_changed = [filepath.strip() for filepath in match.group(3).split('\n')] + #assert filecount == len(self.files_changed) + next_vi_string_regexp = re.compile("(^|\\n)(volume\\s.*?)" "(\\nvolume\\s|$)", re.I | re.S) starting_s_index = 0 @@ -200,6 +219,9 @@ class Manifest: self.del_volume_info(i) return self + def get_files_changed(self): + return self.files_changed + def __eq__(self, other): """ Two manifests are equal if they contain the same volume infos diff --git a/duplicity/statistics.py b/duplicity/statistics.py index 2ec115f..c8cf8b2 100644 --- a/duplicity/statistics.py +++ b/duplicity/statistics.py @@ -306,6 +306,7 @@ class StatsDeltaProcess(StatsObj): self.__dict__[attr] = 0 self.Errors = 0 self.StartTime = time.time() + self.files_changed = [] def add_new_file(self, path): """Add stats of new file path to statistics""" @@ -315,6 +316,7 @@ class StatsDeltaProcess(StatsObj): self.NewFiles += 1 self.NewFileSize += filesize self.DeltaEntries += 1 + self.add_delta_entries_file(path) def add_changed_file(self, path): """Add stats of file that has changed since last backup""" @@ -324,11 +326,13 @@ class StatsDeltaProcess(StatsObj): self.ChangedFiles += 1 self.ChangedFileSize += filesize self.DeltaEntries += 1 + self.add_delta_entries_file(path) - def add_deleted_file(self): + def add_deleted_file(self, path): """Add stats of file no longer in source directory""" self.DeletedFiles += 1 # can't add size since not available self.DeltaEntries += 1 + self.add_delta_entries_file(path) def add_unchanged_file(self, path): """Add stats of file that hasn't changed since last backup""" @@ -339,3 +343,10 @@ class StatsDeltaProcess(StatsObj): def close(self): """End collection of data, set EndTime""" self.EndTime = time.time() + + def add_delta_entries_file(self, path): + if path.isreg(): + self.files_changed.append(path.get_relative_path()) + + def get_delta_entries_file(self): + return self.files_changed -- 1.9.1