output progress indication from inner loop, fixes #500

- so it shows progress while it backups a bigger file
- so it announces the filename earlier

also: move rate limiting code to show_progress()

borg/archive.py

@@ -143,7 +143,6 @@ class Archive:
         self.hard_links = {}
         self.stats = Statistics()
         self.show_progress = progress
-        self.last_progress = time.time()
         self.name = name
         self.checkpoint_interval = checkpoint_interval
         self.numeric_owner = numeric_owner
@@ -215,9 +214,8 @@ Number of files: {0.stats.nfiles}'''.format(self)
         unknown_keys = set(item) - ITEM_KEYS
         assert not unknown_keys, ('unknown item metadata keys detected, please update ITEM_KEYS: %s',
                                   ','.join(k.decode('ascii') for k in unknown_keys))
-        if self.show_progress and time.time() - self.last_progress > 0.2:
-            self.stats.show_progress(item=item)
-            self.last_progress = time.time()
+        if self.show_progress:
+            self.stats.show_progress(item=item, dt=0.2)
         self.items_buffer.add(item)
         if time.time() - self.last_checkpoint > self.checkpoint_interval:
             self.write_checkpoint()
@@ -526,6 +524,7 @@ Number of files: {0.stats.nfiles}'''.format(self)
                 status = 'U'  # regular file, unchanged
         else:
             status = 'A'  # regular file, added
+        item = {b'path': safe_path}
         # Only chunkify the file if needed
         if chunks is None:
             fh = Archive._open_rb(path, st)
@@ -533,9 +532,11 @@ Number of files: {0.stats.nfiles}'''.format(self)
                 chunks = []
                 for chunk in self.chunker.chunkify(fd, fh):
                     chunks.append(cache.add_chunk(self.key.id_hash(chunk), chunk, self.stats))
+                    if self.show_progress:
+                        self.stats.show_progress(item=item, dt=0.2)
             cache.memorize_file(path_hash, st, [c[0] for c in chunks])
             status = status or 'M'  # regular file, modified (if not 'A' already)
-        item = {b'path': safe_path, b'chunks': chunks}
+        item[b'chunks'] = chunks
         item.update(self.stat_attrs(st, path))
         self.stats.nfiles += 1
         self.add_item(item)

borg/helpers.py

@@ -162,6 +162,7 @@ class Statistics:
 
     def __init__(self):
         self.osize = self.csize = self.usize = self.nfiles = 0
+        self.last_progress = 0  # timestamp when last progress was shown
 
     def update(self, size, csize, unique):
         self.osize += size
@@ -191,19 +192,22 @@ class Statistics:
     def csize_fmt(self):
         return format_file_size(self.csize)
 
-    def show_progress(self, item=None, final=False, stream=None):
-        columns, lines = get_terminal_size()
-        if not final:
-            msg = '{0.osize_fmt} O {0.csize_fmt} C {0.usize_fmt} D {0.nfiles} N '.format(self)
-            path = remove_surrogates(item[b'path']) if item else ''
-            space = columns - len(msg)
-            if space < len('...') + len(path):
-                path = '%s...%s' % (path[:(space//2)-len('...')], path[-space//2:])
-            msg += "{0:<{space}}".format(path, space=space)
-        else:
-            msg = ' ' * columns
-        print(msg, file=stream or sys.stderr, end="\r")
-        (stream or sys.stderr).flush()
+    def show_progress(self, item=None, final=False, stream=None, dt=None):
+        now = time.time()
+        if dt is None or now - self.last_progress > dt:
+            self.last_progress = now
+            columns, lines = get_terminal_size()
+            if not final:
+                msg = '{0.osize_fmt} O {0.csize_fmt} C {0.usize_fmt} D {0.nfiles} N '.format(self)
+                path = remove_surrogates(item[b'path']) if item else ''
+                space = columns - len(msg)
+                if space < len('...') + len(path):
+                    path = '%s...%s' % (path[:(space//2)-len('...')], path[-space//2:])
+                msg += "{0:<{space}}".format(path, space=space)
+            else:
+                msg = ' ' * columns
+            print(msg, file=stream or sys.stderr, end="\r")
+            (stream or sys.stderr).flush()
 
 
 def get_keys_dir():