implement ProgressIndicators, use it for repo check and segment replay, fixes #195, fixes #188

borg/archiver.py

@@ -93,10 +93,7 @@ class Archiver:
                        env_var_override='BORG_CHECK_I_KNOW_WHAT_I_AM_DOING', truish=('YES', )):
                 return EXIT_ERROR
         if not args.archives_only:
-            logger.info('Starting repository check...')
-            if repository.check(repair=args.repair):
-                logger.info('Repository check complete, no problems found.')
-            else:
+            if not repository.check(repair=args.repair):
                 return EXIT_WARNING
         if not args.repo_only and not ArchiveChecker().check(
                 repository, repair=args.repair, archive=args.repository.archive, last=args.last):

borg/helpers.py

@@ -890,6 +890,82 @@ def yes(msg=None, retry_msg=None, false_msg=None, true_msg=None,
             ofile.flush()
 
 
+class ProgressIndicatorPercent:
+    def __init__(self, total, step=5, start=0, same_line=False, msg="%3.0f%%", file=sys.stderr):
+        """
+        Percentage-based progress indicator
+
+        :param total: total amount of items
+        :param step: step size in percent
+        :param start: at which percent value to start
+        :param file: output file, default: sys.stderr
+        """
+        self.counter = 0  # 0 .. (total-1)
+        self.total = total
+        self.trigger_at = start  # output next percentage value when reaching (at least) this
+        self.step = step
+        self.file = file
+        self.msg = msg
+        self.same_line = same_line
+
+    def progress(self, current=None):
+        if current is None:
+            current = self.counter
+        else:
+            self.counter = current
+        self.counter += 1
+        pct = current * 100 / self.total
+        if pct >= self.trigger_at:
+            self.trigger_at += self.step
+            return pct
+
+    def show(self, current=None):
+        pct = self.progress(current)
+        if pct is not None:
+            return self.output(pct)
+
+    def output(self, percent):
+        print(self.msg % percent, file=self.file, end='\r' if self.same_line else '\n')
+
+    def finish(self):
+        if self.same_line:
+            print(" " * len(self.msg % 100.0), file=self.file, end='\r')
+
+
+
+class ProgressIndicatorEndless:
+    def __init__(self, step=10, file=sys.stderr):
+        """
+        Progress indicator (long row of dots)
+
+        :param step: every Nth call, call the func
+        :param file: output file, default: sys.stderr
+        """
+        self.counter = 0  # call counter
+        self.triggered = 0  # increases 1 per trigger event
+        self.step = step  # trigger every <step> calls
+        self.file = file
+
+    def progress(self):
+        self.counter += 1
+        trigger = self.counter % self.step == 0
+        if trigger:
+            self.triggered += 1
+        return trigger
+
+    def show(self):
+        trigger = self.progress()
+        if trigger:
+            return self.output(self.triggered)
+
+    def output(self, triggered):
+        print('.', end='', file=self.file)  # python 3.3 gives us flush=True
+        self.file.flush()
+
+    def finish(self):
+        print(file=self.file)
+
+
 def sysinfo():
     info = []
     info.append('Platform: %s' % (' '.join(platform.uname()), ))

borg/repository.py

@@ -10,7 +10,8 @@ import shutil
 import struct
 from zlib import crc32
 
-from .helpers import Error, ErrorWithTraceback, IntegrityError, read_msgpack, write_msgpack, unhexlify
+from .helpers import Error, ErrorWithTraceback, IntegrityError, read_msgpack, write_msgpack, \
+                     unhexlify, ProgressIndicatorPercent
 from .hashindex import NSIndex
 from .locking import UpgradableLock, LockError, LockErrorT
 from .lrucache import LRUCache
@@ -243,13 +244,17 @@ class Repository:
     def replay_segments(self, index_transaction_id, segments_transaction_id):
         self.prepare_txn(index_transaction_id, do_cleanup=False)
         try:
-            for segment, filename in self.io.segment_iterator():
+            segment_count = sum(1 for _ in self.io.segment_iterator())
+            pi = ProgressIndicatorPercent(total=segment_count, msg="Replaying segments %3.0f%%", same_line=True)
+            for i, (segment, filename) in enumerate(self.io.segment_iterator()):
+                pi.show(i)
                 if index_transaction_id is not None and segment <= index_transaction_id:
                     continue
                 if segment > segments_transaction_id:
                     break
                 objects = self.io.iter_objects(segment)
                 self._update_index(segment, objects)
+            pi.finish()
             self.write_index()
         finally:
             self.rollback()
@@ -299,6 +304,7 @@ class Repository:
             error_found = True
             logger.error(msg)
 
+        logger.info('Starting repository check')
         assert not self._active_txn
         try:
             transaction_id = self.get_transaction_id()
@@ -314,7 +320,10 @@ class Repository:
             self.io.cleanup(transaction_id)
         segments_transaction_id = self.io.get_segments_transaction_id()
         self.prepare_txn(None)  # self.index, self.compact, self.segments all empty now!
-        for segment, filename in self.io.segment_iterator():
+        segment_count = sum(1 for _ in self.io.segment_iterator())
+        pi = ProgressIndicatorPercent(total=segment_count, msg="Checking segments %3.0f%%", same_line=True)
+        for i, (segment, filename) in enumerate(self.io.segment_iterator()):
+            pi.show(i)
             if segment > transaction_id:
                 continue
             try:
@@ -326,6 +335,7 @@ class Repository:
                     self.io.recover_segment(segment, filename)
                     objects = list(self.io.iter_objects(segment))
             self._update_index(segment, objects, report_error)
+        pi.finish()
         # self.index, self.segments, self.compact now reflect the state of the segment files up to <transaction_id>
         # We might need to add a commit tag if no committed segment is found
         if repair and segments_transaction_id is None:
@@ -345,6 +355,13 @@ class Repository:
             self.compact_segments()
             self.write_index()
         self.rollback()
+        if error_found:
+            if repair:
+                logger.info('Completed repository check, errors found and repaired.')
+            else:
+                logger.info('Completed repository check, errors found.')
+        else:
+            logger.info('Completed repository check, no problems found.')
         return not error_found or repair
 
     def rollback(self):