add msgid to progress output

docs/internals/frontends.rst

@@ -44,7 +44,9 @@ progress_message
     saying what is currently worked on.
 
     operation
-        integer ID of the operation
+        unique, opaque integer ID of the operation
+    msgid
+        Message ID of the operation (may be *none*)
     finished
         boolean indicating whether the operation has finished, only the last object for an *operation*
         can have this property set to *true*.
@@ -52,10 +54,12 @@ progress_message
         current progress message (may be empty/absent)
 
 progress_percent
-    Absolute progress display with defined end/total and current value.
+    Absolute progress information with defined end/total and current value.
 
     operation
-        integer ID of the operation
+        unique, opaque integer ID of the operation
+    msgid
+        Message ID of the operation (may be *none*)
     finished
         boolean indicating whether the operation has finished, only the last object for an *operation*
         can have this property set to *true*.
@@ -81,7 +85,7 @@ log_message
     created
         Unix timestamp (float)
     levelname
-        Upper-case log level name (also called severity). Defined levels are: DEBUG, INFO, WARNING, CRITICAL
+        Upper-case log level name (also called severity). Defined levels are: DEBUG, INFO, WARNING, ERROR, CRITICAL
     name
         Name of the emitting entity
     message

src/borg/archive.py

@@ -734,7 +734,7 @@ Utilization of max. archive size: {csize_max:.0%}
         try:
             unpacker = msgpack.Unpacker(use_list=False)
             items_ids = self.metadata.items
-            pi = ProgressIndicatorPercent(total=len(items_ids), msg="Decrementing references %3.0f%%")
+            pi = ProgressIndicatorPercent(total=len(items_ids), msg="Decrementing references %3.0f%%", msgid='archive.delete')
             for (i, (items_id, data)) in enumerate(zip(items_ids, self.repository.get_many(items_ids))):
                 if progress:
                     pi.show(i)
@@ -1153,7 +1153,8 @@ class ArchiveChecker:
         chunks_count_segments = 0
         errors = 0
         defect_chunks = []
-        pi = ProgressIndicatorPercent(total=chunks_count_index, msg="Verifying data %6.2f%%", step=0.01)
+        pi = ProgressIndicatorPercent(total=chunks_count_index, msg="Verifying data %6.2f%%", step=0.01,
+                                      msgid='check.verify_data')
         marker = None
         while True:
             chunk_ids = self.repository.scan(limit=100, marker=marker)

src/borg/archiver.py

@@ -555,7 +555,7 @@ class Archiver:
 
         filter = self.build_filter(matcher, peek_and_store_hardlink_masters, strip_components)
         if progress:
-            pi = ProgressIndicatorPercent(msg='%5.1f%% Extracting: %s', step=0.1)
+            pi = ProgressIndicatorPercent(msg='%5.1f%% Extracting: %s', step=0.1, msgid='extract')
             pi.output('Calculating size')
             extracted_size = sum(item.get_size(hardlink_masters) for item in archive.iter_items(filter))
             pi.total = extracted_size
@@ -589,7 +589,8 @@ class Archiver:
                 self.print_warning('%s: %s', remove_surrogates(orig_path), e)
 
         if not args.dry_run:
-            pi = ProgressIndicatorPercent(total=len(dirs), msg='Setting directory permissions %3.0f%%')
+            pi = ProgressIndicatorPercent(total=len(dirs), msg='Setting directory permissions %3.0f%%',
+                                          msgid='extract.permissions')
             while dirs:
                 pi.show()
                 dir_item = dirs.pop(-1)

src/borg/cache.py

@@ -320,7 +320,7 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
 
     def begin_txn(self):
         # Initialize transaction snapshot
-        pi = ProgressIndicatorMessage()
+        pi = ProgressIndicatorMessage(msgid='cache.begin_transaction')
         txn_dir = os.path.join(self.path, 'txn.tmp')
         os.mkdir(txn_dir)
         pi.output('Initializing cache transaction: Reading config')
@@ -340,7 +340,7 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
         if not self.txn_active:
             return
         self.security_manager.save(self.manifest, self.key, self)
-        pi = ProgressIndicatorMessage()
+        pi = ProgressIndicatorMessage(msgid='cache.commit')
         if self.files is not None:
             if self._newest_mtime is None:
                 # was never set because no files were modified/added
@@ -468,7 +468,8 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
                 chunk_idx = None
                 if self.progress:
                     pi = ProgressIndicatorPercent(total=len(archive_ids), step=0.1,
-                                                  msg='%3.0f%% Syncing chunks cache. Processing archive %s')
+                                                  msg='%3.0f%% Syncing chunks cache. Processing archive %s',
+                                                  msgid='cache.sync')
                 for archive_id in archive_ids:
                     archive_name = lookup_name(archive_id)
                     if self.progress:

src/borg/helpers.py

@@ -1384,13 +1384,15 @@ class ProgressIndicatorBase:
 
     @classmethod
     def operation_id(cls):
+        """Unique number, can be used by receiving applications to distinguish different operations."""
         cls.operation_id_counter += 1
         return cls.operation_id_counter
 
-    def __init__(self):
+    def __init__(self, msgid=None):
         self.handler = None
         self.logger = logging.getLogger(self.LOGGER)
         self.id = self.operation_id()
+        self.msgid = msgid
 
         # If there are no handlers, set one up explicitly because the
         # terminator and propagation needs to be set.  If there are,
@@ -1426,12 +1428,13 @@ class ProgressIndicatorBase:
         assert self.json
         if not self.emit:
             return
-        print(json.dumps(dict(
+        kwargs.update(dict(
             operation=self.id,
+            msgid=self.msgid,
             type=self.JSON_TYPE,
-            finished=finished,
-            **kwargs,
-        )), file=sys.stderr)
+            finished=finished
+        ))
+        print(json.dumps(kwargs), file=sys.stderr)
 
     def finish(self):
         if self.json:
@@ -1461,7 +1464,7 @@ class ProgressIndicatorMessage(ProgressIndicatorBase):
 class ProgressIndicatorPercent(ProgressIndicatorBase):
     JSON_TYPE = 'progress_percent'
 
-    def __init__(self, total=0, step=5, start=0, msg="%3.0f%%"):
+    def __init__(self, total=0, step=5, start=0, msg="%3.0f%%", msgid=None):
         """
         Percentage-based progress indicator
 
@@ -1476,7 +1479,7 @@ class ProgressIndicatorPercent(ProgressIndicatorBase):
         self.step = step
         self.msg = msg
 
-        super().__init__()
+        super().__init__(msgid=msgid)
 
     def progress(self, current=None, increase=1):
         if current is not None:

src/borg/repository.py

@@ -483,7 +483,8 @@ class Repository:
             unused = []
 
         logger.debug('compaction started.')
-        pi = ProgressIndicatorPercent(total=len(self.compact), msg='Compacting segments %3.0f%%', step=1)
+        pi = ProgressIndicatorPercent(total=len(self.compact), msg='Compacting segments %3.0f%%', step=1,
+                                      msgid='repository.compact_segments')
         for segment, freeable_space in sorted(self.compact.items()):
             if not self.io.segment_exists(segment):
                 logger.warning('segment %d not found, but listed in compaction data', segment)
@@ -584,7 +585,8 @@ class Repository:
         self.prepare_txn(index_transaction_id, do_cleanup=False)
         try:
             segment_count = sum(1 for _ in self.io.segment_iterator())
-            pi = ProgressIndicatorPercent(total=segment_count, msg="Replaying segments %3.0f%%")
+            pi = ProgressIndicatorPercent(total=segment_count, msg='Replaying segments %3.0f%%',
+                                          msgid='repository.replay_segments')
             for i, (segment, filename) in enumerate(self.io.segment_iterator()):
                 pi.show(i)
                 if index_transaction_id is not None and segment <= index_transaction_id:
@@ -694,7 +696,8 @@ class Repository:
         self.prepare_txn(None)  # self.index, self.compact, self.segments all empty now!
         segment_count = sum(1 for _ in self.io.segment_iterator())
         logger.debug('Found %d segments', segment_count)
-        pi = ProgressIndicatorPercent(total=segment_count, msg="Checking segments %3.1f%%", step=0.1)
+        pi = ProgressIndicatorPercent(total=segment_count, msg='Checking segments %3.1f%%', step=0.1,
+                                      msgid='repository.check')
         for i, (segment, filename) in enumerate(self.io.segment_iterator()):
             pi.show(i)
             if segment > transaction_id: