Merge pull request #1841 from enkore/issue/1837

Fix check incorrectly reporting attic 0.13 and earlier archives as corrupt

borg/archive.py

@@ -1017,11 +1017,21 @@ class ArchiveChecker:
                 self.error_found = True
                 logger.error(msg)
 
+            def list_keys_safe(keys):
+                return ', '.join((k.decode() if isinstance(k, bytes) else str(k) for k in keys))
+
             def valid_item(obj):
                 if not isinstance(obj, StableDict):
-                    return False
+                    return False, 'not a dictionary'
+                # A bug in Attic up to and including release 0.13 added a (meaningless) b'acl' key to every item.
+                # We ignore it here, should it exist. See test_attic013_acl_bug for details.
+                obj.pop(b'acl', None)
                 keys = set(obj)
-                return REQUIRED_ITEM_KEYS.issubset(keys) and keys.issubset(item_keys)
+                if not REQUIRED_ITEM_KEYS.issubset(keys):
+                    return False, 'missing required keys: ' + list_keys_safe(REQUIRED_ITEM_KEYS - keys)
+                if not keys.issubset(item_keys):
+                    return False, 'invalid keys: ' + list_keys_safe(keys - item_keys)
+                return True, ''
 
             i = 0
             for state, items in groupby(archive[b'items'], missing_chunk_detector):
@@ -1037,10 +1047,11 @@ class ArchiveChecker:
                     unpacker.feed(self.key.decrypt(chunk_id, cdata))
                     try:
                         for item in unpacker:
-                            if valid_item(item):
+                            valid, reason = valid_item(item)
+                            if valid:
                                 yield item
                             else:
-                                report('Did not get expected metadata dict when unpacking item metadata', chunk_id, i)
+                                report('Did not get expected metadata dict when unpacking item metadata (%s)' % reason, chunk_id, i)
                     except RobustUnpacker.UnpackerCrashed as err:
                         report('Unpacker crashed while unpacking item metadata, trying to resync...', chunk_id, i)
                         unpacker.resync()

borg/testsuite/archiver.py

@@ -1451,6 +1451,25 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
             repository.commit()
         self.cmd('check', self.repository_location, exit_code=1)
 
+    def test_attic013_acl_bug(self):
+        # Attic up to release 0.13 contained a bug where every item unintentionally received
+        # a b'acl'=None key-value pair.
+        # This bug can still live on in Borg repositories (through borg upgrade).
+        archive, repository = self.open_archive('archive1')
+        with repository:
+            manifest, key = Manifest.load(repository)
+            with Cache(repository, key, manifest) as cache:
+                archive = Archive(repository, key, manifest, '0.13', cache=cache, create=True)
+                archive.items_buffer.add({
+                    # path and mtime are required.
+                    b'path': '1234',
+                    b'mtime': 0,
+                    # acl is the offending key.
+                    b'acl': None
+                })
+                archive.save()
+        self.cmd('check', self.repository_location, exit_code=0)
+
 
 @pytest.mark.skipif(sys.platform == 'cygwin', reason='remote is broken on cygwin and hangs')
 class RemoteArchiverTestCase(ArchiverTestCase):