Merge branch '1.0-maint' into merge-1.0-maint

# Conflicts:
#	AUTHORS
#	src/borg/archive.py
#	src/borg/key.py

docs/installation.rst

@@ -299,7 +299,7 @@ Cygwin
 
 Use the Cygwin installer to install the dependencies::
 
-    python3 python3-setuptools
+    python3 python3-devel python3-setuptools
     binutils gcc-g++
     libopenssl openssl-devel
     liblz4_1 liblz4-devel

src/borg/archive.py

@@ -1013,7 +1013,13 @@ class ArchiveChecker:
             self.error_found = True
             self.manifest = self.rebuild_manifest()
         else:
-            self.manifest, _ = Manifest.load(repository, key=self.key)
+            try:
+                self.manifest, _ = Manifest.load(repository, key=self.key)
+            except IntegrityError as exc:
+                logger.error('Repository manifest is corrupted: %s', exc)
+                self.error_found = True
+                del self.chunks[Manifest.MANIFEST_ID]
+                self.manifest = self.rebuild_manifest()
         self.rebuild_refcounts(archive=archive, first=first, last=last, sort_by=sort_by, prefix=prefix)
         self.orphan_chunks_check()
         self.finish(save_space=save_space)
@@ -1153,7 +1159,12 @@ class ArchiveChecker:
         archive_keys_serialized = [msgpack.packb(name.encode()) for name in ARCHIVE_KEYS]
         for chunk_id, _ in self.chunks.iteritems():
             cdata = self.repository.get(chunk_id)
-            _, data = self.key.decrypt(chunk_id, cdata)
+            try:
+                _, data = self.key.decrypt(chunk_id, cdata)
+            except IntegrityError as exc:
+                logger.error('Skipping corrupted chunk: %s', exc)
+                self.error_found = True
+                continue
             if not valid_msgpacked_dict(data, archive_keys_serialized):
                 continue
             if b'cmdline' not in data or b'\xa7version\x01' not in data:

src/borg/key.py

@@ -152,7 +152,8 @@ class PlaintextKey(KeyBase):
 
     def decrypt(self, id, data, decompress=True):
         if data[0] != self.TYPE:
-            raise IntegrityError('Chunk %s: Invalid encryption envelope' % bin_to_hex(id))
+            id_str = bin_to_hex(id) if id is not None else '(unknown)'
+            raise IntegrityError('Chunk %s: Invalid encryption envelope' % id_str)
         payload = memoryview(data)[1:]
         if not decompress:
             return Chunk(payload)
@@ -209,12 +210,14 @@ class AESKeyBase(KeyBase):
     def decrypt(self, id, data, decompress=True):
         if not (data[0] == self.TYPE or
             data[0] == PassphraseKey.TYPE and isinstance(self, RepoKey)):
-            raise IntegrityError('Chunk %s: Invalid encryption envelope' % bin_to_hex(id))
+            id_str = bin_to_hex(id) if id is not None else '(unknown)'
+            raise IntegrityError('Chunk %s: Invalid encryption envelope' % id_str)
         data_view = memoryview(data)
         hmac_given = data_view[1:33]
         hmac_computed = memoryview(hmac_sha256(self.enc_hmac_key, data_view[33:]))
         if not compare_digest(hmac_computed, hmac_given):
-            raise IntegrityError('Chunk %s: Encryption envelope checksum mismatch' % bin_to_hex(id))
+            id_str = bin_to_hex(id) if id is not None else '(unknown)'
+            raise IntegrityError('Chunk %s: Encryption envelope checksum mismatch' % id_str)
         self.dec_cipher.reset(iv=PREFIX + data[33:41])
         payload = self.dec_cipher.decrypt(data_view[41:])
         if not decompress:

src/borg/testsuite/archiver.py

@@ -2145,6 +2145,35 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
         self.assert_in('archive2', output)
         self.cmd('check', self.repository_location, exit_code=0)
 
+    def test_corrupted_manifest(self):
+        archive, repository = self.open_archive('archive1')
+        with repository:
+            manifest = repository.get(Manifest.MANIFEST_ID)
+            corrupted_manifest = manifest + b'corrupted!'
+            repository.put(Manifest.MANIFEST_ID, corrupted_manifest)
+            repository.commit()
+        self.cmd('check', self.repository_location, exit_code=1)
+        output = self.cmd('check', '-v', '--repair', self.repository_location, exit_code=0)
+        self.assert_in('archive1', output)
+        self.assert_in('archive2', output)
+        self.cmd('check', self.repository_location, exit_code=0)
+
+    def test_manifest_rebuild_corrupted_chunk(self):
+        archive, repository = self.open_archive('archive1')
+        with repository:
+            manifest = repository.get(Manifest.MANIFEST_ID)
+            corrupted_manifest = manifest + b'corrupted!'
+            repository.put(Manifest.MANIFEST_ID, corrupted_manifest)
+
+            chunk = repository.get(archive.id)
+            corrupted_chunk = chunk + b'corrupted!'
+            repository.put(archive.id, corrupted_chunk)
+            repository.commit()
+        self.cmd('check', self.repository_location, exit_code=1)
+        output = self.cmd('check', '-v', '--repair', self.repository_location, exit_code=0)
+        self.assert_in('archive2', output)
+        self.cmd('check', self.repository_location, exit_code=0)
+
     def test_extra_chunks(self):
         self.cmd('check', self.repository_location, exit_code=0)
         with Repository(self.repository_location, exclusive=True) as repository:

src/borg/testsuite/helpers.py

@@ -642,6 +642,7 @@ class TestParseTimestamp(BaseTestCase):
 
 def test_get_cache_dir(monkeypatch):
     """test that get_cache_dir respects environment"""
+    monkeypatch.delenv('BORG_CACHE_DIR', raising=False)
     monkeypatch.delenv('XDG_CACHE_HOME', raising=False)
     assert get_cache_dir() == os.path.join(os.path.expanduser('~'), '.cache', 'borg')
     monkeypatch.setenv('XDG_CACHE_HOME', '/var/tmp/.cache')
@@ -652,6 +653,7 @@ def test_get_cache_dir(monkeypatch):
 
 def test_get_keys_dir(monkeypatch):
     """test that get_keys_dir respects environment"""
+    monkeypatch.delenv('BORG_KEYS_DIR', raising=False)
     monkeypatch.delenv('XDG_CONFIG_HOME', raising=False)
     assert get_keys_dir() == os.path.join(os.path.expanduser('~'), '.config', 'borg', 'keys')
     monkeypatch.setenv('XDG_CONFIG_HOME', '/var/tmp/.config')