Merge pull request #5263 from ThomasWaldmann/persist-shadow-index

persist shadow_index in between borg runs, fixes #4830

src/borg/repository.py

@@ -160,6 +160,7 @@ class Repository:
         # This is an index of shadowed log entries during this transaction. Consider the following sequence:
         # segment_n PUT A, segment_x DELETE A
         # After the "DELETE A" in segment_x the shadow index will contain "A -> [n]".
+        # .delete() is updating this index, it is persisted into "hints" file and is later used by .compact_segments().
         self.shadow_index = {}
         self._active_txn = False
         self.lock_wait = lock_wait
@@ -570,6 +571,7 @@ class Repository:
                 self.segments = hints[b'segments']
                 self.compact = FreeSpace()
                 self.storage_quota_use = 0
+                self.shadow_index = {}
                 for segment in sorted(hints[b'compact']):
                     logger.debug('Rebuilding sparse info for segment %d', segment)
                     self._rebuild_sparse(segment)
@@ -580,6 +582,7 @@ class Repository:
                 self.segments = hints[b'segments']
                 self.compact = FreeSpace(hints[b'compact'])
                 self.storage_quota_use = hints.get(b'storage_quota_use', 0)
+                self.shadow_index = hints.get(b'shadow_index', {})
             self.log_storage_quota()
             # Drop uncommitted segments in the shadow index
             for key, shadowed_segments in self.shadow_index.items():
@@ -600,6 +603,7 @@ class Repository:
             b'segments': self.segments,
             b'compact': self.compact,
             b'storage_quota_use': self.storage_quota_use,
+            b'shadow_index': self.shadow_index,
         }
         integrity = {
             # Integrity version started at 2, the current hints version.
@@ -975,7 +979,7 @@ class Repository:
         segments_transaction_id = self.io.get_segments_transaction_id()
         logger.debug('Segment transaction is    %s', segments_transaction_id)
         logger.debug('Determined transaction is %s', transaction_id)
-        self.prepare_txn(None)  # self.index, self.compact, self.segments all empty now!
+        self.prepare_txn(None)  # self.index, self.compact, self.segments, self.shadow_index all empty now!
         segment_count = sum(1 for _ in self.io.segment_iterator())
         logger.debug('Found %d segments', segment_count)
 

src/borg/testsuite/repository.py

@@ -792,6 +792,46 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
             self.assert_equal(self.repository.get(H(0)), b'data2')
 
 
+class RepositoryHintsTestCase(RepositoryTestCaseBase):
+
+    def test_hints_persistence(self):
+        self.repository.put(H(0), b'data')
+        self.repository.delete(H(0))
+        self.repository.commit(compact=False)
+        shadow_index_expected = self.repository.shadow_index
+        compact_expected = self.repository.compact
+        segments_expected = self.repository.segments
+        # close and re-open the repository (create fresh Repository instance) to
+        # check whether hints were persisted to / reloaded from disk
+        self.reopen()
+        with self.repository:
+            # see also do_compact()
+            self.repository.put(H(42), b'foobar')  # this will call prepare_txn() and load the hints data
+            # check if hints persistence worked:
+            self.assert_equal(shadow_index_expected, self.repository.shadow_index)
+            self.assert_equal(compact_expected, self.repository.compact)
+            del self.repository.segments[2]  # ignore the segment created by put(H(42), ...)
+            self.assert_equal(segments_expected, self.repository.segments)
+
+    def test_hints_behaviour(self):
+        self.repository.put(H(0), b'data')
+        self.assert_equal(self.repository.shadow_index, {})
+        self.assert_true(len(self.repository.compact) == 0)
+        self.repository.delete(H(0))
+        self.repository.commit(compact=False)
+        # now there should be an entry for H(0) in shadow_index
+        self.assert_in(H(0), self.repository.shadow_index)
+        self.assert_equal(len(self.repository.shadow_index[H(0)]), 1)
+        self.assert_in(0, self.repository.compact)  # segment 0 can be compacted
+        self.repository.put(H(42), b'foobar')  # see also do_compact()
+        self.repository.commit(compact=True, threshold=0.0)  # compact completely!
+        # nothing to compact any more! no info left about stuff that does not exist any more:
+        self.assert_equal(self.repository.shadow_index[H(0)], [])
+        # segment 0 was compacted away, no info about it left:
+        self.assert_not_in(0, self.repository.compact)
+        self.assert_not_in(0, self.repository.segments)
+
+
 class RemoteRepositoryTestCase(RepositoryTestCase):
     repository = None  # type: RemoteRepository