Track entire session

Note how this enables the much stricter check in test_moved_deletes_are_tracked

src/borg/repository.py

@@ -111,7 +111,12 @@ class Repository:
         self.io = None
         self.lock = None
         self.index = None
+        # This is an index of shadowed log entries during this transaction. Consider the following sequence:
+        # segment_n PUT A, segment_x DELETE A
+        # After the "DELETE A" in segment_x the shadow index will contain "A -> (n,)".
+        self.shadow_index = defaultdict(list)
         self._active_txn = False
+
         self.lock_wait = lock_wait
         self.do_lock = lock
         self.do_create = create
@@ -306,13 +311,10 @@ class Repository:
             except RuntimeError:
                 self.check_transaction()
                 self.index = self.open_index(transaction_id, False)
-        # This is an index of shadowed log entries during this transaction. Consider the following sequence:
-        # segment_n PUT A, segment_x DELETE A
-        # After the "DELETE A" in segment_x the shadow index will contain "A -> (n,)".
-        self.shadow_index = defaultdict(list)
         if transaction_id is None:
             self.segments = {}  # XXX bad name: usage_count_of_segment_x = self.segments[x]
             self.compact = FreeSpace()  # XXX bad name: freeable_space_of_segment_x = self.compact[x]
+            self.shadow_index.clear()
         else:
             if do_cleanup:
                 self.io.cleanup(transaction_id)
@@ -343,6 +345,11 @@ class Repository:
             else:
                 self.segments = hints[b'segments']
                 self.compact = FreeSpace(hints[b'compact'])
+            # Drop uncommitted segments in the shadow index
+            for key, shadowed_segments in self.shadow_index.items():
+                for segment in list(shadowed_segments):
+                    if segment > transaction_id:
+                        shadowed_segments.remove(segment)
 
     def write_index(self):
         hints = {b'version': 2,

src/borg/testsuite/repository.py

@@ -293,8 +293,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
         assert not self.repository.io.segment_exists(last_segment)
         for segment, _ in self.repository.io.segment_iterator():
             for tag, key, offset, size in self.repository.io.iter_objects(segment):
-                if tag == TAG_DELETE:
-                    assert segment in self.repository.compact
+                assert tag != TAG_DELETE
 
     def test_shadowed_entries_are_preserved(self):
         get_latest_segment = self.repository.io.get_latest_segment
@@ -330,6 +329,22 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
         # Must not reappear
         assert H(1) not in self.repository
 
+    def test_shadow_index_rollback(self):
+        self.repository.put(H(1), b'1')
+        self.repository.delete(H(1))
+        assert self.repository.shadow_index[H(1)] == [0]
+        self.repository.commit()
+        # note how an empty list means that nothing is shadowed for sure
+        assert self.repository.shadow_index[H(1)] == []
+        self.repository.put(H(1), b'1')
+        self.repository.delete(H(1))
+        # 0 put/delete; 1 commit; 2 compacted; 3 commit; 4 put/delete
+        assert self.repository.shadow_index[H(1)] == [4]
+        self.repository.rollback()
+        self.repository.put(H(2), b'1')
+        # After the rollback segment 4 shouldn't be considered anymore
+        assert self.repository.shadow_index[H(1)] == []
+
 
 class RepositoryAppendOnlyTestCase(RepositoryTestCaseBase):
     def open(self, create=False):