test the shadowing-by-double-put behaviour, see #5661

the new test is currently failing due to a bug in the
repository code.

src/borg/testsuite/repository.py

@@ -516,7 +516,8 @@ def test_moved_deletes_are_tracked(repository):
         assert H(1) not in repository.shadow_index
 
 
-def test_shadowed_entries_are_preserved(repository):
+def test_shadowed_entries_are_preserved1(repository):
+    # this tests the shadowing-by-del behaviour
     with repository:
         get_latest_segment = repository.io.get_latest_segment
         repository.put(H(1), fchunk(b"1"))
@@ -546,6 +547,50 @@ def test_shadowed_entries_are_preserved(repository):
         assert H(1) not in repository
 
 
+def test_shadowed_entries_are_preserved2(repository):
+    # this tests the shadowing-by-double-put behaviour, see issue #5661
+    # assume this repo state:
+    # seg1: PUT H1
+    # seg2: COMMIT
+    # seg3: DEL H1, PUT H1, DEL H1, PUT H2
+    # seg4: COMMIT
+    # Note how due to the final DEL H1 in seg3, H1 is effectively deleted.
+    #
+    # compaction of only seg3:
+    # PUT H1 gets dropped because it is not needed any more.
+    # DEL H1 must be kept, because there is still a PUT H1 in seg1 which must not
+    # "reappear" in the index if the index gets rebuilt.
+    with repository:
+        get_latest_segment = repository.io.get_latest_segment
+        repository.put(H(1), fchunk(b"1"))
+        # This is the segment with our original PUT of interest
+        put_segment = get_latest_segment()
+        repository.commit(compact=False)
+        # We now put H(1) again (which implicitly does DEL(H(1)) followed by PUT(H(1), ...)),
+        # delete H(1) afterwards, and force this segment to not be compacted, which can happen
+        # if it's not sparse enough (symbolized by H(2) here).
+        repository.put(H(1), fchunk(b"1"))
+        repository.delete(H(1))
+        repository.put(H(2), fchunk(b"1"))
+        delete_segment = get_latest_segment()
+        # We pretend these are mostly dense (not sparse) and won't be compacted
+        del repository.compact[put_segment]
+        del repository.compact[delete_segment]
+        repository.commit(compact=True)
+        # Now we perform an unrelated operation on the segment containing the DELETE,
+        # causing it to be compacted.
+        repository.delete(H(2))
+        repository.commit(compact=True)
+        assert repository.io.segment_exists(put_segment)
+        assert not repository.io.segment_exists(delete_segment)
+        # Basic case, since the index survived this must be ok
+        assert H(1) not in repository
+        # Nuke index, force replay
+        os.unlink(os.path.join(repository.path, "index.%d" % get_latest_segment()))
+        # Must not reappear
+        assert H(1) not in repository  # F
+
+
 def test_shadow_index_rollback(repository):
     with repository:
         repository.put(H(1), fchunk(b"1"))