Merge pull request #6181 from KN4CK3R/backport-6119

Backport repository: quota computation fixes for delayed compaction (#6119)

src/borg/repository.py

@@ -786,6 +786,7 @@ class Repository:
                         # do not remove entry with empty shadowed_segments list here,
                         # it is needed for shadowed_put_exists code (see below)!
                         pass
+                    self.storage_quota_use -= len(data) + self.io.put_header_fmt.size
                 elif tag == TAG_DELETE and not in_index:
                     # If the shadow index doesn't contain this key, then we can't say if there's a shadowed older tag,
                     # therefore we do not drop the delete, but write it to a current segment.
@@ -885,7 +886,7 @@ class Repository:
                     pass
                 self.index[key] = segment, offset
                 self.segments[segment] += 1
-                self.storage_quota_use += size
+                self.storage_quota_use += size  # note: size already includes the put_header_fmt overhead
             elif tag == TAG_DELETE:
                 try:
                     # if the deleted PUT is not in the index, there is nothing to clean up
@@ -898,7 +899,6 @@ class Repository:
                         # is already gone, then it was already compacted.
                         self.segments[s] -= 1
                         size = self.io.read(s, offset, key, read_data=False)
-                        self.storage_quota_use -= size
                         self.compact[s] += size
             elif tag == TAG_COMMIT:
                 continue
@@ -909,7 +909,7 @@ class Repository:
                 else:
                     report(msg)
         if self.segments[segment] == 0:
-            self.compact[segment] += self.io.segment_size(segment)
+            self.compact[segment] = self.io.segment_size(segment)
 
     def _rebuild_sparse(self, segment):
         """Rebuild sparse bytes count for a single segment relative to the current index."""
@@ -1191,7 +1191,6 @@ class Repository:
             self.shadow_index.setdefault(id, []).append(segment)
         self.segments[segment] -= 1
         size = self.io.read(segment, offset, id, read_data=False)
-        self.storage_quota_use -= size
         self.compact[segment] += size
         segment, size = self.io.write_delete(id)
         self.compact[segment] += size

src/borg/testsuite/repository.py

@@ -461,13 +461,16 @@ class QuotaTestCase(RepositoryTestCaseBase):
         self.repository.put(H(2), bytes(5678))
         assert self.repository.storage_quota_use == 1234 + 5678 + 2 * 41
         self.repository.delete(H(1))
-        assert self.repository.storage_quota_use == 5678 + 41
+        assert self.repository.storage_quota_use == 1234 + 5678 + 2 * 41  # we have not compacted yet
         self.repository.commit()
+        assert self.repository.storage_quota_use == 5678 + 41
         self.reopen()
         with self.repository:
             # Open new transaction; hints and thus quota data is not loaded unless needed.
             self.repository.put(H(3), b'')
             self.repository.delete(H(3))
+            assert self.repository.storage_quota_use == 5678 + 2 * 41  # we have not compacted yet
+            self.repository.commit()
             assert self.repository.storage_quota_use == 5678 + 41
 
     def test_exceed_quota(self):
@@ -484,10 +487,12 @@ class QuotaTestCase(RepositoryTestCaseBase):
         assert self.repository.storage_quota_use == 82
         self.reopen()
         with self.repository:
-            self.repository.storage_quota = 50
+            self.repository.storage_quota = 100
             # Open new transaction; hints and thus quota data is not loaded unless needed.
             self.repository.put(H(1), b'')
-            assert self.repository.storage_quota_use == 41
+            assert self.repository.storage_quota_use == 82  # we have 2 puts for H(1) here and not yet compacted.
+            self.repository.commit()
+            assert self.repository.storage_quota_use == 41  # now we have compacted.
 
 
 class NonceReservation(RepositoryTestCaseBase):