Merge pull request #1052 from enkore/fix/commitcheck

Repository: fix commit tags being seen in data

borg/repository.py

@@ -536,7 +536,7 @@ class LoggedIO:
         """Verify that the transaction id is consistent with the index transaction id
         """
         for segment, filename in self.segment_iterator(reverse=True):
-            if self.is_committed_segment(filename):
+            if self.is_committed_segment(segment):
                 return segment
         return None
 
@@ -550,10 +550,14 @@ class LoggedIO:
             else:
                 break
 
-    def is_committed_segment(self, filename):
+    def is_committed_segment(self, segment):
         """Check if segment ends with a COMMIT_TAG tag
         """
-        with open(filename, 'rb') as fd:
+        try:
+            iterator = self.iter_objects(segment)
+        except IntegrityError:
+            return False
+        with open(self.segment_filename(segment), 'rb') as fd:
             try:
                 fd.seek(-self.header_fmt.size, os.SEEK_END)
             except OSError as e:
@@ -561,7 +565,22 @@ class LoggedIO:
                 if e.errno == errno.EINVAL:
                     return False
                 raise e
-            return fd.read(self.header_fmt.size) == self.COMMIT
+            if fd.read(self.header_fmt.size) != self.COMMIT:
+                return False
+        seen_commit = False
+        while True:
+            try:
+                tag, key, offset = next(iterator)
+            except IntegrityError:
+                return False
+            except StopIteration:
+                break
+            if tag == TAG_COMMIT:
+                seen_commit = True
+                continue
+            if seen_commit:
+                return False
+        return seen_commit
 
     def segment_filename(self, segment):
         return os.path.join(self.path, 'data', str(segment // self.segments_per_dir), str(segment))

borg/testsuite/repository.py

@@ -8,7 +8,7 @@ from ..hashindex import NSIndex
 from ..helpers import Location, IntegrityError
 from ..locking import UpgradableLock, LockFailed
 from ..remote import RemoteRepository, InvalidRPCMethod
-from ..repository import Repository
+from ..repository import Repository, LoggedIO, TAG_COMMIT
 from . import BaseTestCase
 
 
@@ -192,6 +192,13 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
             self.assert_equal(self.repository.check(), True)
             self.assert_equal(len(self.repository), 3)
 
+    def test_ignores_commit_tag_in_data(self):
+        self.repository.put(b'0' * 32, LoggedIO.COMMIT)
+        self.reopen()
+        with self.repository:
+            io = self.repository.io
+            assert not io.is_committed_segment(io.get_latest_segment())
+
 
 class RepositoryAppendOnlyTestCase(RepositoryTestCaseBase):
     def test_destroy_append_only(self):
@@ -268,7 +275,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
         return set(int(key) for key in self.repository.list())
 
     def test_repair_corrupted_segment(self):
-        self.add_objects([[1, 2, 3], [4, 5, 6]])
+        self.add_objects([[1, 2, 3], [4, 5], [6]])
         self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects())
         self.check(status=True)
         self.corrupt_object(5)