repository api: flags support, fixes #6982

- .list: only return IDs for objects where flags & mask == value.
- .flags(_many) (new) to set/query flags

src/borg/remote.py

@@ -146,6 +146,8 @@ class RepositoryServer:  # pragma: no cover
         "commit",
         "delete",
         "destroy",
+        "flags",
+        "flags_many",
         "get",
         "list",
         "scan",
@@ -979,14 +981,26 @@ This problem will go away as soon as the server has been upgraded to 1.0.7+.
     def __len__(self):
         """actual remoting is done via self.call in the @api decorator"""
 
-    @api(since=parse_version("1.0.0"))
-    def list(self, limit=None, marker=None):
+    @api(
+        since=parse_version("1.0.0"),
+        mask={"since": parse_version("2.0.0b2"), "previously": 0},
+        value={"since": parse_version("2.0.0b2"), "previously": 0},
+    )
+    def list(self, limit=None, marker=None, mask=0, value=0):
         """actual remoting is done via self.call in the @api decorator"""
 
     @api(since=parse_version("1.1.0b3"))
     def scan(self, limit=None, marker=None):
         """actual remoting is done via self.call in the @api decorator"""
 
+    @api(since=parse_version("2.0.0b2"))
+    def flags(self, id, mask=0xFFFFFFFF, value=None):
+        """actual remoting is done via self.call in the @api decorator"""
+
+    @api(since=parse_version("2.0.0b2"))
+    def flags_many(self, ids, mask=0xFFFFFFFF, value=None):
+        """actual remoting is done via self.call in the @api decorator"""
+
     def get(self, id):
         for resp in self.get_many([id]):
             return resp

src/borg/repository.py

@@ -1197,13 +1197,15 @@ class Repository:
             self.index = self.open_index(self.get_transaction_id())
         return id in self.index
 
-    def list(self, limit=None, marker=None):
+    def list(self, limit=None, marker=None, mask=0, value=0):
         """
         list <limit> IDs starting from after id <marker> - in index (pseudo-random) order.
+
+        if mask and value are given, only return IDs where flags & mask == value (default: all IDs).
         """
         if not self.index:
             self.index = self.open_index(self.get_transaction_id())
-        return [id_ for id_, _ in islice(self.index.iteritems(marker=marker), limit)]
+        return [id_ for id_, _ in islice(self.index.iteritems(marker=marker, mask=mask, value=value), limit)]
 
     def scan(self, limit=None, marker=None):
         """
@@ -1250,6 +1252,22 @@ class Repository:
                             return result
         return result
 
+    def flags(self, id, mask=0xFFFFFFFF, value=None):
+        """
+        query and optionally set flags
+
+        :param id: id (key) of object
+        :param mask: bitmask for flags (default: operate on all 32 bits)
+        :param value: value to set masked bits to (default: do not change any flags)
+        :return: (previous) flags value (only masked bits)
+        """
+        if not self.index:
+            self.index = self.open_index(self.get_transaction_id())
+        return self.index.flags(id, mask, value)
+
+    def flags_many(self, ids, mask=0xFFFFFFFF, value=None):
+        return [self.flags(id_, mask, value) for id_ in ids]
+
     def get(self, id):
         if not self.index:
             self.index = self.open_index(self.get_transaction_id())

src/borg/testsuite/repository.py

@@ -173,6 +173,54 @@ class RepositoryTestCase(RepositoryTestCaseBase):
         self.assert_equal(self.repository.get(H(0)), max_data)
         self.assert_raises(IntegrityError, lambda: self.repository.put(H(1), max_data + b"x"))
 
+    def test_set_flags(self):
+        id = H(0)
+        self.repository.put(id, b"")
+        self.assert_equal(self.repository.flags(id), 0x00000000)  # init == all zero
+        self.repository.flags(id, mask=0x00000001, value=0x00000001)
+        self.assert_equal(self.repository.flags(id), 0x00000001)
+        self.repository.flags(id, mask=0x00000002, value=0x00000002)
+        self.assert_equal(self.repository.flags(id), 0x00000003)
+        self.repository.flags(id, mask=0x00000001, value=0x00000000)
+        self.assert_equal(self.repository.flags(id), 0x00000002)
+        self.repository.flags(id, mask=0x00000002, value=0x00000000)
+        self.assert_equal(self.repository.flags(id), 0x00000000)
+
+    def test_get_flags(self):
+        id = H(0)
+        self.repository.put(id, b"")
+        self.assert_equal(self.repository.flags(id), 0x00000000)  # init == all zero
+        self.repository.flags(id, mask=0xC0000003, value=0x80000001)
+        self.assert_equal(self.repository.flags(id, mask=0x00000001), 0x00000001)
+        self.assert_equal(self.repository.flags(id, mask=0x00000002), 0x00000000)
+        self.assert_equal(self.repository.flags(id, mask=0x40000008), 0x00000000)
+        self.assert_equal(self.repository.flags(id, mask=0x80000000), 0x80000000)
+
+    def test_flags_many(self):
+        ids_flagged = [H(0), H(1)]
+        ids_default_flags = [H(2), H(3)]
+        [self.repository.put(id, b"") for id in ids_flagged + ids_default_flags]
+        self.repository.flags_many(ids_flagged, mask=0xFFFFFFFF, value=0xDEADBEEF)
+        self.assert_equal(list(self.repository.flags_many(ids_default_flags)), [0x00000000, 0x00000000])
+        self.assert_equal(list(self.repository.flags_many(ids_flagged)), [0xDEADBEEF, 0xDEADBEEF])
+        self.assert_equal(list(self.repository.flags_many(ids_flagged, mask=0xFFFF0000)), [0xDEAD0000, 0xDEAD0000])
+        self.assert_equal(list(self.repository.flags_many(ids_flagged, mask=0x0000FFFF)), [0x0000BEEF, 0x0000BEEF])
+
+    def test_flags_persistence(self):
+        self.repository.put(H(0), b"default")
+        self.repository.put(H(1), b"one one zero")
+        # we do not set flags for H(0), so we can later check their default state.
+        self.repository.flags(H(1), mask=0x00000007, value=0x00000006)
+        self.repository.commit(compact=False)
+        self.repository.close()
+
+        self.repository = self.open()
+        with self.repository:
+            # we query all flags to check if the initial flags were all zero and
+            # only the ones we explicitly set to one are as expected.
+            self.assert_equal(self.repository.flags(H(0), mask=0xFFFFFFFF), 0x00000000)
+            self.assert_equal(self.repository.flags(H(1), mask=0xFFFFFFFF), 0x00000006)
+
 
 class LocalRepositoryTestCase(RepositoryTestCaseBase):
     # test case that doesn't work with remote repositories