Merge pull request #7556 from ThomasWaldmann/kill-nonce-manager

remove nonce management, related repo methods

src/borg/crypto/key.py

@@ -25,7 +25,6 @@ from ..platform import SaveFile
 from ..repoobj import RepoObj
 
 
-from .nonces import NonceManager
 from .low_level import AES, bytes_to_int, num_cipher_blocks, hmac_sha256, blake2b_256, hkdf_hmac_sha512
 from .low_level import AES256_CTR_HMAC_SHA256, AES256_CTR_BLAKE2b, AES256_OCB, CHACHA20_POLY1305
 from . import low_level
@@ -372,7 +371,8 @@ class AESKeyBase(KeyBase):
     logically_encrypted = True
 
     def encrypt(self, id, data):
-        next_iv = self.nonce_manager.ensure_reservation(self.cipher.next_iv(), self.cipher.block_count(len(data)))
+        # legacy, this is only used by the tests.
+        next_iv = self.cipher.next_iv()
         return self.cipher.encrypt(data, header=self.TYPE_STR, iv=next_iv)
 
     def decrypt(self, id, data):
@@ -411,7 +411,6 @@ class AESKeyBase(KeyBase):
             manifest_blocks = num_cipher_blocks(len(manifest_data))
             nonce = self.cipher.extract_iv(manifest_data) + manifest_blocks
         self.cipher.set_iv(nonce)
-        self.nonce_manager = NonceManager(self.repository, nonce)
 
 
 class FlexiKey:

src/borg/crypto/nonces.py

@@ -1,91 +0,0 @@
-import os
-import sys
-from binascii import unhexlify
-
-from ..helpers import get_security_dir
-from ..helpers import bin_to_hex
-from ..platform import SaveFile
-from ..remote import InvalidRPCMethod
-
-from .low_level import bytes_to_long, long_to_bytes
-
-MAX_REPRESENTABLE_NONCE = 2**64 - 1
-NONCE_SPACE_RESERVATION = 2**28  # This in units of AES blocksize (16 bytes)
-
-
-class NonceManager:
-    def __init__(self, repository, manifest_nonce):
-        self.repository = repository
-        self.end_of_nonce_reservation = None
-        self.manifest_nonce = manifest_nonce
-        self.nonce_file = os.path.join(get_security_dir(self.repository.id_str), "nonce")
-
-    def get_local_free_nonce(self):
-        try:
-            with open(self.nonce_file) as fd:
-                return bytes_to_long(unhexlify(fd.read()))
-        except FileNotFoundError:
-            return None
-
-    def commit_local_nonce_reservation(self, next_unreserved, start_nonce):
-        if self.get_local_free_nonce() != start_nonce:
-            raise Exception("nonce space reservation with mismatched previous state")
-        with SaveFile(self.nonce_file, binary=False) as fd:
-            fd.write(bin_to_hex(long_to_bytes(next_unreserved)))
-
-    def get_repo_free_nonce(self):
-        try:
-            return self.repository.get_free_nonce()
-        except InvalidRPCMethod:
-            # old server version, suppress further calls
-            sys.stderr.write("Please upgrade to borg version 1.1+ on the server for safer AES-CTR nonce handling.\n")
-            self.get_repo_free_nonce = lambda: None
-            self.commit_repo_nonce_reservation = lambda next_unreserved, start_nonce: None
-            return None
-
-    def commit_repo_nonce_reservation(self, next_unreserved, start_nonce):
-        self.repository.commit_nonce_reservation(next_unreserved, start_nonce)
-
-    def ensure_reservation(self, nonce, nonce_space_needed):
-        """
-        Call this before doing encryption, give current, yet unused, integer IV as <nonce>
-        and the amount of subsequent (counter-like) IVs needed as <nonce_space_needed>.
-        Return value is the IV (counter) integer you shall use for encryption.
-
-        Note: this method may return the <nonce> you gave, if a reservation for it exists or
-              can be established, so make sure you give a unused nonce.
-        """
-        # Nonces may never repeat, even if a transaction aborts or the system crashes.
-        # Therefore a part of the nonce space is reserved before any nonce is used for encryption.
-        # As these reservations are committed to permanent storage before any nonce is used, this protects
-        # against nonce reuse in crashes and transaction aborts. In that case the reservation still
-        # persists and the whole reserved space is never reused.
-        #
-        # Local storage on the client is used to protect against an attacker that is able to rollback the
-        # state of the server or can do arbitrary modifications to the repository.
-        # Storage on the server is used for the multi client use case where a transaction on client A is
-        # aborted and later client B writes to the repository.
-        #
-        # This scheme does not protect against attacker who is able to rollback the state of the server
-        # or can do arbitrary modifications to the repository in the multi client usecase.
-
-        if self.end_of_nonce_reservation:
-            # we already got a reservation, if nonce_space_needed still fits everything is ok
-            next_nonce = nonce
-            assert next_nonce <= self.end_of_nonce_reservation
-            if next_nonce + nonce_space_needed <= self.end_of_nonce_reservation:
-                return next_nonce
-
-        repo_free_nonce = self.get_repo_free_nonce()
-        local_free_nonce = self.get_local_free_nonce()
-        free_nonce_space = max(
-            x
-            for x in (repo_free_nonce, local_free_nonce, self.manifest_nonce, self.end_of_nonce_reservation)
-            if x is not None
-        )
-        reservation_end = free_nonce_space + nonce_space_needed + NONCE_SPACE_RESERVATION
-        assert reservation_end < MAX_REPRESENTABLE_NONCE
-        self.commit_repo_nonce_reservation(reservation_end, repo_free_nonce)
-        self.commit_local_nonce_reservation(reservation_end, local_free_nonce)
-        self.end_of_nonce_reservation = reservation_end
-        return free_nonce_space

src/borg/remote.py

@@ -134,8 +134,6 @@ compatMap = {
     "negotiate": ("client_data",),
     "open": ("path", "create", "lock_wait", "lock", "exclusive", "append_only"),
     "info": (),
-    "get_free_nonce": (),
-    "commit_nonce_reservation": ("next_unreserved", "start_nonce"),
 }
 
 
@@ -159,8 +157,6 @@ class RepositoryServer:  # pragma: no cover
         "save_key",
         "load_key",
         "break_lock",
-        "get_free_nonce",
-        "commit_nonce_reservation",
         "inject_exception",
     )
 
@@ -1024,14 +1020,6 @@ This problem will go away as soon as the server has been upgraded to 1.0.7+.
     def load_key(self):
         """actual remoting is done via self.call in the @api decorator"""
 
-    @api(since=parse_version("1.0.0"))
-    def get_free_nonce(self):
-        """actual remoting is done via self.call in the @api decorator"""
-
-    @api(since=parse_version("1.0.0"))
-    def commit_nonce_reservation(self, next_unreserved, start_nonce):
-        """actual remoting is done via self.call in the @api decorator"""
-
     @api(since=parse_version("1.0.0"))
     def break_lock(self):
         """actual remoting is done via self.call in the @api decorator"""

src/borg/repository.py

@@ -369,36 +369,6 @@ class Repository:
         # note: if we return an empty string, it means there is no repo key
         return keydata.encode("utf-8")  # remote repo: msgpack issue #99, returning bytes
 
-    def get_free_nonce(self):
-        if self.do_lock and not self.lock.got_exclusive_lock():
-            raise AssertionError("bug in code, exclusive lock should exist here")
-
-        nonce_path = os.path.join(self.path, "nonce")
-        try:
-            with open(nonce_path) as fd:
-                return int.from_bytes(unhexlify(fd.read()), byteorder="big")
-        except FileNotFoundError:
-            return None
-
-    def commit_nonce_reservation(self, next_unreserved, start_nonce):
-        if self.do_lock and not self.lock.got_exclusive_lock():
-            raise AssertionError("bug in code, exclusive lock should exist here")
-
-        if self.get_free_nonce() != start_nonce:
-            raise Exception("nonce space reservation with mismatched previous state")
-        nonce_path = os.path.join(self.path, "nonce")
-        try:
-            with SaveFile(nonce_path, binary=False) as fd:
-                fd.write(bin_to_hex(next_unreserved.to_bytes(8, byteorder="big")))
-        except PermissionError as e:
-            # error is only a problem if we even had a lock
-            if self.do_lock:
-                raise
-            logger.warning(
-                "%s: Failed writing to '%s'. This is expected when working on "
-                "read-only repositories." % (e.strerror, e.filename)
-            )
-
     def destroy(self):
         """Destroy the repository at `self.path`"""
         if self.append_only:

src/borg/testsuite/key.py

@@ -116,12 +116,6 @@ class TestKey:
         id = bytes(32)
         id_str = bin_to_hex(id)
 
-        def get_free_nonce(self):
-            return None
-
-        def commit_nonce_reservation(self, next_unreserved, start_nonce):
-            pass
-
         def save_key(self, data):
             self.key_data = data
 

src/borg/testsuite/nonces.py

@@ -1,197 +0,0 @@
-import os.path
-
-import pytest
-
-from ..crypto import nonces
-from ..crypto.nonces import NonceManager
-from ..crypto.key import bin_to_hex
-from ..helpers import get_security_dir
-from ..remote import InvalidRPCMethod
-
-
-class TestNonceManager:
-    class MockRepository:
-        class _Location:
-            orig = "/some/place"
-
-        _location = _Location()
-        id = bytes(32)
-        id_str = bin_to_hex(id)
-
-        def get_free_nonce(self):
-            return self.next_free
-
-        def commit_nonce_reservation(self, next_unreserved, start_nonce):
-            assert start_nonce == self.next_free
-            self.next_free = next_unreserved
-
-    class MockOldRepository(MockRepository):
-        def get_free_nonce(self):
-            raise InvalidRPCMethod("")
-
-        def commit_nonce_reservation(self, next_unreserved, start_nonce):
-            pytest.fail("commit_nonce_reservation should never be called on an old repository")
-
-    def setUp(self):
-        self.repository = None
-
-    def cache_nonce(self):
-        with open(os.path.join(get_security_dir(self.repository.id_str), "nonce")) as fd:
-            return fd.read()
-
-    def set_cache_nonce(self, nonce):
-        with open(os.path.join(get_security_dir(self.repository.id_str), "nonce"), "w") as fd:
-            assert fd.write(nonce)
-
-    def test_empty_cache_and_old_server(self, monkeypatch):
-        monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
-
-        self.repository = self.MockOldRepository()
-        manager = NonceManager(self.repository, 0x2000)
-        next_nonce = manager.ensure_reservation(0x2000, 19)
-        assert next_nonce == 0x2000
-
-        assert self.cache_nonce() == "0000000000002033"
-
-    def test_empty_cache(self, monkeypatch):
-        monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
-
-        self.repository = self.MockRepository()
-        self.repository.next_free = 0x2000
-        manager = NonceManager(self.repository, 0x2000)
-        next_nonce = manager.ensure_reservation(0x2000, 19)
-        assert next_nonce == 0x2000
-
-        assert self.cache_nonce() == "0000000000002033"
-
-    def test_empty_nonce(self, monkeypatch):
-        monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
-
-        self.repository = self.MockRepository()
-        self.repository.next_free = None
-        manager = NonceManager(self.repository, 0x2000)
-        next_nonce = manager.ensure_reservation(0x2000, 19)
-        assert next_nonce == 0x2000
-
-        assert self.cache_nonce() == "0000000000002033"
-        assert self.repository.next_free == 0x2033
-
-        # enough space in reservation
-        next_nonce = manager.ensure_reservation(0x2013, 13)
-        assert next_nonce == 0x2013
-        assert self.cache_nonce() == "0000000000002033"
-        assert self.repository.next_free == 0x2033
-
-        # just barely enough space in reservation
-        next_nonce = manager.ensure_reservation(0x2020, 19)
-        assert next_nonce == 0x2020
-        assert self.cache_nonce() == "0000000000002033"
-        assert self.repository.next_free == 0x2033
-
-        # no space in reservation
-        next_nonce = manager.ensure_reservation(0x2033, 16)
-        assert next_nonce == 0x2033
-        assert self.cache_nonce() == "0000000000002063"
-        assert self.repository.next_free == 0x2063
-
-        # spans reservation boundary
-        next_nonce = manager.ensure_reservation(0x2043, 64)
-        assert next_nonce == 0x2063
-        assert self.cache_nonce() == "00000000000020c3"
-        assert self.repository.next_free == 0x20C3
-
-    def test_sync_nonce(self, monkeypatch):
-        monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
-
-        self.repository = self.MockRepository()
-        self.repository.next_free = 0x2000
-        self.set_cache_nonce("0000000000002000")
-
-        manager = NonceManager(self.repository, 0x2000)
-        next_nonce = manager.ensure_reservation(0x2000, 19)
-        assert next_nonce == 0x2000
-
-        assert self.cache_nonce() == "0000000000002033"
-        assert self.repository.next_free == 0x2033
-
-    def test_server_just_upgraded(self, monkeypatch):
-        monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
-
-        self.repository = self.MockRepository()
-        self.repository.next_free = None
-        self.set_cache_nonce("0000000000002000")
-
-        manager = NonceManager(self.repository, 0x2000)
-        next_nonce = manager.ensure_reservation(0x2000, 19)
-        assert next_nonce == 0x2000
-
-        assert self.cache_nonce() == "0000000000002033"
-        assert self.repository.next_free == 0x2033
-
-    def test_transaction_abort_no_cache(self, monkeypatch):
-        monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
-
-        self.repository = self.MockRepository()
-        self.repository.next_free = 0x2000
-
-        manager = NonceManager(self.repository, 0x2000)
-        next_nonce = manager.ensure_reservation(0x1000, 19)
-        assert next_nonce == 0x2000
-
-        assert self.cache_nonce() == "0000000000002033"
-        assert self.repository.next_free == 0x2033
-
-    def test_transaction_abort_old_server(self, monkeypatch):
-        monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
-
-        self.repository = self.MockOldRepository()
-        self.set_cache_nonce("0000000000002000")
-
-        manager = NonceManager(self.repository, 0x2000)
-        next_nonce = manager.ensure_reservation(0x1000, 19)
-        assert next_nonce == 0x2000
-
-        assert self.cache_nonce() == "0000000000002033"
-
-    def test_transaction_abort_on_other_client(self, monkeypatch):
-        monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
-
-        self.repository = self.MockRepository()
-        self.repository.next_free = 0x2000
-        self.set_cache_nonce("0000000000001000")
-
-        manager = NonceManager(self.repository, 0x2000)
-        next_nonce = manager.ensure_reservation(0x1000, 19)
-        assert next_nonce == 0x2000
-
-        assert self.cache_nonce() == "0000000000002033"
-        assert self.repository.next_free == 0x2033
-
-    def test_interleaved(self, monkeypatch):
-        monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
-
-        self.repository = self.MockRepository()
-        self.repository.next_free = 0x2000
-        self.set_cache_nonce("0000000000002000")
-
-        manager = NonceManager(self.repository, 0x2000)
-        next_nonce = manager.ensure_reservation(0x2000, 19)
-        assert next_nonce == 0x2000
-
-        assert self.cache_nonce() == "0000000000002033"
-        assert self.repository.next_free == 0x2033
-
-        # somehow the clients unlocks, another client reserves and this client relocks
-        self.repository.next_free = 0x4000
-
-        # enough space in reservation
-        next_nonce = manager.ensure_reservation(0x2013, 12)
-        assert next_nonce == 0x2013
-        assert self.cache_nonce() == "0000000000002033"
-        assert self.repository.next_free == 0x4000
-
-        # spans reservation boundary
-        next_nonce = manager.ensure_reservation(0x201F, 21)
-        assert next_nonce == 0x4000
-        assert self.cache_nonce() == "0000000000004035"
-        assert self.repository.next_free == 0x4035

src/borg/testsuite/repository.py

@@ -613,48 +613,6 @@ class QuotaTestCase(RepositoryTestCaseBase):
             assert self.repository.storage_quota_use == len(ch1) + 41 + 8  # now we have compacted.
 
 
-class NonceReservation(RepositoryTestCaseBase):
-    def test_get_free_nonce_asserts(self):
-        self.reopen(exclusive=False)
-        with pytest.raises(AssertionError):
-            with self.repository:
-                self.repository.get_free_nonce()
-
-    def test_get_free_nonce(self):
-        with self.repository:
-            assert self.repository.get_free_nonce() is None
-
-            with open(os.path.join(self.repository.path, "nonce"), "w") as fd:
-                fd.write("0000000000000000")
-            assert self.repository.get_free_nonce() == 0
-
-            with open(os.path.join(self.repository.path, "nonce"), "w") as fd:
-                fd.write("5000000000000000")
-            assert self.repository.get_free_nonce() == 0x5000000000000000
-
-    def test_commit_nonce_reservation_asserts(self):
-        self.reopen(exclusive=False)
-        with pytest.raises(AssertionError):
-            with self.repository:
-                self.repository.commit_nonce_reservation(0x200, 0x100)
-
-    def test_commit_nonce_reservation(self):
-        with self.repository:
-            with pytest.raises(Exception):
-                self.repository.commit_nonce_reservation(0x200, 15)
-
-            self.repository.commit_nonce_reservation(0x200, None)
-            with open(os.path.join(self.repository.path, "nonce")) as fd:
-                assert fd.read() == "0000000000000200"
-
-            with pytest.raises(Exception):
-                self.repository.commit_nonce_reservation(0x200, 15)
-
-            self.repository.commit_nonce_reservation(0x400, 0x200)
-            with open(os.path.join(self.repository.path, "nonce")) as fd:
-                assert fd.read() == "0000000000000400"
-
-
 class RepositoryAuxiliaryCorruptionTestCase(RepositoryTestCaseBase):
     def setUp(self):
         super().setUp()