Merge pull request #6660 from ThomasWaldmann/reuse-key

init --other-location=OTHER_REPO: reuse key material from OTHER_REPO

src/borg/archiver.py

@@ -112,6 +112,35 @@ def argument(args, str_or_bool):
     return str_or_bool
 
 
+def get_repository(location, *, create, exclusive, lock_wait, lock, append_only,
+                   make_parent_dirs, storage_quota, args):
+    if location.proto == 'ssh':
+        repository = RemoteRepository(location.omit_archive(), create=create, exclusive=exclusive,
+                                      lock_wait=lock_wait, lock=lock, append_only=append_only,
+                                      make_parent_dirs=make_parent_dirs, args=args)
+
+    else:
+        repository = Repository(location.path, create=create, exclusive=exclusive,
+                                lock_wait=lock_wait, lock=lock, append_only=append_only,
+                                make_parent_dirs=make_parent_dirs, storage_quota=storage_quota)
+    return repository
+
+
+def compat_check(*, create, manifest, key, cache, compatibility, decorator_name):
+    if not create and (manifest or key or cache):
+        if compatibility is None:
+            raise AssertionError(f"{decorator_name} decorator used without compatibility argument")
+        if type(compatibility) is not tuple:
+            raise AssertionError(f"{decorator_name} decorator compatibility argument must be of type tuple")
+    else:
+        if compatibility is not None:
+            raise AssertionError(f"{decorator_name} called with compatibility argument, "
+                                 f"but would not check {compatibility!r}")
+        if create:
+            compatibility = Manifest.NO_OPERATION_CHECK
+    return compatibility
+
+
 def with_repository(fake=False, invert_fake=False, create=False, lock=True,
                     exclusive=False, manifest=True, cache=False, secure=True,
                     compatibility=None):
@@ -128,17 +157,9 @@ def with_repository(fake=False, invert_fake=False, create=False, lock=True,
     :param secure: do assert_secure after loading manifest
     :param compatibility: mandatory if not create and (manifest or cache), specifies mandatory feature categories to check
     """
-
-    if not create and (manifest or cache):
-        if compatibility is None:
-            raise AssertionError("with_repository decorator used without compatibility argument")
-        if type(compatibility) is not tuple:
-            raise AssertionError("with_repository decorator compatibility argument must be of type tuple")
-    else:
-        if compatibility is not None:
-            raise AssertionError("with_repository called with compatibility argument but would not check" + repr(compatibility))
-        if create:
-            compatibility = Manifest.NO_OPERATION_CHECK
+    # Note: with_repository decorator does not have a "key" argument (yet?)
+    compatibility = compat_check(create=create, manifest=manifest, key=manifest, cache=cache,
+                                 compatibility=compatibility, decorator_name='with_repository')
 
     # To process the `--bypass-lock` option if specified, we need to
     # modify `lock` inside `wrapper`. Therefore we cannot use the
@@ -159,14 +180,12 @@ def with_repository(fake=False, invert_fake=False, create=False, lock=True,
             make_parent_dirs = getattr(args, 'make_parent_dirs', False)
             if argument(args, fake) ^ invert_fake:
                 return method(self, args, repository=None, **kwargs)
-            elif location.proto == 'ssh':
-                repository = RemoteRepository(location.omit_archive(), create=create, exclusive=argument(args, exclusive),
-                                              lock_wait=self.lock_wait, lock=lock, append_only=append_only,
-                                              make_parent_dirs=make_parent_dirs, args=args)
-            else:
-                repository = Repository(location.path, create=create, exclusive=argument(args, exclusive),
+
+            repository = get_repository(location, create=create, exclusive=argument(args, exclusive),
                                         lock_wait=self.lock_wait, lock=lock, append_only=append_only,
-                                        storage_quota=storage_quota, make_parent_dirs=make_parent_dirs)
+                                        make_parent_dirs=make_parent_dirs, storage_quota=storage_quota,
+                                        args=args)
+
             with repository:
                 if manifest or cache:
                     kwargs['manifest'], kwargs['key'] = Manifest.load(repository, compatibility)
@@ -187,6 +206,51 @@ def with_repository(fake=False, invert_fake=False, create=False, lock=True,
     return decorator
 
 
+def with_other_repository(manifest=False, key=False, cache=False, compatibility=None):
+    """
+    this is a simplified version of "with_repository", just for the "other location".
+
+    the repository at the "other location" is intended to get used as a **source** (== read operations).
+    """
+
+    compatibility = compat_check(create=False, manifest=manifest, key=key, cache=cache,
+                                 compatibility=compatibility, decorator_name='with_other_repository')
+
+    def decorator(method):
+        @functools.wraps(method)
+        def wrapper(self, args, **kwargs):
+            location = getattr(args, 'other_location', None)
+            if location is None:  # nothing to do
+                return method(self, args, **kwargs)
+
+            repository = get_repository(location, create=False, exclusive=True,
+                                        lock_wait=self.lock_wait, lock=True, append_only=False,
+                                        make_parent_dirs=False, storage_quota=None,
+                                        args=args)
+
+            with repository:
+                kwargs['other_repository'] = repository
+                if manifest or key or cache:
+                    manifest_, key_ = Manifest.load(repository, compatibility)
+                    assert_secure(repository, manifest_, self.lock_wait)
+                    if manifest:
+                        kwargs['other_manifest'] = manifest_
+                    if key:
+                        kwargs['other_key'] = key_
+                if cache:
+                    with Cache(repository, key_, manifest_,
+                               progress=False, lock_wait=self.lock_wait,
+                               cache_mode=getattr(args, 'files_cache_mode', DEFAULT_FILES_CACHE_MODE),
+                               consider_part_files=getattr(args, 'consider_part_files', False),
+                               iec=getattr(args, 'iec', False)) as cache_:
+                        kwargs['other_cache'] = cache_
+                        return method(self, args, **kwargs)
+                else:
+                    return method(self, args, **kwargs)
+        return wrapper
+    return decorator
+
+
 def with_archive(method):
     @functools.wraps(method)
     def wrapper(self, args, repository, key, manifest, **kwargs):
@@ -275,12 +339,13 @@ class Archiver:
         return EXIT_SUCCESS
 
     @with_repository(create=True, exclusive=True, manifest=False)
-    def do_init(self, args, repository):
+    @with_other_repository(key=True, compatibility=(Manifest.Operation.READ, ))
+    def do_init(self, args, repository, *, other_repository=None, other_key=None):
         """Initialize an empty repository"""
         path = args.location.canonical_path()
         logger.info('Initializing repository at "%s"' % path)
         try:
-            key = key_creator(repository, args)
+            key = key_creator(repository, args, other_key=other_key)
         except (EOFError, KeyboardInterrupt):
             repository.destroy()
             return EXIT_WARNING
@@ -4361,6 +4426,9 @@ class Archiver:
         subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
                                type=location_validator(archive=False),
                                help='repository to create')
+        subparser.add_argument('--other-location', metavar='OTHER_REPOSITORY', dest='other_location',
+                               type=location_validator(archive=False, other=True),
+                               help='reuse the key material from the other repository')
         subparser.add_argument('-e', '--encryption', metavar='MODE', dest='encryption', required=True,
                                choices=key_argument_names(),
                                help='select encryption key mode **(required)**')

src/borg/crypto/key.py

@@ -85,11 +85,11 @@ class TAMUnsupportedSuiteError(IntegrityError):
     traceback = False
 
 
-def key_creator(repository, args):
+def key_creator(repository, args, *, other_key=None):
     for key in AVAILABLE_KEY_TYPES:
         if key.ARG_NAME == args.encryption:
             assert key.ARG_NAME is not None
-            return key.create(repository, args)
+            return key.create(repository, args, other_key=other_key)
     else:
         raise ValueError('Invalid encryption mode "%s"' % args.encryption)
 
@@ -262,7 +262,7 @@ class PlaintextKey(KeyBase):
         self.tam_required = False
 
     @classmethod
-    def create(cls, repository, args):
+    def create(cls, repository, args, **kw):
         logger.info('Encryption NOT enabled.\nUse the "--encryption=repokey|keyfile" to enable encryption.')
         return cls(repository)
 
@@ -367,15 +367,27 @@ class AESKeyBase(KeyBase):
         self.assert_id(id, data)
         return data
 
+    def init_from_given_data(self, *, enc_key, enc_hmac_key, id_key, chunk_seed):
+        assert len(enc_key) >= 32
+        assert len(enc_hmac_key) >= 32
+        assert len(id_key) >= 32
+        assert isinstance(chunk_seed, int)
+        self.enc_key = enc_key
+        self.enc_hmac_key = enc_hmac_key
+        self.id_key = id_key
+        self.chunk_seed = chunk_seed
+
     def init_from_random_data(self):
         data = os.urandom(100)
-        self.enc_key = data[0:32]
-        self.enc_hmac_key = data[32:64]
-        self.id_key = data[64:96]
-        self.chunk_seed = bytes_to_int(data[96:100])
+        chunk_seed = bytes_to_int(data[96:100])
         # Convert to signed int32
-        if self.chunk_seed & 0x80000000:
-            self.chunk_seed = self.chunk_seed - 0xffffffff - 1
+        if chunk_seed & 0x80000000:
+            chunk_seed = chunk_seed - 0xffffffff - 1
+        self.init_from_given_data(
+            enc_key=data[0:32],
+            enc_hmac_key=data[32:64],
+            id_key=data[64:96],
+            chunk_seed=chunk_seed)
 
     def init_ciphers(self, manifest_data=None):
         self.cipher = self.CIPHERSUITE(mac_key=self.enc_hmac_key, enc_key=self.enc_key, header_len=1, aad_offset=1)
@@ -572,11 +584,41 @@ class FlexiKey:
         self.save(self.target, passphrase, algorithm=self._encrypted_key_algorithm)
 
     @classmethod
-    def create(cls, repository, args):
-        passphrase = Passphrase.new(allow_empty=True)
+    def create(cls, repository, args, *, other_key=None):
         key = cls(repository)
         key.repository_id = repository.id
-        key.init_from_random_data()
+        if other_key is not None:
+            if isinstance(other_key, PlaintextKey):
+                raise Error("Copying key material from an unencrypted repository is not possible.")
+            if isinstance(key, AESKeyBase):
+                # user must use an AEADKeyBase subclass (AEAD modes with session keys)
+                raise Error("Copying key material to an AES-CTR based mode is insecure and unsupported.")
+            # avoid breaking the deduplication by changing the id hash
+            same_ids = (
+                # these use HMAC-SHA256 IDs:
+                isinstance(other_key, (RepoKey, KeyfileKey))
+                and
+                isinstance(key, (AESOCBRepoKey, AESOCBKeyfileKey,
+                                 CHPORepoKey, CHPOKeyfileKey))
+                or
+                # these use BLAKE2b IDs:
+                isinstance(other_key, (Blake2RepoKey, Blake2KeyfileKey))
+                and
+                isinstance(key, (Blake2AESOCBRepoKey, Blake2AESOCBKeyfileKey,
+                                 Blake2CHPORepoKey, Blake2CHPOKeyfileKey))
+            )
+            if not same_ids:
+                # either keep HMAC-SHA256 or keep BLAKE2b!
+                raise Error("You must keep the same ID hash (HMAC-SHA256 or BLAKE2b) or deduplication will break.")
+            key.init_from_given_data(
+                enc_key=other_key.enc_key,
+                enc_hmac_key=other_key.enc_hmac_key,
+                id_key=other_key.id_key,
+                chunk_seed=other_key.chunk_seed)
+            passphrase = other_key._passphrase
+        else:
+            key.init_from_random_data()
+            passphrase = Passphrase.new(allow_empty=True)
         key.init_ciphers()
         target = key.get_new_target(args)
         key.save(target, passphrase, create=True, algorithm=KEY_ALGORITHMS[args.key_algorithm])
@@ -851,15 +893,27 @@ class AEADKeyBase(KeyBase):
         self.assert_id(id, data)
         return data
 
+    def init_from_given_data(self, *, enc_key, enc_hmac_key, id_key, chunk_seed):
+        assert len(enc_key) >= 32
+        assert len(enc_hmac_key) >= 32
+        assert len(id_key) >= 32
+        assert isinstance(chunk_seed, int)
+        self.enc_key = enc_key
+        self.enc_hmac_key = enc_hmac_key
+        self.id_key = id_key
+        self.chunk_seed = chunk_seed
+
     def init_from_random_data(self):
         data = os.urandom(100)
-        self.enc_key = data[0:32]
-        self.enc_hmac_key = data[32:64]
-        self.id_key = data[64:96]
-        self.chunk_seed = bytes_to_int(data[96:100])
+        chunk_seed = bytes_to_int(data[96:100])
         # Convert to signed int32
-        if self.chunk_seed & 0x80000000:
-            self.chunk_seed = self.chunk_seed - 0xffffffff - 1
+        if chunk_seed & 0x80000000:
+            chunk_seed = chunk_seed - 0xffffffff - 1
+        self.init_from_given_data(
+            enc_key=data[0:32],
+            enc_hmac_key=data[32:64],
+            id_key=data[64:96],
+            chunk_seed=chunk_seed)
 
     def _get_session_key(self, sessionid):
         assert len(sessionid) == 24  # 192bit

src/borg/helpers/parseformat.py

@@ -386,7 +386,8 @@ class Location:
         )
         """ + optional_archive_re, re.VERBOSE)              # archive name (optional, may be empty)
 
-    def __init__(self, text='', overrides={}):
+    def __init__(self, text='', overrides={}, other=False):
+        self.repo_env_var = 'BORG_OTHER_REPO' if other else 'BORG_REPO'
         if not self.parse(text, overrides):
             raise ValueError('Invalid location format: "%s"' % self.processed)
 
@@ -399,7 +400,7 @@ class Location:
         m = self.env_re.match(text)
         if not m:
             return False
-        repo_raw = os.environ.get('BORG_REPO')
+        repo_raw = os.environ.get(self.repo_env_var)
         if repo_raw is None:
             return False
         repo = replace_placeholders(repo_raw, overrides)
@@ -512,10 +513,10 @@ class Location:
         return loc
 
 
-def location_validator(archive=None, proto=None):
+def location_validator(archive=None, proto=None, other=False):
     def validator(text):
         try:
-            loc = Location(text)
+            loc = Location(text, other=other)
         except ValueError as err:
             raise argparse.ArgumentTypeError(str(err)) from None
         if archive is True and not loc.archive:

src/borg/testsuite/archiver.py

@@ -2878,7 +2878,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         assert "is invalid" in output
 
     def test_init_interrupt(self):
-        def raise_eof(*args):
+        def raise_eof(*args, **kwargs):
             raise EOFError
 
         with patch.object(FlexiKey, 'create', raise_eof):