repo key mode (and deprecate passphrase mode), fixes #85

see usage.rst change for a description and why this is needed

borg/archiver.py

@@ -539,7 +539,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
                                type=location_validator(archive=False),
                                help='repository to create')
         subparser.add_argument('-e', '--encryption', dest='encryption',
-                               choices=('none', 'passphrase', 'keyfile'), default='none',
+                               choices=('none', 'passphrase', 'keyfile', 'repokey'), default='none',
                                help='select encryption method')
 
         check_epilog = textwrap.dedent("""

borg/key.py

@@ -1,5 +1,6 @@
 from binascii import hexlify, a2b_base64, b2a_base64
-from getpass import getpass
+import configparser
+import getpass
 import os
 import msgpack
 import textwrap
@@ -23,6 +24,11 @@ class KeyfileNotFoundError(Error):
     """
 
 
+class RepoKeyNotFoundError(Error):
+    """No key entry found in the config of repository {}.
+    """
+
+
 class HMAC(hmac.HMAC):
     """Workaround a bug in Python < 3.4 Where HMAC does not accept memoryviews
     """
@@ -33,27 +39,35 @@ class HMAC(hmac.HMAC):
 def key_creator(repository, args):
     if args.encryption == 'keyfile':
         return KeyfileKey.create(repository, args)
-    elif args.encryption == 'passphrase':
+    elif args.encryption == 'repokey':
+        return RepoKey.create(repository, args)
+    elif args.encryption == 'passphrase':  # deprecated, kill in 1.x
         return PassphraseKey.create(repository, args)
     else:
         return PlaintextKey.create(repository, args)
 
 
 def key_factory(repository, manifest_data):
-    if manifest_data[0] == KeyfileKey.TYPE:
+    key_type = manifest_data[0]
+    if key_type == KeyfileKey.TYPE:
         return KeyfileKey.detect(repository, manifest_data)
-    elif manifest_data[0] == PassphraseKey.TYPE:
+    elif key_type == RepoKey.TYPE:
+        return RepoKey.detect(repository, manifest_data)
+    elif key_type == PassphraseKey.TYPE:  # deprecated, kill in 1.x
         return PassphraseKey.detect(repository, manifest_data)
-    elif manifest_data[0] == PlaintextKey.TYPE:
+    elif key_type == PlaintextKey.TYPE:
         return PlaintextKey.detect(repository, manifest_data)
     else:
-        raise UnsupportedPayloadError(manifest_data[0])
+        raise UnsupportedPayloadError(key_type)
 
 
 class KeyBase:
+    TYPE = None  # override in subclasses
 
-    def __init__(self):
+    def __init__(self, repository):
         self.TYPE_STR = bytes([self.TYPE])
+        self.repository = repository
+        self.target = None  # key location file path / repo obj
         self.compression_level = 0
 
     def id_hash(self, data):
@@ -74,12 +88,12 @@ class PlaintextKey(KeyBase):
 
     @classmethod
     def create(cls, repository, args):
-        print('Encryption NOT enabled.\nUse the "--encryption=passphrase|keyfile" to enable encryption.')
-        return cls()
+        print('Encryption NOT enabled.\nUse the "--encryption=repokey|keyfile|passphrase" to enable encryption.')
+        return cls(repository)
 
     @classmethod
     def detect(cls, repository, manifest_data):
-        return cls()
+        return cls(repository)
 
     def id_hash(self, data):
         return sha256(data).digest()
@@ -155,38 +169,65 @@ class AESKeyBase(KeyBase):
         self.dec_cipher = AES(is_encrypt=False, key=self.enc_key)
 
 
+class Passphrase(str):
+    @classmethod
+    def env_passphrase(cls, default=None):
+        passphrase = os.environ.get('BORG_PASSPHRASE', default)
+        if passphrase is not None:
+            return cls(passphrase)
+
+    @classmethod
+    def getpass(cls, prompt):
+        return cls(getpass.getpass(prompt))
+
+    @classmethod
+    def new(cls, allow_empty=False):
+        passphrase = cls.env_passphrase()
+        if passphrase is not None:
+            return passphrase
+        while True:
+            passphrase = cls.getpass('Enter new passphrase: ')
+            if allow_empty or passphrase:
+                passphrase2 = cls.getpass('Enter same passphrase again: ')
+                if passphrase == passphrase2:
+                    print('Remember your passphrase. Your data will be inaccessible without it.')
+                    return passphrase
+                else:
+                    print('Passphrases do not match')
+            else:
+                print('Passphrase must not be blank')
+
+    def __repr__(self):
+        return '<Passphrase "***hidden***">'
+
+    def kdf(self, salt, iterations, length):
+        return pbkdf2_sha256(self.encode('utf-8'), salt, iterations, length)
+
+
 class PassphraseKey(AESKeyBase):
+    # This mode is DEPRECATED and will be killed at 1.0 release.
+    # With this mode:
+    # - you can never ever change your passphrase for existing repos.
+    # - you can never ever use a different iterations count for existing repos.
     TYPE = 0x01
-    iterations = 100000
+    iterations = 100000  # must not be changed ever!
 
     @classmethod
     def create(cls, repository, args):
-        key = cls()
-        passphrase = os.environ.get('BORG_PASSPHRASE')
-        if passphrase is not None:
-            passphrase2 = passphrase
-        else:
-            passphrase, passphrase2 = 1, 2
-        while passphrase != passphrase2:
-            passphrase = getpass('Enter passphrase: ')
-            if not passphrase:
-                print('Passphrase must not be blank')
-                continue
-            passphrase2 = getpass('Enter same passphrase again: ')
-            if passphrase != passphrase2:
-                print('Passphrases do not match')
+        key = cls(repository)
+        print('WARNING: "passphrase" mode is deprecated and will be removed in 1.0.')
+        print('If you want something similar (but with less issues), use "repokey" mode.')
+        passphrase = Passphrase.new(allow_empty=False)
         key.init(repository, passphrase)
-        if passphrase:
-            print('Remember your passphrase. Your data will be inaccessible without it.')
         return key
 
     @classmethod
     def detect(cls, repository, manifest_data):
         prompt = 'Enter passphrase for %s: ' % repository._location.orig
-        key = cls()
-        passphrase = os.environ.get('BORG_PASSPHRASE')
+        key = cls(repository)
+        passphrase = Passphrase.env_passphrase()
         if passphrase is None:
-            passphrase = getpass(prompt)
+            passphrase = Passphrase.getpass(prompt)
         while True:
             key.init(repository, passphrase)
             try:
@@ -195,7 +236,7 @@ class PassphraseKey(AESKeyBase):
                 key.init_ciphers(PREFIX + long_to_bytes(key.extract_nonce(manifest_data) + num_blocks))
                 return key
             except IntegrityError:
-                passphrase = getpass(prompt)
+                passphrase = Passphrase.getpass(prompt)
 
     def change_passphrase(self):
         class ImmutablePassphraseError(Error):
@@ -204,41 +245,31 @@ class PassphraseKey(AESKeyBase):
         raise ImmutablePassphraseError
 
     def init(self, repository, passphrase):
-        self.init_from_random_data(pbkdf2_sha256(passphrase.encode('utf-8'), repository.id, self.iterations, 100))
+        self.init_from_random_data(passphrase.kdf(repository.id, self.iterations, 100))
         self.init_ciphers()
 
 
-class KeyfileKey(AESKeyBase):
-    FILE_ID = 'BORG_KEY'
-    TYPE = 0x00
-
+class KeyfileKeyBase(AESKeyBase):
     @classmethod
     def detect(cls, repository, manifest_data):
-        key = cls()
-        path = cls.find_key_file(repository)
-        prompt = 'Enter passphrase for key file %s: ' % path
-        passphrase = os.environ.get('BORG_PASSPHRASE', '')
-        while not key.load(path, passphrase):
-            passphrase = getpass(prompt)
+        key = cls(repository)
+        target = key.find_key()
+        prompt = 'Enter passphrase for key %s: ' % target
+        passphrase = Passphrase.env_passphrase(default='')
+        while not key.load(target, passphrase):
+            passphrase = Passphrase.getpass(prompt)
         num_blocks = num_aes_blocks(len(manifest_data) - 41)
         key.init_ciphers(PREFIX + long_to_bytes(key.extract_nonce(manifest_data) + num_blocks))
         return key
 
-    @classmethod
-    def find_key_file(cls, repository):
-        id = hexlify(repository.id).decode('ascii')
-        keys_dir = get_keys_dir()
-        for name in os.listdir(keys_dir):
-            filename = os.path.join(keys_dir, name)
-            with open(filename, 'r') as fd:
-                line = fd.readline().strip()
-                if line and line.startswith(cls.FILE_ID) and line[len(cls.FILE_ID)+1:] == id:
-                    return filename
-        raise KeyfileNotFoundError(repository._location.canonical_path(), get_keys_dir())
+    def find_key(self):
+        raise NotImplementedError
+
+    def load(self, target, passphrase):
+        raise NotImplementedError
 
-    def load(self, filename, passphrase):
-        with open(filename, 'r') as fd:
-            cdata = a2b_base64(''.join(fd.readlines()[1:]).encode('ascii'))  # .encode needed for Python 3.[0-2]
+    def _load(self, key_data, passphrase):
+        cdata = a2b_base64(key_data.encode('ascii'))  # .encode needed for Python 3.[0-2]
         data = self.decrypt_key_file(cdata, passphrase)
         if data:
             key = msgpack.unpackb(data)
@@ -249,23 +280,22 @@ class KeyfileKey(AESKeyBase):
             self.enc_hmac_key = key[b'enc_hmac_key']
             self.id_key = key[b'id_key']
             self.chunk_seed = key[b'chunk_seed']
-            self.path = filename
             return True
+        return False
 
     def decrypt_key_file(self, data, passphrase):
         d = msgpack.unpackb(data)
         assert d[b'version'] == 1
         assert d[b'algorithm'] == b'sha256'
-        key = pbkdf2_sha256(passphrase.encode('utf-8'), d[b'salt'], d[b'iterations'], 32)
+        key = passphrase.kdf(d[b'salt'], d[b'iterations'], 32)
         data = AES(is_encrypt=False, key=key).decrypt(d[b'data'])
-        if HMAC(key, data, sha256).digest() != d[b'hash']:
-            return None
-        return data
+        if HMAC(key, data, sha256).digest() == d[b'hash']:
+            return data
 
     def encrypt_key_file(self, data, passphrase):
         salt = get_random_bytes(32)
         iterations = 100000
-        key = pbkdf2_sha256(passphrase.encode('utf-8'), salt, iterations, 32)
+        key = passphrase.kdf(salt, iterations, 32)
         hash = HMAC(key, data, sha256).digest()
         cdata = AES(is_encrypt=True, key=key).encrypt(data)
         d = {
@@ -278,7 +308,7 @@ class KeyfileKey(AESKeyBase):
         }
         return msgpack.packb(d)
 
-    def save(self, path, passphrase):
+    def _save(self, passphrase):
         key = {
             'version': 1,
             'repository_id': self.repository_id,
@@ -288,45 +318,101 @@ class KeyfileKey(AESKeyBase):
             'chunk_seed': self.chunk_seed,
         }
         data = self.encrypt_key_file(msgpack.packb(key), passphrase)
-        with open(path, 'w') as fd:
-            fd.write('%s %s\n' % (self.FILE_ID, hexlify(self.repository_id).decode('ascii')))
-            fd.write('\n'.join(textwrap.wrap(b2a_base64(data).decode('ascii'))))
-            fd.write('\n')
-        self.path = path
+        key_data = '\n'.join(textwrap.wrap(b2a_base64(data).decode('ascii')))
+        return key_data
 
     def change_passphrase(self):
-        passphrase, passphrase2 = 1, 2
-        while passphrase != passphrase2:
-            passphrase = getpass('New passphrase: ')
-            passphrase2 = getpass('Enter same passphrase again: ')
-            if passphrase != passphrase2:
-                print('Passphrases do not match')
-        self.save(self.path, passphrase)
-        print('Key file "%s" updated' % self.path)
+        passphrase = Passphrase.new(allow_empty=True)
+        self.save(self.target, passphrase)
+        print('Key updated')
 
     @classmethod
     def create(cls, repository, args):
+        passphrase = Passphrase.new(allow_empty=True)
+        key = cls(repository)
+        key.repository_id = repository.id
+        key.init_from_random_data(get_random_bytes(100))
+        key.init_ciphers()
+        target = key.get_new_target(args)
+        key.save(target, passphrase)
+        print('Key in "%s" created.' % target)
+        print('Keep this key safe. Your data will be inaccessible without it.')
+        return key
+
+    def save(self, target, passphrase):
+        raise NotImplementedError
+
+    def get_new_target(self, args):
+        raise NotImplementedError
+
+
+class KeyfileKey(KeyfileKeyBase):
+    TYPE = 0x00
+    FILE_ID = 'BORG_KEY'
+
+    def find_key(self):
+        id = hexlify(self.repository.id).decode('ascii')
+        keys_dir = get_keys_dir()
+        for name in os.listdir(keys_dir):
+            filename = os.path.join(keys_dir, name)
+            with open(filename, 'r') as fd:
+                line = fd.readline().strip()
+                if line.startswith(self.FILE_ID) and line[len(self.FILE_ID)+1:] == id:
+                    return filename
+        raise KeyfileNotFoundError(self.repository._location.canonical_path(), get_keys_dir())
+
+    def get_new_target(self, args):
         filename = args.repository.to_key_filename()
         path = filename
         i = 1
         while os.path.exists(path):
             i += 1
             path = filename + '.%d' % i
-        passphrase = os.environ.get('BORG_PASSPHRASE')
-        if passphrase is not None:
-            passphrase2 = passphrase
-        else:
-            passphrase, passphrase2 = 1, 2
-        while passphrase != passphrase2:
-            passphrase = getpass('Enter passphrase (empty for no passphrase):')
-            passphrase2 = getpass('Enter same passphrase again: ')
-            if passphrase != passphrase2:
-                print('Passphrases do not match')
-        key = cls()
-        key.repository_id = repository.id
-        key.init_from_random_data(get_random_bytes(100))
-        key.init_ciphers()
-        key.save(path, passphrase)
-        print('Key file "%s" created.' % key.path)
-        print('Keep this file safe. Your data will be inaccessible without it.')
-        return key
+        return path
+
+    def load(self, target, passphrase):
+        with open(target, 'r') as fd:
+            key_data = ''.join(fd.readlines()[1:])
+        success = self._load(key_data, passphrase)
+        if success:
+            self.target = target
+        return success
+
+    def save(self, target, passphrase):
+        key_data = self._save(passphrase)
+        with open(target, 'w') as fd:
+            fd.write('%s %s\n' % (self.FILE_ID, hexlify(self.repository_id).decode('ascii')))
+            fd.write(key_data)
+            fd.write('\n')
+        self.target = target
+
+
+class RepoKey(KeyfileKeyBase):
+    TYPE = 0x03
+
+    def find_key(self):
+        loc = self.repository._location.canonical_path()
+        try:
+            self.repository.load_key()
+            return loc
+        except configparser.NoOptionError:
+            raise RepoKeyNotFoundError(loc)
+
+    def get_new_target(self, args):
+        return self.repository
+
+    def load(self, target, passphrase):
+        # what we get in target is just a repo location, but we already have the repo obj:
+        target = self.repository
+        key_data = target.load_key()
+        key_data = key_data.decode('utf-8')  # remote repo: msgpack issue #99, getting bytes
+        success = self._load(key_data, passphrase)
+        if success:
+            self.target = target
+        return success
+
+    def save(self, target, passphrase):
+        key_data = self._save(passphrase)
+        key_data = key_data.encode('utf-8')  # remote repo: msgpack issue #99, giving bytes
+        target.save_key(key_data)
+        self.target = target

borg/remote.py

@@ -41,6 +41,8 @@ class RepositoryServer:
         'put',
         'repair',
         'rollback',
+        'save_key',
+        'load_key',
     )
 
     def __init__(self, restrict_to_paths):
@@ -151,6 +153,9 @@ class RemoteRepository:
     def __del__(self):
         self.close()
 
+    def __repr__(self):
+        return '<%s %s>' % (self.__class__.__name__, self.location.canonical_path())
+
     def call(self, cmd, *args, **kw):
         for resp in self.call_many(cmd, [args], **kw):
             return resp
@@ -276,6 +281,12 @@ class RemoteRepository:
     def delete(self, id_, wait=True):
         return self.call('delete', id_, wait=wait)
 
+    def save_key(self, keydata):
+        return self.call('save_key', keydata)
+
+    def load_key(self):
+        return self.call('load_key')
+
     def close(self):
         if self.p:
             self.p.stdin.close()

borg/repository.py

@@ -62,6 +62,9 @@ class Repository:
     def __del__(self):
         self.close()
 
+    def __repr__(self):
+        return '<%s %s>' % (self.__class__.__name__, self.path)
+
     def create(self, path):
         """Create a new empty repository at `path`
         """
@@ -78,9 +81,23 @@ class Repository:
         config.set('repository', 'segments_per_dir', self.DEFAULT_SEGMENTS_PER_DIR)
         config.set('repository', 'max_segment_size', self.DEFAULT_MAX_SEGMENT_SIZE)
         config.set('repository', 'id', hexlify(os.urandom(32)).decode('ascii'))
-        with open(os.path.join(path, 'config'), 'w') as fd:
+        self.save_config(path, config)
+
+    def save_config(self, path, config):
+        config_path = os.path.join(path, 'config')
+        with open(config_path, 'w') as fd:
             config.write(fd)
 
+    def save_key(self, keydata):
+        assert self.config
+        keydata = keydata.decode('utf-8')  # remote repo: msgpack issue #99, getting bytes
+        self.config.set('repository', 'key', keydata)
+        self.save_config(self.path, self.config)
+
+    def load_key(self):
+        keydata = self.config.get('repository', 'key')
+        return keydata.encode('utf-8')  # remote repo: msgpack issue #99, returning bytes
+
     def destroy(self):
         """Destroy the repository at `self.path`
         """

borg/testsuite/archive.py

@@ -23,7 +23,7 @@ class ArchiveTimestampTestCase(BaseTestCase):
 
     def _test_timestamp_parsing(self, isoformat, expected):
         repository = Mock()
-        key = PlaintextKey()
+        key = PlaintextKey(repository)
         manifest = Manifest(repository, key)
         a = Archive(repository, key, manifest, 'test', create=True)
         a.metadata = {b'time': isoformat}
@@ -45,7 +45,7 @@ class ChunkBufferTestCase(BaseTestCase):
     def test(self):
         data = [{b'foo': 1}, {b'bar': 2}]
         cache = MockCache()
-        key = PlaintextKey()
+        key = PlaintextKey(None)
         chunks = CacheChunkBuffer(cache, key, None)
         for d in data:
             chunks.add(d)

docs/usage.rst

@@ -78,8 +78,43 @@ Examples
     # Remote repository (accesses a remote borg via ssh)
     $ borg init user@hostname:backup
 
-    # Encrypted remote repository
-    $ borg init --encryption=passphrase user@hostname:backup
+    # Encrypted remote repository, store the key in the repo
+    $ borg init --encryption=repokey user@hostname:backup
+
+    # Encrypted remote repository, store the key your home dir
+    $ borg init --encryption=keyfile user@hostname:backup
+
+Important notes about encryption:
+
+Use encryption! Repository encryption protects you e.g. against the case that
+an attacker has access to your backup repository.
+
+But be careful with the key / the passphrase:
+
+``--encryption=passphrase`` is DEPRECATED and will be removed in next major release.
+This mode has very fundamental, unfixable problems (like you can never change
+your passphrase or the pbkdf2 iteration count for an existing repository, because
+the encryption / decryption key is directly derived from the passphrase).
+
+If you want "passphrase-only" security, just use the ``repokey`` mode. The key will
+be stored inside the repository (in its "config" file). In above mentioned
+attack scenario, the attacker will have the key (but not the passphrase).
+
+If you want "passphrase and having-the-key" security, use the ``keyfile`` mode.
+The key will be stored in your home directory (in ``.borg/keys``). In the attack
+scenario, the attacker who has just access to your repo won't have the key (and
+also not the passphrase).
+
+Make a backup copy of the key file (``keyfile`` mode) or repo config file
+(``repokey`` mode) and keep it at a safe place, so you still have the key in
+case it gets corrupted or lost.
+The backup that is encrypted with that key won't help you with that, of course.
+
+Make sure you use a good passphrase. Not too short, not too simple. The real
+encryption / decryption key is encrypted with / locked by your passphrase.
+If an attacker gets your key, he can't unlock and use it without knowing the
+passphrase. In ``repokey`` and ``keyfile`` modes, you can change your passphrase
+for existing repos.
 
 
 .. include:: usage/create.rst.inc