move converter code out of test suite

borg/converter.py

@@ -0,0 +1,147 @@
+from binascii import hexlify
+import os
+
+from .helpers import IntegrityError, get_keys_dir
+from .repository import Repository, MAGIC
+from .key import KeyfileKey, KeyfileNotFoundError
+
+class NotImplementedException(Exception):
+    pass
+
+class AtticRepositoryConverter(Repository):
+    def convert(self):
+        '''convert an attic repository to a borg repository
+
+        those are the files that need to be converted here, from most
+        important to least important: segments, key files, and various
+        caches, the latter being optional, as they will be rebuilt if
+        missing.'''
+        print("reading segments from attic repository using borg")
+        segments = [ filename for i, filename in self.io.segment_iterator() ]
+        try:
+            keyfile = self.find_attic_keyfile()
+        except KeyfileNotFoundError:
+            print("no key file found for repository")
+        else:
+            self.convert_keyfiles(keyfile)
+        self.close()
+        self.convert_segments(segments)
+        self.convert_cache()
+
+    @staticmethod
+    def convert_segments(segments):
+        '''convert repository segments from attic to borg
+
+        replacement pattern is `s/ATTICSEG/BORG_SEG/` in files in
+        `$ATTIC_REPO/data/**`.
+
+        luckily the segment length didn't change so we can just
+        replace the 8 first bytes of all regular files in there.'''
+        for filename in segments:
+            print("converting segment %s in place" % filename)
+            with open(filename, 'r+b') as segment:
+                segment.seek(0)
+                segment.write(MAGIC)
+
+    def find_attic_keyfile(self):
+        '''find the attic keyfiles
+
+        the keyfiles are loaded by `KeyfileKey.find_key_file()`. that
+        finds the keys with the right identifier for the repo
+
+        this is expected to look into $HOME/.attic/keys or
+        $ATTIC_KEYS_DIR for key files matching the given Borg
+        repository.
+
+        it is expected to raise an exception (KeyfileNotFoundError) if
+        no key is found. whether that exception is from Borg or Attic
+        is unclear.
+
+        this is split in a separate function in case we want to use
+        the attic code here directly, instead of our local
+        implementation.'''
+        return AtticKeyfileKey.find_key_file(self)
+
+    @staticmethod
+    def convert_keyfiles(keyfile):
+
+        '''convert key files from attic to borg
+
+        replacement pattern is `s/ATTIC KEY/BORG_KEY/` in
+        `get_keys_dir()`, that is `$ATTIC_KEYS_DIR` or
+        `$HOME/.attic/keys`, and moved to `$BORG_KEYS_DIR` or
+        `$HOME/.borg/keys`.
+
+        no need to decrypt to convert. we need to rewrite the whole
+        key file because magic number length changed, but that's not a
+        problem because the keyfiles are small (compared to, say,
+        all the segments).'''
+        print("converting keyfile %s" % keyfile)
+        with open(keyfile, 'r') as f:
+            data = f.read()
+        data = data.replace(AtticKeyfileKey.FILE_ID,
+                            KeyfileKey.FILE_ID,
+                            1)
+        keyfile = os.path.join(get_keys_dir(),
+                               os.path.basename(keyfile))
+        print("writing borg keyfile to %s" % keyfile)
+        with open(keyfile, 'w') as f:
+            f.write(data)
+        with open(keyfile, 'r') as f:
+            data = f.read()
+        assert data.startswith(KeyfileKey.FILE_ID)
+
+    def convert_cache(self):
+        '''convert caches from attic to borg
+
+        those are all hash indexes, so we need to
+        `s/ATTICIDX/BORG_IDX/` in a few locations:
+        
+        * the repository index (in `$ATTIC_REPO/index.%d`, where `%d`
+          is the `Repository.get_index_transaction_id()`), which we
+          should probably update, with a lock, see
+          `Repository.open()`, which i'm not sure we should use
+          because it may write data on `Repository.close()`...
+
+        * the `files` and `chunks` cache (in
+          `$HOME/.cache/attic/<repoid>/`), which we could just drop,
+          but if we'd want to convert, we could open it with the
+          `Cache.open()`, edit in place and then `Cache.close()` to
+          make sure we have locking right
+        '''
+        raise NotImplementedException('not implemented')
+
+class AtticKeyfileKey(KeyfileKey):
+    '''backwards compatible Attick key file parser'''
+    FILE_ID = 'ATTIC KEY'
+
+    # verbatim copy from attic
+    @staticmethod
+    def get_keys_dir():
+        """Determine where to repository keys and cache"""
+        return os.environ.get('ATTIC_KEYS_DIR',
+                              os.path.join(os.path.expanduser('~'), '.attic', 'keys'))
+
+    @classmethod
+    def find_key_file(cls, repository):
+        '''copy of attic's `find_key_file`_
+
+        this has two small modifications:
+
+        1. it uses the above `get_keys_dir`_ instead of the global one,
+           assumed to be borg's
+
+        2. it uses `repository.path`_ instead of
+           `repository._location.canonical_path`_ because we can't
+           assume the repository has been opened by the archiver yet
+        '''
+        get_keys_dir = cls.get_keys_dir
+        id = hexlify(repository.id).decode('ascii')
+        keys_dir = get_keys_dir()
+        for name in os.listdir(keys_dir):
+            filename = os.path.join(keys_dir, name)
+            with open(filename, 'r') as fd:
+                line = fd.readline().strip()
+                if line and line.startswith(cls.FILE_ID) and line[10:] == id:
+                    return filename
+        raise KeyfileNotFoundError(repository.path, get_keys_dir())

borg/testsuite/convert.py

@@ -1,4 +1,3 @@
-from binascii import hexlify
 import os
 import pytest
 import shutil
@@ -8,14 +7,12 @@ import attic.repository
 import attic.key
 import attic.helpers
 
-from ..helpers import IntegrityError, get_keys_dir
+from ..converter import AtticRepositoryConverter, NotImplementedException
+from ..helpers import get_keys_dir
+from ..key import KeyfileKey
 from ..repository import Repository, MAGIC
-from ..key import KeyfileKey, KeyfileNotFoundError
 from . import BaseTestCase
 
-class NotImplementedException(Exception):
-    pass
-
 class ConversionTestCase(BaseTestCase):
 
     class MockArgs:
@@ -56,7 +53,8 @@ class ConversionTestCase(BaseTestCase):
         assert not self.repository.check() # can't check raises() because check() handles the error
         self.repository.close()
         print("opening attic repository with borg and converting")
-        self.open(self.tmppath, repo_type = AtticRepositoryConverter).convert()
+        with pytest.raises(NotImplementedException):
+            self.open(self.tmppath, repo_type = AtticRepositoryConverter).convert()
         # check that the new keyfile is alright
         keyfile = os.path.join(get_keys_dir(),
                                os.path.basename(self.key.path))
@@ -65,142 +63,3 @@ class ConversionTestCase(BaseTestCase):
         self.repository = self.open(self.tmppath)
         assert self.repository.check()
         self.repository.close()
-
-class AtticRepositoryConverter(Repository):
-    def convert(self):
-        '''convert an attic repository to a borg repository
-
-        those are the files that need to be converted here, from most
-        important to least important: segments, key files, and various
-        caches, the latter being optional, as they will be rebuilt if
-        missing.'''
-        print("reading segments from attic repository using borg")
-        segments = [ filename for i, filename in self.io.segment_iterator() ]
-        try:
-            keyfile = self.find_attic_keyfile()
-        except KeyfileNotFoundError:
-            print("no key file found for repository")
-        else:
-            self.convert_keyfiles(keyfile)
-        self.close()
-        self.convert_segments(segments)
-        with pytest.raises(NotImplementedException):
-            self.convert_cache()
-
-    @staticmethod
-    def convert_segments(segments):
-        '''convert repository segments from attic to borg
-
-        replacement pattern is `s/ATTICSEG/BORG_SEG/` in files in
-        `$ATTIC_REPO/data/**`.
-
-        luckily the segment length didn't change so we can just
-        replace the 8 first bytes of all regular files in there.'''
-        for filename in segments:
-            print("converting segment %s in place" % filename)
-            with open(filename, 'r+b') as segment:
-                segment.seek(0)
-                segment.write(MAGIC)
-
-    def find_attic_keyfile(self):
-        '''find the attic keyfiles
-
-        the keyfiles are loaded by `KeyfileKey.find_key_file()`. that
-        finds the keys with the right identifier for the repo
-
-        this is expected to look into $HOME/.attic/keys or
-        $ATTIC_KEYS_DIR for key files matching the given Borg
-        repository.
-
-        it is expected to raise an exception (KeyfileNotFoundError) if
-        no key is found. whether that exception is from Borg or Attic
-        is unclear.
-
-        this is split in a separate function in case we want to use
-        the attic code here directly, instead of our local
-        implementation.'''
-        return AtticKeyfileKey.find_key_file(self)
-
-    @staticmethod
-    def convert_keyfiles(keyfile):
-
-        '''convert key files from attic to borg
-
-        replacement pattern is `s/ATTIC KEY/BORG_KEY/` in
-        `get_keys_dir()`, that is `$ATTIC_KEYS_DIR` or
-        `$HOME/.attic/keys`, and moved to `$BORG_KEYS_DIR` or
-        `$HOME/.borg/keys`.
-
-        no need to decrypt to convert. we need to rewrite the whole
-        key file because magic number length changed, but that's not a
-        problem because the keyfiles are small (compared to, say,
-        all the segments).'''
-        print("converting keyfile %s" % keyfile)
-        with open(keyfile, 'r') as f:
-            data = f.read()
-        data = data.replace(AtticKeyfileKey.FILE_ID,
-                            KeyfileKey.FILE_ID,
-                            1)
-        keyfile = os.path.join(get_keys_dir(),
-                               os.path.basename(keyfile))
-        print("writing borg keyfile to %s" % keyfile)
-        with open(keyfile, 'w') as f:
-            f.write(data)
-        with open(keyfile, 'r') as f:
-            data = f.read()
-        assert data.startswith(KeyfileKey.FILE_ID)
-
-    def convert_cache(self):
-        '''convert caches from attic to borg
-
-        those are all hash indexes, so we need to
-        `s/ATTICIDX/BORG_IDX/` in a few locations:
-        
-        * the repository index (in `$ATTIC_REPO/index.%d`, where `%d`
-          is the `Repository.get_index_transaction_id()`), which we
-          should probably update, with a lock, see
-          `Repository.open()`, which i'm not sure we should use
-          because it may write data on `Repository.close()`...
-
-        * the `files` and `chunks` cache (in
-          `$HOME/.cache/attic/<repoid>/`), which we could just drop,
-          but if we'd want to convert, we could open it with the
-          `Cache.open()`, edit in place and then `Cache.close()` to
-          make sure we have locking right
-        '''
-        raise NotImplementedException('not implemented')
-
-class AtticKeyfileKey(KeyfileKey):
-    '''backwards compatible Attick key file parser'''
-    FILE_ID = 'ATTIC KEY'
-
-    # verbatim copy from attic
-    @staticmethod
-    def get_keys_dir():
-        """Determine where to repository keys and cache"""
-        return os.environ.get('ATTIC_KEYS_DIR',
-                              os.path.join(os.path.expanduser('~'), '.attic', 'keys'))
-
-    @classmethod
-    def find_key_file(cls, repository):
-        '''copy of attic's `find_key_file`_
-
-        this has two small modifications:
-
-        1. it uses the above `get_keys_dir`_ instead of the global one,
-           assumed to be borg's
-
-        2. it uses `repository.path`_ instead of
-           `repository._location.canonical_path`_ because we can't
-           assume the repository has been opened by the archiver yet
-        '''
-        get_keys_dir = cls.get_keys_dir
-        id = hexlify(repository.id).decode('ascii')
-        keys_dir = get_keys_dir()
-        for name in os.listdir(keys_dir):
-            filename = os.path.join(keys_dir, name)
-            with open(filename, 'r') as fd:
-                line = fd.readline().strip()
-                if line and line.startswith(cls.FILE_ID) and line[10:] == id:
-                    return filename
-        raise KeyfileNotFoundError(repository.path, get_keys_dir())