create crypto package with key, keymanager, low_level

.gitignore

@@ -7,7 +7,7 @@ borg-env
 hashindex.c
 chunker.c
 compress.c
-crypto.c
+low_level.c
 item.c
 src/borg/algorithms/crc32.c
 src/borg/platform/darwin.c

setup.py

@@ -50,7 +50,7 @@ from setuptools import setup, find_packages, Extension
 from setuptools.command.sdist import sdist
 
 compress_source = 'src/borg/compress.pyx'
-crypto_source = 'src/borg/crypto.pyx'
+crypto_ll_source = 'src/borg/crypto/low_level.pyx'
 chunker_source = 'src/borg/algorithms/chunker.pyx'
 hashindex_source = 'src/borg/hashindex.pyx'
 item_source = 'src/borg/item.pyx'
@@ -62,7 +62,7 @@ platform_freebsd_source = 'src/borg/platform/freebsd.pyx'
 
 cython_sources = [
     compress_source,
-    crypto_source,
+    crypto_ll_source,
     chunker_source,
     hashindex_source,
     item_source,
@@ -87,7 +87,7 @@ try:
         def make_distribution(self):
             self.filelist.extend([
                 'src/borg/compress.c',
-                'src/borg/crypto.c',
+                'src/borg/crypto/low_level.c',
                 'src/borg/algorithms/chunker.c', 'src/borg/algorithms/buzhash.c',
                 'src/borg/hashindex.c', 'src/borg/_hashindex.c',
                 'src/borg/item.c',
@@ -106,7 +106,7 @@ except ImportError:
             raise Exception('Cython is required to run sdist')
 
     compress_source = compress_source.replace('.pyx', '.c')
-    crypto_source = crypto_source.replace('.pyx', '.c')
+    crypto_ll_source = crypto_ll_source.replace('.pyx', '.c')
     chunker_source = chunker_source.replace('.pyx', '.c')
     hashindex_source = hashindex_source.replace('.pyx', '.c')
     item_source = item_source.replace('.pyx', '.c')
@@ -117,7 +117,7 @@ except ImportError:
     platform_darwin_source = platform_darwin_source.replace('.pyx', '.c')
     from distutils.command.build_ext import build_ext
     if not on_rtd and not all(os.path.exists(path) for path in [
-        compress_source, crypto_source, chunker_source, hashindex_source, item_source, crc32_source,
+        compress_source, crypto_ll_source, chunker_source, hashindex_source, item_source, crc32_source,
         platform_posix_source, platform_linux_source, platform_freebsd_source, platform_darwin_source]):
         raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.')
 
@@ -578,7 +578,8 @@ ext_modules = []
 if not on_rtd:
     ext_modules += [
     Extension('borg.compress', [compress_source], libraries=['lz4'], include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
-    Extension('borg.crypto', [crypto_source], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
+    Extension('borg.crypto', [crypto_ll_source], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
+    Extension('borg.crypto.low_level', [crypto_ll_source], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
     Extension('borg.hashindex', [hashindex_source]),
     Extension('borg.item', [item_source]),
     Extension('borg.algorithms.chunker', [chunker_source]),

src/borg/archive.py

@@ -20,8 +20,9 @@ from .logger import create_logger
 logger = create_logger()
 
 from . import xattr
-from .cache import ChunkListEntry
 from .algorithms.chunker import Chunker
+from .cache import ChunkListEntry
+from .crypto.key import key_factory
 from .compress import Compressor, CompressionSpec
 from .constants import *  # NOQA
 from .hashindex import ChunkIndex, ChunkIndexEntry
@@ -39,7 +40,6 @@ from .helpers import safe_ns
 from .helpers import ellipsis_truncate, ProgressIndicatorPercent, log_multi
 from .patterns import PathPrefixPattern, FnmatchPattern, IECommand
 from .item import Item, ArchiveItem
-from .key import key_factory
 from .platform import acl_get, acl_set, set_flags, get_flags, swidth
 from .remote import cache_if_remote
 from .repository import Repository, LIST_SCAN_LIMIT

src/borg/archiver.py

@@ -24,6 +24,7 @@ from datetime import datetime, timedelta
 from itertools import zip_longest
 
 from .logger import create_logger, setup_logging
+
 logger = create_logger()
 
 import msgpack
@@ -37,15 +38,17 @@ from .archive import BackupOSError, backup_io
 from .cache import Cache
 from .constants import *  # NOQA
 from .compress import CompressionSpec
+from .crypto.key import key_creator, tam_required_file, tam_required, RepoKey, PassphraseKey
+from .crypto.keymanager import KeyManager
 from .helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
 from .helpers import Error, NoManifestError, set_ec
 from .helpers import location_validator, archivename_validator, ChunkerParams
 from .helpers import PrefixSpec, SortBySpec, HUMAN_SORT_KEYS
 from .helpers import BaseFormatter, ItemFormatter, ArchiveFormatter
-from .helpers import format_time, format_timedelta, format_file_size, format_archive
+from .helpers import format_timedelta, format_file_size, format_archive
 from .helpers import safe_encode, remove_surrogates, bin_to_hex, prepare_dump_dict
 from .helpers import prune_within, prune_split
-from .helpers import to_localtime, timestamp
+from .helpers import timestamp
 from .helpers import get_cache_dir
 from .helpers import Manifest
 from .helpers import hardlinkable
@@ -61,8 +64,6 @@ from .helpers import replace_placeholders
 from .patterns import ArgparsePatternAction, ArgparseExcludeFileAction, ArgparsePatternFileAction, parse_exclude_pattern
 from .patterns import PatternMatcher
 from .item import Item
-from .key import key_creator, tam_required_file, tam_required, RepoKey, PassphraseKey
-from .keymanager import KeyManager
 from .platform import get_flags, umount, get_process_id, SyncFile
 from .remote import RepositoryServer, RemoteRepository, cache_if_remote
 from .repository import Repository, LIST_SCAN_LIMIT

src/borg/cache.py

@@ -1,13 +1,14 @@
 import configparser
 import os
-import stat
 import shutil
+import stat
 from binascii import unhexlify
 from collections import namedtuple
 
 import msgpack
 
 from .logger import create_logger
+
 logger = create_logger()
 
 from .constants import CACHE_README
@@ -21,8 +22,8 @@ from .helpers import safe_ns
 from .helpers import yes, hostname_is_unique
 from .helpers import remove_surrogates
 from .helpers import ProgressIndicatorPercent, ProgressIndicatorMessage
-from .item import Item, ArchiveItem, ChunkListEntry
-from .key import PlaintextKey
+from .item import ArchiveItem, ChunkListEntry
+from .crypto.key import PlaintextKey
 from .locking import Lock
 from .platform import SaveFile
 from .remote import cache_if_remote

src/borg/key.py → src/borg/crypto/key.py

@@ -3,27 +3,27 @@ import getpass
 import os
 import sys
 import textwrap
-from binascii import a2b_base64, b2a_base64, hexlify, unhexlify
+from binascii import a2b_base64, b2a_base64, hexlify
 from hashlib import sha256, sha512, pbkdf2_hmac
 from hmac import HMAC, compare_digest
 
 import msgpack
 
-from .logger import create_logger
+from borg.logger import create_logger
+
 logger = create_logger()
 
-from .constants import *  # NOQA
-from .compress import Compressor
-from .crypto import AES, bytes_to_long, bytes_to_int, num_aes_blocks, hmac_sha256, blake2b_256, hkdf_hmac_sha512
-from .helpers import StableDict
-from .helpers import Error, IntegrityError
-from .helpers import yes
-from .helpers import get_keys_dir, get_security_dir
-from .helpers import bin_to_hex
-from .item import Key, EncryptedKey
-from .platform import SaveFile
+from ..constants import *  # NOQA
+from ..compress import Compressor
+from ..helpers import StableDict
+from ..helpers import Error, IntegrityError
+from ..helpers import yes
+from ..helpers import get_keys_dir, get_security_dir
+from ..helpers import bin_to_hex
+from ..item import Key, EncryptedKey
+from ..platform import SaveFile
 from .nonces import NonceManager
-
+from .low_level import AES, bytes_to_long, bytes_to_int, num_aes_blocks, hmac_sha256, blake2b_256, hkdf_hmac_sha512
 
 PREFIX = b'\0' * 8
 

src/borg/keymanager.py → src/borg/crypto/keymanager.py

@@ -1,12 +1,13 @@
-from binascii import unhexlify, a2b_base64, b2a_base64
 import binascii
+import pkgutil
 import textwrap
+from binascii import unhexlify, a2b_base64, b2a_base64
 from hashlib import sha256
-import pkgutil
+
+from ..helpers import Manifest, NoManifestError, Error, yes, bin_to_hex
+from ..repository import Repository
 
 from .key import KeyfileKey, KeyfileNotFoundError, KeyBlobStorage, identify_key
-from .helpers import Manifest, NoManifestError, Error, yes, bin_to_hex
-from .repository import Repository
 
 
 class UnencryptedRepo(Error):

src/borg/crypto.pyx → src/borg/crypto/low_level.pyx

@@ -10,7 +10,7 @@ from cpython.buffer cimport PyBUF_SIMPLE, PyObject_GetBuffer, PyBuffer_Release
 API_VERSION = '1.1_01'
 
 
-cdef extern from "algorithms/blake2-libselect.h":
+cdef extern from "../algorithms/blake2-libselect.h":
     ctypedef struct blake2b_state:
         pass
 

src/borg/nonces.py → src/borg/crypto/nonces.py

@@ -2,12 +2,12 @@ import os
 import sys
 from binascii import unhexlify
 
-from .crypto import bytes_to_long, long_to_bytes
-from .helpers import get_security_dir
-from .helpers import bin_to_hex
-from .platform import SaveFile
-from .remote import InvalidRPCMethod
+from ..helpers import get_security_dir
+from ..helpers import bin_to_hex
+from ..platform import SaveFile
+from ..remote import InvalidRPCMethod
 
+from .low_level import bytes_to_long, long_to_bytes
 
 MAX_REPRESENTABLE_NONCE = 2**64 - 1
 NONCE_SPACE_RESERVATION = 2**28  # This in units of AES blocksize (16 bytes)

src/borg/helpers.py

@@ -35,9 +35,9 @@ from .logger import create_logger
 
 logger = create_logger()
 
+import borg.crypto.low_level
 from . import __version__ as borg_version
 from . import __version_tuple__ as borg_version_tuple
-from . import crypto
 from . import hashindex
 from .algorithms import chunker
 from .constants import *  # NOQA
@@ -120,7 +120,7 @@ def check_extension_modules():
         raise ExtensionModuleError
     if compress.API_VERSION != '1.1_03':
         raise ExtensionModuleError
-    if crypto.API_VERSION != '1.1_01':
+    if borg.crypto.low_level.API_VERSION != '1.1_01':
         raise ExtensionModuleError
     if platform.API_VERSION != platform.OS_API_VERSION != '1.1_01':
         raise ExtensionModuleError
@@ -233,7 +233,7 @@ class Manifest:
     @classmethod
     def load(cls, repository, key=None, force_tam_not_required=False):
         from .item import ManifestItem
-        from .key import key_factory, tam_required_file, tam_required
+        from .crypto.key import key_factory, tam_required_file, tam_required
         from .repository import Repository
         try:
             cdata = repository.get(cls.MANIFEST_ID)

src/borg/testsuite/archive.py

@@ -1,18 +1,17 @@
-import os
 from collections import OrderedDict
 from datetime import datetime, timezone
 from io import StringIO
 from unittest.mock import Mock
 
-import pytest
 import msgpack
+import pytest
 
+from . import BaseTestCase
+from ..crypto.key import PlaintextKey
 from ..archive import Archive, CacheChunkBuffer, RobustUnpacker, valid_msgpacked_dict, ITEM_KEYS, Statistics
 from ..archive import BackupOSError, backup_io, backup_io_iter
-from ..item import Item, ArchiveItem
-from ..key import PlaintextKey
 from ..helpers import Manifest
-from . import BaseTestCase
+from ..item import Item, ArchiveItem
 
 
 @pytest.fixture()

src/borg/testsuite/archiver.py

@@ -1,38 +1,40 @@
-from binascii import unhexlify, b2a_base64
-from configparser import ConfigParser
 import errno
-import os
-import inspect
 import json
-from datetime import datetime
-from datetime import timedelta
-from io import StringIO
 import logging
+import os
 import random
+import shutil
 import socket
 import stat
 import subprocess
 import sys
-import shutil
 import tempfile
 import time
 import unittest
-from unittest.mock import patch
+from binascii import unhexlify, b2a_base64
+from configparser import ConfigParser
+from datetime import datetime
+from datetime import timedelta
 from hashlib import sha256
+from io import StringIO
+from unittest.mock import patch
 
 import msgpack
 import pytest
+
 try:
     import llfuse
 except ImportError:
     pass
 
 from .. import xattr, helpers, platform
-from ..archive import Archive, ChunkBuffer, ArchiveRecreater, flags_noatime, flags_normal
+from ..archive import Archive, ChunkBuffer, flags_noatime, flags_normal
 from ..archiver import Archiver
 from ..cache import Cache
 from ..constants import *  # NOQA
-from ..crypto import bytes_to_long, num_aes_blocks
+from ..crypto.low_level import bytes_to_long, num_aes_blocks
+from ..crypto.key import KeyfileKeyBase, RepoKey, KeyfileKey, Passphrase, TAMRequiredError
+from ..crypto.keymanager import RepoIdMismatch, NotABorgKeyFile
 from ..helpers import Location, get_security_dir
 from ..helpers import Manifest
 from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
@@ -40,8 +42,6 @@ from ..helpers import bin_to_hex
 from ..helpers import MAX_S
 from ..patterns import IECommand, PatternMatcher, parse_pattern
 from ..item import Item
-from ..key import KeyfileKeyBase, RepoKey, KeyfileKey, Passphrase, TAMRequiredError
-from ..keymanager import RepoIdMismatch, NotABorgKeyFile
 from ..remote import RemoteRepository, PathNotAllowed
 from ..repository import Repository
 from . import has_lchflags, has_llfuse

src/borg/testsuite/crypto.py

@@ -1,8 +1,8 @@
 from binascii import hexlify, unhexlify
 
-from ..crypto import AES, bytes_to_long, bytes_to_int, long_to_bytes, hmac_sha256, blake2b_256
-from ..crypto import increment_iv, bytes16_to_int, int_to_bytes16
-from ..crypto import hkdf_hmac_sha512
+from ..crypto.low_level import AES, bytes_to_long, bytes_to_int, long_to_bytes, hmac_sha256, blake2b_256
+from ..crypto.low_level import increment_iv, bytes16_to_int, int_to_bytes16
+from ..crypto.low_level import hkdf_hmac_sha512
 from . import BaseTestCase
 
 # Note: these tests are part of the self test, do not use or import py.test functionality here.

src/borg/testsuite/key.py

@@ -1,20 +1,21 @@
 import getpass
+import os.path
 import re
 import tempfile
-import os.path
 from binascii import hexlify, unhexlify
 
-import pytest
 import msgpack
+import pytest
 
-from ..crypto import bytes_to_long, num_aes_blocks
+from ..crypto.key import Passphrase, PasswordRetriesExceeded, bin_to_hex
+from ..crypto.key import PlaintextKey, PassphraseKey, KeyfileKey, RepoKey, Blake2KeyfileKey, Blake2RepoKey, \
+    AuthenticatedKey
+from ..crypto.key import TAMRequiredError, TAMInvalid, TAMUnsupportedSuiteError, UnsupportedManifestError
+from ..crypto.low_level import bytes_to_long, num_aes_blocks
+from ..helpers import IntegrityError
 from ..helpers import Location
 from ..helpers import StableDict
-from ..helpers import IntegrityError
 from ..helpers import get_security_dir
-from ..key import PlaintextKey, PassphraseKey, KeyfileKey, RepoKey, Blake2KeyfileKey, Blake2RepoKey, AuthenticatedKey
-from ..key import Passphrase, PasswordRetriesExceeded, bin_to_hex
-from ..key import TAMRequiredError, TAMInvalid, TAMUnsupportedSuiteError, UnsupportedManifestError
 
 
 class TestKey:

src/borg/testsuite/nonces.py

@@ -2,13 +2,12 @@ import os.path
 
 import pytest
 
+from ..crypto import nonces
+from ..crypto.nonces import NonceManager
+from ..crypto.key import bin_to_hex
 from ..helpers import get_security_dir
-from ..key import bin_to_hex
-from ..nonces import NonceManager
 from ..remote import InvalidRPCMethod
 
-from .. import nonces  # for monkey patching NONCE_SPACE_RESERVATION
-
 
 class TestNonceManager:
 

src/borg/testsuite/upgrader.py

@@ -10,9 +10,9 @@ except ImportError:
     attic = None
 
 from ..constants import *  # NOQA
+from ..crypto.key import KeyfileKey
 from ..upgrader import AtticRepositoryUpgrader, AtticKeyfileKey
 from ..helpers import get_keys_dir
-from ..key import KeyfileKey
 from ..repository import Repository
 from . import are_hardlinks_supported
 

src/borg/upgrader.py

@@ -3,13 +3,13 @@ import os
 import shutil
 import time
 
+from .crypto.key import KeyfileKey, KeyfileNotFoundError
 from .constants import REPOSITORY_README
-from .helpers import get_home_dir, get_keys_dir, get_cache_dir
 from .helpers import ProgressIndicatorPercent
-from .key import KeyfileKey, KeyfileNotFoundError
+from .helpers import get_home_dir, get_keys_dir, get_cache_dir
 from .locking import Lock
-from .repository import Repository, MAGIC
 from .logger import create_logger
+from .repository import Repository, MAGIC
 
 logger = create_logger(__name__)