Merge pull request #1112 from ThomasWaldmann/refactor-helper-imports

split helpers import into multiple ones

src/borg/archive.py

@@ -1,36 +1,42 @@
-from datetime import datetime, timezone
-from getpass import getuser
-from itertools import groupby
 import errno
-
-from .logger import create_logger
-logger = create_logger()
-from .key import key_factory
-from .remote import cache_if_remote
-
 import os
-from shutil import get_terminal_size
 import socket
 import stat
 import sys
 import time
+from datetime import datetime, timezone
+from getpass import getuser
 from io import BytesIO
+from itertools import groupby
+from shutil import get_terminal_size
+
+import msgpack
+
+from .logger import create_logger
+logger = create_logger()
+
 from . import xattr
-from .compress import COMPR_BUFFER
-from .constants import *  # NOQA
-from .helpers import Chunk, Error, uid2user, user2uid, gid2group, group2gid, \
-    parse_timestamp, to_localtime, format_time, format_timedelta, safe_encode, safe_decode, \
-    Manifest, decode_dict, make_path_safe, StableDict, int_to_bigint, bigint_to_int, bin_to_hex, \
-    ProgressIndicatorPercent, ChunkIteratorFileWrapper, remove_surrogates, log_multi, \
-    PathPrefixPattern, FnmatchPattern, open_item, file_status, format_file_size, consume, \
-    CompressionDecider1, CompressionDecider2, CompressionSpec, \
-    IntegrityError
-from .repository import Repository
-from .platform import acl_get, acl_set, set_flags, get_flags, swidth
+from .cache import ChunkListEntry
 from .chunker import Chunker
+from .constants import *  # NOQA
 from .hashindex import ChunkIndex, ChunkIndexEntry
-from .cache import ChunkListEntry
-import msgpack
+from .helpers import Manifest
+from .helpers import Chunk, ChunkIteratorFileWrapper, open_item
+from .helpers import Error, IntegrityError
+from .helpers import uid2user, user2uid, gid2group, group2gid
+from .helpers import parse_timestamp, to_localtime
+from .helpers import format_time, format_timedelta, format_file_size, file_status
+from .helpers import safe_encode, safe_decode, make_path_safe, remove_surrogates
+from .helpers import decode_dict, StableDict
+from .helpers import int_to_bigint, bigint_to_int, bin_to_hex
+from .helpers import ProgressIndicatorPercent, log_multi
+from .helpers import PathPrefixPattern, FnmatchPattern
+from .helpers import consume
+from .helpers import CompressionDecider1, CompressionDecider2, CompressionSpec
+from .key import key_factory
+from .platform import acl_get, acl_set, set_flags, get_flags, swidth
+from .remote import cache_if_remote
+from .repository import Repository
 
 has_lchmod = hasattr(os, 'lchmod')
 

src/borg/archiver.py

@@ -1,7 +1,3 @@
-from binascii import unhexlify
-from datetime import datetime
-from itertools import zip_longest
-from operator import attrgetter
 import argparse
 import collections
 import functools
@@ -18,28 +14,36 @@ import subprocess
 import sys
 import textwrap
 import traceback
+from binascii import unhexlify
+from datetime import datetime
+from itertools import zip_longest
 
-from . import __version__
-from .helpers import Error, location_validator, archivename_validator, format_time, format_file_size, \
-    parse_pattern, PathPrefixPattern, to_localtime, timestamp, \
-    get_cache_dir, prune_within, prune_split, bin_to_hex, safe_encode, \
-    Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, \
-    dir_is_tagged, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, sysinfo, \
-    log_multi, PatternMatcher, ItemFormatter
 from .logger import create_logger, setup_logging
 logger = create_logger()
+
+from . import __version__
 from . import helpers
-from .compress import Compressor, COMPR_BUFFER
-from .upgrader import AtticRepositoryUpgrader, BorgRepositoryUpgrader
-from .repository import Repository
+from .archive import Archive, ArchiveChecker, ArchiveRecreater, Statistics
 from .cache import Cache
 from .constants import *  # NOQA
+from .helpers import Error
+from .helpers import location_validator, archivename_validator, ChunkerParams, CompressionSpec
+from .helpers import ItemFormatter, format_time, format_file_size, format_archive
+from .helpers import safe_encode, remove_surrogates, bin_to_hex
+from .helpers import prune_within, prune_split
+from .helpers import to_localtime, timestamp
+from .helpers import get_cache_dir
+from .helpers import Manifest
+from .helpers import update_excludes, check_extension_modules
+from .helpers import dir_is_tagged, is_slow_msgpack, yes, sysinfo
+from .helpers import log_multi
+from .helpers import parse_pattern, PatternMatcher, PathPrefixPattern
 from .key import key_creator, RepoKey, PassphraseKey
-from .archive import Archive, ArchiveChecker, ArchiveRecreater, Statistics
+from .platform import get_flags
 from .remote import RepositoryServer, RemoteRepository, cache_if_remote
+from .repository import Repository
 from .selftest import selftest
-from .hashindex import ChunkIndexEntry
-from .platform import get_flags
+from .upgrader import AtticRepositoryUpgrader, BorgRepositoryUpgrader
 
 
 def argument(args, str_or_bool):

src/borg/cache.py

@@ -1,20 +1,24 @@
 import configparser
-from .remote import cache_if_remote
-from collections import namedtuple
 import os
 import stat
-from binascii import unhexlify
 import shutil
+from binascii import unhexlify
+from collections import namedtuple
+
+import msgpack
 
-from .key import PlaintextKey
 from .logger import create_logger
 logger = create_logger()
-from .helpers import Error, get_cache_dir, decode_dict, int_to_bigint, \
-    bigint_to_int, bin_to_hex, format_file_size, yes
-from .locking import UpgradableLock
-from .hashindex import ChunkIndex, ChunkIndexEntry
 
-import msgpack
+from .hashindex import ChunkIndex, ChunkIndexEntry
+from .helpers import Error
+from .helpers import get_cache_dir
+from .helpers import decode_dict, int_to_bigint, bigint_to_int, bin_to_hex
+from .helpers import format_file_size
+from .helpers import yes
+from .key import PlaintextKey
+from .locking import UpgradableLock
+from .remote import cache_if_remote
 
 ChunkListEntry = namedtuple('ChunkListEntry', 'id size csize')
 FileCacheEntry = namedtuple('FileCacheEntry', 'age inode size mtime chunk_ids')

src/borg/fuse.py

@@ -1,20 +1,23 @@
-from collections import defaultdict
 import errno
 import io
-import llfuse
 import os
 import stat
 import tempfile
 import time
-from .archive import Archive
-from .helpers import daemonize, bigint_to_int
-from .logger import create_logger
-from .lrucache import LRUCache
+from collections import defaultdict
 from distutils.version import LooseVersion
+
+import llfuse
 import msgpack
 
+from .logger import create_logger
 logger = create_logger()
 
+from .archive import Archive
+from .helpers import daemonize
+from .helpers import bigint_to_int
+from .lrucache import LRUCache
+
 # Does this version of llfuse support ns precision?
 have_fuse_xtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns')
 

src/borg/helpers.py

@@ -1,42 +1,40 @@
 import argparse
-from binascii import hexlify
-from collections import namedtuple, deque
-from functools import wraps, partial
 import grp
 import hashlib
-from itertools import islice
+import logging
 import os
 import os.path
-import stat
-import textwrap
+import platform
 import pwd
 import re
+import socket
 import sys
-from string import Formatter
-import platform
+import stat
+import textwrap
 import time
 import unicodedata
-import logging
-
-from .logger import create_logger
-logger = create_logger()
-
+from binascii import hexlify
+from collections import namedtuple, deque
 from datetime import datetime, timezone, timedelta
 from fnmatch import translate
+from functools import wraps, partial
+from itertools import islice
 from operator import attrgetter
+from string import Formatter
 
-from . import __version__ as borg_version
-from . import hashindex
-from . import chunker
-from .constants import *  # NOQA
-from . import crypto
-from .compress import COMPR_BUFFER, get_compressor
-from . import shellpattern
 import msgpack
 import msgpack.fallback
 
-import socket
+from .logger import create_logger
+logger = create_logger()
 
+from . import __version__ as borg_version
+from . import chunker
+from . import crypto
+from . import hashindex
+from . import shellpattern
+from .constants import *  # NOQA
+from .compress import COMPR_BUFFER, get_compressor
 
 # meta dict, data bytes
 _Chunk = namedtuple('_Chunk', 'meta data')

src/borg/item.py

@@ -1,5 +1,7 @@
 from .constants import ITEM_KEYS
-from .helpers import safe_encode, safe_decode, bigint_to_int, int_to_bigint, StableDict
+from .helpers import safe_encode, safe_decode
+from .helpers import bigint_to_int, int_to_bigint
+from .helpers import StableDict
 
 
 class PropDict:

src/borg/key.py

@@ -1,20 +1,27 @@
-from binascii import a2b_base64, b2a_base64, hexlify
 import configparser
 import getpass
 import os
 import sys
 import textwrap
-from hmac import compare_digest
+from binascii import a2b_base64, b2a_base64, hexlify
 from hashlib import sha256, pbkdf2_hmac
+from hmac import compare_digest
+
+import msgpack
 
-from .helpers import Chunk, IntegrityError, get_keys_dir, Error, yes, bin_to_hex, CompressionDecider2, CompressionSpec
 from .logger import create_logger
 logger = create_logger()
 
 from .constants import *  # NOQA
-from .crypto import AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks, hmac_sha256
 from .compress import Compressor, COMPR_BUFFER, get_compressor
-import msgpack
+from .crypto import AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks, hmac_sha256
+from .helpers import Chunk
+from .helpers import Error, IntegrityError
+from .helpers import yes
+from .helpers import get_keys_dir
+from .helpers import bin_to_hex
+from .helpers import CompressionDecider2, CompressionSpec
+
 
 PREFIX = b'\0' * 8
 

src/borg/locking.py

@@ -3,7 +3,7 @@ import os
 import socket
 import time
 
-from borg.helpers import Error, ErrorWithTraceback
+from .helpers import Error, ErrorWithTraceback
 
 ADD, REMOVE = 'add', 'remove'
 SHARED, EXCLUSIVE = 'shared', 'exclusive'

src/borg/platform_darwin.pyx

@@ -1,5 +1,7 @@
 import os
-from .helpers import user2uid, group2gid, safe_decode, safe_encode
+
+from .helpers import user2uid, group2gid
+from .helpers import safe_decode, safe_encode
 from .platform_posix import swidth
 
 API_VERSION = 3

src/borg/platform_freebsd.pyx

@@ -1,5 +1,7 @@
 import os
-from .helpers import posix_acl_use_stored_uid_gid, safe_encode, safe_decode
+
+from .helpers import posix_acl_use_stored_uid_gid
+from .helpers import safe_encode, safe_decode
 from .platform_posix import swidth
 
 API_VERSION = 3

src/borg/platform_linux.pyx

@@ -3,7 +3,9 @@ import re
 import resource
 import stat
 
-from .helpers import posix_acl_use_stored_uid_gid, user2uid, group2gid, safe_decode, safe_encode
+from .helpers import posix_acl_use_stored_uid_gid
+from .helpers import user2uid, group2gid
+from .helpers import safe_decode, safe_encode
 from .platform_base import SyncFile as BaseSyncFile
 from .platform_posix import swidth
 

src/borg/remote.py

@@ -4,17 +4,19 @@ import logging
 import os
 import select
 import shlex
-from subprocess import Popen, PIPE
 import sys
 import tempfile
+from subprocess import Popen, PIPE
 
-from . import __version__
+import msgpack
 
-from .helpers import Error, IntegrityError, get_home_dir, sysinfo, bin_to_hex
+from . import __version__
+from .helpers import Error, IntegrityError
+from .helpers import get_home_dir
+from .helpers import sysinfo
+from .helpers import bin_to_hex
 from .repository import Repository
 
-import msgpack
-
 RPC_PROTOCOL_VERSION = 2
 
 BUFSIZE = 10 * 1024 * 1024

src/borg/repository.py

@@ -1,23 +1,26 @@
-from configparser import ConfigParser
-from binascii import unhexlify
-from datetime import datetime
-from itertools import islice
 import errno
-import logging
-logger = logging.getLogger(__name__)
-
 import os
 import shutil
 import struct
+from binascii import unhexlify
 from collections import defaultdict
+from configparser import ConfigParser
+from datetime import datetime
 from functools import partial
+from itertools import islice
 from zlib import crc32
 
 import msgpack
+
+import logging
+logger = logging.getLogger(__name__)
+
 from .constants import *  # NOQA
-from .helpers import Error, ErrorWithTraceback, IntegrityError, InternalOSError, Location, ProgressIndicatorPercent, \
-    bin_to_hex
 from .hashindex import NSIndex
+from .helpers import Error, ErrorWithTraceback, IntegrityError, InternalOSError
+from .helpers import Location
+from .helpers import ProgressIndicatorPercent
+from .helpers import bin_to_hex
 from .locking import UpgradableLock, LockError, LockErrorT
 from .lrucache import LRUCache
 from .platform import SyncFile, sync_dir

src/borg/shellpattern.py

@@ -1,5 +1,5 @@
-import re
 import os
+import re
 
 
 def translate(pat):

src/borg/testsuite/archiver.py

@@ -17,6 +17,10 @@ from unittest.mock import patch
 from hashlib import sha256
 
 import pytest
+try:
+    import llfuse
+except ImportError:
+    pass
 
 from .. import xattr, helpers, platform
 from ..archive import Archive, ChunkBuffer, ArchiveRecreater
@@ -24,18 +28,15 @@ from ..archiver import Archiver
 from ..cache import Cache
 from ..constants import *  # NOQA
 from ..crypto import bytes_to_long, num_aes_blocks
-from ..helpers import Chunk, Manifest, EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, bin_to_hex
+from ..helpers import Chunk, Manifest
+from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
+from ..helpers import bin_to_hex
 from ..key import KeyfileKeyBase
 from ..remote import RemoteRepository, PathNotAllowed
 from ..repository import Repository
 from . import has_lchflags, has_llfuse
 from . import BaseTestCase, changedir, environment_variable
 
-try:
-    import llfuse
-except ImportError:
-    pass
-
 src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 
 

src/borg/testsuite/helpers.py

@@ -1,23 +1,27 @@
 import hashlib
-import io
 import logging
-from time import mktime, strptime
-from datetime import datetime, timezone, timedelta
 import os
+import sys
+from datetime import datetime, timezone, timedelta
+from time import mktime, strptime, sleep
 
 import pytest
-import sys
 import msgpack
 import msgpack.fallback
-import time
-
-from ..helpers import Location, format_file_size, format_timedelta, make_path_safe, clean_lines, \
-    prune_within, prune_split, get_cache_dir, get_keys_dir, is_slow_msgpack, \
-    yes, TRUISH, FALSISH, DEFAULTISH, \
-    StableDict, int_to_bigint, bigint_to_int, bin_to_hex, parse_timestamp, ChunkerParams, Chunk, \
-    ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes, parse_pattern, \
-    CompressionSpec, CompressionDecider1, CompressionDecider2, \
-    PatternMatcher, RegexPattern, PathPrefixPattern, FnmatchPattern, ShellPattern, partial_format, ChunkIteratorFileWrapper
+
+from ..helpers import Location
+from ..helpers import partial_format, format_file_size, format_timedelta
+from ..helpers import make_path_safe, clean_lines
+from ..helpers import prune_within, prune_split
+from ..helpers import get_cache_dir, get_keys_dir
+from ..helpers import is_slow_msgpack
+from ..helpers import yes, TRUISH, FALSISH, DEFAULTISH
+from ..helpers import StableDict, int_to_bigint, bigint_to_int, bin_to_hex
+from ..helpers import parse_timestamp, ChunkIteratorFileWrapper, ChunkerParams, Chunk
+from ..helpers import ProgressIndicatorPercent, ProgressIndicatorEndless
+from ..helpers import load_excludes
+from ..helpers import CompressionSpec, CompressionDecider1, CompressionDecider2
+from ..helpers import parse_pattern, PatternMatcher, RegexPattern, PathPrefixPattern, FnmatchPattern, ShellPattern
 from . import BaseTestCase, environment_variable, FakeInputs
 
 
@@ -115,7 +119,7 @@ class TestLocationWithoutEnv:
         assert repr(Location('/some/path::archive{pid}')) == \
             "Location(proto='file', user=None, host=None, port=None, path='/some/path', archive='archive{}')".format(test_pid)
         location_time1 = Location('/some/path::archive{now:%s}')
-        time.sleep(1.1)
+        sleep(1.1)
         location_time2 = Location('/some/path::archive{now:%s}')
         assert location_time1.archive != location_time2.archive
 

src/borg/testsuite/key.py

@@ -1,15 +1,15 @@
 import getpass
-import os
 import re
-import shutil
 import tempfile
 from binascii import hexlify, unhexlify
 
 import pytest
 
 from ..crypto import bytes_to_long, num_aes_blocks
+from ..helpers import Location
+from ..helpers import Chunk
+from ..helpers import IntegrityError
 from ..key import PlaintextKey, PassphraseKey, KeyfileKey, Passphrase, PasswordRetriesExceeded, bin_to_hex
-from ..helpers import Location, Chunk, IntegrityError
 
 
 @pytest.fixture(autouse=True)

src/borg/testsuite/lrucache.py

@@ -1,7 +1,9 @@
-from ..lrucache import LRUCache
-import pytest
 from tempfile import TemporaryFile
 
+import pytest
+
+from ..lrucache import LRUCache
+
 
 class TestLRUCache:
 

src/borg/testsuite/repository.py

@@ -7,7 +7,8 @@ import tempfile
 from unittest.mock import patch
 
 from ..hashindex import NSIndex
-from ..helpers import Location, IntegrityError, InternalOSError
+from ..helpers import Location
+from ..helpers import IntegrityError, InternalOSError
 from ..locking import UpgradableLock, LockFailed
 from ..remote import RemoteRepository, InvalidRPCMethod, ConnectionClosedWithHint, handle_remote_line
 from ..repository import Repository, LoggedIO, MAGIC

src/borg/upgrader.py

@@ -1,14 +1,16 @@
 import datetime
-import logging
-logger = logging.getLogger(__name__)
 import os
 import shutil
 import time
 
-from .helpers import get_home_dir, get_keys_dir, get_cache_dir, ProgressIndicatorPercent
+import logging
+logger = logging.getLogger(__name__)
+
+from .helpers import get_home_dir, get_keys_dir, get_cache_dir
+from .helpers import ProgressIndicatorPercent
+from .key import KeyfileKey, KeyfileNotFoundError
 from .locking import UpgradableLock
 from .repository import Repository, MAGIC
-from .key import KeyfileKey, KeyfileNotFoundError
 
 ATTIC_MAGIC = b'ATTICSEG'