use bundled msgpack (optionally use external msgpack)

based on a patch by felix schwarz, thanks!

changes after that:

import msgpack from borg.helpers (as master branch does)

there:
- try importing it from borg.algorithms.msgpack bundled code.
- if that does not work, try importing external msgpack.
- if it is desired to import the external msgpack without ripping
  out the bundled code, set prefer_system_msgpack = True.

fix msgpack.fallback reference, fix msgpack import.

src/borg/archive.py

@@ -13,8 +13,6 @@ from io import BytesIO
 from itertools import groupby
 from shutil import get_terminal_size
 
-import msgpack
-
 from .logger import create_logger
 
 logger = create_logger()
@@ -38,6 +36,7 @@ from .helpers import StableDict
 from .helpers import bin_to_hex
 from .helpers import safe_ns
 from .helpers import ellipsis_truncate, ProgressIndicatorPercent, log_multi
+from .helpers import msgpack
 from .patterns import PathPrefixPattern, FnmatchPattern, IECommand
 from .item import Item, ArchiveItem
 from .platform import acl_get, acl_set, set_flags, get_flags, swidth, hostname

src/borg/archiver.py

@@ -33,8 +33,6 @@ try:
 
     logger = create_logger()
 
-    import msgpack
-
     import borg
     from . import __version__
     from . import helpers
@@ -72,6 +70,7 @@ try:
     from .helpers import popen_with_error_handling, prepare_subprocess_env
     from .helpers import dash_open
     from .helpers import umount
+    from .helpers import msgpack, msgpack_fallback
     from .nanorst import rst_to_terminal
     from .patterns import ArgparsePatternAction, ArgparseExcludeFileAction, ArgparsePatternFileAction, parse_exclude_pattern
     from .patterns import PatternMatcher
@@ -1957,7 +1956,7 @@ class Archiver:
 
         data = key.decrypt(None, repository.get(manifest.MANIFEST_ID))
 
-        meta = prepare_dump_dict(msgpack.fallback.unpackb(data, object_hook=StableDict, unicode_errors='surrogateescape'))
+        meta = prepare_dump_dict(msgpack_fallback.unpackb(data, object_hook=StableDict, unicode_errors='surrogateescape'))
 
         with dash_open(args.path, 'w') as fd:
             json.dump(meta, fd, indent=4)

src/borg/cache.py

@@ -6,8 +6,6 @@ from binascii import unhexlify
 from collections import namedtuple
 from time import perf_counter
 
-import msgpack
-
 from .logger import create_logger
 
 logger = create_logger()
@@ -26,6 +24,7 @@ from .helpers import remove_surrogates
 from .helpers import ProgressIndicatorPercent, ProgressIndicatorMessage
 from .helpers import set_ec, EXIT_WARNING
 from .helpers import truncate_and_unlink
+from .helpers import msgpack
 from .item import ArchiveItem, ChunkListEntry
 from .crypto.key import PlaintextKey
 from .crypto.file_integrity import IntegrityCheckedFile, DetachedIntegrityCheckedFile, FileIntegrityError

src/borg/crypto/key.py

@@ -9,8 +9,6 @@ from binascii import a2b_base64, b2a_base64, hexlify
 from hashlib import sha256, sha512, pbkdf2_hmac
 from hmac import HMAC, compare_digest
 
-import msgpack
-
 from borg.logger import create_logger
 
 logger = create_logger()
@@ -24,6 +22,7 @@ from ..helpers import get_keys_dir, get_security_dir
 from ..helpers import get_limited_unpacker
 from ..helpers import bin_to_hex
 from ..helpers import prepare_subprocess_env
+from ..helpers import msgpack
 from ..item import Key, EncryptedKey
 from ..platform import SaveFile
 from .nonces import NonceManager

src/borg/fuse.py

@@ -11,7 +11,6 @@ from signal import SIGINT
 from distutils.version import LooseVersion
 
 import llfuse
-import msgpack
 
 from .logger import create_logger
 logger = create_logger()
@@ -21,6 +20,7 @@ from .archiver import Archiver
 from .archive import Archive
 from .hashindex import FuseVersionsIndex
 from .helpers import daemonize, hardlinkable, signal_handler, format_file_size
+from .helpers import msgpack
 from .item import Item
 from .lrucache import LRUCache
 from .remote import RemoteRepository

src/borg/helpers.py

@@ -31,8 +31,20 @@ from operator import attrgetter
 from string import Formatter
 from shutil import get_terminal_size
 
-import msgpack
-import msgpack.fallback
+prefer_system_msgpack = False
+
+try:
+    if prefer_system_msgpack:
+        raise ImportError
+    # use the bundled msgpack 0.5.6 known-good version - other code only imports it from here:
+    import borg.algorithms.msgpack as msgpack
+    from borg.algorithms.msgpack import fallback as msgpack_fallback
+except ImportError:
+    # when using a system msgpack, make sure it satisfies the requirements, see setup.py.
+    # also, you must make sure it actually gets installed as a dependency.
+    import msgpack
+    from msgpack import fallback as msgpack_fallback
+
 
 from .logger import create_logger
 logger = create_logger()
@@ -1287,7 +1299,7 @@ def int_to_bigint(value):
 
 
 def is_slow_msgpack():
-    return msgpack.Packer is msgpack.fallback.Packer
+    return msgpack.Packer is msgpack_fallback.Packer
 
 
 def is_supported_msgpack():

src/borg/remote.py

@@ -16,8 +16,6 @@ import time
 import traceback
 from subprocess import Popen, PIPE
 
-import msgpack
-
 from . import __version__
 from .compress import LZ4
 from .constants import *  # NOQA
@@ -32,6 +30,7 @@ from .helpers import format_file_size
 from .helpers import truncate_and_unlink
 from .helpers import prepare_subprocess_env
 from .logger import create_logger, setup_logging
+from .helpers import msgpack
 from .repository import Repository
 from .version import parse_version, format_version
 from .algorithms.checksums import xxh64

src/borg/repository.py

@@ -11,8 +11,6 @@ from datetime import datetime
 from functools import partial
 from itertools import islice
 
-import msgpack
-
 from .constants import *  # NOQA
 from .hashindex import NSIndex
 from .helpers import Error, ErrorWithTraceback, IntegrityError, format_file_size, parse_file_size
@@ -21,6 +19,7 @@ from .helpers import ProgressIndicatorPercent
 from .helpers import bin_to_hex
 from .helpers import hostname_is_unique
 from .helpers import secure_erase, truncate_and_unlink
+from .helpers import msgpack
 from .locking import Lock, LockError, LockErrorT
 from .logger import create_logger
 from .lrucache import LRUCache

src/borg/testsuite/archive.py

@@ -3,7 +3,6 @@ from datetime import datetime, timezone
 from io import StringIO
 from unittest.mock import Mock
 
-import msgpack
 import pytest
 
 from . import BaseTestCase
@@ -11,6 +10,7 @@ from ..crypto.key import PlaintextKey
 from ..archive import Archive, CacheChunkBuffer, RobustUnpacker, valid_msgpacked_dict, ITEM_KEYS, Statistics
 from ..archive import BackupOSError, backup_io, backup_io_iter
 from ..helpers import Manifest
+from ..helpers import msgpack
 from ..item import Item, ArchiveItem
 
 

src/borg/testsuite/archiver.py

@@ -22,7 +22,6 @@ from hashlib import sha256
 from io import BytesIO, StringIO
 from unittest.mock import patch
 
-import msgpack
 import pytest
 
 try:
@@ -45,6 +44,7 @@ from ..helpers import Manifest, MandatoryFeatureUnsupported
 from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
 from ..helpers import bin_to_hex
 from ..helpers import MAX_S
+from ..helpers import msgpack
 from ..nanorst import RstToTextLazy, rst_to_terminal
 from ..patterns import IECommand, PatternMatcher, parse_pattern
 from ..item import Item

src/borg/testsuite/cache.py

@@ -1,8 +1,6 @@
 import io
 import os.path
 
-from msgpack import packb
-
 import pytest
 
 from .hashindex import H
@@ -13,8 +11,11 @@ from ..compress import CompressionSpec
 from ..crypto.key import RepoKey
 from ..hashindex import ChunkIndex, CacheSynchronizer
 from ..helpers import Manifest
+from ..helpers import msgpack
 from ..repository import Repository
 
+packb = msgpack.packb
+
 
 class TestCacheSynchronizer:
     @pytest.fixture

src/borg/testsuite/helpers.py

@@ -9,9 +9,6 @@ from time import mktime, strptime, sleep
 
 import pytest
 
-import msgpack
-import msgpack.fallback
-
 from .. import platform
 from ..helpers import Location
 from ..helpers import Buffer
@@ -20,6 +17,7 @@ from ..helpers import make_path_safe, clean_lines
 from ..helpers import interval, prune_within, prune_split
 from ..helpers import get_base_dir, get_cache_dir, get_keys_dir, get_security_dir, get_config_dir
 from ..helpers import is_slow_msgpack
+from ..helpers import msgpack, msgpack_fallback
 from ..helpers import yes, TRUISH, FALSISH, DEFAULTISH
 from ..helpers import StableDict, int_to_bigint, bigint_to_int, bin_to_hex
 from ..helpers import parse_timestamp, ChunkIteratorFileWrapper, ChunkerParams
@@ -576,7 +574,7 @@ def test_parse_file_size_invalid(string):
 def test_is_slow_msgpack():
     saved_packer = msgpack.Packer
     try:
-        msgpack.Packer = msgpack.fallback.Packer
+        msgpack.Packer = msgpack_fallback.Packer
         assert is_slow_msgpack()
     finally:
         msgpack.Packer = saved_packer

src/borg/testsuite/key.py

@@ -4,7 +4,6 @@ import re
 import tempfile
 from binascii import hexlify, unhexlify
 
-import msgpack
 import pytest
 
 from ..crypto.key import Passphrase, PasswordRetriesExceeded, bin_to_hex
@@ -18,6 +17,7 @@ from ..helpers import IntegrityError
 from ..helpers import Location
 from ..helpers import StableDict
 from ..helpers import get_security_dir
+from ..helpers import msgpack
 
 
 class TestKey:

src/borg/testsuite/repository.py

@@ -6,13 +6,12 @@ import sys
 import tempfile
 from unittest.mock import patch
 
-import msgpack
-
 import pytest
 
 from ..hashindex import NSIndex
 from ..helpers import Location
 from ..helpers import IntegrityError
+from ..helpers import msgpack
 from ..locking import Lock, LockFailed
 from ..remote import RemoteRepository, InvalidRPCMethod, PathNotAllowed, ConnectionClosedWithHint, handle_remote_line
 from ..repository import Repository, LoggedIO, MAGIC, MAX_DATA_SIZE, TAG_DELETE