Merge pull request #4560 from ThomasWaldmann/bundle-msgpack

Bundle msgpack

.gitignore

@@ -14,6 +14,8 @@ src/borg/platform/darwin.c
 src/borg/platform/freebsd.c
 src/borg/platform/linux.c
 src/borg/platform/posix.c
+src/borg/algorithms/msgpack/_packer.cpp
+src/borg/algorithms/msgpack/_unpacker.cpp
 src/borg/_version.py
 *.egg-info
 *.pyc

docs/3rd_party/msgpack/COPYING

@@ -0,0 +1,14 @@
+Copyright (C) 2008-2011 INADA Naoki <songofacandy@gmail.com>
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

setup.cfg

@@ -10,5 +10,5 @@ ignore = E122,E123,E125,E126,E127,E128,E226,E402,E722,E731,E741,
          W504
 # line length long term target: 120
 max-line-length = 255
-exclude = build,dist,.git,.idea,.cache,.tox,docs/conf.py
+exclude = build,dist,.git,.idea,.cache,.tox,docs/conf.py,src/borg/algorithms/msgpack
 

setup.py

@@ -25,6 +25,8 @@ prefer_system_libzstd = True
 # True: use the shared libb2 from the system, False: use the bundled blake2 code
 prefer_system_libb2 = True
 
+# prefer_system_msgpack is another option, but you need to set it in src/borg/helpers.py.
+
 min_python = (3, 4)
 my_python = sys.version_info
 
@@ -35,19 +37,7 @@ if my_python < min_python:
 # Are we building on ReadTheDocs?
 on_rtd = os.environ.get('READTHEDOCS')
 
-install_requires = [
-    # we are rather picky about msgpack versions, because a good working msgpack is
-    # very important for borg, see https://github.com/borgbackup/borg/issues/3753
-    # best versions seem to be 0.4.6, 0.4.7, 0.4.8 and 0.5.6:
-    'msgpack-python >=0.4.6, <=0.5.6, !=0.5.0, !=0.5.1, !=0.5.2, !=0.5.3, !=0.5.4, !=0.5.5',
-    # if you can't satisfy the above requirement, these are versions that might
-    # also work ok, IF you make sure to use the COMPILED version of msgpack-python,
-    # NOT the PURE PYTHON fallback implementation: ==0.5.1, ==0.5.4
-    #
-    # Please note:
-    # using any other version is not supported by borg development and
-    # any feedback related to issues caused by this will be ignored.
-]
+install_requires = []
 
 # note for package maintainers: if you package borgbackup for distribution,
 # please add llfuse as a *requirement* on all platforms that have a working
@@ -88,8 +78,11 @@ platform_posix_source = 'src/borg/platform/posix.pyx'
 platform_linux_source = 'src/borg/platform/linux.pyx'
 platform_darwin_source = 'src/borg/platform/darwin.pyx'
 platform_freebsd_source = 'src/borg/platform/freebsd.pyx'
+msgpack_packer_source = 'src/borg/algorithms/msgpack/_packer.pyx'
+msgpack_unpacker_source = 'src/borg/algorithms/msgpack/_unpacker.pyx'
 
-cython_sources = [
+cython_c_sources = [
+    # these .pyx will get compiled to .c
     compress_source,
     crypto_ll_source,
     chunker_source,
@@ -103,14 +96,22 @@ cython_sources = [
     platform_darwin_source,
 ]
 
+cython_cpp_sources = [
+    # these .pyx will get compiled to .cpp
+    msgpack_packer_source,
+    msgpack_unpacker_source,
+]
+
 try:
     from Cython.Distutils import build_ext
     import Cython.Compiler.Main as cython_compiler
 
     class Sdist(sdist):
         def __init__(self, *args, **kwargs):
-            for src in cython_sources:
+            for src in cython_c_sources:
                 cython_compiler.compile(src, cython_compiler.default_options)
+            for src in cython_cpp_sources:
+                cython_compiler.compile(src, cplus=True)
             super().__init__(*args, **kwargs)
 
         def make_distribution(self):
@@ -129,6 +130,8 @@ try:
                 'src/borg/platform/linux.c',
                 'src/borg/platform/freebsd.c',
                 'src/borg/platform/darwin.c',
+                'src/borg/algorithms/msgpack/_packer.cpp',
+                'src/borg/algorithms/msgpack/_unpacker.cpp',
             ])
             super().make_distribution()
 
@@ -147,10 +150,15 @@ except ImportError:
     platform_linux_source = platform_linux_source.replace('.pyx', '.c')
     platform_freebsd_source = platform_freebsd_source.replace('.pyx', '.c')
     platform_darwin_source = platform_darwin_source.replace('.pyx', '.c')
+
+    msgpack_packer_source = msgpack_packer_source.replace('.pyx', '.cpp')
+    msgpack_unpacker_source = msgpack_unpacker_source.replace('.pyx', '.cpp')
+
     from distutils.command.build_ext import build_ext
     if not on_rtd and not all(os.path.exists(path) for path in [
         compress_source, crypto_ll_source, chunker_source, hashindex_source, item_source, checksums_source,
-        platform_posix_source, platform_linux_source, platform_freebsd_source, platform_darwin_source]):
+        platform_posix_source, platform_linux_source, platform_freebsd_source, platform_darwin_source,
+        msgpack_packer_source, msgpack_unpacker_source]):
         raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.')
 
 
@@ -758,9 +766,13 @@ def rm(file):
 class Clean(clean):
     def run(self):
         super().run()
-        for source in cython_sources:
+        for source in cython_c_sources:
             genc = source.replace('.pyx', '.c')
             rm(genc)
+        for source in cython_cpp_sources:
+            gencpp = source.replace('.pyx', '.cpp')
+            rm(gencpp)
+        for source in cython_c_sources + cython_cpp_sources:
             compiled_glob = source.replace('.pyx', '.cpython*')
             for compiled in sorted(glob(compiled_glob)):
                 rm(compiled)
@@ -788,7 +800,27 @@ if not on_rtd:
     crypto_ext_kwargs = setup_b2.b2_ext_kwargs(bundled_path='src/borg/algorithms/blake2',
                                                system_prefix=libb2_prefix, system=libb2_system,
                                                **crypto_ext_kwargs)
+
+    msgpack_endian = '__BIG_ENDIAN__' if (sys.byteorder == 'big') else '__LITTLE_ENDIAN__'
+    msgpack_macros = [(msgpack_endian, '1')]
+    msgpack_packer_ext_kwargs = dict(
+        sources=[msgpack_packer_source],
+        include_dirs=include_dirs,
+        library_dirs=library_dirs,
+        define_macros=msgpack_macros,
+        language='c++',
+    )
+    msgpack_unpacker_ext_kwargs = dict(
+        sources=[msgpack_unpacker_source],
+        include_dirs=include_dirs,
+        library_dirs=library_dirs,
+        define_macros=msgpack_macros,
+        language='c++',
+    )
+
     ext_modules += [
+        Extension('borg.algorithms.msgpack._packer', **msgpack_packer_ext_kwargs),
+        Extension('borg.algorithms.msgpack._unpacker', **msgpack_unpacker_ext_kwargs),
         Extension('borg.compress', **compress_ext_kwargs),
         Extension('borg.crypto.low_level', **crypto_ext_kwargs),
         Extension('borg.hashindex', [hashindex_source]),

src/borg/algorithms/msgpack/__init__.py

@@ -0,0 +1,66 @@
+# coding: utf-8
+from ._version import version
+from .exceptions import *
+
+from collections import namedtuple
+
+
+class ExtType(namedtuple('ExtType', 'code data')):
+    """ExtType represents ext type in msgpack."""
+    def __new__(cls, code, data):
+        if not isinstance(code, int):
+            raise TypeError("code must be int")
+        if not isinstance(data, bytes):
+            raise TypeError("data must be bytes")
+        if not 0 <= code <= 127:
+            raise ValueError("code must be 0~127")
+        return super(ExtType, cls).__new__(cls, code, data)
+
+
+import os
+if os.environ.get('MSGPACK_PUREPYTHON'):
+    from .fallback import Packer, unpackb, Unpacker
+else:
+    try:
+        from ._packer import Packer
+        from ._unpacker import unpackb, Unpacker
+    except ImportError:
+        from .fallback import Packer, unpackb, Unpacker
+
+
+def pack(o, stream, **kwargs):
+    """
+    Pack object `o` and write it to `stream`
+
+    See :class:`Packer` for options.
+    """
+    packer = Packer(**kwargs)
+    stream.write(packer.pack(o))
+
+
+def packb(o, **kwargs):
+    """
+    Pack object `o` and return packed bytes
+
+    See :class:`Packer` for options.
+    """
+    return Packer(**kwargs).pack(o)
+
+
+def unpack(stream, **kwargs):
+    """
+    Unpack an object from `stream`.
+
+    Raises `ExtraData` when `stream` contains extra bytes.
+    See :class:`Unpacker` for options.
+    """
+    data = stream.read()
+    return unpackb(data, **kwargs)
+
+
+# alias for compatibility to simplejson/marshal/pickle.
+load = unpack
+loads = unpackb
+
+dump = pack
+dumps = packb

src/borg/algorithms/msgpack/_packer.pyx

@@ -0,0 +1,357 @@
+# coding: utf-8
+#cython: embedsignature=True, c_string_encoding=ascii
+
+from cpython cimport *
+from cpython.version cimport PY_MAJOR_VERSION
+from cpython.exc cimport PyErr_WarnEx
+
+from .exceptions import PackValueError, PackOverflowError
+from . import ExtType
+
+
+cdef extern from "Python.h":
+
+    int PyMemoryView_Check(object obj)
+    int PyByteArray_Check(object obj)
+    int PyByteArray_CheckExact(object obj)
+    char* PyUnicode_AsUTF8AndSize(object obj, Py_ssize_t *l) except NULL
+
+
+cdef extern from "pack.h":
+    struct msgpack_packer:
+        char* buf
+        size_t length
+        size_t buf_size
+        bint use_bin_type
+
+    int msgpack_pack_int(msgpack_packer* pk, int d)
+    int msgpack_pack_nil(msgpack_packer* pk)
+    int msgpack_pack_true(msgpack_packer* pk)
+    int msgpack_pack_false(msgpack_packer* pk)
+    int msgpack_pack_long(msgpack_packer* pk, long d)
+    int msgpack_pack_long_long(msgpack_packer* pk, long long d)
+    int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d)
+    int msgpack_pack_float(msgpack_packer* pk, float d)
+    int msgpack_pack_double(msgpack_packer* pk, double d)
+    int msgpack_pack_array(msgpack_packer* pk, size_t l)
+    int msgpack_pack_map(msgpack_packer* pk, size_t l)
+    int msgpack_pack_raw(msgpack_packer* pk, size_t l)
+    int msgpack_pack_bin(msgpack_packer* pk, size_t l)
+    int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
+    int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l)
+    int msgpack_pack_unicode(msgpack_packer* pk, object o, long long limit)
+
+cdef int DEFAULT_RECURSE_LIMIT=511
+cdef long long ITEM_LIMIT = (2**32)-1
+
+
+cdef inline int PyBytesLike_Check(object o):
+    return PyBytes_Check(o) or PyByteArray_Check(o)
+
+
+cdef inline int PyBytesLike_CheckExact(object o):
+    return PyBytes_CheckExact(o) or PyByteArray_CheckExact(o)
+
+
+cdef class Packer(object):
+    """
+    MessagePack Packer
+
+    usage::
+
+        packer = Packer()
+        astream.write(packer.pack(a))
+        astream.write(packer.pack(b))
+
+    Packer's constructor has some keyword arguments:
+
+    :param callable default:
+        Convert user type to builtin type that Packer supports.
+        See also simplejson's document.
+
+    :param bool use_single_float:
+        Use single precision float type for float. (default: False)
+
+    :param bool autoreset:
+        Reset buffer after each pack and return its content as `bytes`. (default: True).
+        If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
+
+    :param bool use_bin_type:
+        Use bin type introduced in msgpack spec 2.0 for bytes.
+        It also enables str8 type for unicode.
+        Current default value is false, but it will be changed to true
+        in future version.  You should specify it explicitly.
+
+    :param bool strict_types:
+        If set to true, types will be checked to be exact. Derived classes
+        from serializeable types will not be serialized and will be
+        treated as unsupported type and forwarded to default.
+        Additionally tuples will not be serialized as lists.
+        This is useful when trying to implement accurate serialization
+        for python types.
+
+    :param str unicode_errors:
+        Error handler for encoding unicode. (default: 'strict')
+
+    :param str encoding:
+        (deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8')
+    """
+    cdef msgpack_packer pk
+    cdef object _default
+    cdef object _bencoding
+    cdef object _berrors
+    cdef const char *encoding
+    cdef const char *unicode_errors
+    cdef bint strict_types
+    cdef bool use_float
+    cdef bint autoreset
+
+    def __cinit__(self):
+        cdef int buf_size = 1024*1024
+        self.pk.buf = <char*> PyMem_Malloc(buf_size)
+        if self.pk.buf == NULL:
+            raise MemoryError("Unable to allocate internal buffer.")
+        self.pk.buf_size = buf_size
+        self.pk.length = 0
+
+    def __init__(self, default=None, encoding=None, unicode_errors=None,
+                 bint use_single_float=False, bint autoreset=True, bint use_bin_type=False,
+                 bint strict_types=False):
+        if encoding is not None:
+            PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated.", 1)
+        self.use_float = use_single_float
+        self.strict_types = strict_types
+        self.autoreset = autoreset
+        self.pk.use_bin_type = use_bin_type
+        if default is not None:
+            if not PyCallable_Check(default):
+                raise TypeError("default must be a callable.")
+        self._default = default
+
+        self._bencoding = encoding
+        if encoding is None:
+            if PY_MAJOR_VERSION < 3:
+                self.encoding = 'utf-8'
+            else:
+                self.encoding = NULL
+        else:
+            self.encoding = self._bencoding
+
+        self._berrors = unicode_errors
+        if unicode_errors is None:
+            self.unicode_errors = NULL
+        else:
+            self.unicode_errors = self._berrors
+
+    def __dealloc__(self):
+        PyMem_Free(self.pk.buf)
+        self.pk.buf = NULL
+
+    cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1:
+        cdef long long llval
+        cdef unsigned long long ullval
+        cdef long longval
+        cdef float fval
+        cdef double dval
+        cdef char* rawval
+        cdef int ret
+        cdef dict d
+        cdef Py_ssize_t L
+        cdef int default_used = 0
+        cdef bint strict_types = self.strict_types
+        cdef Py_buffer view
+
+        if nest_limit < 0:
+            raise PackValueError("recursion limit exceeded.")
+
+        while True:
+            if o is None:
+                ret = msgpack_pack_nil(&self.pk)
+            elif PyBool_Check(o) if strict_types else isinstance(o, bool):
+                if o:
+                    ret = msgpack_pack_true(&self.pk)
+                else:
+                    ret = msgpack_pack_false(&self.pk)
+            elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o):
+                # PyInt_Check(long) is True for Python 3.
+                # So we should test long before int.
+                try:
+                    if o > 0:
+                        ullval = o
+                        ret = msgpack_pack_unsigned_long_long(&self.pk, ullval)
+                    else:
+                        llval = o
+                        ret = msgpack_pack_long_long(&self.pk, llval)
+                except OverflowError as oe:
+                    if not default_used and self._default is not None:
+                        o = self._default(o)
+                        default_used = True
+                        continue
+                    else:
+                        raise PackOverflowError("Integer value out of range")
+            elif PyInt_CheckExact(o) if strict_types else PyInt_Check(o):
+                longval = o
+                ret = msgpack_pack_long(&self.pk, longval)
+            elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o):
+                if self.use_float:
+                   fval = o
+                   ret = msgpack_pack_float(&self.pk, fval)
+                else:
+                   dval = o
+                   ret = msgpack_pack_double(&self.pk, dval)
+            elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o):
+                L = len(o)
+                if L > ITEM_LIMIT:
+                    raise PackValueError("%s is too large" % type(o).__name__)
+                rawval = o
+                ret = msgpack_pack_bin(&self.pk, L)
+                if ret == 0:
+                    ret = msgpack_pack_raw_body(&self.pk, rawval, L)
+            elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o):
+                if self.encoding == NULL and self.unicode_errors == NULL:
+                    ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT);
+                    if ret == -2:
+                        raise PackValueError("unicode string is too large")
+                else:
+                    o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
+                    L = len(o)
+                    if L > ITEM_LIMIT:
+                        raise PackValueError("unicode string is too large")
+                    ret = msgpack_pack_raw(&self.pk, L)
+                    if ret == 0:
+                        rawval = o
+                        ret = msgpack_pack_raw_body(&self.pk, rawval, L)
+            elif PyDict_CheckExact(o):
+                d = <dict>o
+                L = len(d)
+                if L > ITEM_LIMIT:
+                    raise PackValueError("dict is too large")
+                ret = msgpack_pack_map(&self.pk, L)
+                if ret == 0:
+                    for k, v in d.iteritems():
+                        ret = self._pack(k, nest_limit-1)
+                        if ret != 0: break
+                        ret = self._pack(v, nest_limit-1)
+                        if ret != 0: break
+            elif not strict_types and PyDict_Check(o):
+                L = len(o)
+                if L > ITEM_LIMIT:
+                    raise PackValueError("dict is too large")
+                ret = msgpack_pack_map(&self.pk, L)
+                if ret == 0:
+                    for k, v in o.items():
+                        ret = self._pack(k, nest_limit-1)
+                        if ret != 0: break
+                        ret = self._pack(v, nest_limit-1)
+                        if ret != 0: break
+            elif type(o) is ExtType if strict_types else isinstance(o, ExtType):
+                # This should be before Tuple because ExtType is namedtuple.
+                longval = o.code
+                rawval = o.data
+                L = len(o.data)
+                if L > ITEM_LIMIT:
+                    raise PackValueError("EXT data is too large")
+                ret = msgpack_pack_ext(&self.pk, longval, L)
+                ret = msgpack_pack_raw_body(&self.pk, rawval, L)
+            elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)):
+                L = len(o)
+                if L > ITEM_LIMIT:
+                    raise PackValueError("list is too large")
+                ret = msgpack_pack_array(&self.pk, L)
+                if ret == 0:
+                    for v in o:
+                        ret = self._pack(v, nest_limit-1)
+                        if ret != 0: break
+            elif PyMemoryView_Check(o):
+                if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0:
+                    raise PackValueError("could not get buffer for memoryview")
+                L = view.len
+                if L > ITEM_LIMIT:
+                    PyBuffer_Release(&view);
+                    raise PackValueError("memoryview is too large")
+                ret = msgpack_pack_bin(&self.pk, L)
+                if ret == 0:
+                    ret = msgpack_pack_raw_body(&self.pk, <char*>view.buf, L)
+                PyBuffer_Release(&view);
+            elif not default_used and self._default:
+                o = self._default(o)
+                default_used = 1
+                continue
+            else:
+                raise TypeError("can't serialize %r" % (o,))
+            return ret
+
+    cpdef pack(self, object obj):
+        cdef int ret
+        try:
+            ret = self._pack(obj, DEFAULT_RECURSE_LIMIT)
+        except:
+            self.pk.length = 0
+            raise
+        if ret:  # should not happen.
+            raise RuntimeError("internal error")
+        if self.autoreset:
+            buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
+            self.pk.length = 0
+            return buf
+
+    def pack_ext_type(self, typecode, data):
+        msgpack_pack_ext(&self.pk, typecode, len(data))
+        msgpack_pack_raw_body(&self.pk, data, len(data))
+
+    def pack_array_header(self, long long size):
+        if size > ITEM_LIMIT:
+            raise PackValueError
+        cdef int ret = msgpack_pack_array(&self.pk, size)
+        if ret == -1:
+            raise MemoryError
+        elif ret:  # should not happen
+            raise TypeError
+        if self.autoreset:
+            buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
+            self.pk.length = 0
+            return buf
+
+    def pack_map_header(self, long long size):
+        if size > ITEM_LIMIT:
+            raise PackValueError
+        cdef int ret = msgpack_pack_map(&self.pk, size)
+        if ret == -1:
+            raise MemoryError
+        elif ret:  # should not happen
+            raise TypeError
+        if self.autoreset:
+            buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
+            self.pk.length = 0
+            return buf
+
+    def pack_map_pairs(self, object pairs):
+        """
+        Pack *pairs* as msgpack map type.
+
+        *pairs* should be a sequence of pairs.
+        (`len(pairs)` and `for k, v in pairs:` should be supported.)
+        """
+        cdef int ret = msgpack_pack_map(&self.pk, len(pairs))
+        if ret == 0:
+            for k, v in pairs:
+                ret = self._pack(k)
+                if ret != 0: break
+                ret = self._pack(v)
+                if ret != 0: break
+        if ret == -1:
+            raise MemoryError
+        elif ret:  # should not happen
+            raise TypeError
+        if self.autoreset:
+            buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
+            self.pk.length = 0
+            return buf
+
+    def reset(self):
+        """Clear internal buffer."""
+        self.pk.length = 0
+
+    def bytes(self):
+        """Return buffer content."""
+        return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)

src/borg/algorithms/msgpack/_unpacker.pyx

@@ -0,0 +1,543 @@
+# coding: utf-8
+#cython: embedsignature=True, c_string_encoding=ascii
+
+from cpython.version cimport PY_MAJOR_VERSION
+from cpython.bytes cimport (
+    PyBytes_AsString,
+    PyBytes_FromStringAndSize,
+    PyBytes_Size,
+)
+from cpython.buffer cimport (
+    Py_buffer,
+    PyObject_CheckBuffer,
+    PyObject_GetBuffer,
+    PyBuffer_Release,
+    PyBuffer_IsContiguous,
+    PyBUF_READ,
+    PyBUF_SIMPLE,
+    PyBUF_FULL_RO,
+)
+from cpython.mem cimport PyMem_Malloc, PyMem_Free
+from cpython.object cimport PyCallable_Check
+from cpython.ref cimport Py_DECREF
+from cpython.exc cimport PyErr_WarnEx
+
+cdef extern from "Python.h":
+    ctypedef struct PyObject
+    cdef int PyObject_AsReadBuffer(object o, const void** buff, Py_ssize_t* buf_len) except -1
+    object PyMemoryView_GetContiguous(object obj, int buffertype, char order)
+
+from libc.stdlib cimport *
+from libc.string cimport *
+from libc.limits cimport *
+ctypedef unsigned long long uint64_t
+
+from .exceptions import (
+    BufferFull,
+    OutOfData,
+    UnpackValueError,
+    ExtraData,
+)
+from . import ExtType
+
+
+cdef extern from "unpack.h":
+    ctypedef struct msgpack_user:
+        bint use_list
+        bint raw
+        bint has_pairs_hook # call object_hook with k-v pairs
+        PyObject* object_hook
+        PyObject* list_hook
+        PyObject* ext_hook
+        char *encoding
+        char *unicode_errors
+        Py_ssize_t max_str_len
+        Py_ssize_t max_bin_len
+        Py_ssize_t max_array_len
+        Py_ssize_t max_map_len
+        Py_ssize_t max_ext_len
+
+    ctypedef struct unpack_context:
+        msgpack_user user
+        PyObject* obj
+        Py_ssize_t count
+
+    ctypedef int (*execute_fn)(unpack_context* ctx, const char* data,
+                               Py_ssize_t len, Py_ssize_t* off) except? -1
+    execute_fn unpack_construct
+    execute_fn unpack_skip
+    execute_fn read_array_header
+    execute_fn read_map_header
+    void unpack_init(unpack_context* ctx)
+    object unpack_data(unpack_context* ctx)
+    void unpack_clear(unpack_context* ctx)
+
+cdef inline init_ctx(unpack_context *ctx,
+                     object object_hook, object object_pairs_hook,
+                     object list_hook, object ext_hook,
+                     bint use_list, bint raw,
+                     const char* encoding, const char* unicode_errors,
+                     Py_ssize_t max_str_len, Py_ssize_t max_bin_len,
+                     Py_ssize_t max_array_len, Py_ssize_t max_map_len,
+                     Py_ssize_t max_ext_len):
+    unpack_init(ctx)
+    ctx.user.use_list = use_list
+    ctx.user.raw = raw
+    ctx.user.object_hook = ctx.user.list_hook = <PyObject*>NULL
+    ctx.user.max_str_len = max_str_len
+    ctx.user.max_bin_len = max_bin_len
+    ctx.user.max_array_len = max_array_len
+    ctx.user.max_map_len = max_map_len
+    ctx.user.max_ext_len = max_ext_len
+
+    if object_hook is not None and object_pairs_hook is not None:
+        raise TypeError("object_pairs_hook and object_hook are mutually exclusive.")
+
+    if object_hook is not None:
+        if not PyCallable_Check(object_hook):
+            raise TypeError("object_hook must be a callable.")
+        ctx.user.object_hook = <PyObject*>object_hook
+
+    if object_pairs_hook is None:
+        ctx.user.has_pairs_hook = False
+    else:
+        if not PyCallable_Check(object_pairs_hook):
+            raise TypeError("object_pairs_hook must be a callable.")
+        ctx.user.object_hook = <PyObject*>object_pairs_hook
+        ctx.user.has_pairs_hook = True
+
+    if list_hook is not None:
+        if not PyCallable_Check(list_hook):
+            raise TypeError("list_hook must be a callable.")
+        ctx.user.list_hook = <PyObject*>list_hook
+
+    if ext_hook is not None:
+        if not PyCallable_Check(ext_hook):
+            raise TypeError("ext_hook must be a callable.")
+        ctx.user.ext_hook = <PyObject*>ext_hook
+
+    ctx.user.encoding = encoding
+    ctx.user.unicode_errors = unicode_errors
+
+def default_read_extended_type(typecode, data):
+    raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode)
+
+cdef inline int get_data_from_buffer(object obj,
+                                     Py_buffer *view,
+                                     char **buf,
+                                     Py_ssize_t *buffer_len,
+                                     int *new_protocol) except 0:
+    cdef object contiguous
+    cdef Py_buffer tmp
+    if PyObject_CheckBuffer(obj):
+        new_protocol[0] = 1
+        if PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) == -1:
+            raise
+        if view.itemsize != 1:
+            PyBuffer_Release(view)
+            raise BufferError("cannot unpack from multi-byte object")
+        if PyBuffer_IsContiguous(view, 'A') == 0:
+            PyBuffer_Release(view)
+            # create a contiguous copy and get buffer
+            contiguous = PyMemoryView_GetContiguous(obj, PyBUF_READ, 'C')
+            PyObject_GetBuffer(contiguous, view, PyBUF_SIMPLE)
+            # view must hold the only reference to contiguous,
+            # so memory is freed when view is released
+            Py_DECREF(contiguous)
+        buffer_len[0] = view.len
+        buf[0] = <char*> view.buf
+        return 1
+    else:
+        new_protocol[0] = 0
+        if PyObject_AsReadBuffer(obj, <const void**> buf, buffer_len) == -1:
+            raise BufferError("could not get memoryview")
+        PyErr_WarnEx(RuntimeWarning,
+                     "using old buffer interface to unpack %s; "
+                     "this leads to unpacking errors if slicing is used and "
+                     "will be removed in a future version" % type(obj),
+                     1)
+        return 1
+
+def unpackb(object packed, object object_hook=None, object list_hook=None,
+            bint use_list=True, bint raw=True,
+            encoding=None, unicode_errors=None,
+            object_pairs_hook=None, ext_hook=ExtType,
+            Py_ssize_t max_str_len=2147483647, # 2**32-1
+            Py_ssize_t max_bin_len=2147483647,
+            Py_ssize_t max_array_len=2147483647,
+            Py_ssize_t max_map_len=2147483647,
+            Py_ssize_t max_ext_len=2147483647):
+    """
+    Unpack packed_bytes to object. Returns an unpacked object.
+
+    Raises `ValueError` when `packed` contains extra bytes.
+
+    See :class:`Unpacker` for options.
+    """
+    cdef unpack_context ctx
+    cdef Py_ssize_t off = 0
+    cdef int ret
+
+    cdef Py_buffer view
+    cdef char* buf = NULL
+    cdef Py_ssize_t buf_len
+    cdef const char* cenc = NULL
+    cdef const char* cerr = NULL
+    cdef int new_protocol = 0
+
+    if encoding is not None:
+        PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1)
+        cenc = encoding
+
+    if unicode_errors is not None:
+        cerr = unicode_errors
+
+    get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol)
+    try:
+        init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook,
+                 use_list, raw, cenc, cerr,
+                 max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len)
+        ret = unpack_construct(&ctx, buf, buf_len, &off)
+    finally:
+        if new_protocol:
+            PyBuffer_Release(&view);
+
+    if ret == 1:
+        obj = unpack_data(&ctx)
+        if off < buf_len:
+            raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off))
+        return obj
+    unpack_clear(&ctx)
+    raise UnpackValueError("Unpack failed: error = %d" % (ret,))
+
+
+def unpack(object stream, **kwargs):
+    PyErr_WarnEx(
+        PendingDeprecationWarning,
+        "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", 1)
+    data = stream.read()
+    return unpackb(data, **kwargs)
+
+
+cdef class Unpacker(object):
+    """Streaming unpacker.
+
+    arguments:
+
+    :param file_like:
+        File-like object having `.read(n)` method.
+        If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable.
+
+    :param int read_size:
+        Used as `file_like.read(read_size)`. (default: `min(1024**2, max_buffer_size)`)
+
+    :param bool use_list:
+        If true, unpack msgpack array to Python list.
+        Otherwise, unpack to Python tuple. (default: True)
+
+    :param bool raw:
+        If true, unpack msgpack raw to Python bytes (default).
+        Otherwise, unpack to Python str (or unicode on Python 2) by decoding
+        with UTF-8 encoding (recommended).
+        Currently, the default is true, but it will be changed to false in
+        near future.  So you must specify it explicitly for keeping backward
+        compatibility.
+
+        *encoding* option which is deprecated overrides this option.
+
+    :param callable object_hook:
+        When specified, it should be callable.
+        Unpacker calls it with a dict argument after unpacking msgpack map.
+        (See also simplejson)
+
+    :param callable object_pairs_hook:
+        When specified, it should be callable.
+        Unpacker calls it with a list of key-value pairs after unpacking msgpack map.
+        (See also simplejson)
+
+    :param int max_buffer_size:
+        Limits size of data waiting unpacked.  0 means system's INT_MAX (default).
+        Raises `BufferFull` exception when it is insufficient.
+        You should set this parameter when unpacking data from untrusted source.
+
+    :param int max_str_len:
+        Limits max length of str. (default: 2**31-1)
+
+    :param int max_bin_len:
+        Limits max length of bin. (default: 2**31-1)
+
+    :param int max_array_len:
+        Limits max length of array. (default: 2**31-1)
+
+    :param int max_map_len:
+        Limits max length of map. (default: 2**31-1)
+
+    :param str encoding:
+        Deprecated, use raw instead.
+        Encoding used for decoding msgpack raw.
+        If it is None (default), msgpack raw is deserialized to Python bytes.
+
+    :param str unicode_errors:
+        Error handler used for decoding str type.  (default: `'strict'`)
+
+
+    Example of streaming deserialize from file-like object::
+
+        unpacker = Unpacker(file_like, raw=False)
+        for o in unpacker:
+            process(o)
+
+    Example of streaming deserialize from socket::
+
+        unpacker = Unpacker(raw=False)
+        while True:
+            buf = sock.recv(1024**2)
+            if not buf:
+                break
+            unpacker.feed(buf)
+            for o in unpacker:
+                process(o)
+    """
+    cdef unpack_context ctx
+    cdef char* buf
+    cdef Py_ssize_t buf_size, buf_head, buf_tail
+    cdef object file_like
+    cdef object file_like_read
+    cdef Py_ssize_t read_size
+    # To maintain refcnt.
+    cdef object object_hook, object_pairs_hook, list_hook, ext_hook
+    cdef object encoding, unicode_errors
+    cdef Py_ssize_t max_buffer_size
+    cdef uint64_t stream_offset
+
+    def __cinit__(self):
+        self.buf = NULL
+
+    def __dealloc__(self):
+        PyMem_Free(self.buf)
+        self.buf = NULL
+
+    def __init__(self, file_like=None, Py_ssize_t read_size=0,
+                 bint use_list=True, bint raw=True,
+                 object object_hook=None, object object_pairs_hook=None, object list_hook=None,
+                 encoding=None, unicode_errors=None, int max_buffer_size=0,
+                 object ext_hook=ExtType,
+                 Py_ssize_t max_str_len=2147483647, # 2**32-1
+                 Py_ssize_t max_bin_len=2147483647,
+                 Py_ssize_t max_array_len=2147483647,
+                 Py_ssize_t max_map_len=2147483647,
+                 Py_ssize_t max_ext_len=2147483647):
+        cdef const char *cenc=NULL,
+        cdef const char *cerr=NULL
+
+        self.object_hook = object_hook
+        self.object_pairs_hook = object_pairs_hook
+        self.list_hook = list_hook
+        self.ext_hook = ext_hook
+
+        self.file_like = file_like
+        if file_like:
+            self.file_like_read = file_like.read
+            if not PyCallable_Check(self.file_like_read):
+                raise TypeError("`file_like.read` must be a callable.")
+        if not max_buffer_size:
+            max_buffer_size = INT_MAX
+        if read_size > max_buffer_size:
+            raise ValueError("read_size should be less or equal to max_buffer_size")
+        if not read_size:
+            read_size = min(max_buffer_size, 1024**2)
+        self.max_buffer_size = max_buffer_size
+        self.read_size = read_size
+        self.buf = <char*>PyMem_Malloc(read_size)
+        if self.buf == NULL:
+            raise MemoryError("Unable to allocate internal buffer.")
+        self.buf_size = read_size
+        self.buf_head = 0
+        self.buf_tail = 0
+        self.stream_offset = 0
+
+        if encoding is not None:
+            PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1)
+            self.encoding = encoding
+            cenc = encoding
+
+        if unicode_errors is not None:
+            self.unicode_errors = unicode_errors
+            cerr = unicode_errors
+
+        init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook,
+                 ext_hook, use_list, raw, cenc, cerr,
+                 max_str_len, max_bin_len, max_array_len,
+                 max_map_len, max_ext_len)
+
+    def feed(self, object next_bytes):
+        """Append `next_bytes` to internal buffer."""
+        cdef Py_buffer pybuff
+        cdef int new_protocol = 0
+        cdef char* buf
+        cdef Py_ssize_t buf_len
+
+        if self.file_like is not None:
+            raise AssertionError(
+                    "unpacker.feed() is not be able to use with `file_like`.")
+
+        get_data_from_buffer(next_bytes, &pybuff, &buf, &buf_len, &new_protocol)
+        try:
+            self.append_buffer(buf, buf_len)
+        finally:
+            if new_protocol:
+                PyBuffer_Release(&pybuff)
+
+    cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len):
+        cdef:
+            char* buf = self.buf
+            char* new_buf
+            Py_ssize_t head = self.buf_head
+            Py_ssize_t tail = self.buf_tail
+            Py_ssize_t buf_size = self.buf_size
+            Py_ssize_t new_size
+
+        if tail + _buf_len > buf_size:
+            if ((tail - head) + _buf_len) <= buf_size:
+                # move to front.
+                memmove(buf, buf + head, tail - head)
+                tail -= head
+                head = 0
+            else:
+                # expand buffer.
+                new_size = (tail-head) + _buf_len
+                if new_size > self.max_buffer_size:
+                    raise BufferFull
+                new_size = min(new_size*2, self.max_buffer_size)
+                new_buf = <char*>PyMem_Malloc(new_size)
+                if new_buf == NULL:
+                    # self.buf still holds old buffer and will be freed during
+                    # obj destruction
+                    raise MemoryError("Unable to enlarge internal buffer.")
+                memcpy(new_buf, buf + head, tail - head)
+                PyMem_Free(buf)
+
+                buf = new_buf
+                buf_size = new_size
+                tail -= head
+                head = 0
+
+        memcpy(buf + tail, <char*>(_buf), _buf_len)
+        self.buf = buf
+        self.buf_head = head
+        self.buf_size = buf_size
+        self.buf_tail = tail + _buf_len
+
+    cdef read_from_file(self):
+        next_bytes = self.file_like_read(
+                min(self.read_size,
+                    self.max_buffer_size - (self.buf_tail - self.buf_head)
+                    ))
+        if next_bytes:
+            self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes))
+        else:
+            self.file_like = None
+
+    cdef object _unpack(self, execute_fn execute, object write_bytes, bint iter=0):
+        cdef int ret
+        cdef object obj
+        cdef Py_ssize_t prev_head
+
+        if write_bytes is not None:
+            PyErr_WarnEx(DeprecationWarning, "`write_bytes` option is deprecated. Use `.tell()` instead.", 1)
+
+        if self.buf_head >= self.buf_tail and self.file_like is not None:
+            self.read_from_file()
+
+        while 1:
+            prev_head = self.buf_head
+            if prev_head >= self.buf_tail:
+                if iter:
+                    raise StopIteration("No more data to unpack.")
+                else:
+                    raise OutOfData("No more data to unpack.")
+
+            try:
+                ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
+                self.stream_offset += self.buf_head - prev_head
+                if write_bytes is not None:
+                    write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head))
+
+                if ret == 1:
+                    obj = unpack_data(&self.ctx)
+                    unpack_init(&self.ctx)
+                    return obj
+                elif ret == 0:
+                    if self.file_like is not None:
+                        self.read_from_file()
+                        continue
+                    if iter:
+                        raise StopIteration("No more data to unpack.")
+                    else:
+                        raise OutOfData("No more data to unpack.")
+                else:
+                    raise UnpackValueError("Unpack failed: error = %d" % (ret,))
+            except ValueError as e:
+                raise UnpackValueError(e)
+
+    def read_bytes(self, Py_ssize_t nbytes):
+        """Read a specified number of raw bytes from the stream"""
+        cdef Py_ssize_t nread
+        nread = min(self.buf_tail - self.buf_head, nbytes)
+        ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread)
+        self.buf_head += nread
+        if len(ret) < nbytes and self.file_like is not None:
+            ret += self.file_like.read(nbytes - len(ret))
+        return ret
+
+    def unpack(self, object write_bytes=None):
+        """Unpack one object
+
+        If write_bytes is not None, it will be called with parts of the raw
+        message as it is unpacked.
+
+        Raises `OutOfData` when there are no more bytes to unpack.
+        """
+        return self._unpack(unpack_construct, write_bytes)
+
+    def skip(self, object write_bytes=None):
+        """Read and ignore one object, returning None
+
+        If write_bytes is not None, it will be called with parts of the raw
+        message as it is unpacked.
+
+        Raises `OutOfData` when there are no more bytes to unpack.
+        """
+        return self._unpack(unpack_skip, write_bytes)
+
+    def read_array_header(self, object write_bytes=None):
+        """assuming the next object is an array, return its size n, such that
+        the next n unpack() calls will iterate over its contents.
+
+        Raises `OutOfData` when there are no more bytes to unpack.
+        """
+        return self._unpack(read_array_header, write_bytes)
+
+    def read_map_header(self, object write_bytes=None):
+        """assuming the next object is a map, return its size n, such that the
+        next n * 2 unpack() calls will iterate over its key-value pairs.
+
+        Raises `OutOfData` when there are no more bytes to unpack.
+        """
+        return self._unpack(read_map_header, write_bytes)
+
+    def tell(self):
+        return self.stream_offset
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        return self._unpack(unpack_construct, None, 1)
+
+    # for debug.
+    #def _buf(self):
+    #    return PyString_FromStringAndSize(self.buf, self.buf_tail)
+
+    #def _off(self):
+    #    return self.buf_head

src/borg/algorithms/msgpack/_version.py

@@ -0,0 +1 @@
+version = (0, 5, 6)

src/borg/algorithms/msgpack/exceptions.py

@@ -0,0 +1,41 @@
+class UnpackException(Exception):
+    """Deprecated.  Use Exception instead to catch all exception during unpacking."""
+
+
+class BufferFull(UnpackException):
+    pass
+
+
+class OutOfData(UnpackException):
+    pass
+
+
+class UnpackValueError(UnpackException, ValueError):
+    """Deprecated.  Use ValueError instead."""
+
+
+class ExtraData(UnpackValueError):
+    def __init__(self, unpacked, extra):
+        self.unpacked = unpacked
+        self.extra = extra
+
+    def __str__(self):
+        return "unpack(b) received extra data."
+
+
+class PackException(Exception):
+    """Deprecated.  Use Exception instead to catch all exception during packing."""
+
+
+class PackValueError(PackException, ValueError):
+    """PackValueError is raised when type of input data is supported but it's value is unsupported.
+
+    Deprecated.  Use ValueError instead.
+    """
+
+
+class PackOverflowError(PackValueError, OverflowError):
+    """PackOverflowError is raised when integer value is out of range of msgpack support [-2**31, 2**32).
+
+    Deprecated.  Use ValueError instead.
+    """

src/borg/algorithms/msgpack/fallback.py

@@ -0,0 +1,971 @@
+"""Fallback pure Python implementation of msgpack"""
+
+import sys
+import struct
+import warnings
+
+if sys.version_info[0] == 3:
+    PY3 = True
+    int_types = int
+    Unicode = str
+    xrange = range
+    def dict_iteritems(d):
+        return d.items()
+else:
+    PY3 = False
+    int_types = (int, long)
+    Unicode = unicode
+    def dict_iteritems(d):
+        return d.iteritems()
+
+
+if hasattr(sys, 'pypy_version_info'):
+    # cStringIO is slow on PyPy, StringIO is faster.  However: PyPy's own
+    # StringBuilder is fastest.
+    from __pypy__ import newlist_hint
+    try:
+        from __pypy__.builders import BytesBuilder as StringBuilder
+    except ImportError:
+        from __pypy__.builders import StringBuilder
+    USING_STRINGBUILDER = True
+    class StringIO(object):
+        def __init__(self, s=b''):
+            if s:
+                self.builder = StringBuilder(len(s))
+                self.builder.append(s)
+            else:
+                self.builder = StringBuilder()
+        def write(self, s):
+            if isinstance(s, memoryview):
+                s = s.tobytes()
+            elif isinstance(s, bytearray):
+                s = bytes(s)
+            self.builder.append(s)
+        def getvalue(self):
+            return self.builder.build()
+else:
+    USING_STRINGBUILDER = False
+    from io import BytesIO as StringIO
+    newlist_hint = lambda size: []
+
+
+from .exceptions import (
+    BufferFull,
+    OutOfData,
+    UnpackValueError,
+    PackValueError,
+    PackOverflowError,
+    ExtraData)
+
+from . import ExtType
+
+
+EX_SKIP                 = 0
+EX_CONSTRUCT            = 1
+EX_READ_ARRAY_HEADER    = 2
+EX_READ_MAP_HEADER      = 3
+
+TYPE_IMMEDIATE          = 0
+TYPE_ARRAY              = 1
+TYPE_MAP                = 2
+TYPE_RAW                = 3
+TYPE_BIN                = 4
+TYPE_EXT                = 5
+
+DEFAULT_RECURSE_LIMIT = 511
+
+
+def _check_type_strict(obj, t, type=type, tuple=tuple):
+    if type(t) is tuple:
+        return type(obj) in t
+    else:
+        return type(obj) is t
+
+
+def _get_data_from_buffer(obj):
+    try:
+        view = memoryview(obj)
+    except TypeError:
+        # try to use legacy buffer protocol if 2.7, otherwise re-raise
+        if not PY3:
+            view = memoryview(buffer(obj))
+            warnings.warn("using old buffer interface to unpack %s; "
+                          "this leads to unpacking errors if slicing is used and "
+                          "will be removed in a future version" % type(obj),
+                          RuntimeWarning)
+        else:
+            raise
+    if view.itemsize != 1:
+        raise ValueError("cannot unpack from multi-byte object")
+    return view
+
+
+def unpack(stream, **kwargs):
+    warnings.warn(
+        "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.",
+        PendingDeprecationWarning)
+    data = stream.read()
+    return unpackb(data, **kwargs)
+
+
+def unpackb(packed, **kwargs):
+    """
+    Unpack an object from `packed`.
+
+    Raises `ExtraData` when `packed` contains extra bytes.
+    See :class:`Unpacker` for options.
+    """
+    unpacker = Unpacker(None, **kwargs)
+    unpacker.feed(packed)
+    try:
+        ret = unpacker._unpack()
+    except OutOfData:
+        raise UnpackValueError("Data is not enough.")
+    if unpacker._got_extradata():
+        raise ExtraData(ret, unpacker._get_extradata())
+    return ret
+
+
+class Unpacker(object):
+    """Streaming unpacker.
+
+    arguments:
+
+    :param file_like:
+        File-like object having `.read(n)` method.
+        If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable.
+
+    :param int read_size:
+        Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`)
+
+    :param bool use_list:
+        If true, unpack msgpack array to Python list.
+        Otherwise, unpack to Python tuple. (default: True)
+
+    :param bool raw:
+        If true, unpack msgpack raw to Python bytes (default).
+        Otherwise, unpack to Python str (or unicode on Python 2) by decoding
+        with UTF-8 encoding (recommended).
+        Currently, the default is true, but it will be changed to false in
+        near future.  So you must specify it explicitly for keeping backward
+        compatibility.
+
+        *encoding* option which is deprecated overrides this option.
+
+    :param callable object_hook:
+        When specified, it should be callable.
+        Unpacker calls it with a dict argument after unpacking msgpack map.
+        (See also simplejson)
+
+    :param callable object_pairs_hook:
+        When specified, it should be callable.
+        Unpacker calls it with a list of key-value pairs after unpacking msgpack map.
+        (See also simplejson)
+
+    :param str encoding:
+        Encoding used for decoding msgpack raw.
+        If it is None (default), msgpack raw is deserialized to Python bytes.
+
+    :param str unicode_errors:
+        (deprecated) Used for decoding msgpack raw with *encoding*.
+        (default: `'strict'`)
+
+    :param int max_buffer_size:
+        Limits size of data waiting unpacked.  0 means system's INT_MAX (default).
+        Raises `BufferFull` exception when it is insufficient.
+        You should set this parameter when unpacking data from untrusted source.
+
+    :param int max_str_len:
+        Limits max length of str. (default: 2**31-1)
+
+    :param int max_bin_len:
+        Limits max length of bin. (default: 2**31-1)
+
+    :param int max_array_len:
+        Limits max length of array. (default: 2**31-1)
+
+    :param int max_map_len:
+        Limits max length of map. (default: 2**31-1)
+
+
+    example of streaming deserialize from file-like object::
+
+        unpacker = Unpacker(file_like, raw=False)
+        for o in unpacker:
+            process(o)
+
+    example of streaming deserialize from socket::
+
+        unpacker = Unpacker(raw=False)
+        while True:
+            buf = sock.recv(1024**2)
+            if not buf:
+                break
+            unpacker.feed(buf)
+            for o in unpacker:
+                process(o)
+    """
+
+    def __init__(self, file_like=None, read_size=0, use_list=True, raw=True,
+                 object_hook=None, object_pairs_hook=None, list_hook=None,
+                 encoding=None, unicode_errors=None, max_buffer_size=0,
+                 ext_hook=ExtType,
+                 max_str_len=2147483647, # 2**32-1
+                 max_bin_len=2147483647,
+                 max_array_len=2147483647,
+                 max_map_len=2147483647,
+                 max_ext_len=2147483647):
+
+        if encoding is not None:
+            warnings.warn(
+                "encoding is deprecated, Use raw=False instead.",
+                PendingDeprecationWarning)
+
+        if unicode_errors is None:
+            unicode_errors = 'strict'
+
+        if file_like is None:
+            self._feeding = True
+        else:
+            if not callable(file_like.read):
+                raise TypeError("`file_like.read` must be callable")
+            self.file_like = file_like
+            self._feeding = False
+
+        #: array of bytes fed.
+        self._buffer = bytearray()
+        #: Which position we currently reads
+        self._buff_i = 0
+
+        # When Unpacker is used as an iterable, between the calls to next(),
+        # the buffer is not "consumed" completely, for efficiency sake.
+        # Instead, it is done sloppily.  To make sure we raise BufferFull at
+        # the correct moments, we have to keep track of how sloppy we were.
+        # Furthermore, when the buffer is incomplete (that is: in the case
+        # we raise an OutOfData) we need to rollback the buffer to the correct
+        # state, which _buf_checkpoint records.
+        self._buf_checkpoint = 0
+
+        self._max_buffer_size = max_buffer_size or 2**31-1
+        if read_size > self._max_buffer_size:
+            raise ValueError("read_size must be smaller than max_buffer_size")
+        self._read_size = read_size or min(self._max_buffer_size, 16*1024)
+        self._raw = bool(raw)
+        self._encoding = encoding
+        self._unicode_errors = unicode_errors
+        self._use_list = use_list
+        self._list_hook = list_hook
+        self._object_hook = object_hook
+        self._object_pairs_hook = object_pairs_hook
+        self._ext_hook = ext_hook
+        self._max_str_len = max_str_len
+        self._max_bin_len = max_bin_len
+        self._max_array_len = max_array_len
+        self._max_map_len = max_map_len
+        self._max_ext_len = max_ext_len
+        self._stream_offset = 0
+
+        if list_hook is not None and not callable(list_hook):
+            raise TypeError('`list_hook` is not callable')
+        if object_hook is not None and not callable(object_hook):
+            raise TypeError('`object_hook` is not callable')
+        if object_pairs_hook is not None and not callable(object_pairs_hook):
+            raise TypeError('`object_pairs_hook` is not callable')
+        if object_hook is not None and object_pairs_hook is not None:
+            raise TypeError("object_pairs_hook and object_hook are mutually "
+                            "exclusive")
+        if not callable(ext_hook):
+            raise TypeError("`ext_hook` is not callable")
+
+    def feed(self, next_bytes):
+        assert self._feeding
+        view = _get_data_from_buffer(next_bytes)
+        if (len(self._buffer) - self._buff_i + len(view) > self._max_buffer_size):
+            raise BufferFull
+
+        # Strip buffer before checkpoint before reading file.
+        if self._buf_checkpoint > 0:
+            del self._buffer[:self._buf_checkpoint]
+            self._buff_i -= self._buf_checkpoint
+            self._buf_checkpoint = 0
+
+        self._buffer += view
+
+    def _consume(self):
+        """ Gets rid of the used parts of the buffer. """
+        self._stream_offset += self._buff_i - self._buf_checkpoint
+        self._buf_checkpoint = self._buff_i
+
+    def _got_extradata(self):
+        return self._buff_i < len(self._buffer)
+
+    def _get_extradata(self):
+        return self._buffer[self._buff_i:]
+
+    def read_bytes(self, n):
+        return self._read(n)
+
+    def _read(self, n):
+        # (int) -> bytearray
+        self._reserve(n)
+        i = self._buff_i
+        self._buff_i = i+n
+        return self._buffer[i:i+n]
+
+    def _reserve(self, n):
+        remain_bytes = len(self._buffer) - self._buff_i - n
+
+        # Fast path: buffer has n bytes already
+        if remain_bytes >= 0:
+            return
+
+        if self._feeding:
+            self._buff_i = self._buf_checkpoint
+            raise OutOfData
+
+        # Strip buffer before checkpoint before reading file.
+        if self._buf_checkpoint > 0:
+            del self._buffer[:self._buf_checkpoint]
+            self._buff_i -= self._buf_checkpoint
+            self._buf_checkpoint = 0
+
+        # Read from file
+        remain_bytes = -remain_bytes
+        while remain_bytes > 0:
+            to_read_bytes = max(self._read_size, remain_bytes)
+            read_data = self.file_like.read(to_read_bytes)
+            if not read_data:
+                break
+            assert isinstance(read_data, bytes)
+            self._buffer += read_data
+            remain_bytes -= len(read_data)
+
+        if len(self._buffer) < n + self._buff_i:
+            self._buff_i = 0  # rollback
+            raise OutOfData
+
+    def _read_header(self, execute=EX_CONSTRUCT):
+        typ = TYPE_IMMEDIATE
+        n = 0
+        obj = None
+        self._reserve(1)
+        b = self._buffer[self._buff_i]
+        self._buff_i += 1
+        if b & 0b10000000 == 0:
+            obj = b
+        elif b & 0b11100000 == 0b11100000:
+            obj = -1 - (b ^ 0xff)
+        elif b & 0b11100000 == 0b10100000:
+            n = b & 0b00011111
+            typ = TYPE_RAW
+            if n > self._max_str_len:
+                raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
+            obj = self._read(n)
+        elif b & 0b11110000 == 0b10010000:
+            n = b & 0b00001111
+            typ = TYPE_ARRAY
+            if n > self._max_array_len:
+                raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
+        elif b & 0b11110000 == 0b10000000:
+            n = b & 0b00001111
+            typ = TYPE_MAP
+            if n > self._max_map_len:
+                raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
+        elif b == 0xc0:
+            obj = None
+        elif b == 0xc2:
+            obj = False
+        elif b == 0xc3:
+            obj = True
+        elif b == 0xc4:
+            typ = TYPE_BIN
+            self._reserve(1)
+            n = self._buffer[self._buff_i]
+            self._buff_i += 1
+            if n > self._max_bin_len:
+                raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
+            obj = self._read(n)
+        elif b == 0xc5:
+            typ = TYPE_BIN
+            self._reserve(2)
+            n = struct.unpack_from(">H", self._buffer, self._buff_i)[0]
+            self._buff_i += 2
+            if n > self._max_bin_len:
+                raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
+            obj = self._read(n)
+        elif b == 0xc6:
+            typ = TYPE_BIN
+            self._reserve(4)
+            n = struct.unpack_from(">I", self._buffer, self._buff_i)[0]
+            self._buff_i += 4
+            if n > self._max_bin_len:
+                raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
+            obj = self._read(n)
+        elif b == 0xc7:  # ext 8
+            typ = TYPE_EXT
+            self._reserve(2)
+            L, n = struct.unpack_from('Bb', self._buffer, self._buff_i)
+            self._buff_i += 2
+            if L > self._max_ext_len:
+                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
+            obj = self._read(L)
+        elif b == 0xc8:  # ext 16
+            typ = TYPE_EXT
+            self._reserve(3)
+            L, n = struct.unpack_from('>Hb', self._buffer, self._buff_i)
+            self._buff_i += 3
+            if L > self._max_ext_len:
+                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
+            obj = self._read(L)
+        elif b == 0xc9:  # ext 32
+            typ = TYPE_EXT
+            self._reserve(5)
+            L, n = struct.unpack_from('>Ib', self._buffer, self._buff_i)
+            self._buff_i += 5
+            if L > self._max_ext_len:
+                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
+            obj = self._read(L)
+        elif b == 0xca:
+            self._reserve(4)
+            obj = struct.unpack_from(">f", self._buffer, self._buff_i)[0]
+            self._buff_i += 4
+        elif b == 0xcb:
+            self._reserve(8)
+            obj = struct.unpack_from(">d", self._buffer, self._buff_i)[0]
+            self._buff_i += 8
+        elif b == 0xcc:
+            self._reserve(1)
+            obj = self._buffer[self._buff_i]
+            self._buff_i += 1
+        elif b == 0xcd:
+            self._reserve(2)
+            obj = struct.unpack_from(">H", self._buffer, self._buff_i)[0]
+            self._buff_i += 2
+        elif b == 0xce:
+            self._reserve(4)
+            obj = struct.unpack_from(">I", self._buffer, self._buff_i)[0]
+            self._buff_i += 4
+        elif b == 0xcf:
+            self._reserve(8)
+            obj = struct.unpack_from(">Q", self._buffer, self._buff_i)[0]
+            self._buff_i += 8
+        elif b == 0xd0:
+            self._reserve(1)
+            obj = struct.unpack_from("b", self._buffer, self._buff_i)[0]
+            self._buff_i += 1
+        elif b == 0xd1:
+            self._reserve(2)
+            obj = struct.unpack_from(">h", self._buffer, self._buff_i)[0]
+            self._buff_i += 2
+        elif b == 0xd2:
+            self._reserve(4)
+            obj = struct.unpack_from(">i", self._buffer, self._buff_i)[0]
+            self._buff_i += 4
+        elif b == 0xd3:
+            self._reserve(8)
+            obj = struct.unpack_from(">q", self._buffer, self._buff_i)[0]
+            self._buff_i += 8
+        elif b == 0xd4:  # fixext 1
+            typ = TYPE_EXT
+            if self._max_ext_len < 1:
+                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
+            self._reserve(2)
+            n, obj = struct.unpack_from("b1s", self._buffer, self._buff_i)
+            self._buff_i += 2
+        elif b == 0xd5:  # fixext 2
+            typ = TYPE_EXT
+            if self._max_ext_len < 2:
+                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
+            self._reserve(3)
+            n, obj = struct.unpack_from("b2s", self._buffer, self._buff_i)
+            self._buff_i += 3
+        elif b == 0xd6:  # fixext 4
+            typ = TYPE_EXT
+            if self._max_ext_len < 4:
+                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
+            self._reserve(5)
+            n, obj = struct.unpack_from("b4s", self._buffer, self._buff_i)
+            self._buff_i += 5
+        elif b == 0xd7:  # fixext 8
+            typ = TYPE_EXT
+            if self._max_ext_len < 8:
+                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
+            self._reserve(9)
+            n, obj = struct.unpack_from("b8s", self._buffer, self._buff_i)
+            self._buff_i += 9
+        elif b == 0xd8:  # fixext 16
+            typ = TYPE_EXT
+            if self._max_ext_len < 16:
+                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
+            self._reserve(17)
+            n, obj = struct.unpack_from("b16s", self._buffer, self._buff_i)
+            self._buff_i += 17
+        elif b == 0xd9:
+            typ = TYPE_RAW
+            self._reserve(1)
+            n = self._buffer[self._buff_i]
+            self._buff_i += 1
+            if n > self._max_str_len:
+                raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
+            obj = self._read(n)
+        elif b == 0xda:
+            typ = TYPE_RAW
+            self._reserve(2)
+            n, = struct.unpack_from(">H", self._buffer, self._buff_i)
+            self._buff_i += 2
+            if n > self._max_str_len:
+                raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
+            obj = self._read(n)
+        elif b == 0xdb:
+            typ = TYPE_RAW
+            self._reserve(4)
+            n, = struct.unpack_from(">I", self._buffer, self._buff_i)
+            self._buff_i += 4
+            if n > self._max_str_len:
+                raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
+            obj = self._read(n)
+        elif b == 0xdc:
+            typ = TYPE_ARRAY
+            self._reserve(2)
+            n, = struct.unpack_from(">H", self._buffer, self._buff_i)
+            self._buff_i += 2
+            if n > self._max_array_len:
+                raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
+        elif b == 0xdd:
+            typ = TYPE_ARRAY
+            self._reserve(4)
+            n, = struct.unpack_from(">I", self._buffer, self._buff_i)
+            self._buff_i += 4
+            if n > self._max_array_len:
+                raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
+        elif b == 0xde:
+            self._reserve(2)
+            n, = struct.unpack_from(">H", self._buffer, self._buff_i)
+            self._buff_i += 2
+            if n > self._max_map_len:
+                raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
+            typ = TYPE_MAP
+        elif b == 0xdf:
+            self._reserve(4)
+            n, = struct.unpack_from(">I", self._buffer, self._buff_i)
+            self._buff_i += 4
+            if n > self._max_map_len:
+                raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
+            typ = TYPE_MAP
+        else:
+            raise UnpackValueError("Unknown header: 0x%x" % b)
+        return typ, n, obj
+
+    def _unpack(self, execute=EX_CONSTRUCT):
+        typ, n, obj = self._read_header(execute)
+
+        if execute == EX_READ_ARRAY_HEADER:
+            if typ != TYPE_ARRAY:
+                raise UnpackValueError("Expected array")
+            return n
+        if execute == EX_READ_MAP_HEADER:
+            if typ != TYPE_MAP:
+                raise UnpackValueError("Expected map")
+            return n
+        # TODO should we eliminate the recursion?
+        if typ == TYPE_ARRAY:
+            if execute == EX_SKIP:
+                for i in xrange(n):
+                    # TODO check whether we need to call `list_hook`
+                    self._unpack(EX_SKIP)
+                return
+            ret = newlist_hint(n)
+            for i in xrange(n):
+                ret.append(self._unpack(EX_CONSTRUCT))
+            if self._list_hook is not None:
+                ret = self._list_hook(ret)
+            # TODO is the interaction between `list_hook` and `use_list` ok?
+            return ret if self._use_list else tuple(ret)
+        if typ == TYPE_MAP:
+            if execute == EX_SKIP:
+                for i in xrange(n):
+                    # TODO check whether we need to call hooks
+                    self._unpack(EX_SKIP)
+                    self._unpack(EX_SKIP)
+                return
+            if self._object_pairs_hook is not None:
+                ret = self._object_pairs_hook(
+                    (self._unpack(EX_CONSTRUCT),
+                     self._unpack(EX_CONSTRUCT))
+                    for _ in xrange(n))
+            else:
+                ret = {}
+                for _ in xrange(n):
+                    key = self._unpack(EX_CONSTRUCT)
+                    ret[key] = self._unpack(EX_CONSTRUCT)
+                if self._object_hook is not None:
+                    ret = self._object_hook(ret)
+            return ret
+        if execute == EX_SKIP:
+            return
+        if typ == TYPE_RAW:
+            if self._encoding is not None:
+                obj = obj.decode(self._encoding, self._unicode_errors)
+            elif self._raw:
+                obj = bytes(obj)
+            else:
+                obj = obj.decode('utf_8')
+            return obj
+        if typ == TYPE_EXT:
+            return self._ext_hook(n, bytes(obj))
+        if typ == TYPE_BIN:
+            return bytes(obj)
+        assert typ == TYPE_IMMEDIATE
+        return obj
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        try:
+            ret = self._unpack(EX_CONSTRUCT)
+            self._consume()
+            return ret
+        except OutOfData:
+            self._consume()
+            raise StopIteration
+
+    next = __next__
+
+    def skip(self, write_bytes=None):
+        self._unpack(EX_SKIP)
+        if write_bytes is not None:
+            warnings.warn("`write_bytes` option is deprecated.  Use `.tell()` instead.", DeprecationWarning)
+            write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
+        self._consume()
+
+    def unpack(self, write_bytes=None):
+        ret = self._unpack(EX_CONSTRUCT)
+        if write_bytes is not None:
+            warnings.warn("`write_bytes` option is deprecated.  Use `.tell()` instead.", DeprecationWarning)
+            write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
+        self._consume()
+        return ret
+
+    def read_array_header(self, write_bytes=None):
+        ret = self._unpack(EX_READ_ARRAY_HEADER)
+        if write_bytes is not None:
+            warnings.warn("`write_bytes` option is deprecated.  Use `.tell()` instead.", DeprecationWarning)
+            write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
+        self._consume()
+        return ret
+
+    def read_map_header(self, write_bytes=None):
+        ret = self._unpack(EX_READ_MAP_HEADER)
+        if write_bytes is not None:
+            warnings.warn("`write_bytes` option is deprecated.  Use `.tell()` instead.", DeprecationWarning)
+            write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
+        self._consume()
+        return ret
+
+    def tell(self):
+        return self._stream_offset
+
+
+class Packer(object):
+    """
+    MessagePack Packer
+
+    usage:
+
+        packer = Packer()
+        astream.write(packer.pack(a))
+        astream.write(packer.pack(b))
+
+    Packer's constructor has some keyword arguments:
+
+    :param callable default:
+        Convert user type to builtin type that Packer supports.
+        See also simplejson's document.
+
+    :param bool use_single_float:
+        Use single precision float type for float. (default: False)
+
+    :param bool autoreset:
+        Reset buffer after each pack and return its content as `bytes`. (default: True).
+        If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
+
+    :param bool use_bin_type:
+        Use bin type introduced in msgpack spec 2.0 for bytes.
+        It also enables str8 type for unicode.
+
+    :param bool strict_types:
+        If set to true, types will be checked to be exact. Derived classes
+        from serializeable types will not be serialized and will be
+        treated as unsupported type and forwarded to default.
+        Additionally tuples will not be serialized as lists.
+        This is useful when trying to implement accurate serialization
+        for python types.
+
+    :param str encoding:
+        (deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8')
+
+    :param str unicode_errors:
+        Error handler for encoding unicode. (default: 'strict')
+    """
+    def __init__(self, default=None, encoding=None, unicode_errors=None,
+                 use_single_float=False, autoreset=True, use_bin_type=False,
+                 strict_types=False):
+        if encoding is None:
+            encoding = 'utf_8'
+        else:
+            warnings.warn(
+                "encoding is deprecated, Use raw=False instead.",
+                PendingDeprecationWarning)
+
+        if unicode_errors is None:
+            unicode_errors = 'strict'
+
+        self._strict_types = strict_types
+        self._use_float = use_single_float
+        self._autoreset = autoreset
+        self._use_bin_type = use_bin_type
+        self._encoding = encoding
+        self._unicode_errors = unicode_errors
+        self._buffer = StringIO()
+        if default is not None:
+            if not callable(default):
+                raise TypeError("default must be callable")
+        self._default = default
+
+    def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT,
+              check=isinstance, check_type_strict=_check_type_strict):
+        default_used = False
+        if self._strict_types:
+            check = check_type_strict
+            list_types = list
+        else:
+            list_types = (list, tuple)
+        while True:
+            if nest_limit < 0:
+                raise PackValueError("recursion limit exceeded")
+            if obj is None:
+                return self._buffer.write(b"\xc0")
+            if check(obj, bool):
+                if obj:
+                    return self._buffer.write(b"\xc3")
+                return self._buffer.write(b"\xc2")
+            if check(obj, int_types):
+                if 0 <= obj < 0x80:
+                    return self._buffer.write(struct.pack("B", obj))
+                if -0x20 <= obj < 0:
+                    return self._buffer.write(struct.pack("b", obj))
+                if 0x80 <= obj <= 0xff:
+                    return self._buffer.write(struct.pack("BB", 0xcc, obj))
+                if -0x80 <= obj < 0:
+                    return self._buffer.write(struct.pack(">Bb", 0xd0, obj))
+                if 0xff < obj <= 0xffff:
+                    return self._buffer.write(struct.pack(">BH", 0xcd, obj))
+                if -0x8000 <= obj < -0x80:
+                    return self._buffer.write(struct.pack(">Bh", 0xd1, obj))
+                if 0xffff < obj <= 0xffffffff:
+                    return self._buffer.write(struct.pack(">BI", 0xce, obj))
+                if -0x80000000 <= obj < -0x8000:
+                    return self._buffer.write(struct.pack(">Bi", 0xd2, obj))
+                if 0xffffffff < obj <= 0xffffffffffffffff:
+                    return self._buffer.write(struct.pack(">BQ", 0xcf, obj))
+                if -0x8000000000000000 <= obj < -0x80000000:
+                    return self._buffer.write(struct.pack(">Bq", 0xd3, obj))
+                if not default_used and self._default is not None:
+                    obj = self._default(obj)
+                    default_used = True
+                    continue
+                raise PackOverflowError("Integer value out of range")
+            if check(obj, (bytes, bytearray)):
+                n = len(obj)
+                if n >= 2**32:
+                    raise PackValueError("%s is too large" % type(obj).__name__)
+                self._pack_bin_header(n)
+                return self._buffer.write(obj)
+            if check(obj, Unicode):
+                if self._encoding is None:
+                    raise TypeError(
+                        "Can't encode unicode string: "
+                        "no encoding is specified")
+                obj = obj.encode(self._encoding, self._unicode_errors)
+                n = len(obj)
+                if n >= 2**32:
+                    raise PackValueError("String is too large")
+                self._pack_raw_header(n)
+                return self._buffer.write(obj)
+            if check(obj, memoryview):
+                n = len(obj) * obj.itemsize
+                if n >= 2**32:
+                    raise PackValueError("Memoryview is too large")
+                self._pack_bin_header(n)
+                return self._buffer.write(obj)
+            if check(obj, float):
+                if self._use_float:
+                    return self._buffer.write(struct.pack(">Bf", 0xca, obj))
+                return self._buffer.write(struct.pack(">Bd", 0xcb, obj))
+            if check(obj, ExtType):
+                code = obj.code
+                data = obj.data
+                assert isinstance(code, int)
+                assert isinstance(data, bytes)
+                L = len(data)
+                if L == 1:
+                    self._buffer.write(b'\xd4')
+                elif L == 2:
+                    self._buffer.write(b'\xd5')
+                elif L == 4:
+                    self._buffer.write(b'\xd6')
+                elif L == 8:
+                    self._buffer.write(b'\xd7')
+                elif L == 16:
+                    self._buffer.write(b'\xd8')
+                elif L <= 0xff:
+                    self._buffer.write(struct.pack(">BB", 0xc7, L))
+                elif L <= 0xffff:
+                    self._buffer.write(struct.pack(">BH", 0xc8, L))
+                else:
+                    self._buffer.write(struct.pack(">BI", 0xc9, L))
+                self._buffer.write(struct.pack("b", code))
+                self._buffer.write(data)
+                return
+            if check(obj, list_types):
+                n = len(obj)
+                self._pack_array_header(n)
+                for i in xrange(n):
+                    self._pack(obj[i], nest_limit - 1)
+                return
+            if check(obj, dict):
+                return self._pack_map_pairs(len(obj), dict_iteritems(obj),
+                                               nest_limit - 1)
+            if not default_used and self._default is not None:
+                obj = self._default(obj)
+                default_used = 1
+                continue
+            raise TypeError("Cannot serialize %r" % (obj, ))
+
+    def pack(self, obj):
+        try:
+            self._pack(obj)
+        except:
+            self._buffer = StringIO()  # force reset
+            raise
+        ret = self._buffer.getvalue()
+        if self._autoreset:
+            self._buffer = StringIO()
+        elif USING_STRINGBUILDER:
+            self._buffer = StringIO(ret)
+        return ret
+
+    def pack_map_pairs(self, pairs):
+        self._pack_map_pairs(len(pairs), pairs)
+        ret = self._buffer.getvalue()
+        if self._autoreset:
+            self._buffer = StringIO()
+        elif USING_STRINGBUILDER:
+            self._buffer = StringIO(ret)
+        return ret
+
+    def pack_array_header(self, n):
+        if n >= 2**32:
+            raise PackValueError
+        self._pack_array_header(n)
+        ret = self._buffer.getvalue()
+        if self._autoreset:
+            self._buffer = StringIO()
+        elif USING_STRINGBUILDER:
+            self._buffer = StringIO(ret)
+        return ret
+
+    def pack_map_header(self, n):
+        if n >= 2**32:
+            raise PackValueError
+        self._pack_map_header(n)
+        ret = self._buffer.getvalue()
+        if self._autoreset:
+            self._buffer = StringIO()
+        elif USING_STRINGBUILDER:
+            self._buffer = StringIO(ret)
+        return ret
+
+    def pack_ext_type(self, typecode, data):
+        if not isinstance(typecode, int):
+            raise TypeError("typecode must have int type.")
+        if not 0 <= typecode <= 127:
+            raise ValueError("typecode should be 0-127")
+        if not isinstance(data, bytes):
+            raise TypeError("data must have bytes type")
+        L = len(data)
+        if L > 0xffffffff:
+            raise PackValueError("Too large data")
+        if L == 1:
+            self._buffer.write(b'\xd4')
+        elif L == 2:
+            self._buffer.write(b'\xd5')
+        elif L == 4:
+            self._buffer.write(b'\xd6')
+        elif L == 8:
+            self._buffer.write(b'\xd7')
+        elif L == 16:
+            self._buffer.write(b'\xd8')
+        elif L <= 0xff:
+            self._buffer.write(b'\xc7' + struct.pack('B', L))
+        elif L <= 0xffff:
+            self._buffer.write(b'\xc8' + struct.pack('>H', L))
+        else:
+            self._buffer.write(b'\xc9' + struct.pack('>I', L))
+        self._buffer.write(struct.pack('B', typecode))
+        self._buffer.write(data)
+
+    def _pack_array_header(self, n):
+        if n <= 0x0f:
+            return self._buffer.write(struct.pack('B', 0x90 + n))
+        if n <= 0xffff:
+            return self._buffer.write(struct.pack(">BH", 0xdc, n))
+        if n <= 0xffffffff:
+            return self._buffer.write(struct.pack(">BI", 0xdd, n))
+        raise PackValueError("Array is too large")
+
+    def _pack_map_header(self, n):
+        if n <= 0x0f:
+            return self._buffer.write(struct.pack('B', 0x80 + n))
+        if n <= 0xffff:
+            return self._buffer.write(struct.pack(">BH", 0xde, n))
+        if n <= 0xffffffff:
+            return self._buffer.write(struct.pack(">BI", 0xdf, n))
+        raise PackValueError("Dict is too large")
+
+    def _pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT):
+        self._pack_map_header(n)
+        for (k, v) in pairs:
+            self._pack(k, nest_limit - 1)
+            self._pack(v, nest_limit - 1)
+
+    def _pack_raw_header(self, n):
+        if n <= 0x1f:
+            self._buffer.write(struct.pack('B', 0xa0 + n))
+        elif self._use_bin_type and n <= 0xff:
+            self._buffer.write(struct.pack('>BB', 0xd9, n))
+        elif n <= 0xffff:
+            self._buffer.write(struct.pack(">BH", 0xda, n))
+        elif n <= 0xffffffff:
+            self._buffer.write(struct.pack(">BI", 0xdb, n))
+        else:
+            raise PackValueError('Raw is too large')
+
+    def _pack_bin_header(self, n):
+        if not self._use_bin_type:
+            return self._pack_raw_header(n)
+        elif n <= 0xff:
+            return self._buffer.write(struct.pack('>BB', 0xc4, n))
+        elif n <= 0xffff:
+            return self._buffer.write(struct.pack(">BH", 0xc5, n))
+        elif n <= 0xffffffff:
+            return self._buffer.write(struct.pack(">BI", 0xc6, n))
+        else:
+            raise PackValueError('Bin is too large')
+
+    def bytes(self):
+        return self._buffer.getvalue()
+
+    def reset(self):
+        self._buffer = StringIO()

src/borg/algorithms/msgpack/pack.h

@@ -0,0 +1,119 @@
+/*
+ * MessagePack for Python packing routine
+ *
+ * Copyright (C) 2009 Naoki INADA
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#include <stddef.h>
+#include <stdlib.h>
+#include "sysdep.h"
+#include <limits.h>
+#include <string.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef _MSC_VER
+#define inline __inline
+#endif
+
+typedef struct msgpack_packer {
+    char *buf;
+    size_t length;
+    size_t buf_size;
+    bool use_bin_type;
+} msgpack_packer;
+
+typedef struct Packer Packer;
+
+static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l)
+{
+    char* buf = pk->buf;
+    size_t bs = pk->buf_size;
+    size_t len = pk->length;
+
+    if (len + l > bs) {
+        bs = (len + l) * 2;
+        buf = (char*)PyMem_Realloc(buf, bs);
+        if (!buf) {
+            PyErr_NoMemory();
+            return -1;
+        }
+    }
+    memcpy(buf + len, data, l);
+    len += l;
+
+    pk->buf = buf;
+    pk->buf_size = bs;
+    pk->length = len;
+    return 0;
+}
+
+#define msgpack_pack_append_buffer(user, buf, len) \
+        return msgpack_pack_write(user, (const char*)buf, len)
+
+#include "pack_template.h"
+
+// return -2 when o is too long
+static inline int
+msgpack_pack_unicode(msgpack_packer *pk, PyObject *o, long long limit)
+{
+#if PY_MAJOR_VERSION >= 3
+    assert(PyUnicode_Check(o));
+
+    Py_ssize_t len;
+    const char* buf = PyUnicode_AsUTF8AndSize(o, &len);
+    if (buf == NULL)
+        return -1;
+
+    if (len > limit) {
+        return -2;
+    }
+
+    int ret = msgpack_pack_raw(pk, len);
+    if (ret) return ret;
+
+    return msgpack_pack_raw_body(pk, buf, len);
+#else
+    PyObject *bytes;
+    Py_ssize_t len;
+    int ret;
+
+    // py2
+    bytes = PyUnicode_AsUTF8String(o);
+    if (bytes == NULL)
+        return -1;
+
+    len = PyString_GET_SIZE(bytes);
+    if (len > limit) {
+        Py_DECREF(bytes);
+        return -2;
+    }
+
+    ret = msgpack_pack_raw(pk, len);
+    if (ret) {
+        Py_DECREF(bytes);
+        return -1;
+    }
+    ret = msgpack_pack_raw_body(pk, PyString_AS_STRING(bytes), len);
+    Py_DECREF(bytes);
+    return ret;
+#endif
+}
+
+#ifdef __cplusplus
+}
+#endif

src/borg/algorithms/msgpack/pack_template.h

@@ -0,0 +1,785 @@
+/*
+ * MessagePack packing routine template
+ *
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#if defined(__LITTLE_ENDIAN__)
+#define TAKE8_8(d)  ((uint8_t*)&d)[0]
+#define TAKE8_16(d) ((uint8_t*)&d)[0]
+#define TAKE8_32(d) ((uint8_t*)&d)[0]
+#define TAKE8_64(d) ((uint8_t*)&d)[0]
+#elif defined(__BIG_ENDIAN__)
+#define TAKE8_8(d)  ((uint8_t*)&d)[0]
+#define TAKE8_16(d) ((uint8_t*)&d)[1]
+#define TAKE8_32(d) ((uint8_t*)&d)[3]
+#define TAKE8_64(d) ((uint8_t*)&d)[7]
+#endif
+
+#ifndef msgpack_pack_append_buffer
+#error msgpack_pack_append_buffer callback is not defined
+#endif
+
+
+/*
+ * Integer
+ */
+
+#define msgpack_pack_real_uint8(x, d) \
+do { \
+    if(d < (1<<7)) { \
+        /* fixnum */ \
+        msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \
+    } else { \
+        /* unsigned 8 */ \
+        unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \
+        msgpack_pack_append_buffer(x, buf, 2); \
+    } \
+} while(0)
+
+#define msgpack_pack_real_uint16(x, d) \
+do { \
+    if(d < (1<<7)) { \
+        /* fixnum */ \
+        msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \
+    } else if(d < (1<<8)) { \
+        /* unsigned 8 */ \
+        unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \
+        msgpack_pack_append_buffer(x, buf, 2); \
+    } else { \
+        /* unsigned 16 */ \
+        unsigned char buf[3]; \
+        buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
+        msgpack_pack_append_buffer(x, buf, 3); \
+    } \
+} while(0)
+
+#define msgpack_pack_real_uint32(x, d) \
+do { \
+    if(d < (1<<8)) { \
+        if(d < (1<<7)) { \
+            /* fixnum */ \
+            msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \
+        } else { \
+            /* unsigned 8 */ \
+            unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \
+            msgpack_pack_append_buffer(x, buf, 2); \
+        } \
+    } else { \
+        if(d < (1<<16)) { \
+            /* unsigned 16 */ \
+            unsigned char buf[3]; \
+            buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
+            msgpack_pack_append_buffer(x, buf, 3); \
+        } else { \
+            /* unsigned 32 */ \
+            unsigned char buf[5]; \
+            buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
+            msgpack_pack_append_buffer(x, buf, 5); \
+        } \
+    } \
+} while(0)
+
+#define msgpack_pack_real_uint64(x, d) \
+do { \
+    if(d < (1ULL<<8)) { \
+        if(d < (1ULL<<7)) { \
+            /* fixnum */ \
+            msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \
+        } else { \
+            /* unsigned 8 */ \
+            unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \
+            msgpack_pack_append_buffer(x, buf, 2); \
+        } \
+    } else { \
+        if(d < (1ULL<<16)) { \
+            /* unsigned 16 */ \
+            unsigned char buf[3]; \
+            buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
+            msgpack_pack_append_buffer(x, buf, 3); \
+        } else if(d < (1ULL<<32)) { \
+            /* unsigned 32 */ \
+            unsigned char buf[5]; \
+            buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
+            msgpack_pack_append_buffer(x, buf, 5); \
+        } else { \
+            /* unsigned 64 */ \
+            unsigned char buf[9]; \
+            buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \
+            msgpack_pack_append_buffer(x, buf, 9); \
+        } \
+    } \
+} while(0)
+
+#define msgpack_pack_real_int8(x, d) \
+do { \
+    if(d < -(1<<5)) { \
+        /* signed 8 */ \
+        unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \
+        msgpack_pack_append_buffer(x, buf, 2); \
+    } else { \
+        /* fixnum */ \
+        msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \
+    } \
+} while(0)
+
+#define msgpack_pack_real_int16(x, d) \
+do { \
+    if(d < -(1<<5)) { \
+        if(d < -(1<<7)) { \
+            /* signed 16 */ \
+            unsigned char buf[3]; \
+            buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \
+            msgpack_pack_append_buffer(x, buf, 3); \
+        } else { \
+            /* signed 8 */ \
+            unsigned char buf[2] = {0xd0, TAKE8_16(d)}; \
+            msgpack_pack_append_buffer(x, buf, 2); \
+        } \
+    } else if(d < (1<<7)) { \
+        /* fixnum */ \
+        msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \
+    } else { \
+        if(d < (1<<8)) { \
+            /* unsigned 8 */ \
+            unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \
+            msgpack_pack_append_buffer(x, buf, 2); \
+        } else { \
+            /* unsigned 16 */ \
+            unsigned char buf[3]; \
+            buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
+            msgpack_pack_append_buffer(x, buf, 3); \
+        } \
+    } \
+} while(0)
+
+#define msgpack_pack_real_int32(x, d) \
+do { \
+    if(d < -(1<<5)) { \
+        if(d < -(1<<15)) { \
+            /* signed 32 */ \
+            unsigned char buf[5]; \
+            buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \
+            msgpack_pack_append_buffer(x, buf, 5); \
+        } else if(d < -(1<<7)) { \
+            /* signed 16 */ \
+            unsigned char buf[3]; \
+            buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \
+            msgpack_pack_append_buffer(x, buf, 3); \
+        } else { \
+            /* signed 8 */ \
+            unsigned char buf[2] = {0xd0, TAKE8_32(d)}; \
+            msgpack_pack_append_buffer(x, buf, 2); \
+        } \
+    } else if(d < (1<<7)) { \
+        /* fixnum */ \
+        msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \
+    } else { \
+        if(d < (1<<8)) { \
+            /* unsigned 8 */ \
+            unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \
+            msgpack_pack_append_buffer(x, buf, 2); \
+        } else if(d < (1<<16)) { \
+            /* unsigned 16 */ \
+            unsigned char buf[3]; \
+            buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
+            msgpack_pack_append_buffer(x, buf, 3); \
+        } else { \
+            /* unsigned 32 */ \
+            unsigned char buf[5]; \
+            buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
+            msgpack_pack_append_buffer(x, buf, 5); \
+        } \
+    } \
+} while(0)
+
+#define msgpack_pack_real_int64(x, d) \
+do { \
+    if(d < -(1LL<<5)) { \
+        if(d < -(1LL<<15)) { \
+            if(d < -(1LL<<31)) { \
+                /* signed 64 */ \
+                unsigned char buf[9]; \
+                buf[0] = 0xd3; _msgpack_store64(&buf[1], d); \
+                msgpack_pack_append_buffer(x, buf, 9); \
+            } else { \
+                /* signed 32 */ \
+                unsigned char buf[5]; \
+                buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \
+                msgpack_pack_append_buffer(x, buf, 5); \
+            } \
+        } else { \
+            if(d < -(1<<7)) { \
+                /* signed 16 */ \
+                unsigned char buf[3]; \
+                buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \
+                msgpack_pack_append_buffer(x, buf, 3); \
+            } else { \
+                /* signed 8 */ \
+                unsigned char buf[2] = {0xd0, TAKE8_64(d)}; \
+                msgpack_pack_append_buffer(x, buf, 2); \
+            } \
+        } \
+    } else if(d < (1<<7)) { \
+        /* fixnum */ \
+        msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \
+    } else { \
+        if(d < (1LL<<16)) { \
+            if(d < (1<<8)) { \
+                /* unsigned 8 */ \
+                unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \
+                msgpack_pack_append_buffer(x, buf, 2); \
+            } else { \
+                /* unsigned 16 */ \
+                unsigned char buf[3]; \
+                buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
+                msgpack_pack_append_buffer(x, buf, 3); \
+            } \
+        } else { \
+            if(d < (1LL<<32)) { \
+                /* unsigned 32 */ \
+                unsigned char buf[5]; \
+                buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
+                msgpack_pack_append_buffer(x, buf, 5); \
+            } else { \
+                /* unsigned 64 */ \
+                unsigned char buf[9]; \
+                buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \
+                msgpack_pack_append_buffer(x, buf, 9); \
+            } \
+        } \
+    } \
+} while(0)
+
+
+static inline int msgpack_pack_uint8(msgpack_packer* x, uint8_t d)
+{
+    msgpack_pack_real_uint8(x, d);
+}
+
+static inline int msgpack_pack_uint16(msgpack_packer* x, uint16_t d)
+{
+    msgpack_pack_real_uint16(x, d);
+}
+
+static inline int msgpack_pack_uint32(msgpack_packer* x, uint32_t d)
+{
+    msgpack_pack_real_uint32(x, d);
+}
+
+static inline int msgpack_pack_uint64(msgpack_packer* x, uint64_t d)
+{
+    msgpack_pack_real_uint64(x, d);
+}
+
+static inline int msgpack_pack_int8(msgpack_packer* x, int8_t d)
+{
+    msgpack_pack_real_int8(x, d);
+}
+
+static inline int msgpack_pack_int16(msgpack_packer* x, int16_t d)
+{
+    msgpack_pack_real_int16(x, d);
+}
+
+static inline int msgpack_pack_int32(msgpack_packer* x, int32_t d)
+{
+    msgpack_pack_real_int32(x, d);
+}
+
+static inline int msgpack_pack_int64(msgpack_packer* x, int64_t d)
+{
+    msgpack_pack_real_int64(x, d);
+}
+
+
+//#ifdef msgpack_pack_inline_func_cint
+
+static inline int msgpack_pack_short(msgpack_packer* x, short d)
+{
+#if defined(SIZEOF_SHORT)
+#if SIZEOF_SHORT == 2
+    msgpack_pack_real_int16(x, d);
+#elif SIZEOF_SHORT == 4
+    msgpack_pack_real_int32(x, d);
+#else
+    msgpack_pack_real_int64(x, d);
+#endif
+
+#elif defined(SHRT_MAX)
+#if SHRT_MAX == 0x7fff
+    msgpack_pack_real_int16(x, d);
+#elif SHRT_MAX == 0x7fffffff
+    msgpack_pack_real_int32(x, d);
+#else
+    msgpack_pack_real_int64(x, d);
+#endif
+
+#else
+if(sizeof(short) == 2) {
+    msgpack_pack_real_int16(x, d);
+} else if(sizeof(short) == 4) {
+    msgpack_pack_real_int32(x, d);
+} else {
+    msgpack_pack_real_int64(x, d);
+}
+#endif
+}
+
+static inline int msgpack_pack_int(msgpack_packer* x, int d)
+{
+#if defined(SIZEOF_INT)
+#if SIZEOF_INT == 2
+    msgpack_pack_real_int16(x, d);
+#elif SIZEOF_INT == 4
+    msgpack_pack_real_int32(x, d);
+#else
+    msgpack_pack_real_int64(x, d);
+#endif
+
+#elif defined(INT_MAX)
+#if INT_MAX == 0x7fff
+    msgpack_pack_real_int16(x, d);
+#elif INT_MAX == 0x7fffffff
+    msgpack_pack_real_int32(x, d);
+#else
+    msgpack_pack_real_int64(x, d);
+#endif
+
+#else
+if(sizeof(int) == 2) {
+    msgpack_pack_real_int16(x, d);
+} else if(sizeof(int) == 4) {
+    msgpack_pack_real_int32(x, d);
+} else {
+    msgpack_pack_real_int64(x, d);
+}
+#endif
+}
+
+static inline int msgpack_pack_long(msgpack_packer* x, long d)
+{
+#if defined(SIZEOF_LONG)
+#if SIZEOF_LONG == 2
+    msgpack_pack_real_int16(x, d);
+#elif SIZEOF_LONG == 4
+    msgpack_pack_real_int32(x, d);
+#else
+    msgpack_pack_real_int64(x, d);
+#endif
+
+#elif defined(LONG_MAX)
+#if LONG_MAX == 0x7fffL
+    msgpack_pack_real_int16(x, d);
+#elif LONG_MAX == 0x7fffffffL
+    msgpack_pack_real_int32(x, d);
+#else
+    msgpack_pack_real_int64(x, d);
+#endif
+
+#else
+if(sizeof(long) == 2) {
+    msgpack_pack_real_int16(x, d);
+} else if(sizeof(long) == 4) {
+    msgpack_pack_real_int32(x, d);
+} else {
+    msgpack_pack_real_int64(x, d);
+}
+#endif
+}
+
+static inline int msgpack_pack_long_long(msgpack_packer* x, long long d)
+{
+#if defined(SIZEOF_LONG_LONG)
+#if SIZEOF_LONG_LONG == 2
+    msgpack_pack_real_int16(x, d);
+#elif SIZEOF_LONG_LONG == 4
+    msgpack_pack_real_int32(x, d);
+#else
+    msgpack_pack_real_int64(x, d);
+#endif
+
+#elif defined(LLONG_MAX)
+#if LLONG_MAX == 0x7fffL
+    msgpack_pack_real_int16(x, d);
+#elif LLONG_MAX == 0x7fffffffL
+    msgpack_pack_real_int32(x, d);
+#else
+    msgpack_pack_real_int64(x, d);
+#endif
+
+#else
+if(sizeof(long long) == 2) {
+    msgpack_pack_real_int16(x, d);
+} else if(sizeof(long long) == 4) {
+    msgpack_pack_real_int32(x, d);
+} else {
+    msgpack_pack_real_int64(x, d);
+}
+#endif
+}
+
+static inline int msgpack_pack_unsigned_short(msgpack_packer* x, unsigned short d)
+{
+#if defined(SIZEOF_SHORT)
+#if SIZEOF_SHORT == 2
+    msgpack_pack_real_uint16(x, d);
+#elif SIZEOF_SHORT == 4
+    msgpack_pack_real_uint32(x, d);
+#else
+    msgpack_pack_real_uint64(x, d);
+#endif
+
+#elif defined(USHRT_MAX)
+#if USHRT_MAX == 0xffffU
+    msgpack_pack_real_uint16(x, d);
+#elif USHRT_MAX == 0xffffffffU
+    msgpack_pack_real_uint32(x, d);
+#else
+    msgpack_pack_real_uint64(x, d);
+#endif
+
+#else
+if(sizeof(unsigned short) == 2) {
+    msgpack_pack_real_uint16(x, d);
+} else if(sizeof(unsigned short) == 4) {
+    msgpack_pack_real_uint32(x, d);
+} else {
+    msgpack_pack_real_uint64(x, d);
+}
+#endif
+}
+
+static inline int msgpack_pack_unsigned_int(msgpack_packer* x, unsigned int d)
+{
+#if defined(SIZEOF_INT)
+#if SIZEOF_INT == 2
+    msgpack_pack_real_uint16(x, d);
+#elif SIZEOF_INT == 4
+    msgpack_pack_real_uint32(x, d);
+#else
+    msgpack_pack_real_uint64(x, d);
+#endif
+
+#elif defined(UINT_MAX)
+#if UINT_MAX == 0xffffU
+    msgpack_pack_real_uint16(x, d);
+#elif UINT_MAX == 0xffffffffU
+    msgpack_pack_real_uint32(x, d);
+#else
+    msgpack_pack_real_uint64(x, d);
+#endif
+
+#else
+if(sizeof(unsigned int) == 2) {
+    msgpack_pack_real_uint16(x, d);
+} else if(sizeof(unsigned int) == 4) {
+    msgpack_pack_real_uint32(x, d);
+} else {
+    msgpack_pack_real_uint64(x, d);
+}
+#endif
+}
+
+static inline int msgpack_pack_unsigned_long(msgpack_packer* x, unsigned long d)
+{
+#if defined(SIZEOF_LONG)
+#if SIZEOF_LONG == 2
+    msgpack_pack_real_uint16(x, d);
+#elif SIZEOF_LONG == 4
+    msgpack_pack_real_uint32(x, d);
+#else
+    msgpack_pack_real_uint64(x, d);
+#endif
+
+#elif defined(ULONG_MAX)
+#if ULONG_MAX == 0xffffUL
+    msgpack_pack_real_uint16(x, d);
+#elif ULONG_MAX == 0xffffffffUL
+    msgpack_pack_real_uint32(x, d);
+#else
+    msgpack_pack_real_uint64(x, d);
+#endif
+
+#else
+if(sizeof(unsigned long) == 2) {
+    msgpack_pack_real_uint16(x, d);
+} else if(sizeof(unsigned long) == 4) {
+    msgpack_pack_real_uint32(x, d);
+} else {
+    msgpack_pack_real_uint64(x, d);
+}
+#endif
+}
+
+static inline int msgpack_pack_unsigned_long_long(msgpack_packer* x, unsigned long long d)
+{
+#if defined(SIZEOF_LONG_LONG)
+#if SIZEOF_LONG_LONG == 2
+    msgpack_pack_real_uint16(x, d);
+#elif SIZEOF_LONG_LONG == 4
+    msgpack_pack_real_uint32(x, d);
+#else
+    msgpack_pack_real_uint64(x, d);
+#endif
+
+#elif defined(ULLONG_MAX)
+#if ULLONG_MAX == 0xffffUL
+    msgpack_pack_real_uint16(x, d);
+#elif ULLONG_MAX == 0xffffffffUL
+    msgpack_pack_real_uint32(x, d);
+#else
+    msgpack_pack_real_uint64(x, d);
+#endif
+
+#else
+if(sizeof(unsigned long long) == 2) {
+    msgpack_pack_real_uint16(x, d);
+} else if(sizeof(unsigned long long) == 4) {
+    msgpack_pack_real_uint32(x, d);
+} else {
+    msgpack_pack_real_uint64(x, d);
+}
+#endif
+}
+
+//#undef msgpack_pack_inline_func_cint
+//#endif
+
+
+
+/*
+ * Float
+ */
+
+static inline int msgpack_pack_float(msgpack_packer* x, float d)
+{
+    union { float f; uint32_t i; } mem;
+    mem.f = d;
+    unsigned char buf[5];
+    buf[0] = 0xca; _msgpack_store32(&buf[1], mem.i);
+    msgpack_pack_append_buffer(x, buf, 5);
+}
+
+static inline int msgpack_pack_double(msgpack_packer* x, double d)
+{
+    union { double f; uint64_t i; } mem;
+    mem.f = d;
+    unsigned char buf[9];
+    buf[0] = 0xcb;
+#if defined(__arm__) && !(__ARM_EABI__) // arm-oabi
+    // https://github.com/msgpack/msgpack-perl/pull/1
+    mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL);
+#endif
+    _msgpack_store64(&buf[1], mem.i);
+    msgpack_pack_append_buffer(x, buf, 9);
+}
+
+
+/*
+ * Nil
+ */
+
+static inline int msgpack_pack_nil(msgpack_packer* x)
+{
+    static const unsigned char d = 0xc0;
+    msgpack_pack_append_buffer(x, &d, 1);
+}
+
+
+/*
+ * Boolean
+ */
+
+static inline int msgpack_pack_true(msgpack_packer* x)
+{
+    static const unsigned char d = 0xc3;
+    msgpack_pack_append_buffer(x, &d, 1);
+}
+
+static inline int msgpack_pack_false(msgpack_packer* x)
+{
+    static const unsigned char d = 0xc2;
+    msgpack_pack_append_buffer(x, &d, 1);
+}
+
+
+/*
+ * Array
+ */
+
+static inline int msgpack_pack_array(msgpack_packer* x, unsigned int n)
+{
+    if(n < 16) {
+        unsigned char d = 0x90 | n;
+        msgpack_pack_append_buffer(x, &d, 1);
+    } else if(n < 65536) {
+        unsigned char buf[3];
+        buf[0] = 0xdc; _msgpack_store16(&buf[1], (uint16_t)n);
+        msgpack_pack_append_buffer(x, buf, 3);
+    } else {
+        unsigned char buf[5];
+        buf[0] = 0xdd; _msgpack_store32(&buf[1], (uint32_t)n);
+        msgpack_pack_append_buffer(x, buf, 5);
+    }
+}
+
+
+/*
+ * Map
+ */
+
+static inline int msgpack_pack_map(msgpack_packer* x, unsigned int n)
+{
+    if(n < 16) {
+        unsigned char d = 0x80 | n;
+        msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
+    } else if(n < 65536) {
+        unsigned char buf[3];
+        buf[0] = 0xde; _msgpack_store16(&buf[1], (uint16_t)n);
+        msgpack_pack_append_buffer(x, buf, 3);
+    } else {
+        unsigned char buf[5];
+        buf[0] = 0xdf; _msgpack_store32(&buf[1], (uint32_t)n);
+        msgpack_pack_append_buffer(x, buf, 5);
+    }
+}
+
+
+/*
+ * Raw
+ */
+
+static inline int msgpack_pack_raw(msgpack_packer* x, size_t l)
+{
+    if (l < 32) {
+        unsigned char d = 0xa0 | (uint8_t)l;
+        msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
+    } else if (x->use_bin_type && l < 256) {  // str8 is new format introduced with bin.
+        unsigned char buf[2] = {0xd9, (uint8_t)l};
+        msgpack_pack_append_buffer(x, buf, 2);
+    } else if (l < 65536) {
+        unsigned char buf[3];
+        buf[0] = 0xda; _msgpack_store16(&buf[1], (uint16_t)l);
+        msgpack_pack_append_buffer(x, buf, 3);
+    } else {
+        unsigned char buf[5];
+        buf[0] = 0xdb; _msgpack_store32(&buf[1], (uint32_t)l);
+        msgpack_pack_append_buffer(x, buf, 5);
+    }
+}
+
+/*
+ * bin
+ */
+static inline int msgpack_pack_bin(msgpack_packer *x, size_t l)
+{
+    if (!x->use_bin_type) {
+        return msgpack_pack_raw(x, l);
+    }
+    if (l < 256) {
+        unsigned char buf[2] = {0xc4, (unsigned char)l};
+        msgpack_pack_append_buffer(x, buf, 2);
+    } else if (l < 65536) {
+        unsigned char buf[3] = {0xc5};
+        _msgpack_store16(&buf[1], (uint16_t)l);
+        msgpack_pack_append_buffer(x, buf, 3);
+    } else {
+        unsigned char buf[5] = {0xc6};
+        _msgpack_store32(&buf[1], (uint32_t)l);
+        msgpack_pack_append_buffer(x, buf, 5);
+    }
+}
+
+static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t l)
+{
+    if (l > 0) msgpack_pack_append_buffer(x, (const unsigned char*)b, l);
+    return 0;
+}
+
+/*
+ * Ext
+ */
+static inline int msgpack_pack_ext(msgpack_packer* x, char typecode, size_t l)
+{
+    if (l == 1) {
+        unsigned char buf[2];
+        buf[0] = 0xd4;
+        buf[1] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 2);
+    }
+    else if(l == 2) {
+        unsigned char buf[2];
+        buf[0] = 0xd5;
+        buf[1] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 2);
+    }
+    else if(l == 4) {
+        unsigned char buf[2];
+        buf[0] = 0xd6;
+        buf[1] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 2);
+    }
+    else if(l == 8) {
+        unsigned char buf[2];
+        buf[0] = 0xd7;
+        buf[1] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 2);
+    }
+    else if(l == 16) {
+        unsigned char buf[2];
+        buf[0] = 0xd8;
+        buf[1] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 2);
+    }
+    else if(l < 256) {
+        unsigned char buf[3];
+        buf[0] = 0xc7;
+        buf[1] = l;
+        buf[2] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 3);
+    } else if(l < 65536) {
+        unsigned char buf[4];
+        buf[0] = 0xc8;
+        _msgpack_store16(&buf[1], (uint16_t)l);
+        buf[3] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 4);
+    } else {
+        unsigned char buf[6];
+        buf[0] = 0xc9;
+        _msgpack_store32(&buf[1], (uint32_t)l);
+        buf[5] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 6);
+    }
+
+}
+
+
+
+#undef msgpack_pack_append_buffer
+
+#undef TAKE8_8
+#undef TAKE8_16
+#undef TAKE8_32
+#undef TAKE8_64
+
+#undef msgpack_pack_real_uint8
+#undef msgpack_pack_real_uint16
+#undef msgpack_pack_real_uint32
+#undef msgpack_pack_real_uint64
+#undef msgpack_pack_real_int8
+#undef msgpack_pack_real_int16
+#undef msgpack_pack_real_int32
+#undef msgpack_pack_real_int64

src/borg/algorithms/msgpack/sysdep.h

@@ -0,0 +1,194 @@
+/*
+ * MessagePack system dependencies
+ *
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#ifndef MSGPACK_SYSDEP_H__
+#define MSGPACK_SYSDEP_H__
+
+#include <stdlib.h>
+#include <stddef.h>
+#if defined(_MSC_VER) && _MSC_VER < 1600
+typedef __int8 int8_t;
+typedef unsigned __int8 uint8_t;
+typedef __int16 int16_t;
+typedef unsigned __int16 uint16_t;
+typedef __int32 int32_t;
+typedef unsigned __int32 uint32_t;
+typedef __int64 int64_t;
+typedef unsigned __int64 uint64_t;
+#elif defined(_MSC_VER)  // && _MSC_VER >= 1600
+#include <stdint.h>
+#else
+#include <stdint.h>
+#include <stdbool.h>
+#endif
+
+#ifdef _WIN32
+#define _msgpack_atomic_counter_header <windows.h>
+typedef long _msgpack_atomic_counter_t;
+#define _msgpack_sync_decr_and_fetch(ptr) InterlockedDecrement(ptr)
+#define _msgpack_sync_incr_and_fetch(ptr) InterlockedIncrement(ptr)
+#elif defined(__GNUC__) && ((__GNUC__*10 + __GNUC_MINOR__) < 41)
+#define _msgpack_atomic_counter_header "gcc_atomic.h"
+#else
+typedef unsigned int _msgpack_atomic_counter_t;
+#define _msgpack_sync_decr_and_fetch(ptr) __sync_sub_and_fetch(ptr, 1)
+#define _msgpack_sync_incr_and_fetch(ptr) __sync_add_and_fetch(ptr, 1)
+#endif
+
+#ifdef _WIN32
+
+#ifdef __cplusplus
+/* numeric_limits<T>::min,max */
+#ifdef max
+#undef max
+#endif
+#ifdef min
+#undef min
+#endif
+#endif
+
+#else
+#include <arpa/inet.h>  /* __BYTE_ORDER */
+#endif
+
+#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+#define __LITTLE_ENDIAN__
+#elif __BYTE_ORDER == __BIG_ENDIAN
+#define __BIG_ENDIAN__
+#elif _WIN32
+#define __LITTLE_ENDIAN__
+#endif
+#endif
+
+
+#ifdef __LITTLE_ENDIAN__
+
+#ifdef _WIN32
+#  if defined(ntohs)
+#    define _msgpack_be16(x) ntohs(x)
+#  elif defined(_byteswap_ushort) || (defined(_MSC_VER) && _MSC_VER >= 1400)
+#    define _msgpack_be16(x) ((uint16_t)_byteswap_ushort((unsigned short)x))
+#  else
+#    define _msgpack_be16(x) ( \
+        ((((uint16_t)x) <<  8) ) | \
+        ((((uint16_t)x) >>  8) ) )
+#  endif
+#else
+#  define _msgpack_be16(x) ntohs(x)
+#endif
+
+#ifdef _WIN32
+#  if defined(ntohl)
+#    define _msgpack_be32(x) ntohl(x)
+#  elif defined(_byteswap_ulong) || (defined(_MSC_VER) && _MSC_VER >= 1400)
+#    define _msgpack_be32(x) ((uint32_t)_byteswap_ulong((unsigned long)x))
+#  else
+#    define _msgpack_be32(x) \
+        ( ((((uint32_t)x) << 24)               ) | \
+          ((((uint32_t)x) <<  8) & 0x00ff0000U ) | \
+          ((((uint32_t)x) >>  8) & 0x0000ff00U ) | \
+          ((((uint32_t)x) >> 24)               ) )
+#  endif
+#else
+#  define _msgpack_be32(x) ntohl(x)
+#endif
+
+#if defined(_byteswap_uint64) || (defined(_MSC_VER) && _MSC_VER >= 1400)
+#  define _msgpack_be64(x) (_byteswap_uint64(x))
+#elif defined(bswap_64)
+#  define _msgpack_be64(x) bswap_64(x)
+#elif defined(__DARWIN_OSSwapInt64)
+#  define _msgpack_be64(x) __DARWIN_OSSwapInt64(x)
+#else
+#define _msgpack_be64(x) \
+    ( ((((uint64_t)x) << 56)                         ) | \
+      ((((uint64_t)x) << 40) & 0x00ff000000000000ULL ) | \
+      ((((uint64_t)x) << 24) & 0x0000ff0000000000ULL ) | \
+      ((((uint64_t)x) <<  8) & 0x000000ff00000000ULL ) | \
+      ((((uint64_t)x) >>  8) & 0x00000000ff000000ULL ) | \
+      ((((uint64_t)x) >> 24) & 0x0000000000ff0000ULL ) | \
+      ((((uint64_t)x) >> 40) & 0x000000000000ff00ULL ) | \
+      ((((uint64_t)x) >> 56)                         ) )
+#endif
+
+#define _msgpack_load16(cast, from) ((cast)( \
+        (((uint16_t)((uint8_t*)(from))[0]) << 8) | \
+        (((uint16_t)((uint8_t*)(from))[1])     ) ))
+
+#define _msgpack_load32(cast, from) ((cast)( \
+        (((uint32_t)((uint8_t*)(from))[0]) << 24) | \
+        (((uint32_t)((uint8_t*)(from))[1]) << 16) | \
+        (((uint32_t)((uint8_t*)(from))[2]) <<  8) | \
+        (((uint32_t)((uint8_t*)(from))[3])      ) ))
+
+#define _msgpack_load64(cast, from) ((cast)( \
+        (((uint64_t)((uint8_t*)(from))[0]) << 56) | \
+        (((uint64_t)((uint8_t*)(from))[1]) << 48) | \
+        (((uint64_t)((uint8_t*)(from))[2]) << 40) | \
+        (((uint64_t)((uint8_t*)(from))[3]) << 32) | \
+        (((uint64_t)((uint8_t*)(from))[4]) << 24) | \
+        (((uint64_t)((uint8_t*)(from))[5]) << 16) | \
+        (((uint64_t)((uint8_t*)(from))[6]) << 8)  | \
+        (((uint64_t)((uint8_t*)(from))[7])     )  ))
+
+#else
+
+#define _msgpack_be16(x) (x)
+#define _msgpack_be32(x) (x)
+#define _msgpack_be64(x) (x)
+
+#define _msgpack_load16(cast, from) ((cast)( \
+        (((uint16_t)((uint8_t*)from)[0]) << 8) | \
+        (((uint16_t)((uint8_t*)from)[1])     ) ))
+
+#define _msgpack_load32(cast, from) ((cast)( \
+        (((uint32_t)((uint8_t*)from)[0]) << 24) | \
+        (((uint32_t)((uint8_t*)from)[1]) << 16) | \
+        (((uint32_t)((uint8_t*)from)[2]) <<  8) | \
+        (((uint32_t)((uint8_t*)from)[3])      ) ))
+
+#define _msgpack_load64(cast, from) ((cast)( \
+        (((uint64_t)((uint8_t*)from)[0]) << 56) | \
+        (((uint64_t)((uint8_t*)from)[1]) << 48) | \
+        (((uint64_t)((uint8_t*)from)[2]) << 40) | \
+        (((uint64_t)((uint8_t*)from)[3]) << 32) | \
+        (((uint64_t)((uint8_t*)from)[4]) << 24) | \
+        (((uint64_t)((uint8_t*)from)[5]) << 16) | \
+        (((uint64_t)((uint8_t*)from)[6]) << 8)  | \
+        (((uint64_t)((uint8_t*)from)[7])     )  ))
+#endif
+
+
+#define _msgpack_store16(to, num) \
+    do { uint16_t val = _msgpack_be16(num); memcpy(to, &val, 2); } while(0)
+#define _msgpack_store32(to, num) \
+    do { uint32_t val = _msgpack_be32(num); memcpy(to, &val, 4); } while(0)
+#define _msgpack_store64(to, num) \
+    do { uint64_t val = _msgpack_be64(num); memcpy(to, &val, 8); } while(0)
+
+/*
+#define _msgpack_load16(cast, from) \
+    ({ cast val; memcpy(&val, (char*)from, 2); _msgpack_be16(val); })
+#define _msgpack_load32(cast, from) \
+    ({ cast val; memcpy(&val, (char*)from, 4); _msgpack_be32(val); })
+#define _msgpack_load64(cast, from) \
+    ({ cast val; memcpy(&val, (char*)from, 8); _msgpack_be64(val); })
+*/
+
+
+#endif /* msgpack/sysdep.h */

src/borg/algorithms/msgpack/unpack.h

@@ -0,0 +1,282 @@
+/*
+ * MessagePack for Python unpacking routine
+ *
+ * Copyright (C) 2009 Naoki INADA
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#define MSGPACK_EMBED_STACK_SIZE  (1024)
+#include "unpack_define.h"
+
+typedef struct unpack_user {
+    bool use_list;
+    bool raw;
+    bool has_pairs_hook;
+    PyObject *object_hook;
+    PyObject *list_hook;
+    PyObject *ext_hook;
+    const char *encoding;
+    const char *unicode_errors;
+    Py_ssize_t max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len;
+} unpack_user;
+
+typedef PyObject* msgpack_unpack_object;
+struct unpack_context;
+typedef struct unpack_context unpack_context;
+typedef int (*execute_fn)(unpack_context *ctx, const char* data, Py_ssize_t len, Py_ssize_t* off);
+
+static inline msgpack_unpack_object unpack_callback_root(unpack_user* u)
+{
+    return NULL;
+}
+
+static inline int unpack_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o)
+{
+    PyObject *p = PyInt_FromLong((long)d);
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+static inline int unpack_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o)
+{
+    return unpack_callback_uint16(u, d, o);
+}
+
+
+static inline int unpack_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o)
+{
+    PyObject *p = PyInt_FromSize_t((size_t)d);
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+
+static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o)
+{
+    PyObject *p;
+    if (d > LONG_MAX) {
+        p = PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)d);
+    } else {
+        p = PyInt_FromLong((long)d);
+    }
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+
+static inline int unpack_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o)
+{
+    PyObject *p = PyInt_FromLong(d);
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+
+static inline int unpack_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o)
+{
+    return unpack_callback_int32(u, d, o);
+}
+
+static inline int unpack_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o)
+{
+    return unpack_callback_int32(u, d, o);
+}
+
+static inline int unpack_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o)
+{
+    PyObject *p;
+    if (d > LONG_MAX || d < LONG_MIN) {
+        p = PyLong_FromLongLong((PY_LONG_LONG)d);
+    } else {
+        p = PyInt_FromLong((long)d);
+    }
+    *o = p;
+    return 0;
+}
+
+static inline int unpack_callback_double(unpack_user* u, double d, msgpack_unpack_object* o)
+{
+    PyObject *p = PyFloat_FromDouble(d);
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+
+static inline int unpack_callback_float(unpack_user* u, float d, msgpack_unpack_object* o)
+{
+    return unpack_callback_double(u, d, o);
+}
+
+static inline int unpack_callback_nil(unpack_user* u, msgpack_unpack_object* o)
+{ Py_INCREF(Py_None); *o = Py_None; return 0; }
+
+static inline int unpack_callback_true(unpack_user* u, msgpack_unpack_object* o)
+{ Py_INCREF(Py_True); *o = Py_True; return 0; }
+
+static inline int unpack_callback_false(unpack_user* u, msgpack_unpack_object* o)
+{ Py_INCREF(Py_False); *o = Py_False; return 0; }
+
+static inline int unpack_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
+{
+    if (n > u->max_array_len) {
+        PyErr_Format(PyExc_ValueError, "%u exceeds max_array_len(%zd)", n, u->max_array_len);
+        return -1;
+    }
+    PyObject *p = u->use_list ? PyList_New(n) : PyTuple_New(n);
+
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+
+static inline int unpack_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o)
+{
+    if (u->use_list)
+        PyList_SET_ITEM(*c, current, o);
+    else
+        PyTuple_SET_ITEM(*c, current, o);
+    return 0;
+}
+
+static inline int unpack_callback_array_end(unpack_user* u, msgpack_unpack_object* c)
+{
+    if (u->list_hook) {
+        PyObject *new_c = PyObject_CallFunctionObjArgs(u->list_hook, *c, NULL);
+        if (!new_c)
+            return -1;
+        Py_DECREF(*c);
+        *c = new_c;
+    }
+    return 0;
+}
+
+static inline int unpack_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
+{
+    if (n > u->max_map_len) {
+        PyErr_Format(PyExc_ValueError, "%u exceeds max_map_len(%zd)", n, u->max_map_len);
+        return -1;
+    }
+    PyObject *p;
+    if (u->has_pairs_hook) {
+        p = PyList_New(n); // Or use tuple?
+    }
+    else {
+        p = PyDict_New();
+    }
+    if (!p)
+        return -1;
+    *o = p;
+    return 0;
+}
+
+static inline int unpack_callback_map_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v)
+{
+    if (u->has_pairs_hook) {
+        msgpack_unpack_object item = PyTuple_Pack(2, k, v);
+        if (!item)
+            return -1;
+        Py_DECREF(k);
+        Py_DECREF(v);
+        PyList_SET_ITEM(*c, current, item);
+        return 0;
+    }
+    else if (PyDict_SetItem(*c, k, v) == 0) {
+        Py_DECREF(k);
+        Py_DECREF(v);
+        return 0;
+    }
+    return -1;
+}
+
+static inline int unpack_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
+{
+    if (u->object_hook) {
+        PyObject *new_c = PyObject_CallFunctionObjArgs(u->object_hook, *c, NULL);
+        if (!new_c)
+            return -1;
+
+        Py_DECREF(*c);
+        *c = new_c;
+    }
+    return 0;
+}
+
+static inline int unpack_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
+{
+    if (l > u->max_str_len) {
+        PyErr_Format(PyExc_ValueError, "%u exceeds max_str_len(%zd)", l, u->max_str_len);
+        return -1;
+    }
+
+    PyObject *py;
+
+    if (u->encoding) {
+        py = PyUnicode_Decode(p, l, u->encoding, u->unicode_errors);
+    } else if (u->raw) {
+        py = PyBytes_FromStringAndSize(p, l);
+    } else {
+        py = PyUnicode_DecodeUTF8(p, l, u->unicode_errors);
+    }
+    if (!py)
+        return -1;
+    *o = py;
+    return 0;
+}
+
+static inline int unpack_callback_bin(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
+{
+    if (l > u->max_bin_len) {
+        PyErr_Format(PyExc_ValueError, "%u exceeds max_bin_len(%zd)", l, u->max_bin_len);
+        return -1;
+    }
+
+    PyObject *py = PyBytes_FromStringAndSize(p, l);
+    if (!py)
+        return -1;
+    *o = py;
+    return 0;
+}
+
+static inline int unpack_callback_ext(unpack_user* u, const char* base, const char* pos,
+                                      unsigned int length, msgpack_unpack_object* o)
+{
+    PyObject *py;
+    int8_t typecode = (int8_t)*pos++;
+    if (!u->ext_hook) {
+        PyErr_SetString(PyExc_AssertionError, "u->ext_hook cannot be NULL");
+        return -1;
+    }
+    if (length-1 > u->max_ext_len) {
+        PyErr_Format(PyExc_ValueError, "%u exceeds max_ext_len(%zd)", length, u->max_ext_len);
+        return -1;
+    }
+    // length also includes the typecode, so the actual data is length-1
+#if PY_MAJOR_VERSION == 2
+    py = PyObject_CallFunction(u->ext_hook, "(is#)", (int)typecode, pos, (Py_ssize_t)length-1);
+#else
+    py = PyObject_CallFunction(u->ext_hook, "(iy#)", (int)typecode, pos, (Py_ssize_t)length-1);
+#endif
+    if (!py)
+        return -1;
+    *o = py;
+    return 0;
+}
+
+#include "unpack_template.h"

src/borg/algorithms/msgpack/unpack_define.h

@@ -0,0 +1,95 @@
+/*
+ * MessagePack unpacking routine template
+ *
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#ifndef MSGPACK_UNPACK_DEFINE_H__
+#define MSGPACK_UNPACK_DEFINE_H__
+
+#include "sysdep.h"
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <stdio.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+#ifndef MSGPACK_EMBED_STACK_SIZE
+#define MSGPACK_EMBED_STACK_SIZE 32
+#endif
+
+
+// CS is first byte & 0x1f
+typedef enum {
+    CS_HEADER            = 0x00,  // nil
+
+    //CS_                = 0x01,
+    //CS_                = 0x02,  // false
+    //CS_                = 0x03,  // true
+
+    CS_BIN_8             = 0x04,
+    CS_BIN_16            = 0x05,
+    CS_BIN_32            = 0x06,
+
+    CS_EXT_8             = 0x07,
+    CS_EXT_16            = 0x08,
+    CS_EXT_32            = 0x09,
+
+    CS_FLOAT             = 0x0a,
+    CS_DOUBLE            = 0x0b,
+    CS_UINT_8            = 0x0c,
+    CS_UINT_16           = 0x0d,
+    CS_UINT_32           = 0x0e,
+    CS_UINT_64           = 0x0f,
+    CS_INT_8             = 0x10,
+    CS_INT_16            = 0x11,
+    CS_INT_32            = 0x12,
+    CS_INT_64            = 0x13,
+
+    //CS_FIXEXT1           = 0x14,
+    //CS_FIXEXT2           = 0x15,
+    //CS_FIXEXT4           = 0x16,
+    //CS_FIXEXT8           = 0x17,
+    //CS_FIXEXT16          = 0x18,
+
+    CS_RAW_8             = 0x19,
+    CS_RAW_16            = 0x1a,
+    CS_RAW_32            = 0x1b,
+    CS_ARRAY_16          = 0x1c,
+    CS_ARRAY_32          = 0x1d,
+    CS_MAP_16            = 0x1e,
+    CS_MAP_32            = 0x1f,
+
+    ACS_RAW_VALUE,
+    ACS_BIN_VALUE,
+    ACS_EXT_VALUE,
+} msgpack_unpack_state;
+
+
+typedef enum {
+    CT_ARRAY_ITEM,
+    CT_MAP_KEY,
+    CT_MAP_VALUE,
+} msgpack_container_type;
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* msgpack/unpack_define.h */

src/borg/algorithms/msgpack/unpack_template.h

@@ -0,0 +1,479 @@
+/*
+ * MessagePack unpacking routine template
+ *
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#ifndef USE_CASE_RANGE
+#if !defined(_MSC_VER)
+#define USE_CASE_RANGE
+#endif
+#endif
+
+typedef struct unpack_stack {
+    PyObject* obj;
+    Py_ssize_t size;
+    Py_ssize_t count;
+    unsigned int ct;
+    PyObject* map_key;
+} unpack_stack;
+
+struct unpack_context {
+    unpack_user user;
+    unsigned int cs;
+    unsigned int trail;
+    unsigned int top;
+    /*
+    unpack_stack* stack;
+    unsigned int stack_size;
+    unpack_stack embed_stack[MSGPACK_EMBED_STACK_SIZE];
+    */
+    unpack_stack stack[MSGPACK_EMBED_STACK_SIZE];
+};
+
+
+static inline void unpack_init(unpack_context* ctx)
+{
+    ctx->cs = CS_HEADER;
+    ctx->trail = 0;
+    ctx->top = 0;
+    /*
+    ctx->stack = ctx->embed_stack;
+    ctx->stack_size = MSGPACK_EMBED_STACK_SIZE;
+    */
+    ctx->stack[0].obj = unpack_callback_root(&ctx->user);
+}
+
+/*
+static inline void unpack_destroy(unpack_context* ctx)
+{
+    if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) {
+        free(ctx->stack);
+    }
+}
+*/
+
+static inline PyObject* unpack_data(unpack_context* ctx)
+{
+    return (ctx)->stack[0].obj;
+}
+
+static inline void unpack_clear(unpack_context *ctx)
+{
+    Py_CLEAR(ctx->stack[0].obj);
+}
+
+template <bool construct>
+static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off)
+{
+    assert(len >= *off);
+
+    const unsigned char* p = (unsigned char*)data + *off;
+    const unsigned char* const pe = (unsigned char*)data + len;
+    const void* n = p;
+
+    unsigned int trail = ctx->trail;
+    unsigned int cs = ctx->cs;
+    unsigned int top = ctx->top;
+    unpack_stack* stack = ctx->stack;
+    /*
+    unsigned int stack_size = ctx->stack_size;
+    */
+    unpack_user* user = &ctx->user;
+
+    PyObject* obj = NULL;
+    unpack_stack* c = NULL;
+
+    int ret;
+
+#define construct_cb(name) \
+    construct && unpack_callback ## name
+
+#define push_simple_value(func) \
+    if(construct_cb(func)(user, &obj) < 0) { goto _failed; } \
+    goto _push
+#define push_fixed_value(func, arg) \
+    if(construct_cb(func)(user, arg, &obj) < 0) { goto _failed; } \
+    goto _push
+#define push_variable_value(func, base, pos, len) \
+    if(construct_cb(func)(user, \
+        (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \
+    goto _push
+
+#define again_fixed_trail(_cs, trail_len) \
+    trail = trail_len; \
+    cs = _cs; \
+    goto _fixed_trail_again
+#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \
+    trail = trail_len; \
+    if(trail == 0) { goto ifzero; } \
+    cs = _cs; \
+    goto _fixed_trail_again
+
+#define start_container(func, count_, ct_) \
+    if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \
+    if(construct_cb(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \
+    if((count_) == 0) { obj = stack[top].obj; \
+        if (construct_cb(func##_end)(user, &obj) < 0) { goto _failed; } \
+        goto _push; } \
+    stack[top].ct = ct_; \
+    stack[top].size  = count_; \
+    stack[top].count = 0; \
+    ++top; \
+    /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \
+    /*printf("stack push %d\n", top);*/ \
+    /* FIXME \
+    if(top >= stack_size) { \
+        if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \
+            size_t csize = sizeof(unpack_stack) * MSGPACK_EMBED_STACK_SIZE; \
+            size_t nsize = csize * 2; \
+            unpack_stack* tmp = (unpack_stack*)malloc(nsize); \
+            if(tmp == NULL) { goto _failed; } \
+            memcpy(tmp, ctx->stack, csize); \
+            ctx->stack = stack = tmp; \
+            ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \
+        } else { \
+            size_t nsize = sizeof(unpack_stack) * ctx->stack_size * 2; \
+            unpack_stack* tmp = (unpack_stack*)realloc(ctx->stack, nsize); \
+            if(tmp == NULL) { goto _failed; } \
+            ctx->stack = stack = tmp; \
+            ctx->stack_size = stack_size = stack_size * 2; \
+        } \
+    } \
+    */ \
+    goto _header_again
+
+#define NEXT_CS(p)  ((unsigned int)*p & 0x1f)
+
+#ifdef USE_CASE_RANGE
+#define SWITCH_RANGE_BEGIN     switch(*p) {
+#define SWITCH_RANGE(FROM, TO) case FROM ... TO:
+#define SWITCH_RANGE_DEFAULT   default:
+#define SWITCH_RANGE_END       }
+#else
+#define SWITCH_RANGE_BEGIN     { if(0) {
+#define SWITCH_RANGE(FROM, TO) } else if(FROM <= *p && *p <= TO) {
+#define SWITCH_RANGE_DEFAULT   } else {
+#define SWITCH_RANGE_END       } }
+#endif
+
+    if(p == pe) { goto _out; }
+    do {
+        switch(cs) {
+        case CS_HEADER:
+            SWITCH_RANGE_BEGIN
+            SWITCH_RANGE(0x00, 0x7f)  // Positive Fixnum
+                push_fixed_value(_uint8, *(uint8_t*)p);
+            SWITCH_RANGE(0xe0, 0xff)  // Negative Fixnum
+                push_fixed_value(_int8, *(int8_t*)p);
+            SWITCH_RANGE(0xc0, 0xdf)  // Variable
+                switch(*p) {
+                case 0xc0:  // nil
+                    push_simple_value(_nil);
+                //case 0xc1:  // never used
+                case 0xc2:  // false
+                    push_simple_value(_false);
+                case 0xc3:  // true
+                    push_simple_value(_true);
+                case 0xc4:  // bin 8
+                    again_fixed_trail(NEXT_CS(p), 1);
+                case 0xc5:  // bin 16
+                    again_fixed_trail(NEXT_CS(p), 2);
+                case 0xc6:  // bin 32
+                    again_fixed_trail(NEXT_CS(p), 4);
+                case 0xc7:  // ext 8
+                    again_fixed_trail(NEXT_CS(p), 1);
+                case 0xc8:  // ext 16
+                    again_fixed_trail(NEXT_CS(p), 2);
+                case 0xc9:  // ext 32
+                    again_fixed_trail(NEXT_CS(p), 4);
+                case 0xca:  // float
+                case 0xcb:  // double
+                case 0xcc:  // unsigned int  8
+                case 0xcd:  // unsigned int 16
+                case 0xce:  // unsigned int 32
+                case 0xcf:  // unsigned int 64
+                case 0xd0:  // signed int  8
+                case 0xd1:  // signed int 16
+                case 0xd2:  // signed int 32
+                case 0xd3:  // signed int 64
+                    again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
+                case 0xd4:  // fixext 1
+                case 0xd5:  // fixext 2
+                case 0xd6:  // fixext 4
+                case 0xd7:  // fixext 8
+                    again_fixed_trail_if_zero(ACS_EXT_VALUE, 
+                                              (1 << (((unsigned int)*p) & 0x03))+1,
+                                              _ext_zero);
+                case 0xd8:  // fixext 16
+                    again_fixed_trail_if_zero(ACS_EXT_VALUE, 16+1, _ext_zero);
+                case 0xd9:  // str 8
+                    again_fixed_trail(NEXT_CS(p), 1);
+                case 0xda:  // raw 16
+                case 0xdb:  // raw 32
+                case 0xdc:  // array 16
+                case 0xdd:  // array 32
+                case 0xde:  // map 16
+                case 0xdf:  // map 32
+                    again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01));
+                default:
+                    goto _failed;
+                }
+            SWITCH_RANGE(0xa0, 0xbf)  // FixRaw
+                again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero);
+            SWITCH_RANGE(0x90, 0x9f)  // FixArray
+                start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM);
+            SWITCH_RANGE(0x80, 0x8f)  // FixMap
+                start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY);
+
+            SWITCH_RANGE_DEFAULT
+                goto _failed;
+            SWITCH_RANGE_END
+            // end CS_HEADER
+
+
+        _fixed_trail_again:
+            ++p;
+
+        default:
+            if((size_t)(pe - p) < trail) { goto _out; }
+            n = p;  p += trail - 1;
+            switch(cs) {
+            case CS_EXT_8:
+                again_fixed_trail_if_zero(ACS_EXT_VALUE, *(uint8_t*)n+1, _ext_zero);
+            case CS_EXT_16:
+                again_fixed_trail_if_zero(ACS_EXT_VALUE,
+                                          _msgpack_load16(uint16_t,n)+1,
+                                          _ext_zero);
+            case CS_EXT_32:
+                again_fixed_trail_if_zero(ACS_EXT_VALUE,
+                                          _msgpack_load32(uint32_t,n)+1,
+                                          _ext_zero);
+            case CS_FLOAT: {
+                    union { uint32_t i; float f; } mem;
+                    mem.i = _msgpack_load32(uint32_t,n);
+                    push_fixed_value(_float, mem.f); }
+            case CS_DOUBLE: {
+                    union { uint64_t i; double f; } mem;
+                    mem.i = _msgpack_load64(uint64_t,n);
+#if defined(__arm__) && !(__ARM_EABI__) // arm-oabi
+                    // https://github.com/msgpack/msgpack-perl/pull/1
+                    mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL);
+#endif
+                    push_fixed_value(_double, mem.f); }
+            case CS_UINT_8:
+                push_fixed_value(_uint8, *(uint8_t*)n);
+            case CS_UINT_16:
+                push_fixed_value(_uint16, _msgpack_load16(uint16_t,n));
+            case CS_UINT_32:
+                push_fixed_value(_uint32, _msgpack_load32(uint32_t,n));
+            case CS_UINT_64:
+                push_fixed_value(_uint64, _msgpack_load64(uint64_t,n));
+
+            case CS_INT_8:
+                push_fixed_value(_int8, *(int8_t*)n);
+            case CS_INT_16:
+                push_fixed_value(_int16, _msgpack_load16(int16_t,n));
+            case CS_INT_32:
+                push_fixed_value(_int32, _msgpack_load32(int32_t,n));
+            case CS_INT_64:
+                push_fixed_value(_int64, _msgpack_load64(int64_t,n));
+
+            case CS_BIN_8:
+                again_fixed_trail_if_zero(ACS_BIN_VALUE, *(uint8_t*)n, _bin_zero);
+            case CS_BIN_16:
+                again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load16(uint16_t,n), _bin_zero);
+            case CS_BIN_32:
+                again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load32(uint32_t,n), _bin_zero);
+            case ACS_BIN_VALUE:
+            _bin_zero:
+                push_variable_value(_bin, data, n, trail);
+
+            case CS_RAW_8:
+                again_fixed_trail_if_zero(ACS_RAW_VALUE, *(uint8_t*)n, _raw_zero);
+            case CS_RAW_16:
+                again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero);
+            case CS_RAW_32:
+                again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero);
+            case ACS_RAW_VALUE:
+            _raw_zero:
+                push_variable_value(_raw, data, n, trail);
+
+            case ACS_EXT_VALUE:
+            _ext_zero:
+                push_variable_value(_ext, data, n, trail);
+
+            case CS_ARRAY_16:
+                start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM);
+            case CS_ARRAY_32:
+                /* FIXME security guard */
+                start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM);
+
+            case CS_MAP_16:
+                start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY);
+            case CS_MAP_32:
+                /* FIXME security guard */
+                start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY);
+
+            default:
+                goto _failed;
+            }
+        }
+
+_push:
+    if(top == 0) { goto _finish; }
+    c = &stack[top-1];
+    switch(c->ct) {
+    case CT_ARRAY_ITEM:
+        if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; }
+        if(++c->count == c->size) {
+            obj = c->obj;
+            if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; }
+            --top;
+            /*printf("stack pop %d\n", top);*/
+            goto _push;
+        }
+        goto _header_again;
+    case CT_MAP_KEY:
+        c->map_key = obj;
+        c->ct = CT_MAP_VALUE;
+        goto _header_again;
+    case CT_MAP_VALUE:
+        if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; }
+        if(++c->count == c->size) {
+            obj = c->obj;
+            if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; }
+            --top;
+            /*printf("stack pop %d\n", top);*/
+            goto _push;
+        }
+        c->ct = CT_MAP_KEY;
+        goto _header_again;
+
+    default:
+        goto _failed;
+    }
+
+_header_again:
+        cs = CS_HEADER;
+        ++p;
+    } while(p != pe);
+    goto _out;
+
+
+_finish:
+    if (!construct)
+        unpack_callback_nil(user, &obj);
+    stack[0].obj = obj;
+    ++p;
+    ret = 1;
+    /*printf("-- finish --\n"); */
+    goto _end;
+
+_failed:
+    /*printf("** FAILED **\n"); */
+    ret = -1;
+    goto _end;
+
+_out:
+    ret = 0;
+    goto _end;
+
+_end:
+    ctx->cs = cs;
+    ctx->trail = trail;
+    ctx->top = top;
+    *off = p - (const unsigned char*)data;
+
+    return ret;
+#undef construct_cb
+}
+
+#undef SWITCH_RANGE_BEGIN
+#undef SWITCH_RANGE
+#undef SWITCH_RANGE_DEFAULT
+#undef SWITCH_RANGE_END
+#undef push_simple_value
+#undef push_fixed_value
+#undef push_variable_value
+#undef again_fixed_trail
+#undef again_fixed_trail_if_zero
+#undef start_container
+
+template <unsigned int fixed_offset, unsigned int var_offset>
+static inline int unpack_container_header(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off)
+{
+    assert(len >= *off);
+    uint32_t size;
+    const unsigned char *const p = (unsigned char*)data + *off;
+
+#define inc_offset(inc) \
+    if (len - *off < inc) \
+        return 0; \
+    *off += inc;
+
+    switch (*p) {
+    case var_offset:
+        inc_offset(3);
+        size = _msgpack_load16(uint16_t, p + 1);
+        break;
+    case var_offset + 1:
+        inc_offset(5);
+        size = _msgpack_load32(uint32_t, p + 1);
+        break;
+#ifdef USE_CASE_RANGE
+    case fixed_offset + 0x0 ... fixed_offset + 0xf:
+#else
+    case fixed_offset + 0x0:
+    case fixed_offset + 0x1:
+    case fixed_offset + 0x2:
+    case fixed_offset + 0x3:
+    case fixed_offset + 0x4:
+    case fixed_offset + 0x5:
+    case fixed_offset + 0x6:
+    case fixed_offset + 0x7:
+    case fixed_offset + 0x8:
+    case fixed_offset + 0x9:
+    case fixed_offset + 0xa:
+    case fixed_offset + 0xb:
+    case fixed_offset + 0xc:
+    case fixed_offset + 0xd:
+    case fixed_offset + 0xe:
+    case fixed_offset + 0xf:
+#endif
+        ++*off;
+        size = ((unsigned int)*p) & 0x0f;
+        break;
+    default:
+        PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream");
+        return -1;
+    }
+    unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj);
+    return 1;
+}
+
+#undef SWITCH_RANGE_BEGIN
+#undef SWITCH_RANGE
+#undef SWITCH_RANGE_DEFAULT
+#undef SWITCH_RANGE_END
+
+static const execute_fn unpack_construct = &unpack_execute<true>;
+static const execute_fn unpack_skip = &unpack_execute<false>;
+static const execute_fn read_array_header = &unpack_container_header<0x90, 0xdc>;
+static const execute_fn read_map_header = &unpack_container_header<0x80, 0xde>;
+
+#undef NEXT_CS
+
+/* vim: set ts=4 sw=4 sts=4 expandtab  */

src/borg/archive.py

@@ -13,8 +13,6 @@ from io import BytesIO
 from itertools import groupby
 from shutil import get_terminal_size
 
-import msgpack
-
 from .logger import create_logger
 
 logger = create_logger()
@@ -38,6 +36,7 @@ from .helpers import StableDict
 from .helpers import bin_to_hex
 from .helpers import safe_ns
 from .helpers import ellipsis_truncate, ProgressIndicatorPercent, log_multi
+from .helpers import msgpack
 from .patterns import PathPrefixPattern, FnmatchPattern, IECommand
 from .item import Item, ArchiveItem
 from .platform import acl_get, acl_set, set_flags, get_flags, swidth, hostname

src/borg/archiver.py

@@ -33,8 +33,6 @@ try:
 
     logger = create_logger()
 
-    import msgpack
-
     import borg
     from . import __version__
     from . import helpers
@@ -72,6 +70,7 @@ try:
     from .helpers import popen_with_error_handling, prepare_subprocess_env
     from .helpers import dash_open
     from .helpers import umount
+    from .helpers import msgpack, msgpack_fallback
     from .nanorst import rst_to_terminal
     from .patterns import ArgparsePatternAction, ArgparseExcludeFileAction, ArgparsePatternFileAction, parse_exclude_pattern
     from .patterns import PatternMatcher
@@ -1957,7 +1956,7 @@ class Archiver:
 
         data = key.decrypt(None, repository.get(manifest.MANIFEST_ID))
 
-        meta = prepare_dump_dict(msgpack.fallback.unpackb(data, object_hook=StableDict, unicode_errors='surrogateescape'))
+        meta = prepare_dump_dict(msgpack_fallback.unpackb(data, object_hook=StableDict, unicode_errors='surrogateescape'))
 
         with dash_open(args.path, 'w') as fd:
             json.dump(meta, fd, indent=4)

src/borg/cache.py

@@ -6,8 +6,6 @@ from binascii import unhexlify
 from collections import namedtuple
 from time import perf_counter
 
-import msgpack
-
 from .logger import create_logger
 
 logger = create_logger()
@@ -26,6 +24,7 @@ from .helpers import remove_surrogates
 from .helpers import ProgressIndicatorPercent, ProgressIndicatorMessage
 from .helpers import set_ec, EXIT_WARNING
 from .helpers import truncate_and_unlink
+from .helpers import msgpack
 from .item import ArchiveItem, ChunkListEntry
 from .crypto.key import PlaintextKey
 from .crypto.file_integrity import IntegrityCheckedFile, DetachedIntegrityCheckedFile, FileIntegrityError

src/borg/crypto/key.py

@@ -9,8 +9,6 @@ from binascii import a2b_base64, b2a_base64, hexlify
 from hashlib import sha256, sha512, pbkdf2_hmac
 from hmac import HMAC, compare_digest
 
-import msgpack
-
 from borg.logger import create_logger
 
 logger = create_logger()
@@ -24,6 +22,7 @@ from ..helpers import get_keys_dir, get_security_dir
 from ..helpers import get_limited_unpacker
 from ..helpers import bin_to_hex
 from ..helpers import prepare_subprocess_env
+from ..helpers import msgpack
 from ..item import Key, EncryptedKey
 from ..platform import SaveFile
 from .nonces import NonceManager

src/borg/fuse.py

@@ -11,7 +11,6 @@ from signal import SIGINT
 from distutils.version import LooseVersion
 
 import llfuse
-import msgpack
 
 from .logger import create_logger
 logger = create_logger()
@@ -21,6 +20,7 @@ from .archiver import Archiver
 from .archive import Archive
 from .hashindex import FuseVersionsIndex
 from .helpers import daemonize, hardlinkable, signal_handler, format_file_size
+from .helpers import msgpack
 from .item import Item
 from .lrucache import LRUCache
 from .remote import RemoteRepository

src/borg/helpers.py

@@ -31,8 +31,43 @@ from operator import attrgetter
 from string import Formatter
 from shutil import get_terminal_size
 
-import msgpack
-import msgpack.fallback
+# MSGPACK =====================================================================
+# we are rather picky about msgpack versions, because a good working msgpack is
+# very important for borg, see https://github.com/borgbackup/borg/issues/3753
+#
+# because some linux distributions didn't get their dependency management right
+# and broke borgbackup by upgrading msgpack to incompatible versions, we now
+# bundle msgpack-python 0.5.6, which is the latest and best msgpack that is
+# still compatible with borg 1.1.x and we use the bundled version by default.
+#
+# if you are a package maintainer and don't like bundled library code, feel
+# free to not use the bundled code:
+# - set prefer_system_msgpack = True
+# - make sure that an external msgpack-python gets installed
+# - make sure the external msgpack-python always stays at supported versions.
+# - best versions seem to be 0.4.6, 0.4.7, 0.4.8 and 0.5.6.
+# - if you can't satisfy the above requirement, these are versions that might
+#   also work ok, IF you make sure to use the COMPILED version of
+#   msgpack-python NOT the PURE PYTHON fallback implementation: 0.5.1 and 0.5.4
+#
+# Please note:
+# - using any other version is not supported by borg development and
+#   any feedback related to issues caused by this will be ignored.
+# - especially, it is known that msgpack 0.6.x does NOT work for borg 1.1.x.
+
+prefer_system_msgpack = False
+
+try:
+    if prefer_system_msgpack:
+        raise ImportError
+    # use the bundled msgpack 0.5.6 known-good version - other code only imports it from here:
+    import borg.algorithms.msgpack as msgpack
+    from borg.algorithms.msgpack import fallback as msgpack_fallback
+except ImportError:
+    # use an external msgpack version
+    import msgpack
+    from msgpack import fallback as msgpack_fallback
+
 
 from .logger import create_logger
 logger = create_logger()
@@ -1290,7 +1325,7 @@ def int_to_bigint(value):
 
 
 def is_slow_msgpack():
-    return msgpack.Packer is msgpack.fallback.Packer
+    return msgpack.Packer is msgpack_fallback.Packer
 
 
 def is_supported_msgpack():

src/borg/remote.py

@@ -16,8 +16,6 @@ import time
 import traceback
 from subprocess import Popen, PIPE
 
-import msgpack
-
 from . import __version__
 from .compress import LZ4
 from .constants import *  # NOQA
@@ -32,6 +30,7 @@ from .helpers import format_file_size
 from .helpers import truncate_and_unlink
 from .helpers import prepare_subprocess_env
 from .logger import create_logger, setup_logging
+from .helpers import msgpack
 from .repository import Repository
 from .version import parse_version, format_version
 from .algorithms.checksums import xxh64

src/borg/repository.py

@@ -11,8 +11,6 @@ from datetime import datetime
 from functools import partial
 from itertools import islice
 
-import msgpack
-
 from .constants import *  # NOQA
 from .hashindex import NSIndex
 from .helpers import Error, ErrorWithTraceback, IntegrityError, format_file_size, parse_file_size
@@ -21,6 +19,7 @@ from .helpers import ProgressIndicatorPercent
 from .helpers import bin_to_hex
 from .helpers import hostname_is_unique
 from .helpers import secure_erase, truncate_and_unlink
+from .helpers import msgpack
 from .locking import Lock, LockError, LockErrorT
 from .logger import create_logger
 from .lrucache import LRUCache

src/borg/testsuite/archive.py

@@ -3,7 +3,6 @@ from datetime import datetime, timezone
 from io import StringIO
 from unittest.mock import Mock
 
-import msgpack
 import pytest
 
 from . import BaseTestCase
@@ -11,6 +10,7 @@ from ..crypto.key import PlaintextKey
 from ..archive import Archive, CacheChunkBuffer, RobustUnpacker, valid_msgpacked_dict, ITEM_KEYS, Statistics
 from ..archive import BackupOSError, backup_io, backup_io_iter
 from ..helpers import Manifest
+from ..helpers import msgpack
 from ..item import Item, ArchiveItem
 
 

src/borg/testsuite/archiver.py

@@ -22,7 +22,6 @@ from hashlib import sha256
 from io import BytesIO, StringIO
 from unittest.mock import patch
 
-import msgpack
 import pytest
 
 try:
@@ -45,6 +44,7 @@ from ..helpers import Manifest, MandatoryFeatureUnsupported
 from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
 from ..helpers import bin_to_hex
 from ..helpers import MAX_S
+from ..helpers import msgpack
 from ..nanorst import RstToTextLazy, rst_to_terminal
 from ..patterns import IECommand, PatternMatcher, parse_pattern
 from ..item import Item

src/borg/testsuite/cache.py

@@ -1,8 +1,6 @@
 import io
 import os.path
 
-from msgpack import packb
-
 import pytest
 
 from .hashindex import H
@@ -13,8 +11,11 @@ from ..compress import CompressionSpec
 from ..crypto.key import RepoKey
 from ..hashindex import ChunkIndex, CacheSynchronizer
 from ..helpers import Manifest
+from ..helpers import msgpack
 from ..repository import Repository
 
+packb = msgpack.packb
+
 
 class TestCacheSynchronizer:
     @pytest.fixture

src/borg/testsuite/helpers.py

@@ -9,9 +9,6 @@ from time import mktime, strptime, sleep
 
 import pytest
 
-import msgpack
-import msgpack.fallback
-
 from .. import platform
 from ..helpers import Location
 from ..helpers import Buffer
@@ -20,6 +17,7 @@ from ..helpers import make_path_safe, clean_lines
 from ..helpers import interval, prune_within, prune_split
 from ..helpers import get_base_dir, get_cache_dir, get_keys_dir, get_security_dir, get_config_dir
 from ..helpers import is_slow_msgpack
+from ..helpers import msgpack, msgpack_fallback
 from ..helpers import yes, TRUISH, FALSISH, DEFAULTISH
 from ..helpers import StableDict, int_to_bigint, bigint_to_int, bin_to_hex
 from ..helpers import parse_timestamp, ChunkIteratorFileWrapper, ChunkerParams
@@ -576,7 +574,7 @@ def test_parse_file_size_invalid(string):
 def test_is_slow_msgpack():
     saved_packer = msgpack.Packer
     try:
-        msgpack.Packer = msgpack.fallback.Packer
+        msgpack.Packer = msgpack_fallback.Packer
         assert is_slow_msgpack()
     finally:
         msgpack.Packer = saved_packer

src/borg/testsuite/key.py

@@ -4,7 +4,6 @@ import re
 import tempfile
 from binascii import hexlify, unhexlify
 
-import msgpack
 import pytest
 
 from ..crypto.key import Passphrase, PasswordRetriesExceeded, bin_to_hex
@@ -18,6 +17,7 @@ from ..helpers import IntegrityError
 from ..helpers import Location
 from ..helpers import StableDict
 from ..helpers import get_security_dir
+from ..helpers import msgpack
 
 
 class TestKey:

src/borg/testsuite/repository.py

@@ -6,13 +6,12 @@ import sys
 import tempfile
 from unittest.mock import patch
 
-import msgpack
-
 import pytest
 
 from ..hashindex import NSIndex
 from ..helpers import Location
 from ..helpers import IntegrityError
+from ..helpers import msgpack
 from ..locking import Lock, LockFailed
 from ..remote import RemoteRepository, InvalidRPCMethod, PathNotAllowed, ConnectionClosedWithHint, handle_remote_line
 from ..repository import Repository, LoggedIO, MAGIC, MAX_DATA_SIZE, TAG_DELETE