Merge pull request #8028 from ThomasWaldmann/modernize-msgpack-1.4

modernize msgpack wrapper

src/borg/helpers/msgpack.py

@@ -3,8 +3,9 @@ from ..constants import *  # NOQA
 
 # wrapping msgpack ---------------------------------------------------------------------------------------------------
 #
-# due to the planned breaking api changes in upstream msgpack, we wrap it the way we need it -
+# due to the breaking api changes in upstream msgpack (from 0.x to 1.0), we wrapped it the way we need it -
 # to avoid having lots of clutter in the calling code. see tickets #968 and #3632.
+# as borg 1.4 now requires msgpack > 1.0 anyway, the wrapper has reduced functionality, but was kept.
 #
 # Packing
 # -------
@@ -30,8 +31,6 @@ from msgpack import OutOfData
 
 version = mp_version
 
-_post_100 = version >= (1, 0, 0)
-
 
 class PackException(Exception):
     """Exception while msgpack packing"""
@@ -73,35 +72,19 @@ def pack(o, stream, *, use_bin_type=False, unicode_errors=None, **kwargs):
         raise PackException(e)
 
 
-# Note: after requiring msgpack >= 0.6.1 we can remove the max_*_len args and
-#       rely on msgpack auto-computing DoS-safe max values from len(data) for
-#       unpack(data) or from max_buffer_len for Unpacker(max_buffer_len=N).
-#       maybe we can also use that to simplify get_limited_unpacker().
-
 class Unpacker(mp_Unpacker):
     def __init__(self, file_like=None, *, read_size=0, use_list=True, raw=True,
                  object_hook=None, object_pairs_hook=None, list_hook=None,
                  unicode_errors=None, max_buffer_size=0,
                  ext_hook=ExtType,
-                 strict_map_key=False,
-                 max_str_len=2147483647,  # 2**32-1
-                 max_bin_len=2147483647,
-                 max_array_len=2147483647,
-                 max_map_len=2147483647,
-                 max_ext_len=2147483647):
+                 strict_map_key=False):
         assert raw is True
         assert unicode_errors is None
         kw = dict(file_like=file_like, read_size=read_size, use_list=use_list, raw=raw,
                   object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook,
                   unicode_errors=unicode_errors, max_buffer_size=max_buffer_size,
                   ext_hook=ext_hook,
-                  max_str_len=max_str_len,
-                  max_bin_len=max_bin_len,
-                  max_array_len=max_array_len,
-                  max_map_len=max_map_len,
-                  max_ext_len=max_ext_len)
-        if _post_100:
-            kw['strict_map_key'] = strict_map_key
+                  strict_map_key=strict_map_key)
         super().__init__(**kw)
 
     def unpack(self):
@@ -123,49 +106,21 @@ class Unpacker(mp_Unpacker):
     next = __next__
 
 
-def unpackb(packed, *, raw=True, unicode_errors=None,
-            strict_map_key=False,
-            max_str_len=2147483647,  # 2**32-1
-            max_bin_len=2147483647,
-            max_array_len=2147483647,
-            max_map_len=2147483647,
-            max_ext_len=2147483647,
-            **kwargs):
+def unpackb(packed, *, raw=True, unicode_errors=None, strict_map_key=False, **kwargs):
     assert unicode_errors is None
     try:
-        kw = dict(raw=raw, unicode_errors=unicode_errors,
-                  max_str_len=max_str_len,
-                  max_bin_len=max_bin_len,
-                  max_array_len=max_array_len,
-                  max_map_len=max_map_len,
-                  max_ext_len=max_ext_len)
+        kw = dict(raw=raw, unicode_errors=unicode_errors, strict_map_key=strict_map_key)
         kw.update(kwargs)
-        if _post_100:
-            kw['strict_map_key'] = strict_map_key
         return mp_unpackb(packed, **kw)
     except Exception as e:
         raise UnpackException(e)
 
 
-def unpack(stream, *, raw=True, unicode_errors=None,
-           strict_map_key=False,
-           max_str_len=2147483647,  # 2**32-1
-           max_bin_len=2147483647,
-           max_array_len=2147483647,
-           max_map_len=2147483647,
-           max_ext_len=2147483647,
-           **kwargs):
+def unpack(stream, *, raw=True, unicode_errors=None, strict_map_key=False, **kwargs):
     assert unicode_errors is None
     try:
-        kw = dict(raw=raw, unicode_errors=unicode_errors,
-                  max_str_len=max_str_len,
-                  max_bin_len=max_bin_len,
-                  max_array_len=max_array_len,
-                  max_map_len=max_map_len,
-                  max_ext_len=max_ext_len)
+        kw = dict(raw=raw, unicode_errors=unicode_errors, strict_map_key=strict_map_key)
         kw.update(kwargs)
-        if _post_100:
-            kw['strict_map_key'] = strict_map_key
         return mp_unpack(stream, **kw)
     except Exception as e:
         raise UnpackException(e)
@@ -188,42 +143,15 @@ def is_supported_msgpack():
 
 def get_limited_unpacker(kind):
     """return a limited Unpacker because we should not trust msgpack data received from remote"""
-    args = dict(use_list=False,  # return tuples, not lists
-                max_bin_len=0,  # not used
-                max_ext_len=0,  # not used
-                max_buffer_size=3 * max(BUFSIZE, MAX_OBJECT_SIZE),
-                max_str_len=MAX_OBJECT_SIZE,  # a chunk or other repo object
-                )
-    if kind == 'server':
-        args.update(dict(max_array_len=100,  # misc. cmd tuples
-                         max_map_len=100,  # misc. cmd dicts
-                         ))
-    elif kind == 'client':
-        args.update(dict(max_array_len=LIST_SCAN_LIMIT,  # result list from repo.list() / .scan()
-                         max_map_len=100,  # misc. result dicts
-                         ))
-    elif kind == 'manifest':
-        args.update(dict(use_list=True,  # default value
-                         max_array_len=100,  # ITEM_KEYS ~= 22
-                         max_map_len=MAX_ARCHIVES,  # list of archives
-                         max_str_len=255,  # archive name
-                         object_hook=StableDict,
-                         ))
-    elif kind == 'archive':
-        args.update(dict(use_list=True,  # default value
-                         max_map_len=100,  # ARCHIVE_KEYS ~= 20
-                         max_str_len=10000,  # comment
-                         object_hook=StableDict,
-                         ))
-    elif kind == 'key':
-        args.update(dict(use_list=True,  # default value
-                         max_array_len=0,  # not used
-                         max_map_len=10,  # EncryptedKey dict
-                         max_str_len=4000,  # inner key data
-                         object_hook=StableDict,
-                         ))
+    # Note: msgpack >= 0.6.1 auto-computes DoS-safe max values from len(data) for
+    #       unpack(data) or from max_buffer_size for Unpacker(max_buffer_size=N).
+    args = dict(use_list=False, max_buffer_size=3 * max(BUFSIZE, MAX_OBJECT_SIZE))  # return tuples, not lists
+    if kind in ('server', 'client'):
+        pass  # nothing special
+    elif kind in ('manifest', 'archive', 'key'):
+        args.update(dict(use_list=True, object_hook=StableDict))  # default value
     else:
-        raise ValueError('kind must be "server", "client", "manifest" or "key"')
+        raise ValueError('kind must be "server", "client", "manifest", "archive" or "key"')
     return Unpacker(**args)