Merge pull request #5404 from ThomasWaldmann/libxxh-1.1

enable using libxxhash (1.1-maint only)

setup.py

@@ -11,6 +11,7 @@ from glob import glob
 import setup_lz4
 import setup_zstd
 import setup_b2
+import setup_xxhash
 
 # True: use the shared liblz4 (>= 1.7.0 / r129) from the system, False: use the bundled lz4 code
 prefer_system_liblz4 = True
@@ -21,6 +22,9 @@ prefer_system_libzstd = True
 # True: use the shared libb2 from the system, False: use the bundled blake2 code
 prefer_system_libb2 = True
 
+# True: use the shared libxxhash (>= 0.6.5 [>= 0.7.2 on ARM]) from the system, False: use the bundled xxhash code
+prefer_system_libxxhash = True
+
 # prefer_system_msgpack is another option, but you need to set it in src/borg/helpers.py.
 
 min_python = (3, 5)
@@ -210,6 +214,18 @@ if prefer_system_libzstd and libzstd_prefix:
 else:
     libzstd_system = False
 
+possible_libxxhash_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libxxhash', '/usr/local/libxxhash',
+                           '/usr/local/borg', '/opt/local', '/opt/pkg', ]
+if os.environ.get('BORG_LIBXXHASH_PREFIX'):
+    possible_libxxhash_prefixes.insert(0, os.environ.get('BORG_LIBXXHASH_PREFIX'))
+libxxhash_prefix = setup_xxhash.xxhash_system_prefix(possible_libxxhash_prefixes)
+if prefer_system_libxxhash and libxxhash_prefix:
+    print('Detected and preferring libxxhash over bundled XXHASH')
+    define_macros.append(('BORG_USE_LIBXXHASH', 'YES'))
+    libxxhash_system = True
+else:
+    libxxhash_system = False
+
 
 with open('README.rst', 'r') as fd:
     long_description = fd.read()
@@ -789,6 +805,10 @@ if not on_rtd:
                                                system_prefix=libb2_prefix, system=libb2_system,
                                                **crypto_ext_kwargs)
 
+    crypto_ext_kwargs = setup_xxhash.xxhash_ext_kwargs(bundled_path='src/borg/algorithms/xxh64',
+                                               system_prefix=libxxhash_prefix, system=libxxhash_system,
+                                               **crypto_ext_kwargs)
+
     msgpack_endian = '__BIG_ENDIAN__' if (sys.byteorder == 'big') else '__LITTLE_ENDIAN__'
     msgpack_macros = [(msgpack_endian, '1')]
     msgpack_packer_ext_kwargs = dict(

setup_xxhash.py

@@ -0,0 +1,73 @@
+# Support code for building a C extension with xxhash files
+#
+# Copyright (c) 2016-present, Gregory Szorc (original code for zstd)
+#               2017-present, Thomas Waldmann (mods to make it more generic, code for blake2)
+#               2020-present, Gianfranco Costamagna (code for xxhash)
+# All rights reserved.
+#
+# This software may be modified and distributed under the terms
+# of the BSD license. See the LICENSE file for details.
+
+import os
+
+# xxhash files, structure as seen in XXHASH (reference implementation) project repository:
+
+xxhash_sources = [
+    'xxhash.c',
+]
+
+xxhash_includes = [
+    '.',
+]
+
+
+def xxhash_system_prefix(prefixes):
+    for prefix in prefixes:
+        filename = os.path.join(prefix, 'include', 'xxhash.h')
+        if os.path.exists(filename):
+            with open(filename, 'rb') as fd:
+                if b'XXH64_digest' in fd.read():
+                    return prefix
+
+
+def xxhash_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs):
+    """amend kwargs with xxhash stuff for a distutils.extension.Extension initialization.
+
+    bundled_path: relative (to this file) path to the bundled library source code files
+    system_prefix: where the system-installed library can be found
+    system: True: use the system-installed shared library, False: use the bundled library code
+    kwargs: distutils.extension.Extension kwargs that should be amended
+    returns: amended kwargs
+    """
+    def multi_join(paths, *path_segments):
+        """apply os.path.join on a list of paths"""
+        return [os.path.join(*(path_segments + (path, ))) for path in paths]
+
+    use_system = system and system_prefix is not None
+
+    sources = kwargs.get('sources', [])
+    if not use_system:
+        sources += multi_join(xxhash_sources, bundled_path)
+
+    include_dirs = kwargs.get('include_dirs', [])
+    if use_system:
+        include_dirs += multi_join(['include'], system_prefix)
+    else:
+        include_dirs += multi_join(xxhash_includes, bundled_path)
+
+    library_dirs = kwargs.get('library_dirs', [])
+    if use_system:
+        library_dirs += multi_join(['lib'], system_prefix)
+
+    libraries = kwargs.get('libraries', [])
+    if use_system:
+        libraries += ['xxhash', ]
+
+    extra_compile_args = kwargs.get('extra_compile_args', [])
+    if not use_system:
+        extra_compile_args += []  # not used yet
+
+    ret = dict(**kwargs)
+    ret.update(dict(sources=sources, extra_compile_args=extra_compile_args,
+                    include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries))
+    return ret

src/borg/algorithms/checksums.pyx

@@ -14,7 +14,7 @@ cdef extern from "crc32_dispatch.c":
     int _have_clmul "have_clmul"()
 
 
-cdef extern from "xxh64/xxhash.c":
+cdef extern from "../algorithms/xxhash-libselect.h":
     ctypedef struct XXH64_canonical_t:
         char digest[8]
 

src/borg/algorithms/xxhash-libselect.h

@@ -0,0 +1,5 @@
+#ifdef BORG_USE_LIBXXHASH
+#include <xxhash.h>
+#else
+#include "xxh64/xxhash.c"
+#endif