compress extension: build lz4 like zstd, see #3415

(cherry picked from commit 73a70082c2ffe5a47e8b17124197aef0e3edb072)

setup.py

@@ -12,8 +12,12 @@ from distutils.core import Command
 
 import textwrap
 
+import setup_lz4
 import setup_zstd
 
+# True: use the shared liblz4 (>= TBD) from the system, False: use the bundled lz4 code
+prefer_system_liblz4 = True
+
 # True: use the shared libzstd (>= 1.3.0) from the system, False: use the bundled zstd code
 prefer_system_libzstd = True
 
@@ -142,15 +146,6 @@ def detect_openssl(prefixes):
                     return prefix
 
 
-def detect_lz4(prefixes):
-    for prefix in prefixes:
-        filename = os.path.join(prefix, 'include', 'lz4.h')
-        if os.path.exists(filename):
-            with open(filename, 'r') as fd:
-                if 'LZ4_decompress_safe' in fd.read():
-                    return prefix
-
-
 def detect_libb2(prefixes):
     for prefix in prefixes:
         filename = os.path.join(prefix, 'include', 'blake2.h')
@@ -164,7 +159,6 @@ include_dirs = []
 library_dirs = []
 define_macros = []
 crypto_libraries = ['crypto']
-compression_libraries = ['lz4']
 
 possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl',
                              '/usr/local/borg', '/opt/local', '/opt/pkg', ]
@@ -177,16 +171,17 @@ include_dirs.append(os.path.join(ssl_prefix, 'include'))
 library_dirs.append(os.path.join(ssl_prefix, 'lib'))
 
 
-possible_lz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4',
+possible_liblz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4',
                          '/usr/local/borg', '/opt/local', '/opt/pkg', ]
 if os.environ.get('BORG_LZ4_PREFIX'):
-    possible_lz4_prefixes.insert(0, os.environ.get('BORG_LZ4_PREFIX'))
-lz4_prefix = detect_lz4(possible_lz4_prefixes)
-if lz4_prefix:
-    include_dirs.append(os.path.join(lz4_prefix, 'include'))
-    library_dirs.append(os.path.join(lz4_prefix, 'lib'))
-elif not on_rtd:
-    raise Exception('Unable to find LZ4 headers. (Looked here: {})'.format(', '.join(possible_lz4_prefixes)))
+    possible_liblz4_prefixes.insert(0, os.environ.get('BORG_LZ4_PREFIX'))
+liblz4_prefix = setup_lz4.lz4_system_prefix(possible_liblz4_prefixes)
+if prefer_system_liblz4 and liblz4_prefix:
+    print('Detected and preferring liblz4 over bundled LZ4')
+    define_macros.append(('BORG_USE_LIBLZ4', 'YES'))
+    liblz4_system = True
+else:
+    liblz4_system = False
 
 possible_libb2_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libb2', '/usr/local/libb2',
                            '/usr/local/borg', '/opt/local', '/opt/pkg', ]
@@ -772,7 +767,10 @@ cmdclass = {
 ext_modules = []
 if not on_rtd:
     compress_ext_kwargs = dict(sources=[compress_source], include_dirs=include_dirs, library_dirs=library_dirs,
-                               libraries=compression_libraries, define_macros=define_macros)
+                               define_macros=define_macros)
+    compress_ext_kwargs = setup_lz4.lz4_ext_kwargs(bundled_path='src/borg/algorithms/lz4',
+                                                   system_prefix=liblz4_prefix, system=liblz4_system,
+                                                   **compress_ext_kwargs)
     compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(bundled_path='src/borg/algorithms/zstd',
                                                      system_prefix=libzstd_prefix, system=libzstd_system,
                                                      multithreaded=False, legacy=False, **compress_ext_kwargs)

setup_lz4.py

@@ -0,0 +1,72 @@
+# Support code for building a C extension with lz4 files
+#
+# Copyright (c) 2016-present, Gregory Szorc (original code for zstd)
+#               2017-present, Thomas Waldmann (mods to make it more generic, code for lz4)
+# All rights reserved.
+#
+# This software may be modified and distributed under the terms
+# of the BSD license. See the LICENSE file for details.
+
+import os
+
+# lz4 files, structure as seen in lz4 project repository:
+
+lz4_sources = [
+    'lib/lz4.c',
+]
+
+lz4_includes = [
+    'lib',
+]
+
+
+def lz4_system_prefix(prefixes):
+    for prefix in prefixes:
+        filename = os.path.join(prefix, 'include', 'lz4.h')
+        if os.path.exists(filename):
+            with open(filename, 'r') as fd:
+                if 'LZ4_decompress_safe' in fd.read():
+                    return prefix
+
+
+def lz4_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs):
+    """amend kwargs with lz4 stuff for a distutils.extension.Extension initialization.
+
+    bundled_path: relative (to this file) path to the bundled library source code files
+    system_prefix: where the system-installed library can be found
+    system: True: use the system-installed shared library, False: use the bundled library code
+    kwargs: distutils.extension.Extension kwargs that should be amended
+    returns: amended kwargs
+    """
+    def multi_join(paths, *path_segments):
+        """apply os.path.join on a list of paths"""
+        return [os.path.join(*(path_segments + (path, ))) for path in paths]
+
+    use_system = system and system_prefix is not None
+
+    sources = kwargs.get('sources', [])
+    if not use_system:
+        sources += multi_join(lz4_sources, bundled_path)
+
+    include_dirs = kwargs.get('include_dirs', [])
+    if use_system:
+        include_dirs += multi_join(['include'], system_prefix)
+    else:
+        include_dirs += multi_join(lz4_includes, bundled_path)
+
+    library_dirs = kwargs.get('library_dirs', [])
+    if use_system:
+        library_dirs += multi_join(['lib'], system_prefix)
+
+    libraries = kwargs.get('libraries', [])
+    if use_system:
+        libraries += ['lz4', ]
+
+    extra_compile_args = kwargs.get('extra_compile_args', [])
+    if not use_system:
+        extra_compile_args += []  # not used yet
+
+    ret = dict(**kwargs)
+    ret.update(dict(sources=sources, extra_compile_args=extra_compile_args,
+                    include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries))
+    return ret

src/borg/algorithms/lz4-libselect.h

@@ -0,0 +1,5 @@
+#ifdef BORG_USE_LIBLZ4
+#include <lz4.h>
+#else
+#include "lz4/lib/lz4.h"
+#endif

src/borg/compress.pyx

@@ -25,9 +25,9 @@ except ImportError:
 
 from .helpers import Buffer, DecompressionError
 
-API_VERSION = '1.1_04'
+API_VERSION = '1.1_05'
 
-cdef extern from "lz4.h":
+cdef extern from "algorithms/lz4-libselect.h":
     int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) nogil
     int LZ4_decompress_safe(const char* source, char* dest, int inputSize, int maxOutputSize) nogil
     int LZ4_compressBound(int inputSize) nogil

src/borg/helpers.py

@@ -135,7 +135,7 @@ def check_extension_modules():
         raise ExtensionModuleError
     if chunker.API_VERSION != '1.1_01':
         raise ExtensionModuleError
-    if compress.API_VERSION != '1.1_04':
+    if compress.API_VERSION != '1.1_05':
         raise ExtensionModuleError
     if borg.crypto.low_level.API_VERSION != '1.1_02':
         raise ExtensionModuleError