refactor lib/bundled related code in setup.py

- removed hardcoded lib search pathes
- to find system libs/headers, one must point these env vars to them:
  BORG_LIBLZ4_PREFIX, BORG_LIBZSTD_PREFIX, BORG_LIBB2_PREFIX
- moved some code from setup.py to setup_*.py

setup.py

@@ -124,43 +124,6 @@ include_dirs.append(os.path.join(ssl_prefix, 'include'))
 library_dirs.append(os.path.join(ssl_prefix, 'lib'))
 
 
-possible_liblz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4',
-                         '/usr/local/borg', '/opt/local', '/opt/pkg', ]
-if os.environ.get('BORG_LIBLZ4_PREFIX'):
-    possible_liblz4_prefixes.insert(0, os.environ.get('BORG_LIBLZ4_PREFIX'))
-liblz4_prefix = setup_lz4.lz4_system_prefix(possible_liblz4_prefixes)
-if prefer_system_liblz4 and liblz4_prefix:
-    print('Detected and preferring liblz4 over bundled LZ4')
-    define_macros.append(('BORG_USE_LIBLZ4', 'YES'))
-    liblz4_system = True
-else:
-    liblz4_system = False
-
-possible_libb2_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libb2', '/usr/local/libb2',
-                           '/usr/local/borg', '/opt/local', '/opt/pkg', ]
-if os.environ.get('BORG_LIBB2_PREFIX'):
-    possible_libb2_prefixes.insert(0, os.environ.get('BORG_LIBB2_PREFIX'))
-libb2_prefix = setup_b2.b2_system_prefix(possible_libb2_prefixes)
-if prefer_system_libb2 and libb2_prefix:
-    print('Detected and preferring libb2 over bundled BLAKE2')
-    define_macros.append(('BORG_USE_LIBB2', 'YES'))
-    libb2_system = True
-else:
-    libb2_system = False
-
-possible_libzstd_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libzstd', '/usr/local/libzstd',
-                             '/usr/local/borg', '/opt/local', '/opt/pkg', ]
-if os.environ.get('BORG_LIBZSTD_PREFIX'):
-    possible_libzstd_prefixes.insert(0, os.environ.get('BORG_LIBZSTD_PREFIX'))
-libzstd_prefix = setup_zstd.zstd_system_prefix(possible_libzstd_prefixes)
-if prefer_system_libzstd and libzstd_prefix:
-    print('Detected and preferring libzstd over bundled ZSTD')
-    define_macros.append(('BORG_USE_LIBZSTD', 'YES'))
-    libzstd_system = True
-else:
-    libzstd_system = False
-
-
 with open('README.rst', 'r') as fd:
     long_description = fd.read()
     # remove header, but have one \n before first headline
@@ -205,15 +168,15 @@ if not on_rtd:
     compress_ext_kwargs = dict(sources=[compress_source], include_dirs=include_dirs, library_dirs=library_dirs,
                                define_macros=define_macros)
     compress_ext_kwargs = setup_lz4.lz4_ext_kwargs(bundled_path='src/borg/algorithms/lz4',
-                                                   system_prefix=liblz4_prefix, system=liblz4_system,
+                                                   prefer_system=prefer_system_liblz4,
                                                    **compress_ext_kwargs)
     compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(bundled_path='src/borg/algorithms/zstd',
-                                                     system_prefix=libzstd_prefix, system=libzstd_system,
+                                                     prefer_system=prefer_system_libzstd,
                                                      multithreaded=False, legacy=False, **compress_ext_kwargs)
     crypto_ext_kwargs = dict(sources=[crypto_ll_source, crypto_helpers], libraries=['crypto'],
                              include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros)
     crypto_ext_kwargs = setup_b2.b2_ext_kwargs(bundled_path='src/borg/algorithms/blake2',
-                                               system_prefix=libb2_prefix, system=libb2_system,
+                                               prefer_system=prefer_system_libb2,
                                                **crypto_ext_kwargs)
     ext_modules += [
         Extension('borg.compress', **compress_ext_kwargs),

setup_b2.py

@@ -20,21 +20,11 @@ b2_includes = [
 ]
 
 
-def b2_system_prefix(prefixes):
-    for prefix in prefixes:
-        filename = os.path.join(prefix, 'include', 'blake2.h')
-        if os.path.exists(filename):
-            with open(filename, 'rb') as fd:
-                if b'blake2b_init' in fd.read():
-                    return prefix
-
-
-def b2_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs):
+def b2_ext_kwargs(bundled_path, prefer_system, **kwargs):
     """amend kwargs with b2 stuff for a distutils.extension.Extension initialization.
 
     bundled_path: relative (to this file) path to the bundled library source code files
-    system_prefix: where the system-installed library can be found
-    system: True: use the system-installed shared library, False: use the bundled library code
+    prefer_system: prefer the system-installed library (if found) over the bundled C code
     kwargs: distutils.extension.Extension kwargs that should be amended
     returns: amended kwargs
     """
@@ -42,6 +32,17 @@ def b2_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs):
         """apply os.path.join on a list of paths"""
         return [os.path.join(*(path_segments + (path, ))) for path in paths]
 
+    define_macros = kwargs.get('define_macros', [])
+
+    system_prefix = os.environ.get('BORG_LIBB2_PREFIX')
+    if prefer_system and system_prefix:
+        print('Detected and preferring libb2 over bundled BLAKE2')
+        define_macros.append(('BORG_USE_LIBB2', 'YES'))
+        system = True
+    else:
+        print('Using bundled BLAKE2')
+        system = False
+
     use_system = system and system_prefix is not None
 
     sources = kwargs.get('sources', [])

setup_lz4.py

@@ -20,21 +20,11 @@ lz4_includes = [
 ]
 
 
-def lz4_system_prefix(prefixes):
-    for prefix in prefixes:
-        filename = os.path.join(prefix, 'include', 'lz4.h')
-        if os.path.exists(filename):
-            with open(filename, 'rb') as fd:
-                if b'LZ4_compress_default' in fd.read():  # requires lz4 >= 1.7.0 (r129)
-                    return prefix
-
-
-def lz4_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs):
+def lz4_ext_kwargs(bundled_path, prefer_system, **kwargs):
     """amend kwargs with lz4 stuff for a distutils.extension.Extension initialization.
 
     bundled_path: relative (to this file) path to the bundled library source code files
-    system_prefix: where the system-installed library can be found
-    system: True: use the system-installed shared library, False: use the bundled library code
+    prefer_system: prefer the system-installed library (if found) over the bundled C code
     kwargs: distutils.extension.Extension kwargs that should be amended
     returns: amended kwargs
     """
@@ -42,6 +32,17 @@ def lz4_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs):
         """apply os.path.join on a list of paths"""
         return [os.path.join(*(path_segments + (path, ))) for path in paths]
 
+    define_macros = kwargs.get('define_macros', [])
+
+    system_prefix = os.environ.get('BORG_LIBLZ4_PREFIX')
+    if prefer_system and system_prefix:
+        print('Detected and preferring liblz4 over bundled LZ4')
+        define_macros.append(('BORG_USE_LIBLZ4', 'YES'))
+        system = True
+    else:
+        print('Using bundled LZ4')
+        system = False
+
     use_system = system and system_prefix is not None
 
     sources = kwargs.get('sources', [])

setup_zstd.py

@@ -67,21 +67,11 @@ zstd_includes_legacy = [
 ]
 
 
-def zstd_system_prefix(prefixes):
-    for prefix in prefixes:
-        filename = os.path.join(prefix, 'include', 'zstd.h')
-        if os.path.exists(filename):
-            with open(filename, 'rb') as fd:
-                if b'ZSTD_getFrameContentSize' in fd.read():  # checks for zstd >= 1.3.0
-                    return prefix
-
-
-def zstd_ext_kwargs(bundled_path, system_prefix=None, system=False, multithreaded=False, legacy=False, **kwargs):
+def zstd_ext_kwargs(bundled_path, prefer_system, multithreaded=False, legacy=False, **kwargs):
     """amend kwargs with zstd suff for a distutils.extension.Extension initialization.
 
     bundled_path: relative (to this file) path to the bundled library source code files
-    system_prefix: where the system-installed library can be found
-    system: True: use the system-installed shared library, False: use the bundled library code
+    prefer_system: prefer the system-installed library (if found) over the bundled C code
     multithreaded: True: define ZSTD_MULTITHREAD
     legacy: include legacy API support
     kwargs: distutils.extension.Extension kwargs that should be amended
@@ -91,6 +81,17 @@ def zstd_ext_kwargs(bundled_path, system_prefix=None, system=False, multithreade
         """apply os.path.join on a list of paths"""
         return [os.path.join(*(path_segments + (path, ))) for path in paths]
 
+    define_macros = kwargs.get('define_macros', [])
+
+    system_prefix = os.environ.get('BORG_LIBZSTD_PREFIX')
+    if prefer_system and system_prefix:
+        print('Detected and preferring libzstd over bundled ZSTD')
+        define_macros.append(('BORG_USE_LIBZSTD', 'YES'))
+        system = True
+    else:
+        print('Using bundled ZSTD')
+        system = False
+
     use_system = system and system_prefix is not None
 
     sources = kwargs.get('sources', [])