add pkg-config support, fixes #1925

1. BORG_*_PREFIX is checked (avoids lib detection via pkg-config).
2. pkg-config is tried
3. fallback to bundled C code (or failure in case of OpenSSL)

also:
- simplified code again
- removed (c) headers, nothing left of original code

.travis/install.sh

@@ -42,7 +42,8 @@ else
     sudo apt-get install -y fakeroot
     sudo apt-get install -y liblz4-dev
     sudo apt-get install -y libacl1-dev
-    sudo apt-get install -y libfuse-dev fuse pkg-config  # optional, for FUSE support
+    sudo apt-get install -y pkg-config
+    sudo apt-get install -y libfuse-dev fuse  # optional, for FUSE support
 fi
 
 python -m virtualenv ~/.venv

requirements.d/development.txt

@@ -3,6 +3,7 @@ setuptools_scm
 pip
 virtualenv
 tox
+pkgconfig
 pytest
 pytest-xdist
 pytest-cov

setup.py

@@ -34,7 +34,7 @@ prefer_system_liblz4 = True
 # True: use the shared libzstd (>= 1.3.0) from the system, False: use the bundled zstd code
 prefer_system_libzstd = True
 
-# True: use the shared libb2 from the system, False: use the bundled blake2 code
+# True: use the shared libb2 (>= 0.98.1) from the system, False: use the bundled blake2 code
 prefer_system_libb2 = True
 
 cpu_threads = multiprocessing.cpu_count() if multiprocessing else 1

setup_b2.py

@@ -1,11 +1,4 @@
-# Support code for building a C extension with blake2 files
-#
-# Copyright (c) 2016-present, Gregory Szorc (original code for zstd)
-#               2017-present, Thomas Waldmann (mods to make it more generic, code for blake2)
-# All rights reserved.
-#
-# This software may be modified and distributed under the terms
-# of the BSD license. See the LICENSE file for details.
+# Support code for building a C extension with blake2
 
 import os
 
@@ -23,41 +16,28 @@ b2_includes = [
 ]
 
 
+def multi_join(paths, *path_segments):
+    """apply os.path.join on a list of paths"""
+    return [os.path.join(*(path_segments + (path,))) for path in paths]
+
+
 def b2_ext_kwargs(prefer_system):
-    """return kwargs with b2 stuff for a distutils.extension.Extension initialization.
-
-    prefer_system: prefer the system-installed library (if found) over the bundled C code
-    returns: kwargs for this lib
-    """
-    def multi_join(paths, *path_segments):
-        """apply os.path.join on a list of paths"""
-        return [os.path.join(*(path_segments + (path, ))) for path in paths]
-
-    define_macros = []
-
-    system_prefix = os.environ.get('BORG_LIBB2_PREFIX')
-    if prefer_system and system_prefix:
-        print('Detected and preferring libb2 over bundled BLAKE2')
-        define_macros.append(('BORG_USE_LIBB2', 'YES'))
-        system = True
-    else:
-        print('Using bundled BLAKE2')
-        system = False
-
-    use_system = system and system_prefix is not None
-
-    if use_system:
-        sources = []
-        include_dirs = multi_join(['include'], system_prefix)
-        library_dirs = multi_join(['lib'], system_prefix)
-        libraries = ['b2', ]
-    else:
-        sources = multi_join(b2_sources, bundled_path)
-        include_dirs = multi_join(b2_includes, bundled_path)
-        library_dirs = []
-        libraries = []
-
-    extra_compile_args = []
-
-    return dict(sources=sources, define_macros=define_macros, extra_compile_args=extra_compile_args,
-                include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries)
+    if prefer_system:
+        system_prefix = os.environ.get('BORG_LIBB2_PREFIX')
+        if system_prefix:
+            print('Detected and preferring libb2 [via BORG_LIBB2_PREFIX]')
+            return dict(include_dirs=[os.path.join(system_prefix, 'include')],
+                        library_dirs=[os.path.join(system_prefix, 'lib')],
+                        libraries=['b2'])
+
+        import pkgconfig
+
+        if pkgconfig.installed('libb2', '>= 0.98.1'):
+            print('Detected and preferring libb2 [via pkg-config]')
+            return pkgconfig.parse('libb2')
+
+    print('Using bundled BLAKE2')
+    sources = multi_join(b2_sources, bundled_path)
+    include_dirs = multi_join(b2_includes, bundled_path)
+    define_macros = [('BORG_USE_BUNDLED_B2', 'YES')]
+    return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros)

setup_crypto.py

@@ -1,32 +1,20 @@
-# Support code for building a C extension with crypto from OpenSSL
-#
-# Copyright (c) 2016-present, Gregory Szorc (original code for zstd)
-#               2017-present, Thomas Waldmann (mods to make it more generic, code for openssl)
-# All rights reserved.
-#
-# This software may be modified and distributed under the terms
-# of the BSD license. See the LICENSE file for details.
+# Support code for building a C extension with crypto from OpenSSL / LibreSSL
 
 import os
 
 
 def crypto_ext_kwargs():
-    """return kwargs with crypto stuff for a distutils.extension.Extension initialization.
-
-    returns: kwargs for this lib
-    """
-    def multi_join(paths, *path_segments):
-        """apply os.path.join on a list of paths"""
-        return [os.path.join(*(path_segments + (path, ))) for path in paths]
-
     system_prefix = os.environ.get('BORG_OPENSSL_PREFIX')
     if system_prefix:
-        print('Detected system OpenSSL')
-    else:
-        raise Exception('Could not find OpenSSL lib/headers, please set BORG_OPENSSL_PREFIX')
+        print('Detected OpenSSL [via BORG_OPENSSL_PREFIX]')
+        return dict(include_dirs=[os.path.join(system_prefix, 'include')],
+                    library_dirs=[os.path.join(system_prefix, 'lib')],
+                    libraries=['crypto'])
+
+    import pkgconfig
 
-    include_dirs = multi_join(['include'], system_prefix)
-    library_dirs = multi_join(['lib'], system_prefix)
-    libraries = ['crypto', ]
+    if pkgconfig.exists('libcrypto'):
+        print('Detected OpenSSL [via pkg-config]')
+        return pkgconfig.parse('libcrypto')
 
-    return dict(include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries)
+    raise Exception('Could not find OpenSSL lib/headers, please set BORG_OPENSSL_PREFIX')

setup_lz4.py

@@ -1,11 +1,4 @@
 # Support code for building a C extension with lz4 files
-#
-# Copyright (c) 2016-present, Gregory Szorc (original code for zstd)
-#               2017-present, Thomas Waldmann (mods to make it more generic, code for lz4)
-# All rights reserved.
-#
-# This software may be modified and distributed under the terms
-# of the BSD license. See the LICENSE file for details.
 
 import os
 
@@ -23,41 +16,28 @@ lz4_includes = [
 ]
 
 
+def multi_join(paths, *path_segments):
+    """apply os.path.join on a list of paths"""
+    return [os.path.join(*(path_segments + (path,))) for path in paths]
+
+
 def lz4_ext_kwargs(prefer_system):
-    """return kwargs with lz4 stuff for a distutils.extension.Extension initialization.
-
-    prefer_system: prefer the system-installed library (if found) over the bundled C code
-    returns: kwargs for this lib
-    """
-    def multi_join(paths, *path_segments):
-        """apply os.path.join on a list of paths"""
-        return [os.path.join(*(path_segments + (path, ))) for path in paths]
-
-    define_macros = []
-
-    system_prefix = os.environ.get('BORG_LIBLZ4_PREFIX')
-    if prefer_system and system_prefix:
-        print('Detected and preferring liblz4 over bundled LZ4')
-        define_macros.append(('BORG_USE_LIBLZ4', 'YES'))
-        system = True
-    else:
-        print('Using bundled LZ4')
-        system = False
-
-    use_system = system and system_prefix is not None
-
-    if use_system:
-        sources = []
-        include_dirs = multi_join(['include'], system_prefix)
-        library_dirs = multi_join(['lib'], system_prefix)
-        libraries = ['lz4', ]
-    else:
-        sources = multi_join(lz4_sources, bundled_path)
-        include_dirs = multi_join(lz4_includes, bundled_path)
-        library_dirs = []
-        libraries = []
-
-    extra_compile_args = []
-
-    return dict(sources=sources, define_macros=define_macros, extra_compile_args=extra_compile_args,
-               include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries)
+    if prefer_system:
+        system_prefix = os.environ.get('BORG_LIBLZ4_PREFIX')
+        if system_prefix:
+            print('Detected and preferring liblz4 [via BORG_LIBLZ4_PREFIX]')
+            return dict(include_dirs=[os.path.join(system_prefix, 'include')],
+                        library_dirs=[os.path.join(system_prefix, 'lib')],
+                        libraries=['lz4'])
+
+        import pkgconfig
+
+        if pkgconfig.installed('liblz4', '>= 1.7.0'):
+            print('Detected and preferring liblz4 [via pkg-config]')
+            return pkgconfig.parse('liblz4')
+
+    print('Using bundled LZ4')
+    sources = multi_join(lz4_sources, bundled_path)
+    include_dirs = multi_join(lz4_includes, bundled_path)
+    define_macros = [('BORG_USE_BUNDLED_LZ4', 'YES')]
+    return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros)

setup_zstd.py

@@ -1,11 +1,4 @@
-# Support code for building a C extension with zstd files
-#
-# Copyright (c) 2016-present, Gregory Szorc
-#               2017-present, Thomas Waldmann (mods to make it more generic)
-# All rights reserved.
-#
-# This software may be modified and distributed under the terms
-# of the BSD license. See the LICENSE file for details.
+# Support code for building a C extension with zstd
 
 import os
 
@@ -70,54 +63,39 @@ zstd_includes_legacy = [
 ]
 
 
-def zstd_ext_kwargs(prefer_system, multithreaded=False, legacy=False):
-    """return kwargs with zstd suff for a distutils.extension.Extension initialization.
-
-    prefer_system: prefer the system-installed library (if found) over the bundled C code
-    multithreaded: True: define ZSTD_MULTITHREAD
-    legacy: include legacy API support
-    returns: kwargs for this lib
-    """
-    def multi_join(paths, *path_segments):
-        """apply os.path.join on a list of paths"""
-        return [os.path.join(*(path_segments + (path, ))) for path in paths]
-
-    define_macros = []
-
-    system_prefix = os.environ.get('BORG_LIBZSTD_PREFIX')
-    if prefer_system and system_prefix:
-        print('Detected and preferring libzstd over bundled ZSTD')
-        define_macros.append(('BORG_USE_LIBZSTD', 'YES'))
-        system = True
-    else:
-        print('Using bundled ZSTD')
-        system = False
+def multi_join(paths, *path_segments):
+    """apply os.path.join on a list of paths"""
+    return [os.path.join(*(path_segments + (path,))) for path in paths]
 
-    use_system = system and system_prefix is not None
 
-    if use_system:
-        sources = []
-        include_dirs = multi_join(['include'], system_prefix)
-        library_dirs = multi_join(['lib'], system_prefix)
-        libraries = ['zstd', ]
-    else:
-        sources = multi_join(zstd_sources, bundled_path)
-        if legacy:
-            sources += multi_join(zstd_sources_legacy, bundled_path)
-        include_dirs = multi_join(zstd_includes, bundled_path)
-        if legacy:
-            include_dirs += multi_join(zstd_includes_legacy, bundled_path)
-        library_dirs = []
-        libraries = []
-
-    extra_compile_args = []
+def zstd_ext_kwargs(prefer_system, multithreaded=False, legacy=False):
+    if prefer_system:
+        system_prefix = os.environ.get('BORG_LIBZSTD_PREFIX')
+        if system_prefix:
+            print('Detected and preferring libzstd [via BORG_LIBZSTD_PREFIX]')
+            return dict(include_dirs=[os.path.join(system_prefix, 'include')],
+                        library_dirs=[os.path.join(system_prefix, 'lib')],
+                        libraries=['zstd'])
+
+        import pkgconfig
+
+        if pkgconfig.installed('libzstd', '>= 1.3.0'):
+            print('Detected and preferring libzstd [via pkg-config]')
+            return pkgconfig.parse('libzstd')
+
+    print('Using bundled ZSTD')
+    sources = multi_join(zstd_sources, bundled_path)
+    if legacy:
+        sources += multi_join(zstd_sources_legacy, bundled_path)
+    include_dirs = multi_join(zstd_includes, bundled_path)
+    if legacy:
+        include_dirs += multi_join(zstd_includes_legacy, bundled_path)
+    extra_compile_args = ['-DZSTDLIB_VISIBILITY=', '-DZDICTLIB_VISIBILITY=', '-DZSTDERRORLIB_VISIBILITY=', ]
+    # '-fvisibility=hidden' does not work, doesn't find PyInit_compress then
+    if legacy:
+        extra_compile_args += ['-DZSTD_LEGACY_SUPPORT=1', ]
     if multithreaded:
         extra_compile_args += ['-DZSTD_MULTITHREAD', ]
-    if not use_system:
-        extra_compile_args += ['-DZSTDLIB_VISIBILITY=', '-DZDICTLIB_VISIBILITY=', '-DZSTDERRORLIB_VISIBILITY=', ]
-                               # '-fvisibility=hidden' does not work, doesn't find PyInit_compress then
-        if legacy:
-            extra_compile_args += ['-DZSTD_LEGACY_SUPPORT=1', ]
-
-    return dict(sources=sources, define_macros=define_macros, extra_compile_args=extra_compile_args,
-                include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries)
+    define_macros = [('BORG_USE_BUNDLED_ZSTD', 'YES')]
+    return dict(sources=sources, include_dirs=include_dirs,
+                extra_compile_args=extra_compile_args, define_macros=define_macros)

src/borg/algorithms/blake2-libselect.h

@@ -1,5 +1,5 @@
-#ifdef BORG_USE_LIBB2
-#include <blake2.h>
-#else
+#ifdef BORG_USE_BUNDLED_B2
 #include "blake2/ref/blake2.h"
+#else
+#include <blake2.h>
 #endif

src/borg/algorithms/lz4-libselect.h

@@ -1,5 +1,5 @@
-#ifdef BORG_USE_LIBLZ4
-#include <lz4.h>
-#else
+#ifdef BORG_USE_BUNDLED_LZ4
 #include "lz4/lib/lz4.h"
+#else
+#include <lz4.h>
 #endif

src/borg/algorithms/zstd-libselect.h

@@ -1,5 +1,5 @@
-#ifdef BORG_USE_LIBZSTD
+#ifdef BORG_USE_BUNDLED_ZSTD
 #include <zstd.h>
 #else
-#include "zstd/lib/zstd.h"
+#include <zstd.h>
 #endif

tox.ini

@@ -14,5 +14,7 @@ passenv = *
 
 [testenv:flake8]
 changedir =
-deps = flake8
+deps =
+    flake8
+    pkgconfig
 commands = flake8 src scripts conftest.py