Merge pull request #4437 from ThomasWaldmann/pkg-config

use pkg-config / refactor setup code

.travis/install.sh

@@ -4,26 +4,29 @@ set -e
 set -x
 
 if [[ "$(uname -s)" == 'Darwin' ]]; then
-    # HOMEBREW_NO_AUTO_UPDATE=1
+    export HOMEBREW_NO_AUTO_UPDATE=1
     export HOMEBREW_LOGS=~/brew-logs
     export HOMEBREW_TEMP=~/brew-temp
     mkdir $HOMEBREW_LOGS
     mkdir $HOMEBREW_TEMP
-    # brew update
-    if [[ "${OPENSSL}" != "0.9.8" ]]; then
-        brew outdated openssl || brew upgrade openssl
-    fi
+    brew update > /dev/null
+    brew cleanup > /dev/null  # do this here, so it won't automatically trigger in the middle of other stuff
+    brew outdated pkg-config || brew upgrade pkg-config
+    # do NOT update openssl 1.0.x, brew will also update a lot of dependent pkgs (and their dependencies) then!
+    #brew outdated openssl || brew upgrade openssl
+    export PKG_CONFIG_PATH="/usr/local/opt/openssl/lib/pkgconfig:$PKG_CONFIG_PATH"
+    brew install readline
+    export PKG_CONFIG_PATH="/usr/local/opt/readline/lib/pkgconfig:$PKG_CONFIG_PATH"
+    brew install zstd
+    brew install lz4
+    brew install xz  # required for python lzma module
+    brew install Caskroom/cask/osxfuse
 
+    brew outdated pyenv || brew upgrade pyenv
     if which pyenv > /dev/null; then
         eval "$(pyenv init -)"
     fi
 
-    brew install lz4
-    brew install xz  # required for python lzma module
-    brew outdated pyenv || brew upgrade pyenv
-    brew install pkg-config
-    brew install Caskroom/cask/osxfuse
-
     case "${TOXENV}" in
         py35)
             pyenv install 3.5.2
@@ -39,10 +42,11 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then
 else
     pip install virtualenv
     sudo apt-get update
-    sudo apt-get install -y fakeroot
-    sudo apt-get install -y liblz4-dev
+    sudo apt-get install -y pkg-config fakeroot
+    #sudo apt-get install -y liblz4-dev  # too old on trusty and xenial
+    #sudo apt-get install -y libzstd-dev  # too old on trusty and xenial
     sudo apt-get install -y libacl1-dev
-    sudo apt-get install -y libfuse-dev fuse pkg-config  # optional, for FUSE support
+    sudo apt-get install -y libfuse-dev fuse  # optional, for FUSE support
 fi
 
 python -m virtualenv ~/.venv

.travis/run.sh

@@ -5,14 +5,11 @@ set -x
 
 if [[ "$(uname -s)" == "Darwin" ]]; then
     eval "$(pyenv init -)"
-    if [[ "${OPENSSL}" != "0.9.8" ]]; then
-        # set our flags to use homebrew openssl
-        export ARCHFLAGS="-arch x86_64"
-        export LDFLAGS="-L/usr/local/opt/openssl/lib"
-        export CFLAGS="-I/usr/local/opt/openssl/include"
-    fi
+    # set our flags to use homebrew openssl
+    export ARCHFLAGS="-arch x86_64"
+    export PKG_CONFIG_PATH="/usr/local/opt/openssl/lib/pkgconfig:$PKG_CONFIG_PATH"
 fi
 
 # do not use fakeroot, but run as root on travis.
 # avoids the dreaded EISDIR sporadic failures. see #2482.
-sudo bash -c "source ~/.venv/bin/activate ; tox -e $TOXENV -r"
+sudo -E bash -c "source ~/.venv/bin/activate ; tox -e $TOXENV -r"

requirements.d/development.txt

@@ -3,6 +3,7 @@ setuptools_scm
 pip
 virtualenv
 tox
+pkgconfig
 pytest
 pytest-xdist
 pytest-cov

setup.py

@@ -1,9 +1,8 @@
+# borgbackup - main setup code (see also other setup_*.py files)
+
 import os
-import io
-import re
 import sys
-from collections import OrderedDict
-from datetime import datetime
+from collections import defaultdict
 from glob import glob
 
 try:
@@ -21,19 +20,41 @@ try:
 except ImportError:
     cythonize = None
 
-import setup_lz4
-import setup_zstd
-import setup_b2
+import setup_compress
+import setup_crypto
 import setup_docs
 
-# True: use the shared liblz4 (>= 1.7.0 / r129) from the system, False: use the bundled lz4 code
-prefer_system_liblz4 = True
-
-# True: use the shared libzstd (>= 1.3.0) from the system, False: use the bundled zstd code
-prefer_system_libzstd = True
-
-# True: use the shared libb2 from the system, False: use the bundled blake2 code
-prefer_system_libb2 = True
+# How the build process finds the system libs / uses the bundled code:
+#
+# 1. it will try to use (system) libs (see 1.1. and 1.2.),
+#    except if you use these env vars to force using the bundled code:
+#    BORG_USE_BUNDLED_XXX undefined  -->  try using system lib
+#    BORG_USE_BUNDLED_XXX=YES        -->  use the bundled code
+#    Note: do not use =NO, that is not supported!
+# 1.1. if BORG_LIBXXX_PREFIX is set, it will use headers and libs from there.
+# 1.2. if not and pkg-config can locate the lib, the lib located by
+#      pkg-config will be used. We use the pkg-config tool via the pkgconfig
+#      python package, which must be installed before invoking setup.py.
+#      if pkgconfig is not installed, this step is skipped.
+# 2. if no system lib could be located via 1.1. or 1.2., it will fall back
+#    to using the bundled code.
+
+# OpenSSL is required as a (system) lib in any case as we do not bundle it.
+# Thus, only step 1.1. and 1.2. apply to openssl (but not 1. and 2.).
+# needed: openssl >=1.0.2 or >=1.1.0 (or compatible)
+system_prefix_openssl = os.environ.get('BORG_OPENSSL_PREFIX')
+
+# needed: blake2 (>= 0.98.1)
+prefer_system_libb2 = not bool(os.environ.get('BORG_USE_BUNDLED_B2'))
+system_prefix_libb2 = os.environ.get('BORG_LIBB2_PREFIX')
+
+# needed: lz4 (>= 1.7.0 / r129)
+prefer_system_liblz4 = not bool(os.environ.get('BORG_USE_BUNDLED_LZ4'))
+system_prefix_liblz4 = os.environ.get('BORG_LIBLZ4_PREFIX')
+
+# needed: zstd (>= 1.3.0)
+prefer_system_libzstd = not bool(os.environ.get('BORG_USE_BUNDLED_ZSTD'))
+system_prefix_libzstd = os.environ.get('BORG_LIBZSTD_PREFIX')
 
 cpu_threads = multiprocessing.cpu_count() if multiprocessing else 1
 
@@ -100,79 +121,6 @@ else:
         raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.')
 
 
-def detect_openssl(prefixes):
-    for prefix in prefixes:
-        filename = os.path.join(prefix, 'include', 'openssl', 'evp.h')
-        if os.path.exists(filename):
-            with open(filename, 'rb') as fd:
-                if b'PKCS5_PBKDF2_HMAC(' in fd.read():
-                    return prefix
-
-
-include_dirs = []
-library_dirs = []
-define_macros = []
-
-possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl',
-                             '/usr/local/borg', '/opt/local', '/opt/pkg', ]
-if os.environ.get('BORG_OPENSSL_PREFIX'):
-    possible_openssl_prefixes.insert(0, os.environ.get('BORG_OPENSSL_PREFIX'))
-ssl_prefix = detect_openssl(possible_openssl_prefixes)
-if not ssl_prefix:
-    raise Exception('Unable to find OpenSSL >= 1.0 headers. (Looked here: {})'.format(', '.join(possible_openssl_prefixes)))
-include_dirs.append(os.path.join(ssl_prefix, 'include'))
-library_dirs.append(os.path.join(ssl_prefix, 'lib'))
-
-
-possible_liblz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4',
-                         '/usr/local/borg', '/opt/local', '/opt/pkg', ]
-if os.environ.get('BORG_LIBLZ4_PREFIX'):
-    possible_liblz4_prefixes.insert(0, os.environ.get('BORG_LIBLZ4_PREFIX'))
-liblz4_prefix = setup_lz4.lz4_system_prefix(possible_liblz4_prefixes)
-if prefer_system_liblz4 and liblz4_prefix:
-    print('Detected and preferring liblz4 over bundled LZ4')
-    define_macros.append(('BORG_USE_LIBLZ4', 'YES'))
-    liblz4_system = True
-else:
-    liblz4_system = False
-
-possible_libb2_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libb2', '/usr/local/libb2',
-                           '/usr/local/borg', '/opt/local', '/opt/pkg', ]
-if os.environ.get('BORG_LIBB2_PREFIX'):
-    possible_libb2_prefixes.insert(0, os.environ.get('BORG_LIBB2_PREFIX'))
-libb2_prefix = setup_b2.b2_system_prefix(possible_libb2_prefixes)
-if prefer_system_libb2 and libb2_prefix:
-    print('Detected and preferring libb2 over bundled BLAKE2')
-    define_macros.append(('BORG_USE_LIBB2', 'YES'))
-    libb2_system = True
-else:
-    libb2_system = False
-
-possible_libzstd_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libzstd', '/usr/local/libzstd',
-                             '/usr/local/borg', '/opt/local', '/opt/pkg', ]
-if os.environ.get('BORG_LIBZSTD_PREFIX'):
-    possible_libzstd_prefixes.insert(0, os.environ.get('BORG_LIBZSTD_PREFIX'))
-libzstd_prefix = setup_zstd.zstd_system_prefix(possible_libzstd_prefixes)
-if prefer_system_libzstd and libzstd_prefix:
-    print('Detected and preferring libzstd over bundled ZSTD')
-    define_macros.append(('BORG_USE_LIBZSTD', 'YES'))
-    libzstd_system = True
-else:
-    libzstd_system = False
-
-
-with open('README.rst', 'r') as fd:
-    long_description = fd.read()
-    # remove header, but have one \n before first headline
-    start = long_description.find('What is BorgBackup?')
-    assert start >= 0
-    long_description = '\n' + long_description[start:]
-    # remove badges
-    long_description = re.compile(r'^\.\. start-badges.*^\.\. end-badges', re.M | re.S).sub('', long_description)
-    # remove unknown directives
-    long_description = re.compile(r'^\.\. highlight:: \w+$', re.M).sub('', long_description)
-
-
 def rm(file):
     try:
         os.unlink(file)
@@ -202,22 +150,37 @@ cmdclass = {
 
 ext_modules = []
 if not on_rtd:
-    compress_ext_kwargs = dict(sources=[compress_source], include_dirs=include_dirs, library_dirs=library_dirs,
-                               define_macros=define_macros)
-    compress_ext_kwargs = setup_lz4.lz4_ext_kwargs(bundled_path='src/borg/algorithms/lz4',
-                                                   system_prefix=liblz4_prefix, system=liblz4_system,
-                                                   **compress_ext_kwargs)
-    compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(bundled_path='src/borg/algorithms/zstd',
-                                                     system_prefix=libzstd_prefix, system=libzstd_system,
-                                                     multithreaded=False, legacy=False, **compress_ext_kwargs)
-    crypto_ext_kwargs = dict(sources=[crypto_ll_source, crypto_helpers], libraries=['crypto'],
-                             include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros)
-    crypto_ext_kwargs = setup_b2.b2_ext_kwargs(bundled_path='src/borg/algorithms/blake2',
-                                               system_prefix=libb2_prefix, system=libb2_system,
-                                               **crypto_ext_kwargs)
+
+    def members_appended(*ds):
+        result = defaultdict(list)
+        for d in ds:
+            for k, v in d.items():
+                assert isinstance(v, list)
+                result[k].extend(v)
+        return result
+
+    try:
+        import pkgconfig as pc
+    except ImportError:
+        print('Warning: can not import pkgconfig python package.')
+        pc = None
+
+    crypto_ext_kwargs = members_appended(
+        dict(sources=[crypto_ll_source, crypto_helpers]),
+        setup_crypto.crypto_ext_kwargs(pc, system_prefix_openssl),
+        setup_crypto.b2_ext_kwargs(pc, prefer_system_libb2, system_prefix_libb2),
+    )
+
+    compress_ext_kwargs = members_appended(
+        dict(sources=[compress_source]),
+        setup_compress.lz4_ext_kwargs(pc, prefer_system_liblz4, system_prefix_liblz4),
+        setup_compress.zstd_ext_kwargs(pc, prefer_system_libzstd, system_prefix_libzstd,
+                                       multithreaded=False, legacy=False),
+    )
+
     ext_modules += [
-        Extension('borg.compress', **compress_ext_kwargs),
         Extension('borg.crypto.low_level', **crypto_ext_kwargs),
+        Extension('borg.compress', **compress_ext_kwargs),
         Extension('borg.hashindex', [hashindex_source]),
         Extension('borg.item', [item_source]),
         Extension('borg.chunker', [chunker_source]),
@@ -254,6 +217,7 @@ if not on_rtd:
         cythonize([posix_ext, linux_ext, freebsd_ext, darwin_ext], **cython_opts)
         ext_modules = cythonize(ext_modules, **cython_opts)
 
+
 setup(
     name='borgbackup',
     use_scm_version={
@@ -263,7 +227,7 @@ setup(
     author_email='borgbackup@python.org',
     url='https://borgbackup.readthedocs.io/',
     description='Deduplicated, encrypted, authenticated and compressed backups',
-    long_description=long_description,
+    long_description=setup_docs.long_desc_from_readme(),
     license='BSD',
     platforms=['Linux', 'MacOS X', 'FreeBSD', 'OpenBSD', 'NetBSD', ],
     classifiers=[

setup_b2.py

@@ -1,72 +0,0 @@
-# Support code for building a C extension with blake2 files
-#
-# Copyright (c) 2016-present, Gregory Szorc (original code for zstd)
-#               2017-present, Thomas Waldmann (mods to make it more generic, code for blake2)
-# All rights reserved.
-#
-# This software may be modified and distributed under the terms
-# of the BSD license. See the LICENSE file for details.
-
-import os
-
-# b2 files, structure as seen in BLAKE2 (reference implementation) project repository:
-
-b2_sources = [
-    'ref/blake2b-ref.c',
-]
-
-b2_includes = [
-    'ref',
-]
-
-
-def b2_system_prefix(prefixes):
-    for prefix in prefixes:
-        filename = os.path.join(prefix, 'include', 'blake2.h')
-        if os.path.exists(filename):
-            with open(filename, 'rb') as fd:
-                if b'blake2b_init' in fd.read():
-                    return prefix
-
-
-def b2_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs):
-    """amend kwargs with b2 stuff for a distutils.extension.Extension initialization.
-
-    bundled_path: relative (to this file) path to the bundled library source code files
-    system_prefix: where the system-installed library can be found
-    system: True: use the system-installed shared library, False: use the bundled library code
-    kwargs: distutils.extension.Extension kwargs that should be amended
-    returns: amended kwargs
-    """
-    def multi_join(paths, *path_segments):
-        """apply os.path.join on a list of paths"""
-        return [os.path.join(*(path_segments + (path, ))) for path in paths]
-
-    use_system = system and system_prefix is not None
-
-    sources = kwargs.get('sources', [])
-    if not use_system:
-        sources += multi_join(b2_sources, bundled_path)
-
-    include_dirs = kwargs.get('include_dirs', [])
-    if use_system:
-        include_dirs += multi_join(['include'], system_prefix)
-    else:
-        include_dirs += multi_join(b2_includes, bundled_path)
-
-    library_dirs = kwargs.get('library_dirs', [])
-    if use_system:
-        library_dirs += multi_join(['lib'], system_prefix)
-
-    libraries = kwargs.get('libraries', [])
-    if use_system:
-        libraries += ['b2', ]
-
-    extra_compile_args = kwargs.get('extra_compile_args', [])
-    if not use_system:
-        extra_compile_args += []  # not used yet
-
-    ret = dict(**kwargs)
-    ret.update(dict(sources=sources, extra_compile_args=extra_compile_args,
-                    include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries))
-    return ret

setup_compress.py

@@ -0,0 +1,132 @@
+# Support code for building a C extension with compression code
+
+import os
+
+
+def multi_join(paths, *path_segments):
+    """apply os.path.join on a list of paths"""
+    return [os.path.join(*(path_segments + (path,))) for path in paths]
+
+
+# zstd files, structure as seen in zstd project repository:
+
+# path relative (to this file) to the bundled library source code files
+zstd_bundled_path = 'src/borg/algorithms/zstd'
+
+zstd_sources = [
+    'lib/common/debug.c',
+    'lib/common/entropy_common.c',
+    'lib/common/error_private.c',
+    'lib/common/fse_decompress.c',
+    'lib/common/pool.c',
+    'lib/common/threading.c',
+    'lib/common/xxhash.c',
+    'lib/common/zstd_common.c',
+    'lib/compress/fse_compress.c',
+    'lib/compress/hist.c',
+    'lib/compress/huf_compress.c',
+    'lib/compress/zstd_compress.c',
+    'lib/compress/zstd_double_fast.c',
+    'lib/compress/zstd_fast.c',
+    'lib/compress/zstd_lazy.c',
+    'lib/compress/zstd_ldm.c',
+    'lib/compress/zstd_opt.c',
+    'lib/compress/zstdmt_compress.c',
+    'lib/decompress/huf_decompress.c',
+    'lib/decompress/zstd_ddict.c',
+    'lib/decompress/zstd_decompress.c',
+    'lib/decompress/zstd_decompress_block.c',
+    'lib/dictBuilder/cover.c',
+    'lib/dictBuilder/divsufsort.c',
+    'lib/dictBuilder/fastcover.c',
+    'lib/dictBuilder/zdict.c',
+]
+
+zstd_sources_legacy = [
+    'lib/deprecated/zbuff_common.c',
+    'lib/deprecated/zbuff_compress.c',
+    'lib/deprecated/zbuff_decompress.c',
+    'lib/legacy/zstd_v01.c',
+    'lib/legacy/zstd_v02.c',
+    'lib/legacy/zstd_v03.c',
+    'lib/legacy/zstd_v04.c',
+    'lib/legacy/zstd_v05.c',
+    'lib/legacy/zstd_v06.c',
+    'lib/legacy/zstd_v07.c',
+]
+
+zstd_includes = [
+    'lib',
+    'lib/common',
+    'lib/compress',
+    'lib/decompress',
+    'lib/dictBuilder',
+]
+
+zstd_includes_legacy = [
+    'lib/deprecated',
+    'lib/legacy',
+]
+
+
+def zstd_ext_kwargs(pc, prefer_system, system_prefix, multithreaded=False, legacy=False):
+    if prefer_system:
+        if system_prefix:
+            print('Detected and preferring libzstd [via BORG_LIBZSTD_PREFIX]')
+            return dict(include_dirs=[os.path.join(system_prefix, 'include')],
+                        library_dirs=[os.path.join(system_prefix, 'lib')],
+                        libraries=['zstd'])
+
+        if pc and pc.installed('libzstd', '>= 1.3.0'):
+            print('Detected and preferring libzstd [via pkg-config]')
+            return pc.parse('libzstd')
+
+    print('Using bundled ZSTD')
+    sources = multi_join(zstd_sources, zstd_bundled_path)
+    if legacy:
+        sources += multi_join(zstd_sources_legacy, zstd_bundled_path)
+    include_dirs = multi_join(zstd_includes, zstd_bundled_path)
+    if legacy:
+        include_dirs += multi_join(zstd_includes_legacy, zstd_bundled_path)
+    extra_compile_args = ['-DZSTDLIB_VISIBILITY=', '-DZDICTLIB_VISIBILITY=', '-DZSTDERRORLIB_VISIBILITY=', ]
+    # '-fvisibility=hidden' does not work, doesn't find PyInit_compress then
+    if legacy:
+        extra_compile_args += ['-DZSTD_LEGACY_SUPPORT=1', ]
+    if multithreaded:
+        extra_compile_args += ['-DZSTD_MULTITHREAD', ]
+    define_macros = [('BORG_USE_BUNDLED_ZSTD', 'YES')]
+    return dict(sources=sources, include_dirs=include_dirs,
+                extra_compile_args=extra_compile_args, define_macros=define_macros)
+
+
+# lz4 files, structure as seen in lz4 project repository:
+
+# path relative (to this file) to the bundled library source code files
+lz4_bundled_path = 'src/borg/algorithms/lz4'
+
+lz4_sources = [
+    'lib/lz4.c',
+]
+
+lz4_includes = [
+    'lib',
+]
+
+
+def lz4_ext_kwargs(pc, prefer_system, system_prefix):
+    if prefer_system:
+        if system_prefix:
+            print('Detected and preferring liblz4 [via BORG_LIBLZ4_PREFIX]')
+            return dict(include_dirs=[os.path.join(system_prefix, 'include')],
+                        library_dirs=[os.path.join(system_prefix, 'lib')],
+                        libraries=['lz4'])
+
+        if pc and pc.installed('liblz4', '>= 1.7.0'):
+            print('Detected and preferring liblz4 [via pkg-config]')
+            return pc.parse('liblz4')
+
+    print('Using bundled LZ4')
+    sources = multi_join(lz4_sources, lz4_bundled_path)
+    include_dirs = multi_join(lz4_includes, lz4_bundled_path)
+    define_macros = [('BORG_USE_BUNDLED_LZ4', 'YES')]
+    return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros)

setup_crypto.py

@@ -0,0 +1,55 @@
+# Support code for building a C extension with crypto code
+
+import os
+
+
+def multi_join(paths, *path_segments):
+    """apply os.path.join on a list of paths"""
+    return [os.path.join(*(path_segments + (path,))) for path in paths]
+
+
+def crypto_ext_kwargs(pc, system_prefix):
+    if system_prefix:
+        print('Detected OpenSSL [via BORG_OPENSSL_PREFIX]')
+        return dict(include_dirs=[os.path.join(system_prefix, 'include')],
+                    library_dirs=[os.path.join(system_prefix, 'lib')],
+                    libraries=['crypto'])
+
+    if pc and pc.exists('libcrypto'):
+        print('Detected OpenSSL [via pkg-config]')
+        return pc.parse('libcrypto')
+
+    raise Exception('Could not find OpenSSL lib/headers, please set BORG_OPENSSL_PREFIX')
+
+
+# b2 files, structure as seen in BLAKE2 (reference implementation) project repository:
+
+# path relative (to this file) to the bundled library source code files
+b2_bundled_path = 'src/borg/algorithms/blake2'
+
+b2_sources = [
+    'ref/blake2b-ref.c',
+]
+
+b2_includes = [
+    'ref',
+]
+
+
+def b2_ext_kwargs(pc, prefer_system, system_prefix):
+    if prefer_system:
+        if system_prefix:
+            print('Detected and preferring libb2 [via BORG_LIBB2_PREFIX]')
+            return dict(include_dirs=[os.path.join(system_prefix, 'include')],
+                        library_dirs=[os.path.join(system_prefix, 'lib')],
+                        libraries=['b2'])
+
+        if pc and pc.installed('libb2', '>= 0.98.1'):
+            print('Detected and preferring libb2 [via pkg-config]')
+            return pc.parse('libb2')
+
+    print('Using bundled BLAKE2')
+    sources = multi_join(b2_sources, b2_bundled_path)
+    include_dirs = multi_join(b2_includes, b2_bundled_path)
+    define_macros = [('BORG_USE_BUNDLED_B2', 'YES')]
+    return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros)

setup_docs.py

@@ -11,6 +11,20 @@ from datetime import datetime
 from setuptools import Command
 
 
+def long_desc_from_readme():
+    with open('README.rst', 'r') as fd:
+        long_description = fd.read()
+        # remove header, but have one \n before first headline
+        start = long_description.find('What is BorgBackup?')
+        assert start >= 0
+        long_description = '\n' + long_description[start:]
+        # remove badges
+        long_description = re.compile(r'^\.\. start-badges.*^\.\. end-badges', re.M | re.S).sub('', long_description)
+        # remove unknown directives
+        long_description = re.compile(r'^\.\. highlight:: \w+$', re.M).sub('', long_description)
+        return long_description
+
+
 def format_metavar(option):
     if option.nargs in ('*', '...'):
         return '[%s...]' % option.metavar

setup_lz4.py

@@ -1,72 +0,0 @@
-# Support code for building a C extension with lz4 files
-#
-# Copyright (c) 2016-present, Gregory Szorc (original code for zstd)
-#               2017-present, Thomas Waldmann (mods to make it more generic, code for lz4)
-# All rights reserved.
-#
-# This software may be modified and distributed under the terms
-# of the BSD license. See the LICENSE file for details.
-
-import os
-
-# lz4 files, structure as seen in lz4 project repository:
-
-lz4_sources = [
-    'lib/lz4.c',
-]
-
-lz4_includes = [
-    'lib',
-]
-
-
-def lz4_system_prefix(prefixes):
-    for prefix in prefixes:
-        filename = os.path.join(prefix, 'include', 'lz4.h')
-        if os.path.exists(filename):
-            with open(filename, 'rb') as fd:
-                if b'LZ4_compress_default' in fd.read():  # requires lz4 >= 1.7.0 (r129)
-                    return prefix
-
-
-def lz4_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs):
-    """amend kwargs with lz4 stuff for a distutils.extension.Extension initialization.
-
-    bundled_path: relative (to this file) path to the bundled library source code files
-    system_prefix: where the system-installed library can be found
-    system: True: use the system-installed shared library, False: use the bundled library code
-    kwargs: distutils.extension.Extension kwargs that should be amended
-    returns: amended kwargs
-    """
-    def multi_join(paths, *path_segments):
-        """apply os.path.join on a list of paths"""
-        return [os.path.join(*(path_segments + (path, ))) for path in paths]
-
-    use_system = system and system_prefix is not None
-
-    sources = kwargs.get('sources', [])
-    if not use_system:
-        sources += multi_join(lz4_sources, bundled_path)
-
-    include_dirs = kwargs.get('include_dirs', [])
-    if use_system:
-        include_dirs += multi_join(['include'], system_prefix)
-    else:
-        include_dirs += multi_join(lz4_includes, bundled_path)
-
-    library_dirs = kwargs.get('library_dirs', [])
-    if use_system:
-        library_dirs += multi_join(['lib'], system_prefix)
-
-    libraries = kwargs.get('libraries', [])
-    if use_system:
-        libraries += ['lz4', ]
-
-    extra_compile_args = kwargs.get('extra_compile_args', [])
-    if not use_system:
-        extra_compile_args += []  # not used yet
-
-    ret = dict(**kwargs)
-    ret.update(dict(sources=sources, extra_compile_args=extra_compile_args,
-                    include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries))
-    return ret

setup_zstd.py

@@ -1,130 +0,0 @@
-# Support code for building a C extension with zstd files
-#
-# Copyright (c) 2016-present, Gregory Szorc
-#               2017-present, Thomas Waldmann (mods to make it more generic)
-# All rights reserved.
-#
-# This software may be modified and distributed under the terms
-# of the BSD license. See the LICENSE file for details.
-
-import os
-
-# zstd files, structure as seen in zstd project repository:
-
-zstd_sources = [
-    'lib/common/debug.c',
-    'lib/common/entropy_common.c',
-    'lib/common/error_private.c',
-    'lib/common/fse_decompress.c',
-    'lib/common/pool.c',
-    'lib/common/threading.c',
-    'lib/common/xxhash.c',
-    'lib/common/zstd_common.c',
-    'lib/compress/fse_compress.c',
-    'lib/compress/hist.c',
-    'lib/compress/huf_compress.c',
-    'lib/compress/zstd_compress.c',
-    'lib/compress/zstd_double_fast.c',
-    'lib/compress/zstd_fast.c',
-    'lib/compress/zstd_lazy.c',
-    'lib/compress/zstd_ldm.c',
-    'lib/compress/zstd_opt.c',
-    'lib/compress/zstdmt_compress.c',
-    'lib/decompress/huf_decompress.c',
-    'lib/decompress/zstd_ddict.c',
-    'lib/decompress/zstd_decompress.c',
-    'lib/decompress/zstd_decompress_block.c',
-    'lib/dictBuilder/cover.c',
-    'lib/dictBuilder/divsufsort.c',
-    'lib/dictBuilder/fastcover.c',
-    'lib/dictBuilder/zdict.c',
-]
-
-zstd_sources_legacy = [
-    'lib/deprecated/zbuff_common.c',
-    'lib/deprecated/zbuff_compress.c',
-    'lib/deprecated/zbuff_decompress.c',
-    'lib/legacy/zstd_v01.c',
-    'lib/legacy/zstd_v02.c',
-    'lib/legacy/zstd_v03.c',
-    'lib/legacy/zstd_v04.c',
-    'lib/legacy/zstd_v05.c',
-    'lib/legacy/zstd_v06.c',
-    'lib/legacy/zstd_v07.c',
-]
-
-zstd_includes = [
-    'lib',
-    'lib/common',
-    'lib/compress',
-    'lib/decompress',
-    'lib/dictBuilder',
-]
-
-zstd_includes_legacy = [
-    'lib/deprecated',
-    'lib/legacy',
-]
-
-
-def zstd_system_prefix(prefixes):
-    for prefix in prefixes:
-        filename = os.path.join(prefix, 'include', 'zstd.h')
-        if os.path.exists(filename):
-            with open(filename, 'rb') as fd:
-                if b'ZSTD_getFrameContentSize' in fd.read():  # checks for zstd >= 1.3.0
-                    return prefix
-
-
-def zstd_ext_kwargs(bundled_path, system_prefix=None, system=False, multithreaded=False, legacy=False, **kwargs):
-    """amend kwargs with zstd suff for a distutils.extension.Extension initialization.
-
-    bundled_path: relative (to this file) path to the bundled library source code files
-    system_prefix: where the system-installed library can be found
-    system: True: use the system-installed shared library, False: use the bundled library code
-    multithreaded: True: define ZSTD_MULTITHREAD
-    legacy: include legacy API support
-    kwargs: distutils.extension.Extension kwargs that should be amended
-    returns: amended kwargs
-    """
-    def multi_join(paths, *path_segments):
-        """apply os.path.join on a list of paths"""
-        return [os.path.join(*(path_segments + (path, ))) for path in paths]
-
-    use_system = system and system_prefix is not None
-
-    sources = kwargs.get('sources', [])
-    if not use_system:
-        sources += multi_join(zstd_sources, bundled_path)
-        if legacy:
-            sources += multi_join(zstd_sources_legacy, bundled_path)
-
-    include_dirs = kwargs.get('include_dirs', [])
-    if use_system:
-        include_dirs += multi_join(['include'], system_prefix)
-    else:
-        include_dirs += multi_join(zstd_includes, bundled_path)
-        if legacy:
-            include_dirs += multi_join(zstd_includes_legacy, bundled_path)
-
-    library_dirs = kwargs.get('library_dirs', [])
-    if use_system:
-        library_dirs += multi_join(['lib'], system_prefix)
-
-    libraries = kwargs.get('libraries', [])
-    if use_system:
-        libraries += ['zstd', ]
-
-    extra_compile_args = kwargs.get('extra_compile_args', [])
-    if multithreaded:
-        extra_compile_args += ['-DZSTD_MULTITHREAD', ]
-    if not use_system:
-        extra_compile_args += ['-DZSTDLIB_VISIBILITY=', '-DZDICTLIB_VISIBILITY=', '-DZSTDERRORLIB_VISIBILITY=', ]
-                               # '-fvisibility=hidden' does not work, doesn't find PyInit_compress then
-        if legacy:
-            extra_compile_args += ['-DZSTD_LEGACY_SUPPORT=1', ]
-
-    ret = dict(**kwargs)
-    ret.update(dict(sources=sources, extra_compile_args=extra_compile_args,
-                    include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries))
-    return ret

src/borg/algorithms/blake2-libselect.h

@@ -1,5 +1,5 @@
-#ifdef BORG_USE_LIBB2
-#include <blake2.h>
-#else
+#ifdef BORG_USE_BUNDLED_B2
 #include "blake2/ref/blake2.h"
+#else
+#include <blake2.h>
 #endif

src/borg/algorithms/lz4-libselect.h

@@ -1,5 +1,5 @@
-#ifdef BORG_USE_LIBLZ4
-#include <lz4.h>
-#else
+#ifdef BORG_USE_BUNDLED_LZ4
 #include "lz4/lib/lz4.h"
+#else
+#include <lz4.h>
 #endif

src/borg/algorithms/zstd-libselect.h

@@ -1,5 +1,5 @@
-#ifdef BORG_USE_LIBZSTD
+#ifdef BORG_USE_BUNDLED_ZSTD
 #include <zstd.h>
 #else
-#include "zstd/lib/zstd.h"
+#include <zstd.h>
 #endif

tox.ini

@@ -14,5 +14,7 @@ passenv = *
 
 [testenv:flake8]
 changedir =
-deps = flake8
+deps =
+    flake8
+    pkgconfig
 commands = flake8 src scripts conftest.py