support code to build bundled zstd code into the compress extension

setup_zstd.py modified so it is just amending the Extension() kwargs,
but the Extension is initialized by the caller.

this way, amending can happend multiple times (e.g. for multiple
compression algorithms).

also:
- move include/library dirs processing for system-library case
- move system zstd prefix detection to setup_zstd module
- cosmetic: setup.py whitespace fixes
- prefer system zstd option, document zstd min. requirement

setup.py

@@ -12,6 +12,11 @@ from distutils.core import Command
 
 import textwrap
 
+import setup_zstd
+
+# True: use the shared libzstd (>= 1.3.0) from the system, False: use the bundled zstd code
+prefer_system_libzstd = True
+
 min_python = (3, 5)
 my_python = sys.version_info
 
@@ -159,15 +164,6 @@ def detect_libb2(prefixes):
                     return prefix
 
 
-def detect_libzstd(prefixes):
-    for prefix in prefixes:
-        filename = os.path.join(prefix, 'include', 'zstd.h')
-        if os.path.exists(filename):
-            with open(filename, 'r') as fd:
-                if 'ZSTD_getFrameContentSize' in fd.read():
-                    return prefix
-
-
 include_dirs = []
 library_dirs = []
 define_macros = []
@@ -212,13 +208,13 @@ possible_libzstd_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libzstd', '/u
                              '/usr/local/borg', '/opt/local', '/opt/pkg', ]
 if os.environ.get('BORG_LIBZSTD_PREFIX'):
     possible_libzstd_prefixes.insert(0, os.environ.get('BORG_LIBZSTD_PREFIX'))
-libzstd_prefix = detect_libzstd(possible_libzstd_prefixes)
-if libzstd_prefix:
+libzstd_prefix = setup_zstd.zstd_system_prefix(possible_libzstd_prefixes)
+if prefer_system_libzstd and libzstd_prefix:
     print('Detected and preferring libzstd over bundled ZSTD')
-    include_dirs.append(os.path.join(libzstd_prefix, 'include'))
-    library_dirs.append(os.path.join(libzstd_prefix, 'lib'))
-    compression_libraries.append('zstd')
     define_macros.append(('BORG_USE_LIBZSTD', 'YES'))
+    libzstd_system = True
+else:
+    libzstd_system = False
 
 
 with open('README.rst', 'r') as fd:
@@ -777,18 +773,22 @@ cmdclass = {
 
 ext_modules = []
 if not on_rtd:
+    compress_ext_kwargs = dict(sources=[compress_source], include_dirs=include_dirs, library_dirs=library_dirs,
+                               libraries=compression_libraries, define_macros=define_macros)
+    compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(bundled_path='src/borg/algorithms/zstd',
+                                                     system_prefix=libzstd_prefix, system=libzstd_system,
+                                                     multithreaded=False, legacy=False, **compress_ext_kwargs)
     ext_modules += [
-    Extension('borg.compress', [compress_source], libraries=compression_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
-    Extension('borg.crypto.low_level', [crypto_ll_source, crypto_helpers], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
-    Extension('borg.hashindex', [hashindex_source]),
-    Extension('borg.item', [item_source]),
-    Extension('borg.chunker', [chunker_source]),
-    Extension('borg.algorithms.checksums', [checksums_source]),
-
-]
+        Extension('borg.compress', **compress_ext_kwargs),
+        Extension('borg.crypto.low_level', [crypto_ll_source, crypto_helpers], libraries=crypto_libraries,
+                  include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
+        Extension('borg.hashindex', [hashindex_source]),
+        Extension('borg.item', [item_source]),
+        Extension('borg.chunker', [chunker_source]),
+        Extension('borg.algorithms.checksums', [checksums_source]),
+    ]
     if not sys.platform.startswith(('win32', )):
         ext_modules.append(Extension('borg.platform.posix', [platform_posix_source]))
-
     if sys.platform == 'linux':
         ext_modules.append(Extension('borg.platform.linux', [platform_linux_source], libraries=['acl']))
     elif sys.platform.startswith('freebsd'):

setup_zstd.py

@@ -1,125 +1,125 @@
+# Support code for building a C extension with zstd files
+#
 # Copyright (c) 2016-present, Gregory Szorc
+#               2017-present, Thomas Waldmann (mods to make it more generic)
 # All rights reserved.
 #
 # This software may be modified and distributed under the terms
 # of the BSD license. See the LICENSE file for details.
 
 import os
-from distutils.extension import Extension
-
-
-zstd_sources = ['zstd/%s' % p for p in (
-    'common/entropy_common.c',
-    'common/error_private.c',
-    'common/fse_decompress.c',
-    'common/pool.c',
-    'common/threading.c',
-    'common/xxhash.c',
-    'common/zstd_common.c',
-    'compress/fse_compress.c',
-    'compress/huf_compress.c',
-    'compress/zstd_compress.c',
-    'compress/zstdmt_compress.c',
-    'decompress/huf_decompress.c',
-    'decompress/zstd_decompress.c',
-    'dictBuilder/cover.c',
-    'dictBuilder/divsufsort.c',
-    'dictBuilder/zdict.c',
-)]
-
-zstd_sources_legacy = ['zstd/%s' % p for p in (
-    'deprecated/zbuff_common.c',
-    'deprecated/zbuff_compress.c',
-    'deprecated/zbuff_decompress.c',
-    'legacy/zstd_v01.c',
-    'legacy/zstd_v02.c',
-    'legacy/zstd_v03.c',
-    'legacy/zstd_v04.c',
-    'legacy/zstd_v05.c',
-    'legacy/zstd_v06.c',
-    'legacy/zstd_v07.c'
-)]
-
-zstd_includes = [
-    'zstd',
-    'zstd/common',
-    'zstd/compress',
-    'zstd/decompress',
-    'zstd/dictBuilder',
-]
 
-zstd_includes_legacy = [
-    'zstd/deprecated',
-    'zstd/legacy',
+# zstd files, structure as seen in zstd project repository:
+
+zstd_sources = [
+    'lib/common/entropy_common.c',
+    'lib/common/error_private.c',
+    'lib/common/fse_decompress.c',
+    'lib/common/pool.c',
+    'lib/common/threading.c',
+    'lib/common/xxhash.c',
+    'lib/common/zstd_common.c',
+    'lib/compress/fse_compress.c',
+    'lib/compress/huf_compress.c',
+    'lib/compress/zstd_compress.c',
+    'lib/compress/zstd_double_fast.c',
+    'lib/compress/zstd_fast.c',
+    'lib/compress/zstd_lazy.c',
+    'lib/compress/zstd_ldm.c',
+    'lib/compress/zstd_opt.c',
+    'lib/compress/zstdmt_compress.c',
+    'lib/decompress/huf_decompress.c',
+    'lib/decompress/zstd_decompress.c',
+    'lib/dictBuilder/cover.c',
+    'lib/dictBuilder/divsufsort.c',
+    'lib/dictBuilder/zdict.c',
 ]
 
-ext_includes = [
-    'c-ext',
-    'zstd/common',
+zstd_sources_legacy = [
+    'lib/deprecated/zbuff_common.c',
+    'lib/deprecated/zbuff_compress.c',
+    'lib/deprecated/zbuff_decompress.c',
+    'lib/legacy/zstd_v01.c',
+    'lib/legacy/zstd_v02.c',
+    'lib/legacy/zstd_v03.c',
+    'lib/legacy/zstd_v04.c',
+    'lib/legacy/zstd_v05.c',
+    'lib/legacy/zstd_v06.c',
+    'lib/legacy/zstd_v07.c',
 ]
 
-ext_sources = [
-    'zstd/common/pool.c',
-    'zstd/common/threading.c',
-    'zstd.c',
-    'c-ext/bufferutil.c',
-    'c-ext/compressiondict.c',
-    'c-ext/compressobj.c',
-    'c-ext/compressor.c',
-    'c-ext/compressoriterator.c',
-    'c-ext/compressionparams.c',
-    'c-ext/compressionreader.c',
-    'c-ext/compressionwriter.c',
-    'c-ext/constants.c',
-    'c-ext/decompressobj.c',
-    'c-ext/decompressor.c',
-    'c-ext/decompressoriterator.c',
-    'c-ext/decompressionreader.c',
-    'c-ext/decompressionwriter.c',
-    'c-ext/frameparams.c',
+zstd_includes = [
+    'lib',
+    'lib/common',
+    'lib/compress',
+    'lib/decompress',
+    'lib/dictBuilder',
 ]
 
-zstd_depends = [
-    'c-ext/python-zstandard.h',
+zstd_includes_legacy = [
+    'lib/deprecated',
+    'lib/legacy',
 ]
 
 
-def get_c_extension(support_legacy=False, system_zstd=False, name='zstd'):
-    """Obtain a distutils.extension.Extension for the C extension."""
-    root = os.path.abspath(os.path.dirname(__file__))
-
-    sources = set([os.path.join(root, p) for p in ext_sources])
-    if not system_zstd:
-        sources.update([os.path.join(root, p) for p in zstd_sources])
-        if support_legacy:
-            sources.update([os.path.join(root, p) for p in zstd_sources_legacy])
-    sources = list(sources)
-
-    include_dirs = set([os.path.join(root, d) for d in ext_includes])
-    if not system_zstd:
-        include_dirs.update([os.path.join(root, d) for d in zstd_includes])
-        if support_legacy:
-            include_dirs.update([os.path.join(root, d) for d in zstd_includes_legacy])
-    include_dirs = list(include_dirs)
-
-    depends = [os.path.join(root, p) for p in zstd_depends]
-
-    extra_args = ['-DZSTD_MULTITHREAD']
-
-    if not system_zstd:
-        extra_args.append('-DZSTDLIB_VISIBILITY=')
-        extra_args.append('-DZDICTLIB_VISIBILITY=')
-        extra_args.append('-DZSTDERRORLIB_VISIBILITY=')
-        extra_args.append('-fvisibility=hidden')
-
-    if not system_zstd and support_legacy:
-        extra_args.append('-DZSTD_LEGACY_SUPPORT=1')
-
-    libraries = ['zstd'] if system_zstd else []
-
-    # TODO compile with optimizations.
-    return Extension(name, sources,
-                     include_dirs=include_dirs,
-                     depends=depends,
-                     extra_compile_args=extra_args,
-                     libraries=libraries)
+def zstd_system_prefix(prefixes):
+    for prefix in prefixes:
+        filename = os.path.join(prefix, 'include', 'zstd.h')
+        if os.path.exists(filename):
+            with open(filename, 'r') as fd:
+                if 'ZSTD_getFrameContentSize' in fd.read():  # checks for zstd >= 1.3.0
+                    return prefix
+
+
+def zstd_ext_kwargs(bundled_path, system_prefix=None, system=False, multithreaded=False, legacy=False, **kwargs):
+    """amend kwargs with zstd suff for a distutils.extension.Extension initialization.
+
+    bundled_path: relative (to this file) path to the bundled library source code files
+    system_prefix: where the system-installed library can be found
+    system: True: use the system-installed shared library, False: use the bundled library code
+    multithreaded: True: define ZSTD_MULTITHREAD
+    legacy: include legacy API support
+    kwargs: distutils.extension.Extension kwargs that should be amended
+    returns: amended kwargs
+    """
+    def multi_join(paths, *path_segments):
+        """apply os.path.join on a list of paths"""
+        return [os.path.join(*(path_segments + (path, ))) for path in paths]
+
+    use_system = system and system_prefix is not None
+
+    sources = kwargs.get('sources', [])
+    if not use_system:
+        sources += multi_join(zstd_sources, bundled_path)
+        if legacy:
+            sources += multi_join(zstd_sources_legacy, bundled_path)
+
+    include_dirs = kwargs.get('include_dirs', [])
+    if use_system:
+        include_dirs += multi_join(['include'], system_prefix)
+    else:
+        include_dirs += multi_join(zstd_includes, bundled_path)
+        if legacy:
+            include_dirs += multi_join(zstd_includes_legacy, bundled_path)
+
+    library_dirs = kwargs.get('library_dirs', [])
+    if use_system:
+        library_dirs += multi_join(['lib'], system_prefix)
+
+    libraries = kwargs.get('libraries', [])
+    if use_system:
+        libraries += ['zstd', ]
+
+    extra_compile_args = kwargs.get('extra_compile_args', [])
+    if multithreaded:
+        extra_compile_args += ['-DZSTD_MULTITHREAD', ]
+    if not use_system:
+        extra_compile_args += ['-DZSTDLIB_VISIBILITY=', '-DZDICTLIB_VISIBILITY=', '-DZSTDERRORLIB_VISIBILITY=', ]
+                               # '-fvisibility=hidden' does not work, doesn't find PyInit_compress then
+        if legacy:
+            extra_compile_args += ['-DZSTD_LEGACY_SUPPORT=1', ]
+
+    ret = dict(**kwargs)
+    ret.update(dict(sources=sources, extra_compile_args=extra_compile_args,
+                    include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries))
+    return ret

src/borg/algorithms/zstd-libselect.h

@@ -1,5 +1,5 @@
 #ifdef BORG_USE_LIBZSTD
 #include <zstd.h>
 #else
-#error "TODO"
+#include "zstd/lib/zstd.h"
 #endif