Simplify libs setup (#6482)

setup.cfg

@@ -1,3 +1,73 @@
+[metadata]
+name = borgbackup
+author = The Borg Collective (see AUTHORS file)
+description = Deduplicated, encrypted, authenticated and compressed backups
+url = https://github.com/borgbackup/borg
+keywords =
+  backup
+  borgbackup
+classifiers =
+    Development Status :: 4 - Beta
+    Environment :: Console
+    Intended Audience :: System Administrators
+    License :: OSI Approved :: BSD License
+    Operating System :: POSIX :: BSD :: FreeBSD
+    Operating System :: POSIX :: BSD :: OpenBSD
+    Operating System :: POSIX :: BSD :: NetBSD
+    Operating System :: MacOS :: MacOS X
+    Operating System :: POSIX :: Linux
+    Programming Language :: Python
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Topic :: Security :: Cryptography
+    Topic :: System :: Archiving :: Backup
+platforms = Linux, MacOS X, FreeBSD, OpenBSD, NetBSD
+license = BSD
+license_file = LICENSE
+project_urls =
+    Bug Tracker = https://github.com/borgbackup/borg/issues
+    Documentation = https://borgbackup.readthedocs.io
+    Source Code = https://github.com/borgbackup/borg
+
+[options]
+packages = find:
+package_dir =
+    =src
+python_requires = >=3.9
+setup_requires =
+  setuptools_scm>=1.7
+install_requires =
+  msgpack >=1.0.3, <=1.0.3
+  packaging
+  argon2-cffi
+tests_require =
+  pytest
+zip_safe = False
+# See also the MANIFEST.in file.
+# We want to install all the files in the package directories...
+include_package_data = true
+
+# ...except the source files which have been compiled (C extensions):
+[options.exclude_package_data]
+* =
+    *.c
+    *.h
+    *.pyx
+
+[options.packages.find]
+where = src
+
+[options.entry_points]
+console_scripts =
+    borg = borg.archiver:main
+    borgfs = borg.archiver:main
+
+[options.extras_require]
+llfuse = llfuse >= 1.3.8
+pyfuse3 = pyfuse3 >= 3.1.1
+nofuse = 
+
 [tool:pytest]
 python_files = testsuite/*.py
 markers =

setup.py

@@ -20,68 +20,25 @@ except ImportError:
     cythonize = None
 
 sys.path += [os.path.dirname(__file__)]
-import setup_checksums
-import setup_compress
-import setup_crypto
 import setup_docs
 
 is_win32 = sys.platform.startswith('win32')
 
+# Number of threads to use for cythonize, not used on windows
+cpu_threads = multiprocessing.cpu_count() if multiprocessing and multiprocessing.get_start_method() != 'spawn' else None
+
 # How the build process finds the system libs:
 #
-# 1. if BORG_LIBXXX_PREFIX is set, it will use headers and libs from there.
+# 1. if BORG_{LIBXXX,OPENSSL}_PREFIX is set, it will use headers and libs from there.
 # 2. if not and pkg-config can locate the lib, the lib located by
 #    pkg-config will be used. We use the pkg-config tool via the pkgconfig
 #    python package, which must be installed before invoking setup.py.
 #    if pkgconfig is not installed, this step is skipped.
 # 3. otherwise raise a fatal error.
 
-# needed: >=1.1.1 (or compatible)
-system_prefix_openssl = os.environ.get('BORG_OPENSSL_PREFIX')
-
-# needed: lz4 (>= 1.7.0 / r129)
-system_prefix_liblz4 = os.environ.get('BORG_LIBLZ4_PREFIX')
-
-# needed: zstd (>= 1.3.0)
-system_prefix_libzstd = os.environ.get('BORG_LIBZSTD_PREFIX')
-
-# needed: xxhash (>= 0.8.1)
-system_prefix_libxxhash = os.environ.get('BORG_LIBXXHASH_PREFIX')
-
-# needed: deflate (>= 1.5)
-system_prefix_libdeflate = os.environ.get('BORG_LIBDEFLATE_PREFIX')
-
-# Number of threads to use for cythonize, not used on windows
-cpu_threads = multiprocessing.cpu_count() if multiprocessing and multiprocessing.get_start_method() != 'spawn' else None
-
 # Are we building on ReadTheDocs?
 on_rtd = os.environ.get('READTHEDOCS')
 
-install_requires = [
-    # we are rather picky about msgpack versions, because a good working msgpack is
-    # very important for borg, see: https://github.com/borgbackup/borg/issues/3753
-    'msgpack >=1.0.3, <=1.0.3',
-    # Please note:
-    # using any other version is not supported by borg development and
-    # any feedback related to issues caused by this will be ignored.
-    'packaging',
-    'argon2-cffi',
-]
-
-# note for package maintainers: if you package borgbackup for distribution,
-# please (if available) add pyfuse3 (preferably) or llfuse (not maintained any more)
-# as a *requirement*. "borg mount" needs one of them to work.
-# if neither is available, do not require it, most of borgbackup will work.
-extras_require = {
-    'llfuse': [
-        'llfuse >= 1.3.8',
-    ],
-    'pyfuse3': [
-        'pyfuse3 >= 3.1.1',
-    ],
-    'nofuse': [],
-}
-
 # Extra cflags for all extensions, usually just warnings we want to explicitly enable
 cflags = [
     '-Wall',
@@ -165,6 +122,7 @@ cmdclass = {
     'clean2': Clean,
 }
 
+
 ext_modules = []
 if not on_rtd:
 
@@ -182,23 +140,46 @@ if not on_rtd:
         print('Warning: can not import pkgconfig python package.')
         pc = None
 
+    def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_subdir='lib'):
+        system_prefix = os.environ.get(prefix_env_var)
+        if system_prefix:
+            print(f"Detected and preferring {lib_pkg_name} [via {prefix_env_var}]")
+            return dict(include_dirs=[os.path.join(system_prefix, 'include')],
+                        library_dirs=[os.path.join(system_prefix, lib_subdir)],
+                        libraries=[lib_name])
+
+        if pc and pc.installed(lib_pkg_name, pc_version):
+            print(f"Detected and preferring {lib_pkg_name} [via pkg-config]")
+            return pc.parse(lib_pkg_name)
+        raise Exception(
+            f"Could not find {lib_name} lib/headers, please set {prefix_env_var} "
+            f"or ensure {lib_pkg_name}.pc is in PKG_CONFIG_PATH."
+        )
+
+    if is_win32:
+        crypto_ext_lib = lib_ext_kwargs(
+            pc, 'BORG_OPENSSL_PREFIX', 'libcrypto', 'libcrypto', '>=1.1.1', lib_subdir='')
+    else:
+        crypto_ext_lib = lib_ext_kwargs(
+            pc, 'BORG_OPENSSL_PREFIX', 'crypto', 'libcrypto', '>=1.1.1')
+
     crypto_ext_kwargs = members_appended(
         dict(sources=[crypto_ll_source, crypto_helpers]),
-        setup_crypto.crypto_ext_kwargs(pc, system_prefix_openssl),
+        crypto_ext_lib,
         dict(extra_compile_args=cflags),
     )
 
     compress_ext_kwargs = members_appended(
         dict(sources=[compress_source]),
-        setup_compress.lz4_ext_kwargs(pc, system_prefix_liblz4),
-        setup_compress.zstd_ext_kwargs(pc, system_prefix_libzstd),
+        lib_ext_kwargs(pc, 'BORG_LIBLZ4_PREFIX', 'lz4', 'liblz4', '>= 1.7.0'),
+        lib_ext_kwargs(pc, 'BORG_LIBZSTD_PREFIX', 'zstd', 'libzstd', '>= 1.3.0'),
         dict(extra_compile_args=cflags),
     )
 
     checksums_ext_kwargs = members_appended(
         dict(sources=[checksums_source]),
-        setup_checksums.xxhash_ext_kwargs(pc, system_prefix_libxxhash),
-        setup_checksums.deflate_ext_kwargs(pc, system_prefix_libdeflate),
+        lib_ext_kwargs(pc, 'BORG_LIBXXHASH_PREFIX', 'xxhash', 'libxxhash', '>= 0.7.3'),
+        lib_ext_kwargs(pc, 'BORG_LIBDEFLATE_PREFIX', 'deflate', 'libdeflate', '>= 1.5'),
         dict(extra_compile_args=cflags),
     )
 
@@ -253,54 +234,10 @@ if not on_rtd:
 
 
 setup(
-    name='borgbackup',
     use_scm_version={
         'write_to': 'src/borg/_version.py',
     },
-    author='The Borg Collective (see AUTHORS file)',
-    author_email='borgbackup@python.org',
-    url='https://borgbackup.readthedocs.io/',
-    description='Deduplicated, encrypted, authenticated and compressed backups',
-    long_description=setup_docs.long_desc_from_readme(),
-    license='BSD',
-    platforms=['Linux', 'MacOS X', 'FreeBSD', 'OpenBSD', 'NetBSD', ],
-    classifiers=[
-        'Development Status :: 4 - Beta',
-        'Environment :: Console',
-        'Intended Audience :: System Administrators',
-        'License :: OSI Approved :: BSD License',
-        'Operating System :: POSIX :: BSD :: FreeBSD',
-        'Operating System :: POSIX :: BSD :: OpenBSD',
-        'Operating System :: POSIX :: BSD :: NetBSD',
-        'Operating System :: MacOS :: MacOS X',
-        'Operating System :: POSIX :: Linux',
-        'Programming Language :: Python',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.9',
-        'Programming Language :: Python :: 3.10',
-        'Topic :: Security :: Cryptography',
-        'Topic :: System :: Archiving :: Backup',
-    ],
-    packages=find_packages('src'),
-    package_dir={'': 'src'},
-    zip_safe=False,
-    entry_points={
-        'console_scripts': [
-            'borg = borg.archiver:main',
-            'borgfs = borg.archiver:main',
-        ]
-    },
-    # See also the MANIFEST.in file.
-    # We want to install all the files in the package directories...
-    include_package_data=True,
-    # ...except the source files which have been compiled (C extensions):
-    exclude_package_data={
-        '': ['*.c', '*.h', '*.pyx', ],
-    },
     cmdclass=cmdclass,
     ext_modules=ext_modules,
-    setup_requires=['setuptools_scm>=1.7'],
-    install_requires=install_requires,
-    extras_require=extras_require,
-    python_requires='>=3.9',
+    long_description=setup_docs.long_desc_from_readme()
 )

setup_checksums.py

@@ -1,31 +0,0 @@
-# Support code for building a C extension with checksums code
-
-import os
-
-
-def xxhash_ext_kwargs(pc, system_prefix):
-    if system_prefix:
-        print('Detected and preferring libxxhash [via BORG_LIBXXHASH_PREFIX]')
-        return dict(include_dirs=[os.path.join(system_prefix, 'include')],
-                    library_dirs=[os.path.join(system_prefix, 'lib')],
-                    libraries=['xxhash'])
-
-    if pc and pc.installed('libxxhash', '>= 0.7.3'):
-        print('Detected and preferring libxxhash [via pkg-config]')
-        return pc.parse('libxxhash')
-
-    raise Exception('Could not find xxhash lib/headers, please set BORG_LIBXXHASH_PREFIX')
-
-
-def deflate_ext_kwargs(pc, system_prefix):
-    if system_prefix:
-        print('Detected and preferring libdeflate [via BORG_LIBDEFLATE_PREFIX]')
-        return dict(include_dirs=[os.path.join(system_prefix, 'include')],
-                    library_dirs=[os.path.join(system_prefix, 'lib')],
-                    libraries=['deflate'])
-
-    if pc and pc.installed('libdeflate', '>= 1.5'):
-        print('Detected and preferring libdeflate [via pkg-config]')
-        return pc.parse('libdeflate')
-
-    raise Exception('Could not find deflate lib/headers, please set BORG_LIBDEFLATE_PREFIX')

setup_compress.py

@@ -1,31 +0,0 @@
-# Support code for building a C extension with compression code
-
-import os
-
-
-def zstd_ext_kwargs(pc, system_prefix):
-    if system_prefix:
-        print('Detected and preferring libzstd [via BORG_LIBZSTD_PREFIX]')
-        return dict(include_dirs=[os.path.join(system_prefix, 'include')],
-                    library_dirs=[os.path.join(system_prefix, 'lib')],
-                    libraries=['zstd'])
-
-    if pc and pc.installed('libzstd', '>= 1.3.0'):
-        print('Detected and preferring libzstd [via pkg-config]')
-        return pc.parse('libzstd')
-
-    raise Exception('Could not find zstd lib/headers, please set BORG_LIBZSTD_PREFIX')
-
-
-def lz4_ext_kwargs(pc, system_prefix):
-    if system_prefix:
-        print('Detected and preferring liblz4 [via BORG_LIBLZ4_PREFIX]')
-        return dict(include_dirs=[os.path.join(system_prefix, 'include')],
-                    library_dirs=[os.path.join(system_prefix, 'lib')],
-                    libraries=['lz4'])
-
-    if pc and pc.installed('liblz4', '>= 1.7.0'):
-        print('Detected and preferring liblz4 [via pkg-config]')
-        return pc.parse('liblz4')
-
-    raise Exception('Could not find lz4 lib/headers, please set BORG_LIBLZ4_PREFIX')

setup_crypto.py

@@ -1,27 +0,0 @@
-# Support code for building a C extension with crypto code
-
-import os
-import sys
-
-is_win32 = sys.platform.startswith('win32')
-
-
-def crypto_ext_kwargs(pc, system_prefix):
-    if system_prefix:
-        print('Detected OpenSSL [via BORG_OPENSSL_PREFIX]')
-        if is_win32:
-            lib_dir = system_prefix
-            lib_name = 'libcrypto'
-        else:
-            lib_dir = os.path.join(system_prefix, 'lib')
-            lib_name = 'crypto'
-
-        return dict(include_dirs=[os.path.join(system_prefix, 'include')],
-                    library_dirs=[lib_dir],
-                    libraries=[lib_name])
-
-    if pc and pc.exists('libcrypto'):
-        print('Detected OpenSSL [via pkg-config]')
-        return pc.parse('libcrypto')
-
-    raise Exception('Could not find OpenSSL lib/headers, please set BORG_OPENSSL_PREFIX')

tox.ini

@@ -2,7 +2,7 @@
 # fakeroot -u tox --recreate
 
 [tox]
-envlist = py{38,39,310}-{none,fuse2,fuse3}
+envlist = py{39,310}-{none,fuse2,fuse3}
 minversion = 3.2
 requires =
 	pkgconfig