stop using libdeflate

borg2's new repo format does not need computing crc32 over big amounts of
(content) data any more (we now use xxh64 for that).

thus, having a quick crc32 implementation via libdeflate is not important
enough any more to rectify having libdeflate as a requirement.

.github/workflows/ci.yml

@@ -67,7 +67,6 @@ jobs:
     env:
       # Configure pkg-config to use OpenSSL from Homebrew
       PKG_CONFIG_PATH: /usr/local/opt/openssl@1.1/lib/pkgconfig
-      BORG_LIBDEFLATE_PREFIX: /usr  # on ubuntu 20.04 pkgconfig does not find libdeflate
       TOXENV: ${{ matrix.toxenv }}
 
     runs-on: ${{ matrix.os }}
@@ -96,7 +95,7 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y pkg-config build-essential
-        sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev libdeflate-dev liblz4-dev libzstd-dev
+        sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev
         sudo apt-get install -y libfuse-dev fuse || true  # Required for Python llfuse module
         sudo apt-get install -y libfuse3-dev fuse3 || true  # Required for Python pyfuse3 module
 
@@ -106,7 +105,6 @@ jobs:
         brew install pkg-config || brew upgrade pkg-config
         brew install zstd || brew upgrade zstd
         brew install lz4 || brew upgrade lz4
-        brew install libdeflate || brew upgrade libdeflate
         brew install xxhash || brew upgrade xxhash
         brew install openssl@1.1 || brew upgrade openssl@1.1
         brew install --cask macfuse || brew upgrade --cask macfuse  # Required for Python llfuse module

.github/workflows/codeql-analysis.yml

@@ -28,7 +28,7 @@ jobs:
         # Learn more about CodeQL language support at https://git.io/codeql-language-support
 
     env:
-      BORG_LIBDEFLATE_PREFIX: /usr  # on ubuntu 20.04 pkgconfig does not find libdeflate
+      # nothing special right now
 
     steps:
     - name: Checkout repository
@@ -52,7 +52,7 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y pkg-config build-essential
-        sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev libdeflate-dev liblz4-dev libzstd-dev
+        sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v1

Vagrantfile

@@ -15,7 +15,7 @@ def packages_debianoid(user)
     apt-get -y -qq update
     apt-get -y -qq dist-upgrade
     # for building borgbackup and dependencies:
-    apt install -y libssl-dev libacl1-dev libxxhash-dev libdeflate-dev liblz4-dev libzstd-dev pkg-config
+    apt install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev pkg-config
     apt install -y libfuse-dev fuse || true
     apt install -y libfuse3-dev fuse3 || true
     apt install -y locales || true
@@ -29,7 +29,6 @@ def packages_debianoid(user)
     apt install -y zlib1g-dev libbz2-dev libncurses5-dev libreadline-dev liblzma-dev libsqlite3-dev libffi-dev
     # older debian / ubuntu have no .pc file for these, so we need to point at the lib/header location:
     echo 'export BORG_LIBXXHASH_PREFIX=/usr' >> ~vagrant/.bash_profile
-    echo 'export BORG_LIBDEFLATE_PREFIX=/usr' >> ~vagrant/.bash_profile
   EOF
 end
 
@@ -40,7 +39,7 @@ def packages_freebsd
     # install all the (security and other) updates, base system
     freebsd-update --not-running-from-cron fetch install
     # for building borgbackup and dependencies:
-    pkg install -y xxhash libdeflate liblz4 zstd pkgconf
+    pkg install -y xxhash liblz4 zstd pkgconf
     pkg install -y fusefs-libs || true
     pkg install -y fusefs-libs3 || true
     pkg install -y git bash  # fakeroot causes lots of troubles on freebsd
@@ -74,7 +73,6 @@ def packages_openbsd
     pkg_add bash
     chsh -s bash vagrant
     pkg_add xxhash
-    pkg_add libdeflate
     pkg_add lz4
     pkg_add zstd
     pkg_add git  # no fakeroot
@@ -119,7 +117,7 @@ def packages_darwin
     sudo softwareupdate --install --all
     which brew || CI=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
     brew update > /dev/null
-    brew install pkg-config readline openssl@1.1 xxhash libdeflate zstd lz4 xz
+    brew install pkg-config readline openssl@1.1 xxhash zstd lz4 xz
     brew install --cask macfuse
     # brew upgrade  # upgrade everything (takes rather long)
     echo 'export PKG_CONFIG_PATH=/usr/local/opt/openssl@1.1/lib/pkgconfig' >> ~vagrant/.bash_profile

docs/installation.rst

@@ -167,7 +167,6 @@ following dependencies first:
   - liblz4_ >= 1.7.0 (r129)
   - libzstd_ >= 1.3.0
   - libxxhash >= 0.8.1 (0.8.0 might work also)
-  - libdeflate >= 1.5
 * pkg-config (cli tool) and pkgconfig python package (borg uses these to
   discover header and library location - if it can't import pkgconfig and
   is not pointed to header/library locations via env vars [see setup.py],
@@ -202,7 +201,7 @@ Install the dependencies with development headers::
     sudo apt-get install python3 python3-dev python3-pip python3-virtualenv \
     libacl1-dev libacl1 \
     libssl-dev \
-    liblz4-dev libzstd-dev libxxhash-dev libdeflate-dev \
+    liblz4-dev libzstd-dev libxxhash-dev \
     build-essential \
     pkg-config python3-pkgconfig
     sudo apt-get install libfuse-dev fuse    # needed for llfuse
@@ -220,7 +219,7 @@ Install the dependencies with development headers::
     sudo dnf install python3 python3-devel python3-pip python3-virtualenv \
     libacl-devel libacl \
     openssl-devel \
-    lz4-devel libzstd-devel xxhash-devel libdeflate-devel \
+    lz4-devel libzstd-devel xxhash-devel \
     pkgconf python3-pkgconfig
     sudo dnf install gcc gcc-c++ redhat-rpm-config
     sudo dnf install fuse-devel fuse         # needed for llfuse
@@ -236,8 +235,7 @@ Install the dependencies automatically using zypper::
 Alternatively, you can enumerate all build dependencies in the command line::
 
     sudo zypper install python3 python3-devel \
-    libacl-devel openssl-devel \
-    libxxhash-devel libdeflate-devel \
+    libacl-devel openssl-devel libxxhash-devel \
     python3-Cython python3-Sphinx python3-msgpack-python python3-pkgconfig pkgconf \
     python3-pytest python3-setuptools python3-setuptools_scm \
     python3-sphinx_rtd_theme gcc gcc-c++
@@ -249,7 +247,7 @@ macOS
 When installing via Homebrew_, dependencies are installed automatically. To install
 dependencies manually::
 
-    brew install python3 openssl zstd lz4 xxhash libdeflate
+    brew install python3 openssl zstd lz4 xxhash
     brew install pkg-config
     pip3 install virtualenv pkgconfig
 
@@ -286,7 +284,7 @@ and commands to make FUSE work for using the mount command.
 
      pkg install -y python3 pkgconf
      pkg install openssl
-     pkg install liblz4 zstd xxhash libdeflate
+     pkg install liblz4 zstd xxhash
      pkg install fusefs-libs  # needed for llfuse
      pkg install -y git
      python3 -m ensurepip # to install pip for Python3
@@ -316,7 +314,7 @@ Use the Cygwin installer to install the dependencies::
 
     python39 python39-devel python39-pkgconfig
     python39-setuptools python39-pip python39-wheel python39-virtualenv
-    libssl-devel libxxhash-devel libdeflate-devel liblz4-devel libzstd-devel
+    libssl-devel libxxhash-devel liblz4-devel libzstd-devel
     binutils gcc-g++ git make openssh
 
 

setup.py

@@ -186,7 +186,6 @@ if not on_rtd:
     checksums_ext_kwargs = members_appended(
         dict(sources=[checksums_source]),
         lib_ext_kwargs(pc, 'BORG_LIBXXHASH_PREFIX', 'xxhash', 'libxxhash', '>= 0.7.3'),
-        lib_ext_kwargs(pc, 'BORG_LIBDEFLATE_PREFIX', 'deflate', 'libdeflate', '>= 1.5'),
         dict(extra_compile_args=cflags),
     )
 

src/borg/archiver.py

@@ -715,21 +715,13 @@ class Archiver:
         ]:
             print(f"{spec:<24} {size:<10} {timeit(func, number=100):.3f}s")
 
-        import zlib
-        from .checksums import crc32, deflate_crc32, xxh64
+        from .checksums import crc32, xxh64
         print("Non-cryptographic checksums / hashes ===========================")
         size = "1GB"
         tests = [
             ("xxh64", lambda: xxh64(random_10M)),
+            ("crc32 (zlib)", lambda: crc32(random_10M)),
         ]
-        if crc32 is zlib.crc32:
-            tests.insert(0, ("crc32 (zlib, used)", lambda: crc32(random_10M)))
-            tests.insert(1, ("crc32 (libdeflate)", lambda: deflate_crc32(random_10M)))
-        elif crc32 is deflate_crc32:
-            tests.insert(0, ("crc32 (libdeflate, used)", lambda: crc32(random_10M)))
-            tests.insert(1, ("crc32 (zlib)", lambda: zlib.crc32(random_10M)))
-        else:
-            raise Error("crc32 benchmarking code missing")
         for spec, func in tests:
             print(f"{spec:<24} {size:<10} {timeit(func, number=100):.3f}s")
 

src/borg/checksums.pyx

@@ -33,31 +33,15 @@ cdef extern from "xxhash.h":
     XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
 
 
-cdef extern from "libdeflate.h":
-    uint32_t libdeflate_crc32(uint32_t crc, const void *buffer, size_t len)
-
-
 cdef Py_buffer ro_buffer(object data) except *:
     cdef Py_buffer view
     PyObject_GetBuffer(data, &view, PyBUF_SIMPLE)
     return view
 
 
-def deflate_crc32(data, value=0):
-    cdef Py_buffer data_buf = ro_buffer(data)
-    cdef uint32_t val = value
-    try:
-        return libdeflate_crc32(val, data_buf.buf, data_buf.len)
-    finally:
-        PyBuffer_Release(&data_buf)
-
-
-if is_darwin:
-    # macOS (darwin) has a highly optimized zlib.crc32 (Intel as well as Apple Silicon M1)
-    crc32 = zlib.crc32
-else:
-    # on Linux x64 (and maybe others), libdeflate_crc32 is faster than zlib.crc32
-    crc32 = deflate_crc32
+# borg 2.0's new repos do not compute crc32 over big amounts of data,
+# so speed does not matter much any more and we can just use zlib.crc32.
+crc32 = zlib.crc32
 
 
 def xxh64(data, seed=0):

src/borg/testsuite/checksums.py

@@ -1,30 +1,8 @@
-import os
-import zlib
 from binascii import unhexlify
 
-import pytest
-
 from .. import checksums
 from ..helpers import bin_to_hex
 
-crc32_implementations = [checksums.deflate_crc32]
-
-
-@pytest.mark.parametrize('implementation', crc32_implementations)
-def test_crc32(implementation):
-    # This includes many critical values, like misc. length and misc. aligned start addresses.
-    data = os.urandom(300)
-    mv = memoryview(data)
-    initial_crc = 0x12345678
-    for start in range(0, 4):  # 4B / int32 alignment, head processing
-        for length in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
-                       31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41,
-                       63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73,
-                       127, 128, 129, 130, 131, 132, 133, 134, 135,
-                       255, 256, 257, ]:
-            d = mv[start:start+length]
-            assert zlib.crc32(d, initial_crc) == implementation(d, initial_crc)
-
 
 def test_xxh64():
     assert bin_to_hex(checksums.xxh64(b'test', 123)) == '2b81b9401bef86cf'