Merge pull request #9066 from ThomasWaldmann/build-binaries-on-gh-fixes-1.4

binary building on gh: fixes

.github/workflows/build-binary.yml

@@ -1,119 +0,0 @@
-name: Build binaries
-
-on:
-  push:
-    paths:
-      - '**.py'
-      - '**.pyx'
-      - '**.c'
-      - '**.h'
-      - '**.yml'
-      - '**.toml'
-      - '**.cfg'
-      - '**.ini'
-      - 'requirements.d/*'
-      - 'scripts/**'
-  pull_request:
-    paths:
-      - '**.py'
-      - '**.pyx'
-      - '**.c'
-      - '**.h'
-      - '**.yml'
-      - '**.toml'
-      - '**.cfg'
-      - '**.ini'
-      - 'requirements.d/*'
-      - 'scripts/**'
-  workflow_dispatch:
-
-jobs:
-  build-binary-macos:
-    name: Build (macOS ${{ matrix.os }} / ${{ runner.arch }})
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [macos-12, macos-14]
-    runs-on: ${{ matrix.os }}
-    timeout-minutes: 60
-
-    env:
-      # Support both Homebrew locations (Intel and Apple Silicon)
-      PKG_CONFIG_PATH: "/opt/homebrew/opt/openssl@3.0/lib/pkgconfig:/usr/local/opt/openssl@3.0/lib/pkgconfig:$PKG_CONFIG_PATH"
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-            # Just fetching one commit is not enough for setuptools-scm, so we fetch all
-            fetch-depth: 0
-
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.11'
-
-      - name: Cache pip
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.d/development.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-            ${{ runner.os }}-
-
-      - name: Install macOS packages via Homebrew (see Brewfile)
-        run: |
-          brew bundle install
-
-      - name: Install Python requirements
-        run: |
-          python -m pip install --upgrade pip setuptools wheel
-          pip install -r requirements.d/development.txt
-
-      - name: Build Borg to compile extensions
-        env:
-          # Set both paths again to be sure nothing overrides them
-          PKG_CONFIG_PATH: "/opt/homebrew/opt/openssl@3.0/lib/pkgconfig:/usr/local/opt/openssl@3.0/lib/pkgconfig:${{ env.PKG_CONFIG_PATH }}"
-        run: |
-          pip install -ve .
-
-      - name: Build PyInstaller single-file binary
-        run: |
-          python -m pip install 'pyinstaller==6.7.0'
-          mkdir -p dist/binary
-          pyinstaller --clean --distpath=dist/binary scripts/borg.exe.spec
-          pushd dist/binary
-          tar -czvf borg.tgz borg-dir
-          rm -rf borg-dir
-          popd
-
-      - name: Smoke-test the built binary (borg -V)
-        run: |
-          pushd dist/binary
-          ls -l
-          # test single-file binary
-          chmod +x borg.exe
-          ./borg.exe -V
-          # test single-dir binary
-          tar -xzvf borg.tgz
-          chmod +x borg-dir/borg.exe
-          ./borg-dir/borg.exe -V
-          popd
-
-      - name: Prepare artifacts
-        run: |
-          mkdir -p artifacts
-          if [ -f dist/binary/borg.exe ]; then
-            cp dist/binary/borg.exe artifacts/borg-${{ matrix.os }}-${{ runner.arch }}
-          fi
-          if [ -f dist/binary/borg.tgz ]; then
-            cp dist/binary/borg.tgz artifacts/borg-dir-${{ matrix.os }}-${{ runner.arch }}.tgz
-          fi
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: borg-macos-${{ matrix.os }}-${{ runner.arch }}
-          path: artifacts/*
-          if-no-files-found: error

.github/workflows/ci.yml

@@ -45,33 +45,30 @@ jobs:
     needs: lint
     strategy:
       fail-fast: false
-      matrix:
-        include:
-            - os: ubuntu-22.04
-              python-version: '3.9'
-              toxenv: py39-fuse2
-            - os: ubuntu-22.04
-              python-version: '3.10'
-              toxenv: py310-fuse3
-            - os: ubuntu-22.04
-              python-version: '3.11'
-              toxenv: py311-fuse2
-            - os: ubuntu-24.04
-              python-version: '3.12'
-              toxenv: py312-fuse3
-            - os: ubuntu-24.04
-              python-version: '3.13'
-              toxenv: py313-fuse3
-            - os: ubuntu-24.04
-              python-version: '3.14'
-              toxenv: py314-fuse3
-            - os: macos-14
-              python-version: '3.11'
-              toxenv: py311-none  # Note: no FUSE testing due to #6099; see also #6196.
+      # noinspection YAMLSchemaValidation
+      matrix: >-
+        ${{ fromJSON(
+          github.event_name == 'pull_request' && '{
+            "include": [
+              {"os": "ubuntu-22.04", "python-version": "3.9", "toxenv": "py39-fuse2"},
+              {"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-fuse3"}
+            ]
+          }' || '{
+            "include": [
+              {"os": "ubuntu-22.04", "python-version": "3.9", "toxenv": "py39-fuse2"},
+              {"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "py310-fuse3"},
+              {"os": "ubuntu-22.04", "python-version": "3.11", "toxenv": "py311-fuse2", "binary": "borg-linux-glibc235-x86_64-gh"},
+              {"os": "ubuntu-22.04-arm", "python-version": "3.11", "toxenv": "py311-fuse2", "binary": "borg-linux-glibc235-arm64-gh"},
+              {"os": "ubuntu-24.04", "python-version": "3.12", "toxenv": "py312-fuse3"},
+              {"os": "ubuntu-24.04", "python-version": "3.13", "toxenv": "py313-fuse3"},
+              {"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-fuse3"},
+              {"os": "macos-13", "python-version": "3.11", "toxenv": "py311-none", "binary": "borg-macos-13-x86_64-gh"},
+              {"os": "macos-14", "python-version": "3.11", "toxenv": "py311-none", "binary": "borg-macos-14-arm64-gh"}
+            ]
+          }'
+        ) }}
 
     env:
-      # Configure pkg-config to use OpenSSL from Homebrew
-      PKG_CONFIG_PATH: "/opt/homebrew/opt/openssl@3.0/lib/pkgconfig:$PKG_CONFIG_PATH"
       TOXENV: ${{ matrix.toxenv }}
 
     runs-on: ${{ matrix.os }}
@@ -82,10 +79,22 @@ jobs:
       with:
         # Just fetching one commit is not enough for setuptools-scm, so we fetch all
         fetch-depth: 0
+        fetch-tags: true
+
+    - name: Detect if commit is tagged
+      id: detect_tag
+      run: |
+        if git describe --exact-match --tags HEAD >/dev/null 2>&1; then
+          echo "tagged=true" >> "$GITHUB_OUTPUT"
+        else
+          echo "tagged=false" >> "$GITHUB_OUTPUT"
+        fi
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+
     - name: Cache pip
       uses: actions/cache@v4
       with:
@@ -112,22 +121,17 @@ jobs:
       run: |
         python -m pip install --upgrade pip setuptools wheel
         pip install -r requirements.d/development.txt
+
     - name: Install BorgBackup
-      env:
-        # We already have this in the global environment, but something overrides it.
-        # So set it here again.
-        PKG_CONFIG_PATH: "/opt/homebrew/opt/openssl@3.0/lib/pkgconfig:$PKG_CONFIG_PATH"
       run: |
         pip install -ve .
+
     - name: Run pytest via tox
-      env:
-        # We already have this in the global environment, but something overrides it.
-        # So set it here again.
-        PKG_CONFIG_PATH: "/opt/homebrew/opt/openssl@3.0/lib/pkgconfig:$PKG_CONFIG_PATH"
       run: |
         # Do not use fakeroot; run as root. Avoids the dreaded sporadic EISDIR failures; see #2482.
         #sudo -E bash -c "tox -e py"
         tox --skip-missing-interpreters
+
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v4
       env:
@@ -136,3 +140,44 @@ jobs:
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         env_vars: OS, python
+
+    - name: Build Borg fat binaries (${{ matrix.binary }})
+      if: ${{ matrix.binary && steps.detect_tag.outputs.tagged == 'true' }}
+      run: |
+        pip install 'pyinstaller==6.14.2'
+        mkdir -p dist/binary
+        pyinstaller --clean --distpath=dist/binary scripts/borg.exe.spec
+
+    - name: Smoke-test the built binary (${{ matrix.binary }})
+      if: ${{ matrix.binary && steps.detect_tag.outputs.tagged == 'true' }}
+      run: |
+        pushd dist/binary
+        echo "single-file binary"
+        chmod +x borg.exe
+        ./borg.exe -V
+        echo "single-directory binary"
+        chmod +x borg-dir/borg.exe
+        ./borg-dir/borg.exe -V
+        tar czf borg.tgz borg-dir
+        popd
+
+    - name: Prepare binaries (${{ matrix.binary }})
+      if: ${{ matrix.binary && steps.detect_tag.outputs.tagged == 'true' }}
+      run: |
+        mkdir -p artifacts
+        if [ -f dist/binary/borg.exe ]; then
+          cp dist/binary/borg.exe artifacts/${{ matrix.binary }}
+        fi
+        if [ -f dist/binary/borg.tgz ]; then
+          cp dist/binary/borg.tgz artifacts/${{ matrix.binary }}.tgz
+        fi
+        echo "binary files"
+        ls -l artifacts/
+
+    - name: Upload binaries (${{ matrix.binary }})
+      if: ${{ matrix.binary && steps.detect_tag.outputs.tagged == 'true' }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ matrix.binary }}
+        path: artifacts/*
+        if-no-files-found: error

docs/binaries/00_README.txt

@@ -1,36 +1,52 @@
 Binary BorgBackup builds
 ========================
 
-The binaries are supposed to work on the specified platform without installing
-any dependencies.
+General notes
+-------------
+
+The binaries are supposed to work on the specified platform without installing anything else.
+
+There are some limitations, though:
+- for Linux, your system must have the same or newer glibc version as the one used for building
+- for macOS, you need to have the same or newer macOS version as the one used for building
+- for other OSes, there are likely similar limitations
+
+If you don't find something working on your system, check the older borg releases.
+
+*.asc are GnuPG signatures - only provided for locally built binaries.
+*.exe (or no extension) is the single-file fat binary.
+*.tgz is the single-directory fat binary (extract it once with tar -xzf).
+
+Using the single-directory build is faster and does not require as much space
+in the temporary directory as the self-extracting single-file build.
+
+macOS: to avoid issues, download the file via the command line OR remove the
+       "quarantine" attribute after downloading:
+       $ xattr -dr com.apple.quarantine borg-macos1012.tgz
 
 
 Download the correct files
 --------------------------
 
-AMD64/x86_64 architecture
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Binaries built on GitHub servers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+borg-linux-glibc235-x86_64-gh Linux AMD/Intel (built on Ubuntu 22.04 LTS with glibc 2.35)
+borg-linux-glibc235-arm64-gh  Linux ARM (built on Ubuntu 22.04 LTS with glibc 2.35)
 
-borg-linux-glibc241 Linux (built on Debian 13 "Trixie" with glibc 2.41)
-borg-linux-glibc236 Linux (built on Debian 12 "Bookworm" with glibc 2.36)
-borg-linux-glibc231 Linux (built on Debian 11 "Bullseye" with glibc 2.31)
-                    Note: You can also try them on other Linux distributions with different glibc
-                    versions - as long as glibc is compatible, they will work.
-                    If it doesn't work, try a Borg 1.2.x binary.
+borg-macos-14-arm64-gh        macOS Apple Silicon (built on macOS 14 w/o FUSE support)
+borg-macos-13-x86_64-gh       macOS Intel (built on macOS 13 w/o FUSE support)
 
-borg-macos1012      macOS (built on macOS Sierra 10.12 with the latest macFUSE from Homebrew)
-                    To avoid signing issues, download the file via the command line OR
-                    remove the "quarantine" attribute after downloading:
-                    $ xattr -dr com.apple.quarantine borg-macos1012.tgz
 
-borg-freebsd13      FreeBSD (built on FreeBSD 13)
-borg-freebsd14      FreeBSD (built on FreeBSD 14)
+Binaries built locally
+~~~~~~~~~~~~~~~~~~~~~~
 
-*.tgz               Similar to the above, but built as a directory with files,
-                    not as a single self-extracting binary. Using the directory
-                    build is faster and does not require as much space in the temporary
-                    directory as the one-file build.
-*.asc               GnuPG signatures for *
+borg-linux-glibc241-x86_64 Linux (built on Debian 13 "Trixie" with glibc 2.41)
+borg-linux-glibc236-x86_64 Linux (built on Debian 12 "Bookworm" with glibc 2.36)
+borg-linux-glibc231-x86_64 Linux (built on Debian 11 "Bullseye" with glibc 2.31)
+
+borg-freebsd-13-x86_64     FreeBSD (built on FreeBSD 13)
+borg-freebsd-14-x86_64     FreeBSD (built on FreeBSD 14)
 
 
 Verifying your download
@@ -84,4 +100,3 @@ There, please give:
 - a traceback with system info (if you have one)
 - your precise platform (CPU, 32/64-bit?), OS, distribution, release
 - your Python and (g)libc versions
-

scripts/fetch-binaries

@@ -12,11 +12,9 @@ check_and_copy () {
     echo ""
 }
 
-check_and_copy bullseye  borg-linux-glibc231
-check_and_copy bookworm  borg-linux-glibc236
-check_and_copy trixie    borg-linux-glibc241
+check_and_copy bullseye  borg-linux-glibc231-x86_64
+check_and_copy bookworm  borg-linux-glibc236-x86_64
+check_and_copy trixie    borg-linux-glibc241-x86_64
 
-check_and_copy freebsd13 borg-freebsd13
-check_and_copy freebsd14 borg-freebsd14
-
-check_and_copy macos1012 borg-macos1012
+check_and_copy freebsd13 borg-freebsd-13-x86_64
+check_and_copy freebsd14 borg-freebsd-14-x86_64