Merge pull request #9154 from ThomasWaldmann/unify-ci-master

Integrate CI (master)

.github/workflows/ci.yml

@@ -54,7 +54,7 @@ jobs:
 
     runs-on: ubuntu-24.04
     timeout-minutes: 25
-    needs: [security]
+    needs: [lint, security]
 
     steps:
     - uses: actions/checkout@v4
@@ -120,7 +120,7 @@ jobs:
         echo "Using LD_PRELOAD=$LD_PRELOAD"
         pytest -v --benchmark-skip -k "not remote"
 
-  posix_tests:
+  native_tests:
 
     needs: [lint, security]
     permissions:
@@ -328,12 +328,197 @@ jobs:
         path: artifacts/*
         if-no-files-found: error
 
+  vm_tests:
+    permissions:
+      contents: read
+      id-token: write
+      attestations: write
+    runs-on: ubuntu-24.04
+    timeout-minutes: 90
+    needs: [lint, security]
+    continue-on-error: true
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: freebsd
+            version: '14.3'
+            display_name: FreeBSD
+            # Controls binary build and provenance attestation on tags
+            do_binaries: true
+            artifact_prefix: borg-freebsd-14-x86_64-gh
+
+          - os: netbsd
+            version: '10.1'
+            display_name: NetBSD
+            do_binaries: false
+
+          - os: openbsd
+            version: '7.7'
+            display_name: OpenBSD
+            do_binaries: false
+
+          - os: haiku
+            version: 'r1beta5'
+            display_name: Haiku
+            do_binaries: false
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Test on ${{ matrix.display_name }}
+        id: cross_os
+        uses: cross-platform-actions/action@v0.29.0
+        env:
+          DO_BINARIES: ${{ matrix.do_binaries }}
+        with:
+          operating_system: ${{ matrix.os }}
+          version: ${{ matrix.version }}
+          shell: bash
+          run: |
+            set -euxo pipefail
+
+            case "${{ matrix.os }}" in
+              freebsd)
+                export IGNORE_OSVERSION=yes
+                sudo -E pkg update -f
+                sudo -E pkg install -y xxhash liblz4 zstd pkgconf
+                # Install one of the FUSE libraries; fail if neither is available
+                sudo -E pkg install -y fusefs-libs || sudo -E pkg install -y fusefs-libs3
+                sudo -E pkg install -y rust
+                sudo -E pkg install -y git
+                sudo -E pkg install -y python310 py310-sqlite3
+                sudo -E pkg install -y python311 py311-sqlite3 py311-pip py311-virtualenv
+                sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python3
+                sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python
+                sudo ln -sf /usr/local/bin/pip3.11 /usr/local/bin/pip3
+                sudo ln -sf /usr/local/bin/pip3.11 /usr/local/bin/pip
+
+                python -m venv .venv
+                . .venv/bin/activate
+                python -V
+                pip -V
+                python -m pip install --upgrade pip wheel
+                pip install -r requirements.d/development.txt
+                pip install -e .
+                pytest -v -n auto --benchmark-skip
+
+                if [[ "${DO_BINARIES}" == "true" && "${GITHUB_REF:-}" == refs/tags/* ]]; then
+                  python -m pip install 'pyinstaller==6.14.2'
+                  mkdir -p dist/binary
+                  pyinstaller --clean --distpath=dist/binary scripts/borg.exe.spec
+                  pushd dist/binary
+                  echo "single-file binary"
+                  chmod +x borg.exe
+                  ./borg.exe -V
+                  echo "single-directory binary"
+                  chmod +x borg-dir/borg.exe
+                  ./borg-dir/borg.exe -V
+                  tar czf borg.tgz borg-dir
+                  popd
+                  mkdir -p artifacts
+                  if [ -f dist/binary/borg.exe ]; then
+                    cp -v dist/binary/borg.exe artifacts/${{ matrix.artifact_prefix }}
+                  fi
+                  if [ -f dist/binary/borg.tgz ]; then
+                    cp -v dist/binary/borg.tgz artifacts/${{ matrix.artifact_prefix }}.tgz
+                  fi
+                fi
+                ;;
+
+              netbsd)
+                arch="$(uname -m)"
+                sudo -E mkdir -p /usr/pkg/etc/pkgin
+                echo "http://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/${arch}/10.1/All" | sudo tee /usr/pkg/etc/pkgin/repositories.conf > /dev/null
+                sudo -E pkgin update || true
+                sudo -E pkgin -y upgrade || true
+                sudo -E pkgin -y install zstd lz4 xxhash git || true
+                sudo -E pkgin -y install rust || true
+                sudo -E pkgin -y install pkg-config || true
+                sudo -E pkgin -y install py311-pip py311-virtualenv || true
+                sudo -E ln -sf /usr/pkg/bin/python3.11 /usr/pkg/bin/python3 || true
+                sudo -E ln -sf /usr/pkg/bin/pip3.11 /usr/pkg/bin/pip3 || true
+                sudo -E ln -sf /usr/pkg/bin/virtualenv-3.11 /usr/pkg/bin/virtualenv3 || true
+
+                python3 -m virtualenv .venv
+                . .venv/bin/activate
+                python -m pip install --upgrade pip
+                pip install -r requirements.d/development.txt
+                pip install -e .
+                pytest -v -n auto --benchmark-skip
+                ;;
+
+              openbsd)
+                # echo "https://ftp.eu.openbsd.org/pub/OpenBSD" | sudo tee /etc/installurl > /dev/null || true
+                sudo -E pkg_add xxhash || true
+                sudo -E pkg_add lz4 || true
+                sudo -E pkg_add zstd || true
+                sudo -E pkg_add git || true
+                sudo -E pkg_add rust || true
+                sudo -E pkg_add openssl%3.4 || true
+                sudo -E pkg_add py3-pip || true
+                sudo -E pkg_add py3-virtualenv || true
+
+                export BORG_OPENSSL_NAME=eopenssl34
+                python3 -m virtualenv .venv
+                . .venv/bin/activate
+                python -m pip install --upgrade pip
+                pip install -r requirements.d/development.txt
+                pip install -e .
+                pytest -v -n auto --benchmark-skip
+                ;;
+
+              haiku)
+                pkgman refresh || true
+                pkgman install -y git pkgconfig zstd lz4 xxhash || true
+                pkgman install -y openssl3 || true
+                pkgman install -y rust_bin || true
+                pkgman install -y python3.10 || true
+                pkgman install -y cffi || true
+                python3 -V || true
+                pkgman install -y lz4_devel zstd_devel xxhash_devel openssl3_devel libffi_devel || true
+                export PKG_CONFIG_PATH="/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
+                export BORG_LIBLZ4_PREFIX=/system/develop
+                export BORG_LIBZSTD_PREFIX=/system/develop
+                export BORG_LIBXXHASH_PREFIX=/system/develop
+                export BORG_OPENSSL_PREFIX=/system/develop
+
+                python3 -m ensurepip --upgrade || true
+                python3 -m pip install --upgrade pip wheel
+                python3 -m venv .venv
+                . .venv/bin/activate
+                python -V
+                pip install -r requirements.d/development.txt
+                pip install -e .
+                pytest -v --benchmark-skip -k "not remote and not socket"
+                ;;
+            esac
+
+      - name: Upload artifacts
+        if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.os }}-${{ matrix.version }}-dist
+          path: artifacts/*
+          if-no-files-found: ignore
+
+      - name: Attest provenance
+        if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries
+        uses: actions/attest-build-provenance@v3
+        with:
+          subject-path: 'artifacts/*'
+
   windows_tests:
 
     if: false  # can be used to temporarily disable the build
     runs-on: windows-latest
     timeout-minutes: 120
-    needs: posix_tests
+    needs: native_tests
 
     env:
       PY_COLORS: 1

.github/workflows/freebsd.yml

@@ -1,119 +0,0 @@
-name: CI FreeBSD
-
-on:
-  push:
-    branches: [ master ]
-    tags:
-    - '2.*'
-  pull_request:
-    branches: [ master ]
-    paths:
-    - '**.py'
-    - '**.pyx'
-    - '**.c'
-    - '**.h'
-    - '**.yml'
-    - '**.toml'
-    - '**.cfg'
-    - '**.ini'
-    - 'requirements.d/*'
-    - '!docs/**'
-
-jobs:
-  freebsd_tests:
-    name: FreeBSD tests and build
-    permissions:
-      contents: read
-      id-token: write
-      attestations: write
-    runs-on: ubuntu-24.04
-    continue-on-error: true
-    steps:
-      - name: Check out repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          fetch-tags: true
-
-      - name: Run on FreeBSD
-        id: freebsd
-        uses: cross-platform-actions/action@v0.29.0
-        with:
-          operating_system: freebsd
-          version: '14.3'
-          shell: bash
-          # Increase resources a bit for building Cython extensions
-          cpu_count: 3
-          memory: 8G
-          run: |
-            set -euxo pipefail
-            # Update package catalog and install dependencies
-            export IGNORE_OSVERSION=yes
-            sudo -E pkg update -f
-            sudo -E pkg install -y xxhash liblz4 zstd pkgconf
-            sudo -E pkg install -y fusefs-libs || true
-            sudo -E pkg install -y fusefs-libs3 || true
-            sudo -E pkg install -y rust
-            sudo -E pkg install -y git  # fakeroot causes lots of troubles on freebsd
-            sudo -E pkg install -y python310 py310-sqlite3
-            sudo -E pkg install -y python311 py311-sqlite3 py311-pip py311-virtualenv
-
-            # Ensure python3 and pip point to the chosen version
-            sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python3 || true
-            sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python || true
-            sudo ln -sf /usr/local/bin/pip3.11 /usr/local/bin/pip3 || true
-            sudo ln -sf /usr/local/bin/pip3.11 /usr/local/bin/pip || true
-
-            # Create and activate venv
-            python -m venv .venv
-            . .venv/bin/activate
-
-            # Build prerequisites
-            python -V
-            pip -V
-            python -m pip install --upgrade pip wheel
-            # Install dev requirements and project in editable mode
-            pip install -r requirements.d/development.txt
-            pip install -e .
-
-            # Run tests (skip benchmarks which are slow on CI)
-            pytest -v -n auto --benchmark-skip
-
-            # Only build PyInstaller binaries for tags
-            if [[ "${GITHUB_REF:-}" == refs/tags/* ]]; then
-              python -m pip install 'pyinstaller==6.14.2'
-              mkdir -p dist/binary
-              pyinstaller --clean --distpath=dist/binary scripts/borg.exe.spec
-              # Smoke-test the built binaries and package dir-mode as tarball
-              pushd dist/binary
-              echo "single-file binary"
-              chmod +x borg.exe
-              ./borg.exe -V
-              echo "single-directory binary"
-              chmod +x borg-dir/borg.exe
-              ./borg-dir/borg.exe -V
-              tar czf borg.tgz borg-dir
-              popd
-              # Prepare artifacts
-              mkdir -p artifacts
-              if [ -f dist/binary/borg.exe ]; then
-                cp -v dist/binary/borg.exe artifacts/borg-freebsd-14-x86_64-gh
-              fi
-              if [ -f dist/binary/borg.tgz ]; then
-                cp -v dist/binary/borg.tgz artifacts/borg-freebsd-14-x86_64-gh.tgz
-              fi
-            fi
-
-      - name: Upload FreeBSD artifacts
-        if: startsWith(github.ref, 'refs/tags/')
-        uses: actions/upload-artifact@v4
-        with:
-          name: freebsd14-dist
-          path: artifacts/*
-          if-no-files-found: ignore
-
-      - name: Attest provenance for FreeBSD artifacts
-        if: startsWith(github.ref, 'refs/tags/')
-        uses: actions/attest-build-provenance@v3
-        with:
-          subject-path: 'artifacts/*'

.github/workflows/haiku.yml

@@ -1,86 +0,0 @@
-name: CI Haiku
-
-on:
-  push:
-    branches: [ master ]
-    tags:
-    - '2.*'
-  pull_request:
-    branches: [ master ]
-    paths:
-    - '**.py'
-    - '**.pyx'
-    - '**.c'
-    - '**.h'
-    - '**.yml'
-    - '**.toml'
-    - '**.cfg'
-    - '**.ini'
-    - 'requirements.d/*'
-    - '!docs/**'
-
-jobs:
-  haiku_tests:
-    name: Haiku tests
-    runs-on: ubuntu-24.04
-    timeout-minutes: 90
-    continue-on-error: true
-
-    steps:
-    - name: Check out repository
-      uses: actions/checkout@v4
-      with:
-        # setuptools-scm needs full history for versioning
-        fetch-depth: 0
-        fetch-tags: true
-
-    - name: Run on Haiku
-      id: haiku
-      uses: cross-platform-actions/action@v0.29.0
-      with:
-        operating_system: haiku
-        version: 'r1beta5'
-        shell: bash
-        cpu_count: 2
-        memory: 6G
-        run: |
-          set -euxo pipefail
-          # Refresh repositories
-          pkgman refresh || true
-          # pkgman update -y || true
-
-          # Base tools and libraries
-          pkgman install -y git pkgconfig zstd lz4 xxhash || true
-          pkgman install -y openssl3 || true
-          pkgman install -y rust_bin || true
-          pkgman install -y python3.10 || true
-          pkgman install -y cffi || true
-          python3 -V || true
-          # Development headers and pkg-config files needed for building C extensions
-          pkgman install -y lz4_devel zstd_devel xxhash_devel openssl3_devel libffi_devel || true
-
-          # Ensure pkg-config can find .pc files
-          export PKG_CONFIG_PATH="/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
-
-          # Point Borg build to library/include prefixes (avoids pkg-config name mismatches)
-          export BORG_LIBLZ4_PREFIX=/system/develop
-          export BORG_LIBZSTD_PREFIX=/system/develop
-          export BORG_LIBXXHASH_PREFIX=/system/develop
-          export BORG_OPENSSL_PREFIX=/system/develop
-
-          # Ensure pip is available
-          python3 -m ensurepip --upgrade || true
-          python3 -m pip install --upgrade pip wheel
-
-          # Create and activate a virtualenv
-          python3 -m venv .venv
-          . .venv/bin/activate
-
-          python -V
-          # Install development/test dependencies
-          pip install -r requirements.d/development.txt
-          # Build Borg (editable)
-          pip install -e .
-          # Run tests; skip benchmarks
-          # remote archiver tests are broken on Haiku
-          pytest -v --benchmark-skip -k "not remote and not socket"

.github/workflows/netbsd.yml

@@ -1,76 +0,0 @@
-name: CI NetBSD
-
-on:
-  push:
-    branches: [ master ]
-    tags:
-    - '2.*'
-  pull_request:
-    branches: [ master ]
-    paths:
-    - '**.py'
-    - '**.pyx'
-    - '**.c'
-    - '**.h'
-    - '**.yml'
-    - '**.toml'
-    - '**.cfg'
-    - '**.ini'
-    - 'requirements.d/*'
-    - '!docs/**'
-
-jobs:
-  netbsd_tests:
-    name: NetBSD tests
-    runs-on: ubuntu-24.04
-    timeout-minutes: 90
-    continue-on-error: true
-
-    steps:
-    - name: Check out repository
-      uses: actions/checkout@v4
-      with:
-        # setuptools-scm needs full history for versioning
-        fetch-depth: 0
-        fetch-tags: true
-
-    - name: Run on NetBSD
-      id: netbsd
-      uses: cross-platform-actions/action@v0.29.0
-      with:
-        operating_system: netbsd
-        version: '10.1'
-        shell: bash
-        cpu_count: 3
-        memory: 6G
-        run: |
-          set -euxo pipefail
-          # Configure pkgin repository based on architecture.
-          arch="$(uname -m)"
-          sudo -E mkdir -p /usr/pkg/etc/pkgin
-          echo "http://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/${arch}/10.1/All" | sudo tee /usr/pkg/etc/pkgin/repositories.conf > /dev/null
-          sudo -E pkgin update || true
-          sudo -E pkgin -y upgrade || true
-
-          # Base tools and libraries
-          sudo -E pkgin -y install zstd lz4 xxhash git || true
-          sudo -E pkgin -y install rust || true
-          # Work around NetBSD 9.3 bug with .bash_profile in screen sessions
-          # echo "export PROMPT_COMMAND=" >> ~/.bash_profile || true
-          sudo -E pkgin -y install pkg-config || true
-          # Python 3.11 and tools
-          sudo -E pkgin -y install py311-pip py311-virtualenv || true
-          sudo -E ln -sf /usr/pkg/bin/python3.11 /usr/pkg/bin/python3 || true
-          sudo -E ln -sf /usr/pkg/bin/pip3.11 /usr/pkg/bin/pip3 || true
-          sudo -E ln -sf /usr/pkg/bin/virtualenv-3.11 /usr/pkg/bin/virtualenv3 || true
-
-          # Create and activate a virtualenv
-          python3 -m virtualenv .venv
-          . .venv/bin/activate
-          python -m pip install --upgrade pip
-          # Install development/test dependencies
-          pip install -r requirements.d/development.txt
-          # Build Borg (editable)
-          pip install -e .
-          # Run tests; skip benchmarks
-          pytest -v -n auto --benchmark-skip

.github/workflows/openbsd.yml

@@ -1,73 +0,0 @@
-name: CI OpenBSD
-
-on:
-  push:
-    branches: [ master ]
-    tags:
-    - '2.*'
-  pull_request:
-    branches: [ master ]
-    paths:
-    - '**.py'
-    - '**.pyx'
-    - '**.c'
-    - '**.h'
-    - '**.yml'
-    - '**.toml'
-    - '**.cfg'
-    - '**.ini'
-    - 'requirements.d/*'
-    - '!docs/**'
-
-jobs:
-  openbsd_tests:
-    name: OpenBSD tests
-    runs-on: ubuntu-24.04
-    timeout-minutes: 90
-    continue-on-error: true
-
-    steps:
-    - name: Check out repository
-      uses: actions/checkout@v4
-      with:
-        # setuptools-scm needs full history for versioning
-        fetch-depth: 0
-        fetch-tags: true
-
-    - name: Run on OpenBSD
-      id: openbsd
-      uses: cross-platform-actions/action@v0.29.0
-      with:
-        operating_system: openbsd
-        version: '7.7'
-        shell: bash
-        cpu_count: 3
-        memory: 6G
-        run: |
-          set -euxo pipefail
-          # On OpenBSD, use pkg_add and set installurl. Use sudo for privileged operations.
-          # echo "https://ftp.eu.openbsd.org/pub/OpenBSD" | sudo tee /etc/installurl > /dev/null
-          # crypto / compression libs
-          sudo -E pkg_add xxhash || true
-          sudo -E pkg_add lz4 || true
-          sudo -E pkg_add zstd || true
-          # toolchain / VCS
-          sudo -E pkg_add git || true
-          sudo -E pkg_add rust || true
-          # OpenSSL 3.4
-          sudo -E pkg_add openssl%3.4 || true
-          # Python tooling
-          sudo -E pkg_add py3-pip || true
-          sudo -E pkg_add py3-virtualenv || true
-
-          export BORG_OPENSSL_NAME=eopenssl34
-          # Create and activate a virtualenv
-          python3 -m virtualenv .venv
-          . .venv/bin/activate
-          python -m pip install --upgrade pip
-          # Install development/test dependencies
-          pip install -r requirements.d/development.txt
-          # Build Borg (editable)
-          pip install -e .
-          # Run tests; skip benchmarks
-          pytest -v -n auto --benchmark-skip