Merge pull request #9149 from ThomasWaldmann/ci-bsd-master

CI: add *BSD and Haiku OS

.github/workflows/freebsd.yml

@@ -0,0 +1,119 @@
+name: CI FreeBSD
+
+on:
+  push:
+    branches: [ master ]
+    tags:
+    - '2.*'
+  pull_request:
+    branches: [ master ]
+    paths:
+    - '**.py'
+    - '**.pyx'
+    - '**.c'
+    - '**.h'
+    - '**.yml'
+    - '**.toml'
+    - '**.cfg'
+    - '**.ini'
+    - 'requirements.d/*'
+    - '!docs/**'
+
+jobs:
+  freebsd_tests:
+    name: FreeBSD tests and build
+    permissions:
+      contents: read
+      id-token: write
+      attestations: write
+    runs-on: ubuntu-24.04
+    continue-on-error: true
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Run on FreeBSD
+        id: freebsd
+        uses: cross-platform-actions/action@v0.29.0
+        with:
+          operating_system: freebsd
+          version: '14.3'
+          shell: bash
+          # Increase resources a bit for building Cython extensions
+          cpu_count: 3
+          memory: 8G
+          run: |
+            set -euxo pipefail
+            # Update package catalog and install dependencies
+            export IGNORE_OSVERSION=yes
+            sudo -E pkg update -f
+            sudo -E pkg install -y xxhash liblz4 zstd pkgconf
+            sudo -E pkg install -y fusefs-libs || true
+            sudo -E pkg install -y fusefs-libs3 || true
+            sudo -E pkg install -y rust
+            sudo -E pkg install -y git  # fakeroot causes lots of troubles on freebsd
+            sudo -E pkg install -y python310 py310-sqlite3
+            sudo -E pkg install -y python311 py311-sqlite3 py311-pip py311-virtualenv
+
+            # Ensure python3 and pip point to the chosen version
+            sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python3 || true
+            sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python || true
+            sudo ln -sf /usr/local/bin/pip3.11 /usr/local/bin/pip3 || true
+            sudo ln -sf /usr/local/bin/pip3.11 /usr/local/bin/pip || true
+
+            # Create and activate venv
+            python -m venv .venv
+            . .venv/bin/activate
+
+            # Build prerequisites
+            python -V
+            pip -V
+            python -m pip install --upgrade pip wheel
+            # Install dev requirements and project in editable mode
+            pip install -r requirements.d/development.txt
+            pip install -e .
+
+            # Run tests (skip benchmarks which are slow on CI)
+            pytest -v -n auto --benchmark-skip
+
+            # Only build PyInstaller binaries for tags
+            if [[ "${GITHUB_REF:-}" == refs/tags/* ]]; then
+              python -m pip install 'pyinstaller==6.14.2'
+              mkdir -p dist/binary
+              pyinstaller --clean --distpath=dist/binary scripts/borg.exe.spec
+              # Smoke-test the built binaries and package dir-mode as tarball
+              pushd dist/binary
+              echo "single-file binary"
+              chmod +x borg.exe
+              ./borg.exe -V
+              echo "single-directory binary"
+              chmod +x borg-dir/borg.exe
+              ./borg-dir/borg.exe -V
+              tar czf borg.tgz borg-dir
+              popd
+              # Prepare artifacts
+              mkdir -p artifacts
+              if [ -f dist/binary/borg.exe ]; then
+                cp -v dist/binary/borg.exe artifacts/borg-freebsd-14-x86_64-gh
+              fi
+              if [ -f dist/binary/borg.tgz ]; then
+                cp -v dist/binary/borg.tgz artifacts/borg-freebsd-14-x86_64-gh.tgz
+              fi
+            fi
+
+      - name: Upload FreeBSD artifacts
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: actions/upload-artifact@v4
+        with:
+          name: freebsd14-dist
+          path: artifacts/*
+          if-no-files-found: ignore
+
+      - name: Attest provenance for FreeBSD artifacts
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: actions/attest-build-provenance@v3
+        with:
+          subject-path: 'artifacts/*'

.github/workflows/haiku.yml

@@ -0,0 +1,86 @@
+name: CI Haiku
+
+on:
+  push:
+    branches: [ master ]
+    tags:
+    - '2.*'
+  pull_request:
+    branches: [ master ]
+    paths:
+    - '**.py'
+    - '**.pyx'
+    - '**.c'
+    - '**.h'
+    - '**.yml'
+    - '**.toml'
+    - '**.cfg'
+    - '**.ini'
+    - 'requirements.d/*'
+    - '!docs/**'
+
+jobs:
+  haiku_tests:
+    name: Haiku tests
+    runs-on: ubuntu-24.04
+    timeout-minutes: 90
+    continue-on-error: true
+
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v4
+      with:
+        # setuptools-scm needs full history for versioning
+        fetch-depth: 0
+        fetch-tags: true
+
+    - name: Run on Haiku
+      id: haiku
+      uses: cross-platform-actions/action@v0.29.0
+      with:
+        operating_system: haiku
+        version: 'r1beta5'
+        shell: bash
+        cpu_count: 2
+        memory: 6G
+        run: |
+          set -euxo pipefail
+          # Refresh repositories
+          pkgman refresh || true
+          # pkgman update -y || true
+
+          # Base tools and libraries
+          pkgman install -y git pkgconfig zstd lz4 xxhash || true
+          pkgman install -y openssl3 || true
+          pkgman install -y rust_bin || true
+          pkgman install -y python3.10 || true
+          pkgman install -y cffi || true
+          python3 -V || true
+          # Development headers and pkg-config files needed for building C extensions
+          pkgman install -y lz4_devel zstd_devel xxhash_devel openssl3_devel libffi_devel || true
+
+          # Ensure pkg-config can find .pc files
+          export PKG_CONFIG_PATH="/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
+
+          # Point Borg build to library/include prefixes (avoids pkg-config name mismatches)
+          export BORG_LIBLZ4_PREFIX=/system/develop
+          export BORG_LIBZSTD_PREFIX=/system/develop
+          export BORG_LIBXXHASH_PREFIX=/system/develop
+          export BORG_OPENSSL_PREFIX=/system/develop
+
+          # Ensure pip is available
+          python3 -m ensurepip --upgrade || true
+          python3 -m pip install --upgrade pip wheel
+
+          # Create and activate a virtualenv
+          python3 -m venv .venv
+          . .venv/bin/activate
+
+          python -V
+          # Install development/test dependencies
+          pip install -r requirements.d/development.txt
+          # Build Borg (editable)
+          pip install -e .
+          # Run tests; skip benchmarks
+          # remote archiver tests are broken on Haiku
+          pytest -v --benchmark-skip -k "not remote and not socket"

.github/workflows/netbsd.yml

@@ -0,0 +1,76 @@
+name: CI NetBSD
+
+on:
+  push:
+    branches: [ master ]
+    tags:
+    - '2.*'
+  pull_request:
+    branches: [ master ]
+    paths:
+    - '**.py'
+    - '**.pyx'
+    - '**.c'
+    - '**.h'
+    - '**.yml'
+    - '**.toml'
+    - '**.cfg'
+    - '**.ini'
+    - 'requirements.d/*'
+    - '!docs/**'
+
+jobs:
+  netbsd_tests:
+    name: NetBSD tests
+    runs-on: ubuntu-24.04
+    timeout-minutes: 90
+    continue-on-error: true
+
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v4
+      with:
+        # setuptools-scm needs full history for versioning
+        fetch-depth: 0
+        fetch-tags: true
+
+    - name: Run on NetBSD
+      id: netbsd
+      uses: cross-platform-actions/action@v0.29.0
+      with:
+        operating_system: netbsd
+        version: '10.1'
+        shell: bash
+        cpu_count: 3
+        memory: 6G
+        run: |
+          set -euxo pipefail
+          # Configure pkgin repository based on architecture.
+          arch="$(uname -m)"
+          sudo -E mkdir -p /usr/pkg/etc/pkgin
+          echo "http://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/${arch}/10.1/All" | sudo tee /usr/pkg/etc/pkgin/repositories.conf > /dev/null
+          sudo -E pkgin update || true
+          sudo -E pkgin -y upgrade || true
+
+          # Base tools and libraries
+          sudo -E pkgin -y install zstd lz4 xxhash git || true
+          sudo -E pkgin -y install rust || true
+          # Work around NetBSD 9.3 bug with .bash_profile in screen sessions
+          # echo "export PROMPT_COMMAND=" >> ~/.bash_profile || true
+          sudo -E pkgin -y install pkg-config || true
+          # Python 3.11 and tools
+          sudo -E pkgin -y install py311-pip py311-virtualenv || true
+          sudo -E ln -sf /usr/pkg/bin/python3.11 /usr/pkg/bin/python3 || true
+          sudo -E ln -sf /usr/pkg/bin/pip3.11 /usr/pkg/bin/pip3 || true
+          sudo -E ln -sf /usr/pkg/bin/virtualenv-3.11 /usr/pkg/bin/virtualenv3 || true
+
+          # Create and activate a virtualenv
+          python3 -m virtualenv .venv
+          . .venv/bin/activate
+          python -m pip install --upgrade pip
+          # Install development/test dependencies
+          pip install -r requirements.d/development.txt
+          # Build Borg (editable)
+          pip install -e .
+          # Run tests; skip benchmarks
+          pytest -v -n auto --benchmark-skip

.github/workflows/openbsd.yml

@@ -0,0 +1,73 @@
+name: CI OpenBSD
+
+on:
+  push:
+    branches: [ master ]
+    tags:
+    - '2.*'
+  pull_request:
+    branches: [ master ]
+    paths:
+    - '**.py'
+    - '**.pyx'
+    - '**.c'
+    - '**.h'
+    - '**.yml'
+    - '**.toml'
+    - '**.cfg'
+    - '**.ini'
+    - 'requirements.d/*'
+    - '!docs/**'
+
+jobs:
+  openbsd_tests:
+    name: OpenBSD tests
+    runs-on: ubuntu-24.04
+    timeout-minutes: 90
+    continue-on-error: true
+
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v4
+      with:
+        # setuptools-scm needs full history for versioning
+        fetch-depth: 0
+        fetch-tags: true
+
+    - name: Run on OpenBSD
+      id: openbsd
+      uses: cross-platform-actions/action@v0.29.0
+      with:
+        operating_system: openbsd
+        version: '7.7'
+        shell: bash
+        cpu_count: 3
+        memory: 6G
+        run: |
+          set -euxo pipefail
+          # On OpenBSD, use pkg_add and set installurl. Use sudo for privileged operations.
+          # echo "https://ftp.eu.openbsd.org/pub/OpenBSD" | sudo tee /etc/installurl > /dev/null
+          # crypto / compression libs
+          sudo -E pkg_add xxhash || true
+          sudo -E pkg_add lz4 || true
+          sudo -E pkg_add zstd || true
+          # toolchain / VCS
+          sudo -E pkg_add git || true
+          sudo -E pkg_add rust || true
+          # OpenSSL 3.4
+          sudo -E pkg_add openssl%3.4 || true
+          # Python tooling
+          sudo -E pkg_add py3-pip || true
+          sudo -E pkg_add py3-virtualenv || true
+
+          export BORG_OPENSSL_NAME=eopenssl34
+          # Create and activate a virtualenv
+          python3 -m virtualenv .venv
+          . .venv/bin/activate
+          python -m pip install --upgrade pip
+          # Install development/test dependencies
+          pip install -r requirements.d/development.txt
+          # Build Borg (editable)
+          pip install -e .
+          # Run tests; skip benchmarks
+          pytest -v -n auto --benchmark-skip

src/borg/testsuite/__init__.py

@@ -26,6 +26,8 @@ from ..platformflags import is_win32
 # Does this version of llfuse support ns precision?
 have_fuse_mtime_ns = hasattr(llfuse.EntryAttributes, "st_mtime_ns") if llfuse else False
 
+has_mknod = hasattr(os, "mknod")
+
 has_lchflags = hasattr(os, "lchflags") or sys.platform.startswith("linux")
 try:
     with tempfile.NamedTemporaryFile() as file:

src/borg/testsuite/archiver/__init__.py

@@ -26,7 +26,7 @@ from ...manifest import Manifest
 from ...platform import get_flags
 from ...remote import RemoteRepository
 from ...repository import Repository
-from .. import has_lchflags, is_utime_fully_supported, have_fuse_mtime_ns, st_mtime_ns_round, no_selinux
+from .. import has_lchflags, has_mknod, is_utime_fully_supported, have_fuse_mtime_ns, st_mtime_ns_round, no_selinux
 from .. import changedir
 from .. import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported
 from ..platform.platform_test import is_win32
@@ -232,10 +232,11 @@ def create_test_files(input_path, create_hardlinks=True):
         have_root = False
     else:
         try:
-            # Block device
-            os.mknod("input/bdev", 0o600 | stat.S_IFBLK, os.makedev(10, 20))
-            # Char device
-            os.mknod("input/cdev", 0o600 | stat.S_IFCHR, os.makedev(30, 40))
+            if has_mknod:
+                # Block device
+                os.mknod("input/bdev", 0o600 | stat.S_IFBLK, os.makedev(10, 20))
+                # Char device
+                os.mknod("input/cdev", 0o600 | stat.S_IFCHR, os.makedev(30, 40))
             # File owner
             os.chown("input/file1", 100, 200)  # raises OSError invalid argument on cygwin
             # File mode

src/borg/testsuite/archiver/create_cmd_test.py

@@ -17,7 +17,7 @@ from ...manifest import Manifest
 from ...platform import is_win32, is_darwin
 from ...repository import Repository
 from ...helpers import CommandError, BackupPermissionError
-from .. import has_lchflags
+from .. import has_lchflags, has_mknod
 from .. import changedir
 from .. import (
     are_symlinks_supported,
@@ -84,7 +84,7 @@ def test_basic_functionality(archivers, request):
         expected.append("input/link1")
     if are_hardlinks_supported():
         expected.append("input/hardlink")
-    if not have_root:
+    if not have_root or not has_mknod:
         # We could not create these device files without (fake)root.
         expected.remove("input/bdev")
         expected.remove("input/cdev")

src/borg/testsuite/archiver/extract_cmd_test.py

@@ -703,6 +703,7 @@ def test_do_not_fail_when_percent_is_in_file_name(archivers, request):
             cmd(archiver, "extract", "test", exit_code=EXIT_WARNING)
 
 
+@pytest.mark.skipif(not are_hardlinks_supported(), reason="hardlinks not supported")
 def test_extract_continue(archivers, request):
     archiver = request.getfixturevalue(archivers)
     CONTENTS1, CONTENTS2, CONTENTS3 = b"contents1" * 100, b"contents2" * 200, b"contents3" * 300