ci(linux): split to compile, build and reh jobs [skip ci]

.github/workflows/insider-linux.yml

@@ -24,83 +24,127 @@ env:
   APP_NAME: VSCodium
   ASSETS_REPOSITORY: ${{ github.repository }}-insiders
   BINARY_NAME: codium-insiders
+  DISABLE_UPDATE: 'yes'
   GITHUB_BRANCH: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || 'insider' }}
   OS_NAME: linux
   VERSIONS_REPOSITORY: ${{ github.repository_owner }}/versions
   VSCODE_QUALITY: insider
 
 jobs:
-  build:
+  check:
+    runs-on: ubuntu-latest
+    container:
+      image: vscodium/vscodium-linux-build-agent:bionic-x64
+    outputs:
+      MS_COMMIT: ${{ env.MS_COMMIT }}
+      MS_TAG: ${{ env.MS_TAG }}
+      RELEASE_VERSION: ${{ env.RELEASE_VERSION }}
+      SHOULD_BUILD: ${{ env.SHOULD_BUILD }}
+      SHOULD_DEPLOY: ${{ env.SHOULD_DEPLOY }}
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.GITHUB_BRANCH }}
+
+      - name: Clone VSCode repo
+        env:
+          PULL_REQUEST_ID: ${{ github.event.inputs.checkout_pr }}
+        run: ./get_repo.sh
+
+      - name: Check PR or cron
+        env:
+          GENERATE_ASSETS: ${{ github.event.inputs.generate_assets }}
+        run: ./check_cron_or_pr.sh
+
+  compile:
+    needs:
+      - check
     runs-on: ubuntu-20.04
-    # strategy:
-    #   fail-fast: false
-    #   matrix:
-    #     include:
-    #     - vscode_arch: x64
-    #       npm_arch: x64
-    #     - vscode_arch: arm64
-    #       npm_arch: arm64
-    #     - vscode_arch: armhf
-    #       npm_arch: arm
-    #     # - vscode_arch: ppc64le
-    #     #   npm_arch: ppc64
+    env:
+      MS_COMMIT: ${{ needs.check.outputs.MS_COMMIT }}
+      MS_TAG: ${{ needs.check.outputs.MS_TAG }}
+      RELEASE_VERSION: ${{ needs.check.outputs.RELEASE_VERSION }}
+    if: needs.check.outputs.SHOULD_BUILD == 'yes' || needs.check.outputs.SHOULD_DEPLOY == 'yes' || github.event.inputs.generate_assets == 'true'
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.GITHUB_BRANCH }}
+
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v4
+        with:
+          node-version: '18.17'
+
+      - name: Install Yarn
+        run: npm install -g yarn
+
+      - name: Setup Python 3
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install libkrb5-dev
+        run: sudo apt-get install -y libkrb5-dev
+
+      - name: Clone VSCode repo
+        run: ./get_repo.sh
+
+      - name: Build
+        run: ./build.sh
+
+      - name: Compress vscode artifact
+        run: |
+          tar -cz --exclude='.build/node' --exclude='.git' --exclude='**/node_modules' -f vscode.tar.gz vscode
+
+      - name: Upload vscode artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: vscode.tar.gz
+          path: ./vscode.tar.gz
+          retention-days: ${{ needs.check.outputs.SHOULD_DEPLOY == 'yes' && 30 || 1 }}
+
+  build:
+     needs:
+      - compile
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         include:
         - vscode_arch: x64
           npm_arch: x64
-          image: vscodium/vscodium-linux-build-agent:centos7-devtoolset8-x64
+          image: vscodium/vscodium-linux-build-agent:bionic-x64
         - vscode_arch: arm64
           npm_arch: arm64
-          image: vscodium/vscodium-linux-build-agent:centos7-devtoolset8-arm64
+          image: vscodium/vscodium-linux-build-agent:bionic-arm64
         - vscode_arch: armhf
           npm_arch: arm
-          image: vscodium/vscodium-linux-build-agent:bionic-devtoolset-arm32v7
-        # - vscode_arch: ppc64le
-        #   npm_arch: ppc64
-        #   image: vscodium/vscodium-linux-build-agent:bionic-ppc64le
+          image: vscodium/vscodium-linux-build-agent:bionic-armhf
+        - vscode_arch: ppc64le
+          npm_arch: ppc64
+          image: vscodium/vscodium-linux-build-agent:bionic-ppc64le
     container:
       image: ${{ matrix.image }}
     env:
-      DISABLE_UPDATE: 'yes'
+      MS_COMMIT: ${{ needs.check.outputs.MS_COMMIT }}
+      MS_TAG: ${{ needs.check.outputs.MS_TAG }}
+      RELEASE_VERSION: ${{ needs.check.outputs.RELEASE_VERSION }}
+      SHOULD_BUILD: ${{ needs.check.outputs.SHOULD_BUILD }}
+      SHOULD_DEPLOY: ${{ needs.check.outputs.SHOULD_DEPLOY }}
       VSCODE_ARCH: ${{ matrix.vscode_arch }}
     outputs:
-      MS_COMMIT: ${{ env.MS_COMMIT }}
-      MS_TAG: ${{ env.MS_TAG }}
       RELEASE_VERSION: ${{ env.RELEASE_VERSION }}
       SHOULD_BUILD: ${{ env.SHOULD_BUILD }}
       SHOULD_DEPLOY: ${{ env.SHOULD_DEPLOY }}
+    if: needs.check.outputs.SHOULD_BUILD == 'yes' || needs.check.outputs.SHOULD_DEPLOY == 'yes' || github.event.inputs.generate_assets == 'true'
 
     steps:
       - uses: actions/checkout@v3
         with:
           ref: ${{ env.GITHUB_BRANCH }}
 
-      # - name: Setup Node.js environment
-      #   uses: actions/setup-node@v4
-      #   with:
-      #     node-version: '18.17'
-
-      # - name: Install Yarn
-      #   run: npm install -g yarn
-
-      # - name: Setup Python 3
-      #   uses: actions/setup-python@v5
-      #   with:
-      #     python-version: '3.11'
-
-      # - name: Install libkrb5-dev
-      #   run: sudo apt-get install -y libkrb5-dev
-
-      - name: Clone VSCode repo
-        run: ./get_repo.sh
-
-      - name: Check PR or cron
-        env:
-          GENERATE_ASSETS: ${{ github.event.inputs.generate_assets }}
-        run: ./check_cron_or_pr.sh
-
       - name: Install GH
         run: ./install_gh.sh
         if: env.SHOULD_DEPLOY == 'yes'
@@ -111,11 +155,17 @@ jobs:
         run: ./check_tags.sh
         if: env.SHOULD_DEPLOY == 'yes'
 
+      - name: Download vscode artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: vscode.tar.gz
+        if: env.SHOULD_BUILD == 'yes'
+
       - name: Build
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           npm_config_arch: ${{ matrix.npm_arch }}
-        run: ./build.sh
+        run: ./package_linux_bin.sh
         if: env.SHOULD_BUILD == 'yes'
 
       - name: Prepare assets
@@ -139,7 +189,75 @@ jobs:
       - name: Upload assets
         uses: actions/upload-artifact@v3
         with:
-          name: ${{ matrix.vscode_arch }}
+          name: bin-${{ matrix.vscode_arch }}
+          path: assets/
+          retention-days: 3
+        if: env.SHOULD_BUILD == 'yes' && env.SHOULD_DEPLOY == 'no' && github.event.inputs.generate_assets == 'true'
+
+  reh:
+    needs:
+      - compile
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - vscode_arch: x64
+          npm_arch: x64
+        - vscode_arch: arm64
+          npm_arch: arm64
+        - vscode_arch: armhf
+          npm_arch: arm
+        - vscode_arch: ppc64le
+          npm_arch: ppc64
+    env:
+      MS_COMMIT: ${{ needs.check.outputs.MS_COMMIT }}
+      MS_TAG: ${{ needs.check.outputs.MS_TAG }}
+      RELEASE_VERSION: ${{ needs.check.outputs.RELEASE_VERSION }}
+      SHOULD_BUILD: ${{ needs.check.outputs.SHOULD_BUILD }}
+      SHOULD_DEPLOY: ${{ needs.check.outputs.SHOULD_DEPLOY }}
+      VSCODE_ARCH: ${{ matrix.vscode_arch }}
+    if: needs.check.outputs.SHOULD_DEPLOY == 'yes' || github.event.inputs.generate_assets == 'true'
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.GITHUB_BRANCH }}
+
+      - name: Install GH
+        run: ./install_gh.sh
+        if: env.SHOULD_DEPLOY == 'yes'
+
+      - name: Check existing VSCodium tags/releases
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: ./check_tags.sh
+        if: env.SHOULD_DEPLOY == 'yes'
+
+      - name: Download compilation artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: compilation.tar.gz
+        if: env.SHOULD_BUILD_REH != 'no' || github.event.inputs.generate_assets == 'true'
+
+      - name: Build
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          npm_config_arch: ${{ matrix.npm_arch }}
+        run: ./package_linux_reh.sh
+        if: env.SHOULD_BUILD_REH != 'no' || github.event.inputs.generate_assets == 'true'
+
+      - name: Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.STRONGER_GITHUB_TOKEN }}
+          GITHUB_USERNAME: ${{ github.repository_owner }}
+        run: ./release.sh
+        if: env.SHOULD_BUILD == 'yes' && env.SHOULD_DEPLOY == 'yes'
+
+      - name: Upload assets
+        uses: actions/upload-artifact@v3
+        with:
+          name: reh-${{ matrix.vscode_arch }}
           path: assets/
           retention-days: 3
         if: env.SHOULD_BUILD == 'yes' && env.SHOULD_DEPLOY == 'no' && github.event.inputs.generate_assets == 'true'

.nvmrc.linux

@@ -1 +0,0 @@
-16.20

build.sh

@@ -3,10 +3,6 @@
 
 set -ex
 
-if [[ -f  "./remote-dependencies.tar" ]]; then
-  tar -xf ./remote-dependencies.tar ./vscode/remote/node_modules
-fi
-
 . version.sh
 
 if [[ "${SHOULD_BUILD}" == "yes" ]]; then
@@ -40,12 +36,15 @@ if [[ "${SHOULD_BUILD}" == "yes" ]]; then
     fi
 
     VSCODE_PLATFORM="win32"
-  elif [[ "${VSCODE_ARCH}" == "ppc64le" ]]; then # linux-ppc64le
-    VSCODE_PLATFORM="linux"
   else # linux
-    yarn gulp "vscode-linux-${VSCODE_ARCH}-min-ci"
+    # in CI, packaging will be done by a different job
+    if [[ "${CI_BUILD}" != "no" ]]; then
+      yarn gulp "vscode-linux-${VSCODE_ARCH}-min-ci"
 
-    find "../VSCode-linux-${VSCODE_ARCH}" -print0 | xargs -0 touch -c
+      find "../VSCode-linux-${VSCODE_ARCH}" -print0 | xargs -0 touch -c
+
+      SHOULD_BUILD_REH="no"
+    fi
 
     VSCODE_PLATFORM="linux"
   fi

install_remote_dependencies.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-cd vscode || { echo "'vscode' dir not found"; exit 1; }
-
-yarn --cwd remote --frozen-lockfile --check-files
-
-cd ..
-
-tar -cf remote-dependencies.tar ./vscode/remote/node_modules

package_linux_bin.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+
+set -ex
+
+if [[ "${CI_BUILD}" == "no" ]]; then
+  exit 1
+fi
+
+if [[ -f  "./vscode.tar.gz" ]]; then
+  tar -xfz ./vscode.tar.gz .
+fi
+
+cd vscode || { echo "'vscode' dir not found"; exit 1; }
+
+export VSCODE_SYSROOT_PREFIX='-glibc-2.17'
+
+for i in {1..5}; do # try 5 times
+  yarn --cwd build --frozen-lockfile --check-files && break
+  if [[ $i == 3 ]]; then
+    echo "Yarn failed too many times" >&2
+    exit 1
+  fi
+  echo "Yarn failed $i, trying again..."
+done
+
+./build/azure-pipelines/linux/install.sh
+
+EXPECTED_GLIBC_VERSION="2.17" EXPECTED_GLIBCXX_VERSION="3.4.22" ./build/azure-pipelines/linux/verify-glibc-requirements.sh
+
+node build/azure-pipelines/distro/mixin-npm
+
+yarn gulp "vscode-linux-${VSCODE_ARCH}-min-ci"
+
+find "../VSCode-linux-${VSCODE_ARCH}" -print0 | xargs -0 touch -c
+
+cd ..

package_linux_reh.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+
+set -ex
+
+if [[ "${CI_BUILD}" == "no" ]]; then
+  exit 1
+fi
+
+if [[ -f  "./vscode.tar.gz" ]]; then
+  tar -xfz ./vscode.tar.gz .
+fi
+
+cd vscode || { echo "'vscode' dir not found"; exit 1; }
+
+export VSCODE_SYSROOT_PREFIX='-glibc-2.17'
+
+VSCODE_HOST_MOUNT="$( pwd )"
+
+export VSCODE_HOST_MOUNT
+
+if [[ "${VSCODE_ARCH}" == "x64" || "${VSCODE_ARCH}" == "arm64" ]]; then
+  VSCODE_REMOTE_DEPENDENCIES_CONTAINER_NAME="vscodium/vscodium-linux-build-agent:centos7-devtoolset8-${VSCODE_ARCH}"
+elif [[ "${VSCODE_ARCH}" == "armhf" ]]; then
+  VSCODE_REMOTE_DEPENDENCIES_CONTAINER_NAME="vscodium/vscodium-linux-build-agent:bionic-devtoolset-arm32v7"
+elif [[ "${VSCODE_ARCH}" == "ppc64le" ]]; then
+  VSCODE_REMOTE_DEPENDENCIES_CONTAINER_NAME="vscodium/vscodium-linux-build-agent:bionic-devtoolset-ppc64le"
+fi
+
+export VSCODE_REMOTE_DEPENDENCIES_CONTAINER_NAME
+
+for i in {1..5}; do # try 5 times
+  yarn --cwd build --frozen-lockfile --check-files && break
+  if [[ $i == 3 ]]; then
+    echo "Yarn failed too many times" >&2
+    exit 1
+  fi
+  echo "Yarn failed $i, trying again..."
+done
+
+./build/azure-pipelines/linux/install.sh
+
+EXPECTED_GLIBC_VERSION="2.17" EXPECTED_GLIBCXX_VERSION="3.4.22" ./build/azure-pipelines/linux/verify-glibc-requirements.sh
+
+node build/azure-pipelines/distro/mixin-npm
+
+export VSCODE_NODE_GLIBC='-glibc-2.17'
+
+yarn gulp minify-vscode-reh
+yarn gulp "vscode-reh-${VSCODE_PLATFORM}-${VSCODE_ARCH}-min-ci"
+
+cd ..
+
+echo "Building and moving REH"
+cd "vscode-reh-${VSCODE_PLATFORM}-${VSCODE_ARCH}"
+tar czf "../assets/${APP_NAME_LC}-reh-${VSCODE_PLATFORM}-${VSCODE_ARCH}-${RELEASE_VERSION}.tar.gz" .
+cd ..

prepare_vscode.sh

@@ -86,12 +86,6 @@ export PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
 if [[ "${OS_NAME}" == "linux" ]]; then
   export VSCODE_SKIP_NODE_VERSION_CHECK=1
 
-  if [[ -f "../.nvmrc.linux" ]]; then
-    mv ../.nvmrc.linux ../.nvmrc
-  fi
-fi
-
-if [[ "${OS_NAME}" == "linux" ]]; then
    if [[ "${npm_config_arch}" == "arm" ]]; then
     export npm_config_arm_version=7
   fi