parallel CI package builds

.gitea/workflows/build.yml

@@ -9,41 +9,35 @@ on:
   pull_request:
   workflow_dispatch:
 
+env:
+  ALPINE_VERSION: "3.24"
+  ALPINE_ARCHES: "x86_64 aarch64"
+  ALPINE_REGISTRY_BRANCH: "v3.24"
+  ALPINE_REGISTRY_REPOSITORY: "${{ vars.PACKAGE_NAME }}"
+  INSTANCE_URL: "${{ vars.INSTANCE_URL }}"
+  PACKAGE_OWNER: "${{ vars.PACKAGE_OWNER }}"
+  PACKAGE_USER: "${{ secrets.PACKAGE_USER }}"
+  PACKAGE_TOKEN: "${{ secrets.PACKAGE_TOKEN }}"
+  PACKAGER: "${{ gitea.repository_owner }}"
+  ALPINE_RUST_CACHE_ROOT: "/cache/public-alpine-packages/v3.24/rust"
+  TRIVY_VERSION: "0.74.0"
+  TRIVY_SHA256: "2ae6fe3ee734b7fdf11335663e18c75ea12dccc76062f09f164a3b0f8be4371a"
+
 jobs:
-  build-and-publish:
+  preflight:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        include:
-          - alpine_version: "3.24"
-            alpine_registry_branch: "v3.24"
+    timeout-minutes: 180
     container:
-      image: "alpine:${{ matrix.alpine_version }}"
-    env:
-      ALPINE_VERSION: "${{ matrix.alpine_version }}"
-      ALPINE_ARCHES: "x86_64 aarch64"
-      ALPINE_REGISTRY_BRANCH: "${{ matrix.alpine_registry_branch }}"
-      ALPINE_REGISTRY_REPOSITORY: "${{ vars.PACKAGE_NAME }}"
-      INSTANCE_URL: "${{ vars.INSTANCE_URL }}"
-      PACKAGE_OWNER: "${{ vars.PACKAGE_OWNER }}"
-      PACKAGE_USER: "${{ secrets.PACKAGE_USER }}"
-      PACKAGE_TOKEN: "${{ secrets.PACKAGE_TOKEN }}"
-      PACKAGER: "${{ gitea.repository_owner }}"
-      ALPINE_APK_REPO_CACHE: "/cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/apk-repo"
-      ALPINE_RUST_CACHE_ROOT: "/cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/rust"
-      TRIVY_VERSION: "0.74.0"
-      TRIVY_SHA256: "2ae6fe3ee734b7fdf11335663e18c75ea12dccc76062f09f164a3b0f8be4371a"
+      image: "alpine:3.24"
+    outputs:
+      packages_selected: "${{ steps.package-selection.outputs.packages_selected }}"
+      selected_packages: "${{ steps.package-selection.outputs.selected_packages }}"
+      greptimedb_selected: "${{ steps.package-selection.outputs.greptimedb_selected }}"
     steps:
       - name: Prepare Environment
         run: |
           printf '%s\n' 'https://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories
-          apk add --no-cache --update abuild-rootbld alpine-sdk atools-apkbuild-lint bash ca-certificates curl doas git jq nodejs rsync shellcheck sudo tar
-          trivy_archive="trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz"
-          curl -fsSL -o "/tmp/${trivy_archive}" "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/${trivy_archive}"
-          echo "${TRIVY_SHA256}  /tmp/${trivy_archive}" | sha256sum -c -
-          tar -xzf "/tmp/${trivy_archive}" -C /usr/local/bin trivy
-          mkdir -p "$ALPINE_APK_REPO_CACHE" "$ALPINE_RUST_CACHE_ROOT"
-          chown -R 1000:1000 /cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}
+          apk add --no-cache --update atools-apkbuild-lint bash ca-certificates curl git jq nodejs shellcheck tar
 
       - name: Checkout
         uses: actions/checkout@v3
@@ -66,41 +60,128 @@ jobs:
 
       - name: Prepare package cache
         id: package-selection
-        run: |
-          scripts/apk/ci-package-cache.sh prepare
+        run: scripts/apk/ci-package-cache.sh prepare
 
-      - name: Build packages
+  build:
+    needs: preflight
+    if: needs.preflight.outputs.packages_selected == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    strategy:
+      fail-fast: false
+      matrix:
+        package: [seaweedfs, greptimedb, lavinmq, gitea]
+    container:
+      image: "alpine:3.24"
+    env:
+      ALPINE_PACKAGE: "${{ matrix.package }}"
+    steps:
+      - name: Prepare Environment
+        run: |
+          printf '%s\n' 'https://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories
+          apk add --no-cache --update abuild-rootbld alpine-sdk bash ca-certificates doas git nodejs sudo
+          mkdir -p "$ALPINE_RUST_CACHE_ROOT"
+          chown -R 1000:1000 /cache/public-alpine-packages/v3.24
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Build package
+        if: contains(needs.preflight.outputs.selected_packages, matrix.package)
         run: |
           scripts/apk/ci-build.sh
-          if [ -d packages/local ]; then
-            scripts/apk/list-packages.sh
-          fi
-
-      - name: Export GreptimeDB CI assets
-        if: steps.package-selection.outputs.greptimedb_selected == 'true'
-        run: scripts/apk/export-ci-assets.sh
+          scripts/apk/list-packages.sh
 
       - name: Check artifact upload support
-        id: artifact-upload
+        if: contains(needs.preflight.outputs.selected_packages, matrix.package)
         run: |
-          if [ -n "${ACTIONS_RUNTIME_TOKEN:-}" ]; then
-            echo "available=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "available=false" >> "$GITHUB_OUTPUT"
-          fi
+          test -n "${ACTIONS_RUNTIME_TOKEN:-}" || {
+            echo "artifact upload support is required for split package builds" >&2
+            exit 1
+          }
 
-      - name: Upload generated CI assets
-        if: steps.package-selection.outputs.greptimedb_selected == 'true' && steps.artifact-upload.outputs.available == 'true'
+      - name: Upload package artifacts
+        if: contains(needs.preflight.outputs.selected_packages, matrix.package)
         uses: actions/upload-artifact@v3
         with:
-          name: greptimedb-musl-binaries-${{ matrix.alpine_registry_branch }}
+          name: alpine-package-${{ matrix.package }}-${{ github.sha }}
+          path: packages/local/
+          retention-days: 1
+
+  verify-and-publish:
+    needs: [preflight, build]
+    if: needs.preflight.outputs.packages_selected == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    container:
+      image: "alpine:3.24"
+    env:
+      ALPINE_PACKAGES: "${{ needs.preflight.outputs.selected_packages }}"
+    steps:
+      - name: Prepare Environment
+        run: |
+          printf '%s\n' 'https://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories
+          apk add --no-cache --update abuild-rootbld alpine-sdk bash ca-certificates curl doas git jq nodejs rsync shellcheck sudo tar
+          trivy_archive="trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz"
+          curl -fsSL -o "/tmp/${trivy_archive}" "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/${trivy_archive}"
+          echo "${TRIVY_SHA256}  /tmp/${trivy_archive}" | sha256sum -c -
+          tar -xzf "/tmp/${trivy_archive}" -C /usr/local/bin trivy
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Prepare registry fallback
+        run: |
+          selected_packages="$ALPINE_PACKAGES"
+          scripts/apk/ci-package-cache.sh prepare
+          printf 'ALPINE_PACKAGES=%s\n' "$selected_packages" >> "$GITHUB_ENV"
+
+      - name: Download SeaweedFS artifacts
+        if: contains(needs.preflight.outputs.selected_packages, 'seaweedfs')
+        uses: actions/download-artifact@v3
+        with:
+          name: alpine-package-seaweedfs-${{ github.sha }}
+          path: packages/worker-artifacts/seaweedfs
+
+      - name: Download GreptimeDB artifacts
+        if: contains(needs.preflight.outputs.selected_packages, 'greptimedb')
+        uses: actions/download-artifact@v3
+        with:
+          name: alpine-package-greptimedb-${{ github.sha }}
+          path: packages/worker-artifacts/greptimedb
+
+      - name: Download LavinMQ artifacts
+        if: contains(needs.preflight.outputs.selected_packages, 'lavinmq')
+        uses: actions/download-artifact@v3
+        with:
+          name: alpine-package-lavinmq-${{ github.sha }}
+          path: packages/worker-artifacts/lavinmq
+
+      - name: Download Gitea artifacts
+        if: contains(needs.preflight.outputs.selected_packages, 'gitea')
+        uses: actions/download-artifact@v3
+        with:
+          name: alpine-package-gitea-${{ github.sha }}
+          path: packages/worker-artifacts/gitea
+
+      - name: Merge package artifacts
+        run: scripts/apk/merge-ci-artifacts.sh
+
+      - name: Export GreptimeDB CI assets
+        if: needs.preflight.outputs.greptimedb_selected == 'true'
+        run: scripts/apk/export-ci-assets.sh
+
+      - name: Upload generated CI assets
+        if: needs.preflight.outputs.greptimedb_selected == 'true'
+        uses: actions/upload-artifact@v3
+        with:
+          name: greptimedb-musl-binaries-v3.24
           path: packages/ci-assets/
 
       - name: Upload local Alpine registry
-        if: steps.package-selection.outputs.packages_selected == 'true' && steps.artifact-upload.outputs.available == 'true'
         uses: actions/upload-artifact@v3
         with:
-          name: alpine-local-registry-${{ matrix.alpine_registry_branch }}-${{ github.sha }}
+          name: alpine-local-registry-v3.24-${{ github.sha }}
           path: packages/local/
           retention-days: 30
 
@@ -108,7 +189,6 @@ jobs:
         run: scripts/apk/ci-smoke.sh
 
       - name: Generate security reports
-        if: env.ALPINE_SKIP_BUILD != '1'
         run: |
           scripts/apk/security-report.sh
           if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
@@ -118,16 +198,14 @@ jobs:
           fi
 
       - name: Upload security reports
-        if: env.ALPINE_SKIP_BUILD != '1' && steps.artifact-upload.outputs.available == 'true'
         uses: actions/upload-artifact@v3
         with:
-          name: apk-security-reports-${{ matrix.alpine_registry_branch }}
+          name: apk-security-reports-v3.24
           path: packages/security-reports/
 
       - name: Runtime test
-        if: env.ALPINE_SKIP_BUILD != '1'
         run: scripts/apk/runtime-test.sh
 
       - name: Publish Alpine packages
-        if: github.ref == 'refs/heads/main' && env.ALPINE_SKIP_BUILD != '1'
+        if: github.ref == 'refs/heads/main'
         run: scripts/apk/publish-gitea.sh

scripts/apk/ci-package-cache.sh

@@ -138,6 +138,7 @@ prepare() {
     } >> "${GITHUB_ENV}"
   fi
   if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
+    printf 'selected_packages=%s\n' "${ALPINE_PACKAGES}" >> "${GITHUB_OUTPUT}"
     if [[ -n "${ALPINE_PACKAGES}" ]]; then
       printf 'packages_selected=true\n' >> "${GITHUB_OUTPUT}"
     else

scripts/apk/merge-ci-artifacts.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+repo_name="${ALPINE_REPO_NAME:-local}"
+arches="${ALPINE_ARCHES:-x86_64 aarch64}"
+artifact_root="${APK_CI_ARTIFACTS_DIR:-${repo_root}/packages/worker-artifacts}"
+package_root="${APK_CI_PACKAGE_ROOT:-${repo_root}/packages/${repo_name}}"
+
+require_command() {
+  command -v "$1" >/dev/null 2>&1 || {
+    printf 'missing required command: %s\n' "$1" >&2
+    exit 2
+  }
+}
+
+copy_key() {
+  local key="$1"
+  local destination="$2"
+  local digest
+
+  digest="$(sha256sum "${key}" | awk '{print $1}')"
+  cp "${key}" "${destination}/${digest}.rsa.pub"
+}
+
+require_command abuild-keygen
+require_command abuild-sign
+require_command apk
+
+if [[ ! -d "${artifact_root}" ]]; then
+  printf 'missing worker artifacts directory: %s\n' "${artifact_root}" >&2
+  exit 1
+fi
+
+rm -rf "${package_root}"
+mkdir -p "${package_root}" "${HOME}/.abuild"
+
+if ! compgen -G "${HOME}/.abuild/*.rsa" > /dev/null; then
+  abuild-keygen -a -n
+fi
+
+shopt -s nullglob
+merge_private_keys=("${HOME}"/.abuild/*.rsa)
+merge_public_keys=("${HOME}"/.abuild/*.rsa.pub)
+shopt -u nullglob
+
+if [[ "${#merge_private_keys[@]}" -ne 1 || "${#merge_public_keys[@]}" -ne 1 ]]; then
+  printf 'expected exactly one merge signing key in %s/.abuild\n' "${HOME}" >&2
+  exit 1
+fi
+
+for arch in ${arches}; do
+  destination="${package_root}/${arch}"
+  mkdir -p "${destination}"
+
+  while IFS= read -r -d '' apk; do
+    cp "${apk}" "${destination}/"
+  done < <(find "${artifact_root}" -type f -path "*/${arch}/*.apk" -print0)
+
+  shopt -s nullglob
+  apks=("${destination}"/*.apk)
+  shopt -u nullglob
+  if [[ "${#apks[@]}" -eq 0 ]]; then
+    rmdir "${destination}"
+    continue
+  fi
+
+  while IFS= read -r -d '' key; do
+    copy_key "${key}" "${destination}"
+  done < <(find "${artifact_root}" -type f -path "*/${arch}/*.rsa.pub" -print0)
+  copy_key "${merge_public_keys[0]}" "${destination}"
+
+  (
+    cd "${destination}"
+    apk index --rewrite-arch "${arch}" -o APKINDEX.tar.gz ./*.apk
+    abuild-sign -k "${merge_private_keys[0]}" APKINDEX.tar.gz
+  )
+done
+
+if ! compgen -G "${package_root}"/*/*.apk > /dev/null; then
+  printf 'no APK files found in worker artifacts: %s\n' "${artifact_root}" >&2
+  exit 1
+fi