parallel CI package builds

.gitea/workflows/build.yml

@@ -9,41 +9,35 @@ on:
   pull_request:
   workflow_dispatch:
 
+env:
+  ALPINE_VERSION: "3.24"
+  ALPINE_ARCHES: "x86_64 aarch64"
+  ALPINE_REGISTRY_BRANCH: "v3.24"
+  ALPINE_REGISTRY_REPOSITORY: "${{ vars.PACKAGE_NAME }}"
+  INSTANCE_URL: "${{ vars.INSTANCE_URL }}"
+  PACKAGE_OWNER: "${{ vars.PACKAGE_OWNER }}"
+  PACKAGE_USER: "${{ secrets.PACKAGE_USER }}"
+  PACKAGE_TOKEN: "${{ secrets.PACKAGE_TOKEN }}"
+  PACKAGER: "${{ gitea.repository_owner }}"
+  ALPINE_RUST_CACHE_ROOT: "/cache/public-alpine-packages/v3.24/rust"
+  TRIVY_VERSION: "0.74.0"
+  TRIVY_SHA256: "2ae6fe3ee734b7fdf11335663e18c75ea12dccc76062f09f164a3b0f8be4371a"
+
 jobs:
-  build-and-publish:
+  preflight:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        include:
-          - alpine_version: "3.24"
-            alpine_registry_branch: "v3.24"
+    timeout-minutes: 180
     container:
-      image: "alpine:${{ matrix.alpine_version }}"
-    env:
-      ALPINE_VERSION: "${{ matrix.alpine_version }}"
-      ALPINE_ARCHES: "x86_64 aarch64"
-      ALPINE_REGISTRY_BRANCH: "${{ matrix.alpine_registry_branch }}"
-      ALPINE_REGISTRY_REPOSITORY: "${{ vars.PACKAGE_NAME }}"
-      INSTANCE_URL: "${{ vars.INSTANCE_URL }}"
-      PACKAGE_OWNER: "${{ vars.PACKAGE_OWNER }}"
-      PACKAGE_USER: "${{ secrets.PACKAGE_USER }}"
-      PACKAGE_TOKEN: "${{ secrets.PACKAGE_TOKEN }}"
-      PACKAGER: "${{ gitea.repository_owner }}"
-      ALPINE_APK_REPO_CACHE: "/cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/apk-repo"
-      ALPINE_RUST_CACHE_ROOT: "/cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/rust"
-      TRIVY_VERSION: "0.74.0"
-      TRIVY_SHA256: "2ae6fe3ee734b7fdf11335663e18c75ea12dccc76062f09f164a3b0f8be4371a"
+      image: "alpine:3.24"
+    outputs:
+      packages_selected: "${{ steps.package-selection.outputs.packages_selected }}"
+      selected_packages: "${{ steps.package-selection.outputs.selected_packages }}"
+      greptimedb_selected: "${{ steps.package-selection.outputs.greptimedb_selected }}"
     steps:
       - name: Prepare Environment
         run: |
           printf '%s\n' 'https://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories
-          apk add --no-cache --update abuild-rootbld alpine-sdk atools-apkbuild-lint bash ca-certificates curl doas git jq nodejs rsync shellcheck sudo tar
-          trivy_archive="trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz"
-          curl -fsSL -o "/tmp/${trivy_archive}" "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/${trivy_archive}"
-          echo "${TRIVY_SHA256}  /tmp/${trivy_archive}" | sha256sum -c -
-          tar -xzf "/tmp/${trivy_archive}" -C /usr/local/bin trivy
-          mkdir -p "$ALPINE_APK_REPO_CACHE" "$ALPINE_RUST_CACHE_ROOT"
-          chown -R 1000:1000 /cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}
+          apk add --no-cache --update atools-apkbuild-lint bash ca-certificates curl git jq nodejs shellcheck tar
 
       - name: Checkout
         uses: actions/checkout@v3
@@ -66,41 +60,128 @@ jobs:
 
       - name: Prepare package cache
         id: package-selection
-        run: |
-          scripts/apk/ci-package-cache.sh prepare
+        run: scripts/apk/ci-package-cache.sh prepare
 
-      - name: Build packages
+  build:
+    needs: preflight
+    if: needs.preflight.outputs.packages_selected == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    strategy:
+      fail-fast: false
+      matrix:
+        package: [seaweedfs, greptimedb, lavinmq, gitea]
+    container:
+      image: "alpine:3.24"
+    env:
+      ALPINE_PACKAGE: "${{ matrix.package }}"
+    steps:
+      - name: Prepare Environment
+        run: |
+          printf '%s\n' 'https://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories
+          apk add --no-cache --update abuild-rootbld alpine-sdk bash ca-certificates doas git nodejs sudo
+          mkdir -p "$ALPINE_RUST_CACHE_ROOT"
+          chown -R 1000:1000 /cache/public-alpine-packages/v3.24
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Build package
+        if: contains(needs.preflight.outputs.selected_packages, matrix.package)
         run: |
           scripts/apk/ci-build.sh
-          if [ -d packages/local ]; then
-            scripts/apk/list-packages.sh
-          fi
-
-      - name: Export GreptimeDB CI assets
-        if: steps.package-selection.outputs.greptimedb_selected == 'true'
-        run: scripts/apk/export-ci-assets.sh
+          scripts/apk/list-packages.sh
 
       - name: Check artifact upload support
-        id: artifact-upload
+        if: contains(needs.preflight.outputs.selected_packages, matrix.package)
         run: |
-          if [ -n "${ACTIONS_RUNTIME_TOKEN:-}" ]; then
-            echo "available=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "available=false" >> "$GITHUB_OUTPUT"
-          fi
+          test -n "${ACTIONS_RUNTIME_TOKEN:-}" || {
+            echo "artifact upload support is required for split package builds" >&2
+            exit 1
+          }
 
-      - name: Upload generated CI assets
-        if: steps.package-selection.outputs.greptimedb_selected == 'true' && steps.artifact-upload.outputs.available == 'true'
+      - name: Upload package artifacts
+        if: contains(needs.preflight.outputs.selected_packages, matrix.package)
         uses: actions/upload-artifact@v3
         with:
-          name: greptimedb-musl-binaries-${{ matrix.alpine_registry_branch }}
+          name: alpine-package-${{ matrix.package }}-${{ github.sha }}
+          path: packages/local/
+          retention-days: 1
+
+  verify-and-publish:
+    needs: [preflight, build]
+    if: needs.preflight.outputs.packages_selected == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    container:
+      image: "alpine:3.24"
+    env:
+      ALPINE_PACKAGES: "${{ needs.preflight.outputs.selected_packages }}"
+    steps:
+      - name: Prepare Environment
+        run: |
+          printf '%s\n' 'https://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories
+          apk add --no-cache --update abuild-rootbld alpine-sdk bash ca-certificates curl doas git jq nodejs rsync shellcheck sudo tar
+          trivy_archive="trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz"
+          curl -fsSL -o "/tmp/${trivy_archive}" "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/${trivy_archive}"
+          echo "${TRIVY_SHA256}  /tmp/${trivy_archive}" | sha256sum -c -
+          tar -xzf "/tmp/${trivy_archive}" -C /usr/local/bin trivy
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Prepare registry fallback
+        run: |
+          selected_packages="$ALPINE_PACKAGES"
+          scripts/apk/ci-package-cache.sh prepare
+          printf 'ALPINE_PACKAGES=%s\n' "$selected_packages" >> "$GITHUB_ENV"
+
+      - name: Download SeaweedFS artifacts
+        if: contains(needs.preflight.outputs.selected_packages, 'seaweedfs')
+        uses: actions/download-artifact@v3
+        with:
+          name: alpine-package-seaweedfs-${{ github.sha }}
+          path: packages/worker-artifacts/seaweedfs
+
+      - name: Download GreptimeDB artifacts
+        if: contains(needs.preflight.outputs.selected_packages, 'greptimedb')
+        uses: actions/download-artifact@v3
+        with:
+          name: alpine-package-greptimedb-${{ github.sha }}
+          path: packages/worker-artifacts/greptimedb
+
+      - name: Download LavinMQ artifacts
+        if: contains(needs.preflight.outputs.selected_packages, 'lavinmq')
+        uses: actions/download-artifact@v3
+        with:
+          name: alpine-package-lavinmq-${{ github.sha }}
+          path: packages/worker-artifacts/lavinmq
+
+      - name: Download Gitea artifacts
+        if: contains(needs.preflight.outputs.selected_packages, 'gitea')
+        uses: actions/download-artifact@v3
+        with:
+          name: alpine-package-gitea-${{ github.sha }}
+          path: packages/worker-artifacts/gitea
+
+      - name: Merge package artifacts
+        run: scripts/apk/merge-ci-artifacts.sh
+
+      - name: Export GreptimeDB CI assets
+        if: needs.preflight.outputs.greptimedb_selected == 'true'
+        run: scripts/apk/export-ci-assets.sh
+
+      - name: Upload generated CI assets
+        if: needs.preflight.outputs.greptimedb_selected == 'true'
+        uses: actions/upload-artifact@v3
+        with:
+          name: greptimedb-musl-binaries-v3.24
           path: packages/ci-assets/
 
       - name: Upload local Alpine registry
-        if: steps.package-selection.outputs.packages_selected == 'true' && steps.artifact-upload.outputs.available == 'true'
         uses: actions/upload-artifact@v3
         with:
-          name: alpine-local-registry-${{ matrix.alpine_registry_branch }}-${{ github.sha }}
+          name: alpine-local-registry-v3.24-${{ github.sha }}
           path: packages/local/
           retention-days: 30
 
@@ -108,7 +189,6 @@ jobs:
         run: scripts/apk/ci-smoke.sh
 
       - name: Generate security reports
-        if: env.ALPINE_SKIP_BUILD != '1'
         run: |
           scripts/apk/security-report.sh
           if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
@@ -118,16 +198,14 @@ jobs:
           fi
 
       - name: Upload security reports
-        if: env.ALPINE_SKIP_BUILD != '1' && steps.artifact-upload.outputs.available == 'true'
         uses: actions/upload-artifact@v3
         with:
-          name: apk-security-reports-${{ matrix.alpine_registry_branch }}
+          name: apk-security-reports-v3.24
           path: packages/security-reports/
 
       - name: Runtime test
-        if: env.ALPINE_SKIP_BUILD != '1'
         run: scripts/apk/runtime-test.sh
 
       - name: Publish Alpine packages
-        if: github.ref == 'refs/heads/main' && env.ALPINE_SKIP_BUILD != '1'
+        if: github.ref == 'refs/heads/main'
         run: scripts/apk/publish-gitea.sh