diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index 720567f..3cb80ae 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -9,41 +9,35 @@ on: pull_request: workflow_dispatch: +env: + ALPINE_VERSION: "3.24" + ALPINE_ARCHES: "x86_64 aarch64" + ALPINE_REGISTRY_BRANCH: "v3.24" + ALPINE_REGISTRY_REPOSITORY: "${{ vars.PACKAGE_NAME }}" + INSTANCE_URL: "${{ vars.INSTANCE_URL }}" + PACKAGE_OWNER: "${{ vars.PACKAGE_OWNER }}" + PACKAGE_USER: "${{ secrets.PACKAGE_USER }}" + PACKAGE_TOKEN: "${{ secrets.PACKAGE_TOKEN }}" + PACKAGER: "${{ gitea.repository_owner }}" + ALPINE_RUST_CACHE_ROOT: "/cache/public-alpine-packages/v3.24/rust" + TRIVY_VERSION: "0.74.0" + TRIVY_SHA256: "2ae6fe3ee734b7fdf11335663e18c75ea12dccc76062f09f164a3b0f8be4371a" + jobs: - build-and-publish: + preflight: runs-on: ubuntu-latest - strategy: - matrix: - include: - - alpine_version: "3.24" - alpine_registry_branch: "v3.24" + timeout-minutes: 180 container: - image: "alpine:${{ matrix.alpine_version }}" - env: - ALPINE_VERSION: "${{ matrix.alpine_version }}" - ALPINE_ARCHES: "x86_64 aarch64" - ALPINE_REGISTRY_BRANCH: "${{ matrix.alpine_registry_branch }}" - ALPINE_REGISTRY_REPOSITORY: "${{ vars.PACKAGE_NAME }}" - INSTANCE_URL: "${{ vars.INSTANCE_URL }}" - PACKAGE_OWNER: "${{ vars.PACKAGE_OWNER }}" - PACKAGE_USER: "${{ secrets.PACKAGE_USER }}" - PACKAGE_TOKEN: "${{ secrets.PACKAGE_TOKEN }}" - PACKAGER: "${{ gitea.repository_owner }}" - ALPINE_APK_REPO_CACHE: "/cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/apk-repo" - ALPINE_RUST_CACHE_ROOT: "/cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/rust" - TRIVY_VERSION: "0.74.0" - TRIVY_SHA256: "2ae6fe3ee734b7fdf11335663e18c75ea12dccc76062f09f164a3b0f8be4371a" + image: "alpine:3.24" + outputs: + packages_selected: "${{ steps.package-selection.outputs.packages_selected }}" + selected_packages: "${{ steps.package-selection.outputs.selected_packages }}" + greptimedb_selected: "${{ steps.package-selection.outputs.greptimedb_selected }}" steps: - name: Prepare Environment run: | printf '%s\n' 'https://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories - apk add --no-cache --update abuild-rootbld alpine-sdk atools-apkbuild-lint bash ca-certificates curl doas git jq nodejs rsync shellcheck sudo tar - trivy_archive="trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" - curl -fsSL -o "/tmp/${trivy_archive}" "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/${trivy_archive}" - echo "${TRIVY_SHA256} /tmp/${trivy_archive}" | sha256sum -c - - tar -xzf "/tmp/${trivy_archive}" -C /usr/local/bin trivy - mkdir -p "$ALPINE_APK_REPO_CACHE" "$ALPINE_RUST_CACHE_ROOT" - chown -R 1000:1000 /cache/public-alpine-packages/${{ matrix.alpine_registry_branch }} + apk add --no-cache --update atools-apkbuild-lint bash ca-certificates curl git jq nodejs shellcheck tar - name: Checkout uses: actions/checkout@v3 @@ -66,41 +60,128 @@ jobs: - name: Prepare package cache id: package-selection - run: | - scripts/apk/ci-package-cache.sh prepare + run: scripts/apk/ci-package-cache.sh prepare - - name: Build packages + build: + needs: preflight + if: needs.preflight.outputs.packages_selected == 'true' + runs-on: ubuntu-latest + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + package: [seaweedfs, greptimedb, lavinmq, gitea] + container: + image: "alpine:3.24" + env: + ALPINE_PACKAGE: "${{ matrix.package }}" + steps: + - name: Prepare Environment + run: | + printf '%s\n' 'https://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories + apk add --no-cache --update abuild-rootbld alpine-sdk bash ca-certificates doas git nodejs sudo + mkdir -p "$ALPINE_RUST_CACHE_ROOT" + chown -R 1000:1000 /cache/public-alpine-packages/v3.24 + + - name: Checkout + uses: actions/checkout@v3 + + - name: Build package + if: contains(needs.preflight.outputs.selected_packages, matrix.package) run: | scripts/apk/ci-build.sh - if [ -d packages/local ]; then - scripts/apk/list-packages.sh - fi - - - name: Export GreptimeDB CI assets - if: steps.package-selection.outputs.greptimedb_selected == 'true' - run: scripts/apk/export-ci-assets.sh + scripts/apk/list-packages.sh - name: Check artifact upload support - id: artifact-upload + if: contains(needs.preflight.outputs.selected_packages, matrix.package) run: | - if [ -n "${ACTIONS_RUNTIME_TOKEN:-}" ]; then - echo "available=true" >> "$GITHUB_OUTPUT" - else - echo "available=false" >> "$GITHUB_OUTPUT" - fi + test -n "${ACTIONS_RUNTIME_TOKEN:-}" || { + echo "artifact upload support is required for split package builds" >&2 + exit 1 + } - - name: Upload generated CI assets - if: steps.package-selection.outputs.greptimedb_selected == 'true' && steps.artifact-upload.outputs.available == 'true' + - name: Upload package artifacts + if: contains(needs.preflight.outputs.selected_packages, matrix.package) uses: actions/upload-artifact@v3 with: - name: greptimedb-musl-binaries-${{ matrix.alpine_registry_branch }} + name: alpine-package-${{ matrix.package }}-${{ github.sha }} + path: packages/local/ + retention-days: 1 + + verify-and-publish: + needs: [preflight, build] + if: needs.preflight.outputs.packages_selected == 'true' + runs-on: ubuntu-latest + timeout-minutes: 180 + container: + image: "alpine:3.24" + env: + ALPINE_PACKAGES: "${{ needs.preflight.outputs.selected_packages }}" + steps: + - name: Prepare Environment + run: | + printf '%s\n' 'https://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories + apk add --no-cache --update abuild-rootbld alpine-sdk bash ca-certificates curl doas git jq nodejs rsync shellcheck sudo tar + trivy_archive="trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" + curl -fsSL -o "/tmp/${trivy_archive}" "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/${trivy_archive}" + echo "${TRIVY_SHA256} /tmp/${trivy_archive}" | sha256sum -c - + tar -xzf "/tmp/${trivy_archive}" -C /usr/local/bin trivy + + - name: Checkout + uses: actions/checkout@v3 + + - name: Prepare registry fallback + run: | + selected_packages="$ALPINE_PACKAGES" + scripts/apk/ci-package-cache.sh prepare + printf 'ALPINE_PACKAGES=%s\n' "$selected_packages" >> "$GITHUB_ENV" + + - name: Download SeaweedFS artifacts + if: contains(needs.preflight.outputs.selected_packages, 'seaweedfs') + uses: actions/download-artifact@v3 + with: + name: alpine-package-seaweedfs-${{ github.sha }} + path: packages/worker-artifacts/seaweedfs + + - name: Download GreptimeDB artifacts + if: contains(needs.preflight.outputs.selected_packages, 'greptimedb') + uses: actions/download-artifact@v3 + with: + name: alpine-package-greptimedb-${{ github.sha }} + path: packages/worker-artifacts/greptimedb + + - name: Download LavinMQ artifacts + if: contains(needs.preflight.outputs.selected_packages, 'lavinmq') + uses: actions/download-artifact@v3 + with: + name: alpine-package-lavinmq-${{ github.sha }} + path: packages/worker-artifacts/lavinmq + + - name: Download Gitea artifacts + if: contains(needs.preflight.outputs.selected_packages, 'gitea') + uses: actions/download-artifact@v3 + with: + name: alpine-package-gitea-${{ github.sha }} + path: packages/worker-artifacts/gitea + + - name: Merge package artifacts + run: scripts/apk/merge-ci-artifacts.sh + + - name: Export GreptimeDB CI assets + if: needs.preflight.outputs.greptimedb_selected == 'true' + run: scripts/apk/export-ci-assets.sh + + - name: Upload generated CI assets + if: needs.preflight.outputs.greptimedb_selected == 'true' + uses: actions/upload-artifact@v3 + with: + name: greptimedb-musl-binaries-v3.24 path: packages/ci-assets/ - name: Upload local Alpine registry - if: steps.package-selection.outputs.packages_selected == 'true' && steps.artifact-upload.outputs.available == 'true' uses: actions/upload-artifact@v3 with: - name: alpine-local-registry-${{ matrix.alpine_registry_branch }}-${{ github.sha }} + name: alpine-local-registry-v3.24-${{ github.sha }} path: packages/local/ retention-days: 30 @@ -108,7 +189,6 @@ jobs: run: scripts/apk/ci-smoke.sh - name: Generate security reports - if: env.ALPINE_SKIP_BUILD != '1' run: | scripts/apk/security-report.sh if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then @@ -118,16 +198,14 @@ jobs: fi - name: Upload security reports - if: env.ALPINE_SKIP_BUILD != '1' && steps.artifact-upload.outputs.available == 'true' uses: actions/upload-artifact@v3 with: - name: apk-security-reports-${{ matrix.alpine_registry_branch }} + name: apk-security-reports-v3.24 path: packages/security-reports/ - name: Runtime test - if: env.ALPINE_SKIP_BUILD != '1' run: scripts/apk/runtime-test.sh - name: Publish Alpine packages - if: github.ref == 'refs/heads/main' && env.ALPINE_SKIP_BUILD != '1' + if: github.ref == 'refs/heads/main' run: scripts/apk/publish-gitea.sh diff --git a/scripts/apk/ci-package-cache.sh b/scripts/apk/ci-package-cache.sh index 80508a3..8117634 100755 --- a/scripts/apk/ci-package-cache.sh +++ b/scripts/apk/ci-package-cache.sh @@ -138,6 +138,7 @@ prepare() { } >> "${GITHUB_ENV}" fi if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + printf 'selected_packages=%s\n' "${ALPINE_PACKAGES}" >> "${GITHUB_OUTPUT}" if [[ -n "${ALPINE_PACKAGES}" ]]; then printf 'packages_selected=true\n' >> "${GITHUB_OUTPUT}" else diff --git a/scripts/apk/merge-ci-artifacts.sh b/scripts/apk/merge-ci-artifacts.sh new file mode 100755 index 0000000..4deada0 --- /dev/null +++ b/scripts/apk/merge-ci-artifacts.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +repo_name="${ALPINE_REPO_NAME:-local}" +arches="${ALPINE_ARCHES:-x86_64 aarch64}" +artifact_root="${APK_CI_ARTIFACTS_DIR:-${repo_root}/packages/worker-artifacts}" +package_root="${APK_CI_PACKAGE_ROOT:-${repo_root}/packages/${repo_name}}" + +require_command() { + command -v "$1" >/dev/null 2>&1 || { + printf 'missing required command: %s\n' "$1" >&2 + exit 2 + } +} + +copy_key() { + local key="$1" + local destination="$2" + local digest + + digest="$(sha256sum "${key}" | awk '{print $1}')" + cp "${key}" "${destination}/${digest}.rsa.pub" +} + +require_command abuild-keygen +require_command abuild-sign +require_command apk + +if [[ ! -d "${artifact_root}" ]]; then + printf 'missing worker artifacts directory: %s\n' "${artifact_root}" >&2 + exit 1 +fi + +rm -rf "${package_root}" +mkdir -p "${package_root}" "${HOME}/.abuild" + +if ! compgen -G "${HOME}/.abuild/*.rsa" > /dev/null; then + abuild-keygen -a -n +fi + +shopt -s nullglob +merge_private_keys=("${HOME}"/.abuild/*.rsa) +merge_public_keys=("${HOME}"/.abuild/*.rsa.pub) +shopt -u nullglob + +if [[ "${#merge_private_keys[@]}" -ne 1 || "${#merge_public_keys[@]}" -ne 1 ]]; then + printf 'expected exactly one merge signing key in %s/.abuild\n' "${HOME}" >&2 + exit 1 +fi + +for arch in ${arches}; do + destination="${package_root}/${arch}" + mkdir -p "${destination}" + + while IFS= read -r -d '' apk; do + cp "${apk}" "${destination}/" + done < <(find "${artifact_root}" -type f -path "*/${arch}/*.apk" -print0) + + shopt -s nullglob + apks=("${destination}"/*.apk) + shopt -u nullglob + if [[ "${#apks[@]}" -eq 0 ]]; then + rmdir "${destination}" + continue + fi + + while IFS= read -r -d '' key; do + copy_key "${key}" "${destination}" + done < <(find "${artifact_root}" -type f -path "*/${arch}/*.rsa.pub" -print0) + copy_key "${merge_public_keys[0]}" "${destination}" + + ( + cd "${destination}" + apk index --rewrite-arch "${arch}" -o APKINDEX.tar.gz ./*.apk + abuild-sign -k "${merge_private_keys[0]}" APKINDEX.tar.gz + ) +done + +if ! compgen -G "${package_root}"/*/*.apk > /dev/null; then + printf 'no APK files found in worker artifacts: %s\n' "${artifact_root}" >&2 + exit 1 +fi