From 6b8488d7cc30af4809d6d7074b4b23e2878edbaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joachim=20Schl=C3=B6ffel?= Date: Thu, 9 Jul 2026 09:59:35 +0200 Subject: [PATCH] Skip cached Alpine package rebuilds in CI --- .gitea/workflows/build.yml | 12 +- scripts/apk/ci-build.sh | 7 + scripts/apk/ci-package-cache.sh | 225 ++++++++++++++++++++++++++++++++ 3 files changed, 241 insertions(+), 3 deletions(-) create mode 100755 scripts/apk/ci-package-cache.sh diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index f0440a8..1674471 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -30,6 +30,7 @@ jobs: PACKAGE_USER: "${{ secrets.PACKAGE_USER }}" PACKAGE_TOKEN: "${{ secrets.PACKAGE_TOKEN }}" PACKAGER: "Joachim Schlöffel " + ALPINE_APK_REPO_CACHE: "/cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/apk-repo" CARGO_HOME: "/cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/greptimedb/cargo" RUSTUP_HOME: "/cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/greptimedb/rustup" CARGO_TARGET_DIR: "/cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/greptimedb/target" @@ -37,17 +38,22 @@ jobs: - name: Prepare Environment run: | apk add --no-cache --update abuild-rootbld alpine-sdk atools-apkbuild-lint bash ca-certificates curl doas git nodejs sudo tar - mkdir -p "$CARGO_HOME" "$RUSTUP_HOME" "$CARGO_TARGET_DIR" - chown 1000:1000 /cache /cache/public-alpine-packages /cache/public-alpine-packages/${{ matrix.alpine_registry_branch }} /cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/greptimedb "$CARGO_HOME" "$RUSTUP_HOME" "$CARGO_TARGET_DIR" + mkdir -p "$ALPINE_APK_REPO_CACHE" "$CARGO_HOME" "$RUSTUP_HOME" "$CARGO_TARGET_DIR" + chown -R 1000:1000 /cache/public-alpine-packages/${{ matrix.alpine_registry_branch }} - name: Checkout uses: actions/checkout@v3 - - name: Build packages + - name: Select packages run: | scripts/apk/clean.sh find packaging/alpine/local -mindepth 2 -maxdepth 2 -name APKBUILD -exec apkbuild-lint {} + + scripts/apk/ci-package-cache.sh prepare + + - name: Build packages + run: | scripts/apk/ci-build.sh + scripts/apk/ci-package-cache.sh sync scripts/apk/export-ci-assets.sh scripts/apk/list-packages.sh diff --git a/scripts/apk/ci-build.sh b/scripts/apk/ci-build.sh index 692f51e..1efe182 100755 --- a/scripts/apk/ci-build.sh +++ b/scripts/apk/ci-build.sh @@ -2,12 +2,19 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=scripts/apk/package-lib.sh . "${repo_root}/scripts/apk/package-lib.sh" repo_name="${ALPINE_REPO_NAME:-local}" arches="${ALPINE_ARCHES:-x86_64 aarch64}" packager="${PACKAGER:-Joachim Schlöffel }" selected_packages="${ALPINE_PACKAGE:-${ALPINE_PACKAGES:-}}" +skip_build="${ALPINE_SKIP_BUILD:-0}" + +if [[ "${skip_build}" == "1" ]]; then + printf 'Skipping package build: restored package cache is complete\n' + exit 0 +fi if [[ "${1:-}" != "--as-builder" && "$(id -u)" == "0" ]]; then addgroup -g 1000 builder 2>/dev/null || addgroup builder diff --git a/scripts/apk/ci-package-cache.sh b/scripts/apk/ci-package-cache.sh new file mode 100755 index 0000000..22e04b9 --- /dev/null +++ b/scripts/apk/ci-package-cache.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=scripts/apk/package-lib.sh +. "${repo_root}/scripts/apk/package-lib.sh" + +repo_name="${ALPINE_REPO_NAME:-local}" +arches="${ALPINE_ARCHES:-x86_64 aarch64}" +cache_root="${ALPINE_APK_REPO_CACHE:-/cache/public-alpine-packages/${ALPINE_REGISTRY_BRANCH:-local}/apk-repo}" +repo_dest="${repo_root}/packages/${repo_name}" +command_name="${1:-prepare}" + +all_packages() { + while IFS= read -r package_dir; do + apk_package_name "${package_dir}" + done < <(apk_package_dirs "${repo_root}") +} + +mark_all_packages() { + local reason="$1" + + printf 'Rebuilding all packages: %s\n' "${reason}" >&2 + all_packages +} + +diff_base() { + local before="${GITHUB_EVENT_BEFORE:-${CI_COMMIT_BEFORE_SHA:-}}" + local base_ref="${GITHUB_BASE_REF:-}" + + if [[ -n "${base_ref}" ]] && git rev-parse --verify "origin/${base_ref}" >/dev/null 2>&1; then + printf '%s...HEAD\n' "origin/${base_ref}" + return + fi + + if [[ -n "${before}" && ! "${before}" =~ ^0+$ ]] && git rev-parse --verify "${before}^{commit}" >/dev/null 2>&1; then + printf '%s..HEAD\n' "${before}" + return + fi + + if git rev-parse --verify "HEAD^" >/dev/null 2>&1; then + printf 'HEAD^..HEAD\n' + return + fi + + return 1 +} + +changed_packages() { + local base + local changed_paths + local changed_path + local package + declare -A selected=() + + if ! base="$(diff_base)"; then + mark_all_packages "no usable git diff base" + return + fi + + if ! changed_paths="$(git diff --name-only "${base}")"; then + mark_all_packages "git diff failed for ${base}" + return + fi + + while IFS= read -r changed_path; do + [[ -n "${changed_path}" ]] || continue + case "${changed_path}" in + scripts/apk/*|.gitea/workflows/build.yml|mise.toml) + mark_all_packages "shared build logic changed" + return + ;; + packaging/alpine/local/*/APKBUILD) + package="${changed_path#packaging/alpine/local/}" + package="${package%%/*}" + selected["${package}"]=1 + ;; + esac + done <<< "${changed_paths}" + + for package in "${!selected[@]}"; do + printf '%s\n' "${package}" + done | sort +} + +restore_cache() { + if [[ -d "${cache_root}" ]]; then + mkdir -p "${repo_dest}" + cp -a "${cache_root}/." "${repo_dest}/" + printf 'Restored Alpine package repository cache from %s\n' "${cache_root}" + else + printf 'No Alpine package repository cache found at %s\n' "${cache_root}" + fi +} + +package_apk_names() { + local package_dir="$1" + local arch="$2" + + ( + set +u + local pkgname + local pkgver + local pkgrel + local subpackages + export CARCH="${arch}" + cd "${package_dir}" + # shellcheck source=/dev/null + . ./APKBUILD + + printf '%s-%s-r%s.apk\n' "${pkgname}" "${pkgver}" "${pkgrel}" + for subpackage in ${subpackages:-}; do + subpackage="${subpackage%%:*}" + printf '%s-%s-r%s.apk\n' "${subpackage}" "${pkgver}" "${pkgrel}" + done + ) +} + +package_cache_complete() { + local package_dir="$1" + local package_name + local arch + local apk_name + + package_name="$(apk_package_name "${package_dir}")" + + for arch in ${arches}; do + case "${arch}" in + x86_64|aarch64) ;; + *) printf 'unsupported Alpine architecture: %s\n' "${arch}" >&2; exit 2 ;; + esac + + if ! apk_package_supports_arch "${package_dir}" "${arch}"; then + continue + fi + + while IFS= read -r apk_name; do + if [[ ! -f "${repo_dest}/${arch}/${apk_name}" ]]; then + printf 'Cache miss for %s: missing %s/%s\n' "${package_name}" "${arch}" "${apk_name}" >&2 + return 1 + fi + done < <(package_apk_names "${package_dir}" "${arch}") + done + + return 0 +} + +select_missing_packages() { + local package_dir + local package_name + + while IFS= read -r package_dir; do + apk_validate_package_dir "${package_dir}" + package_name="$(apk_package_name "${package_dir}")" + if ! package_cache_complete "${package_dir}"; then + printf '%s\n' "${package_name}" + fi + done < <(apk_package_dirs "${repo_root}") +} + +prepare() { + local selected_packages + local selected + declare -A selected_map=() + + restore_cache + + while IFS= read -r selected; do + [[ -n "${selected}" ]] || continue + selected_map["${selected}"]=1 + done < <(changed_packages) + + while IFS= read -r selected; do + [[ -n "${selected}" ]] || continue + selected_map["${selected}"]=1 + done < <(select_missing_packages) + + selected_packages="$( + for selected in "${!selected_map[@]}"; do + printf '%s\n' "${selected}" + done | sort | xargs + )" + + if [[ -n "${selected_packages}" ]]; then + printf 'Selected packages for rebuild: %s\n' "${selected_packages}" + export ALPINE_PACKAGES="${selected_packages}" + export ALPINE_SKIP_BUILD=0 + else + printf 'No packages selected for rebuild\n' + export ALPINE_PACKAGES="" + export ALPINE_SKIP_BUILD=1 + fi + + if [[ -n "${GITHUB_ENV:-}" ]]; then + { + printf 'ALPINE_PACKAGES=%s\n' "${ALPINE_PACKAGES}" + printf 'ALPINE_SKIP_BUILD=%s\n' "${ALPINE_SKIP_BUILD}" + } >> "${GITHUB_ENV}" + fi +} + +sync_cache() { + if [[ ! -d "${repo_dest}" ]]; then + printf 'missing package repository: %s\n' "${repo_dest}" >&2 + exit 1 + fi + + mkdir -p "${cache_root}" + find "${cache_root}" -mindepth 1 -maxdepth 1 -exec rm -rf {} + + cp -a "${repo_dest}/." "${cache_root}/" + printf 'Updated Alpine package repository cache at %s\n' "${cache_root}" +} + +case "${command_name}" in + prepare) + prepare + ;; + sync) + sync_cache + ;; + *) + printf 'usage: %s [prepare|sync]\n' "$0" >&2 + exit 2 + ;; +esac