Skip cached Alpine package rebuilds in CI

.gitea/workflows/build.yml

@@ -30,6 +30,7 @@ jobs:
       PACKAGE_USER: "${{ secrets.PACKAGE_USER }}"
       PACKAGE_TOKEN: "${{ secrets.PACKAGE_TOKEN }}"
       PACKAGER: "Joachim Schlöffel <me@joachim-schloeffel.com>"
+      ALPINE_APK_REPO_CACHE: "/cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/apk-repo"
       CARGO_HOME: "/cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/greptimedb/cargo"
       RUSTUP_HOME: "/cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/greptimedb/rustup"
       CARGO_TARGET_DIR: "/cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/greptimedb/target"
@@ -37,17 +38,22 @@ jobs:
       - name: Prepare Environment
         run: |
           apk add --no-cache --update abuild-rootbld alpine-sdk atools-apkbuild-lint bash ca-certificates curl doas git nodejs sudo tar
-          mkdir -p "$CARGO_HOME" "$RUSTUP_HOME" "$CARGO_TARGET_DIR"
+          mkdir -p "$ALPINE_APK_REPO_CACHE" "$CARGO_HOME" "$RUSTUP_HOME" "$CARGO_TARGET_DIR"
-          chown 1000:1000 /cache /cache/public-alpine-packages /cache/public-alpine-packages/${{ matrix.alpine_registry_branch }} /cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/greptimedb "$CARGO_HOME" "$RUSTUP_HOME" "$CARGO_TARGET_DIR"
+          chown -R 1000:1000 /cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}
 
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Build packages
+      - name: Select packages
         run: |
           scripts/apk/clean.sh
           find packaging/alpine/local -mindepth 2 -maxdepth 2 -name APKBUILD -exec apkbuild-lint {} +
+          scripts/apk/ci-package-cache.sh prepare
+
+      - name: Build packages
+        run: |
           scripts/apk/ci-build.sh
+          scripts/apk/ci-package-cache.sh sync
           scripts/apk/export-ci-assets.sh
           scripts/apk/list-packages.sh
 

scripts/apk/ci-build.sh

@@ -2,12 +2,19 @@
 set -euo pipefail
 
 repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+# shellcheck source=scripts/apk/package-lib.sh
 . "${repo_root}/scripts/apk/package-lib.sh"
 
 repo_name="${ALPINE_REPO_NAME:-local}"
 arches="${ALPINE_ARCHES:-x86_64 aarch64}"
 packager="${PACKAGER:-Joachim Schlöffel <me@joachim-schloeffel.com>}"
 selected_packages="${ALPINE_PACKAGE:-${ALPINE_PACKAGES:-}}"
+skip_build="${ALPINE_SKIP_BUILD:-0}"
+
+if [[ "${skip_build}" == "1" ]]; then
+  printf 'Skipping package build: restored package cache is complete\n'
+  exit 0
+fi
 
 if [[ "${1:-}" != "--as-builder" && "$(id -u)" == "0" ]]; then
   addgroup -g 1000 builder 2>/dev/null || addgroup builder

scripts/apk/ci-package-cache.sh

@@ -0,0 +1,225 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+# shellcheck source=scripts/apk/package-lib.sh
+. "${repo_root}/scripts/apk/package-lib.sh"
+
+repo_name="${ALPINE_REPO_NAME:-local}"
+arches="${ALPINE_ARCHES:-x86_64 aarch64}"
+cache_root="${ALPINE_APK_REPO_CACHE:-/cache/public-alpine-packages/${ALPINE_REGISTRY_BRANCH:-local}/apk-repo}"
+repo_dest="${repo_root}/packages/${repo_name}"
+command_name="${1:-prepare}"
+
+all_packages() {
+  while IFS= read -r package_dir; do
+    apk_package_name "${package_dir}"
+  done < <(apk_package_dirs "${repo_root}")
+}
+
+mark_all_packages() {
+  local reason="$1"
+
+  printf 'Rebuilding all packages: %s\n' "${reason}" >&2
+  all_packages
+}
+
+diff_base() {
+  local before="${GITHUB_EVENT_BEFORE:-${CI_COMMIT_BEFORE_SHA:-}}"
+  local base_ref="${GITHUB_BASE_REF:-}"
+
+  if [[ -n "${base_ref}" ]] && git rev-parse --verify "origin/${base_ref}" >/dev/null 2>&1; then
+    printf '%s...HEAD\n' "origin/${base_ref}"
+    return
+  fi
+
+  if [[ -n "${before}" && ! "${before}" =~ ^0+$ ]] && git rev-parse --verify "${before}^{commit}" >/dev/null 2>&1; then
+    printf '%s..HEAD\n' "${before}"
+    return
+  fi
+
+  if git rev-parse --verify "HEAD^" >/dev/null 2>&1; then
+    printf 'HEAD^..HEAD\n'
+    return
+  fi
+
+  return 1
+}
+
+changed_packages() {
+  local base
+  local changed_paths
+  local changed_path
+  local package
+  declare -A selected=()
+
+  if ! base="$(diff_base)"; then
+    mark_all_packages "no usable git diff base"
+    return
+  fi
+
+  if ! changed_paths="$(git diff --name-only "${base}")"; then
+    mark_all_packages "git diff failed for ${base}"
+    return
+  fi
+
+  while IFS= read -r changed_path; do
+    [[ -n "${changed_path}" ]] || continue
+    case "${changed_path}" in
+      scripts/apk/*|.gitea/workflows/build.yml|mise.toml)
+        mark_all_packages "shared build logic changed"
+        return
+        ;;
+      packaging/alpine/local/*/APKBUILD)
+        package="${changed_path#packaging/alpine/local/}"
+        package="${package%%/*}"
+        selected["${package}"]=1
+        ;;
+    esac
+  done <<< "${changed_paths}"
+
+  for package in "${!selected[@]}"; do
+    printf '%s\n' "${package}"
+  done | sort
+}
+
+restore_cache() {
+  if [[ -d "${cache_root}" ]]; then
+    mkdir -p "${repo_dest}"
+    cp -a "${cache_root}/." "${repo_dest}/"
+    printf 'Restored Alpine package repository cache from %s\n' "${cache_root}"
+  else
+    printf 'No Alpine package repository cache found at %s\n' "${cache_root}"
+  fi
+}
+
+package_apk_names() {
+  local package_dir="$1"
+  local arch="$2"
+
+  (
+    set +u
+    local pkgname
+    local pkgver
+    local pkgrel
+    local subpackages
+    export CARCH="${arch}"
+    cd "${package_dir}"
+    # shellcheck source=/dev/null
+    . ./APKBUILD
+
+    printf '%s-%s-r%s.apk\n' "${pkgname}" "${pkgver}" "${pkgrel}"
+    for subpackage in ${subpackages:-}; do
+      subpackage="${subpackage%%:*}"
+      printf '%s-%s-r%s.apk\n' "${subpackage}" "${pkgver}" "${pkgrel}"
+    done
+  )
+}
+
+package_cache_complete() {
+  local package_dir="$1"
+  local package_name
+  local arch
+  local apk_name
+
+  package_name="$(apk_package_name "${package_dir}")"
+
+  for arch in ${arches}; do
+    case "${arch}" in
+      x86_64|aarch64) ;;
+      *) printf 'unsupported Alpine architecture: %s\n' "${arch}" >&2; exit 2 ;;
+    esac
+
+    if ! apk_package_supports_arch "${package_dir}" "${arch}"; then
+      continue
+    fi
+
+    while IFS= read -r apk_name; do
+      if [[ ! -f "${repo_dest}/${arch}/${apk_name}" ]]; then
+        printf 'Cache miss for %s: missing %s/%s\n' "${package_name}" "${arch}" "${apk_name}" >&2
+        return 1
+      fi
+    done < <(package_apk_names "${package_dir}" "${arch}")
+  done
+
+  return 0
+}
+
+select_missing_packages() {
+  local package_dir
+  local package_name
+
+  while IFS= read -r package_dir; do
+    apk_validate_package_dir "${package_dir}"
+    package_name="$(apk_package_name "${package_dir}")"
+    if ! package_cache_complete "${package_dir}"; then
+      printf '%s\n' "${package_name}"
+    fi
+  done < <(apk_package_dirs "${repo_root}")
+}
+
+prepare() {
+  local selected_packages
+  local selected
+  declare -A selected_map=()
+
+  restore_cache
+
+  while IFS= read -r selected; do
+    [[ -n "${selected}" ]] || continue
+    selected_map["${selected}"]=1
+  done < <(changed_packages)
+
+  while IFS= read -r selected; do
+    [[ -n "${selected}" ]] || continue
+    selected_map["${selected}"]=1
+  done < <(select_missing_packages)
+
+  selected_packages="$(
+    for selected in "${!selected_map[@]}"; do
+      printf '%s\n' "${selected}"
+    done | sort | xargs
+  )"
+
+  if [[ -n "${selected_packages}" ]]; then
+    printf 'Selected packages for rebuild: %s\n' "${selected_packages}"
+    export ALPINE_PACKAGES="${selected_packages}"
+    export ALPINE_SKIP_BUILD=0
+  else
+    printf 'No packages selected for rebuild\n'
+    export ALPINE_PACKAGES=""
+    export ALPINE_SKIP_BUILD=1
+  fi
+
+  if [[ -n "${GITHUB_ENV:-}" ]]; then
+    {
+      printf 'ALPINE_PACKAGES=%s\n' "${ALPINE_PACKAGES}"
+      printf 'ALPINE_SKIP_BUILD=%s\n' "${ALPINE_SKIP_BUILD}"
+    } >> "${GITHUB_ENV}"
+  fi
+}
+
+sync_cache() {
+  if [[ ! -d "${repo_dest}" ]]; then
+    printf 'missing package repository: %s\n' "${repo_dest}" >&2
+    exit 1
+  fi
+
+  mkdir -p "${cache_root}"
+  find "${cache_root}" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
+  cp -a "${repo_dest}/." "${cache_root}/"
+  printf 'Updated Alpine package repository cache at %s\n' "${cache_root}"
+}
+
+case "${command_name}" in
+  prepare)
+    prepare
+    ;;
+  sync)
+    sync_cache
+    ;;
+  *)
+    printf 'usage: %s [prepare|sync]\n' "$0" >&2
+    exit 2
+    ;;
+esac