diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index 058d49e..05c7db0 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -33,9 +33,7 @@ jobs: PACKAGE_TOKEN: "${{ secrets.PACKAGE_TOKEN }}" PACKAGER: "${{ gitea.repository_owner }}" ALPINE_APK_REPO_CACHE: "/cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/apk-repo" - CARGO_HOME: "/cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/greptimedb/cargo" - RUSTUP_HOME: "/cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/greptimedb/rustup" - CARGO_TARGET_DIR: "/cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/greptimedb/target" + ALPINE_RUST_CACHE_ROOT: "/cache/public-alpine-packages/${{ matrix.alpine_registry_branch }}/rust" TRIVY_VERSION: "0.74.0" TRIVY_SHA256: "2ae6fe3ee734b7fdf11335663e18c75ea12dccc76062f09f164a3b0f8be4371a" steps: @@ -47,7 +45,7 @@ jobs: curl -fsSL -o "/tmp/${trivy_archive}" "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/${trivy_archive}" echo "${TRIVY_SHA256} /tmp/${trivy_archive}" | sha256sum -c - tar -xzf "/tmp/${trivy_archive}" -C /usr/local/bin trivy - mkdir -p "$ALPINE_APK_REPO_CACHE" "$CARGO_HOME" "$RUSTUP_HOME" "$CARGO_TARGET_DIR" + mkdir -p "$ALPINE_APK_REPO_CACHE" "$ALPINE_RUST_CACHE_ROOT" chown -R 1000:1000 /cache/public-alpine-packages/${{ matrix.alpine_registry_branch }} - name: Checkout diff --git a/README.md b/README.md index e13a3c8..192b758 100644 --- a/README.md +++ b/README.md @@ -19,10 +19,10 @@ https://code.factoring.digital/api/packages/public/alpine/v3.24/alpine-packages | Package | APK Name | Package Version | Package Type | Purpose | |-----------------------------------------------------------|--------------|-----------------|----------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------| -| [SeaweedFS](packaging/alpine/local/seaweedfs/README.md) | `seaweedfs` | `4.40` | Binary repackaging of official upstream release tarballs | Ships current SeaweedFS features with Alpine OpenRC role splits. | +| [SeaweedFS](packaging/alpine/local/seaweedfs/README.md) | `seaweedfs` | `4.45` | Binary repackaging of official upstream release tarballs | Ships current SeaweedFS features with Alpine OpenRC role splits. | | [GreptimeDB](packaging/alpine/local/greptimedb/README.md) | `greptimedb` | `1.1.4` | Source build from upstream release | Provides a current GreptimeDB build and component OpenRC packages before it is available from Alpine community. | -| [LavinMQ](packaging/alpine/local/lavinmq/README.md) | `lavinmq` | `2.9.1` | Source build from upstream release | Provides a current LavinMQ build on Alpine 3.24 with matching Crystal support. | -| [Gitea](packaging/alpine/local/gitea/README.md) | `gitea` | `1.27.0` | Binary repackaging of official upstream release binaries | Ships current Gitea releases with the expected Alpine runtime layout and OpenRC service. | +| [LavinMQ](packaging/alpine/local/lavinmq/README.md) | `lavinmq` | `2.9.2` | Source build from upstream release | Provides a current LavinMQ build on Alpine 3.24 with matching Crystal support. | +| [Gitea](packaging/alpine/local/gitea/README.md) | `gitea` | `1.27.3` | Binary repackaging of official upstream release binaries | Ships current Gitea releases with the expected Alpine runtime layout and OpenRC service. | ## Use In Production @@ -251,10 +251,8 @@ available in the signed local repository. ### Gitea Workflow -`.gitea/workflows/build.yml` builds and publishes `v3.24`. Pull requests -build only. CI compares source package versions with the published registry, -then rebuilds only changed or unpublished packages; local build commands keep -using their local caches. +The smoke test installs rebuilt artifacts from the local repository and uses +the published registry as a fallback for unchanged packages. Repository variables: diff --git a/scripts/apk/ci-build.sh b/scripts/apk/ci-build.sh index a4d8b41..0f01c23 100755 --- a/scripts/apk/ci-build.sh +++ b/scripts/apk/ci-build.sh @@ -10,6 +10,7 @@ arches="${ALPINE_ARCHES:-x86_64 aarch64}" packager="${PACKAGER:-local}" selected_packages="${ALPINE_PACKAGE:-${ALPINE_PACKAGES:-}}" skip_build="${ALPINE_SKIP_BUILD:-0}" +rust_cache_root="${ALPINE_RUST_CACHE_ROOT:-}" if [[ "${skip_build}" == "1" ]]; then printf 'Skipping package build: restored package cache is complete\n' @@ -24,7 +25,7 @@ if [[ "${1:-}" != "--as-builder" && "$(id -u)" == "0" ]]; then printf 'permit nopass :wheel\n' > /etc/doas.d/wheel.conf printf '%%wheel ALL=(ALL) NOPASSWD: ALL\n' > /etc/sudoers.d/wheel chown -R builder:builder "${repo_root}" - exec su builder -c "ALPINE_ARCHES='${arches}' ALPINE_REPO_NAME='${repo_name}' ALPINE_PACKAGES='${selected_packages}' PACKAGER='${packager}' '${BASH_SOURCE[0]}' --as-builder" + exec su builder -c "ALPINE_ARCHES='${arches}' ALPINE_REPO_NAME='${repo_name}' ALPINE_PACKAGES='${selected_packages}' PACKAGER='${packager}' ALPINE_RUST_CACHE_ROOT='${rust_cache_root}' '${BASH_SOURCE[0]}' --as-builder" fi export PACKAGER="${packager}" @@ -72,6 +73,21 @@ while IFS= read -r package_dir; do printf 'Building %s for %s\n' "$(basename "${package_dir}")" "${arch}" ( + package_name="$(apk_package_name "${package_dir}")" + if [[ -n "${rust_cache_root}" ]]; then + case "${package_name}" in + greptimedb) + export CARGO_HOME="${rust_cache_root}/greptimedb/cargo" + export RUSTUP_HOME="${rust_cache_root}/greptimedb/rustup" + export CARGO_TARGET_DIR="${rust_cache_root}/greptimedb/target" + ;; + seaweedfs) + export CARGO_HOME="${rust_cache_root}/seaweedfs/cargo" + export CARGO_TARGET_DIR="${rust_cache_root}/seaweedfs/target" + unset RUSTUP_HOME + ;; + esac + fi cd "${package_dir}" ALPINE_ARCH="${arch}" CARCH="${arch}" abuild -r ) diff --git a/scripts/apk/ci-package-cache.sh b/scripts/apk/ci-package-cache.sh index 6b18fd0..80508a3 100755 --- a/scripts/apk/ci-package-cache.sh +++ b/scripts/apk/ci-package-cache.sh @@ -29,66 +29,8 @@ registry_key_url() { printf '%s/api/packages/%s/alpine/key\n' "${instance_url%/}" "${owner}" } -all_packages() { - while IFS= read -r package_dir; do - apk_package_name "${package_dir}" - done < <(apk_all_package_dirs "${repo_root}") -} - -mark_all_packages() { - printf 'Rebuilding all packages: %s\n' "$1" >&2 - all_packages -} - -diff_base() { - local before="${GITHUB_EVENT_BEFORE:-${CI_COMMIT_BEFORE_SHA:-}}" - local base_ref="${GITHUB_BASE_REF:-}" - - if [[ -n "${base_ref}" ]] && git rev-parse --verify "origin/${base_ref}" >/dev/null 2>&1; then - printf '%s...HEAD\n' "origin/${base_ref}" - elif [[ -n "${before}" && ! "${before}" =~ ^0+$ ]] && git rev-parse --verify "${before}^{commit}" >/dev/null 2>&1; then - printf '%s..HEAD\n' "${before}" - elif git rev-parse --verify HEAD^ >/dev/null 2>&1; then - printf 'HEAD^..HEAD\n' - else - return 1 - fi -} - -changed_packages() { - local base changed_paths changed_path package - declare -A selected=() - - if ! base="$(diff_base)"; then - mark_all_packages "no usable git diff base" - return - fi - if ! changed_paths="$(git diff --name-only "${base}")"; then - mark_all_packages "git diff failed for ${base}" - return - fi - - while IFS= read -r changed_path; do - [[ -n "${changed_path}" ]] || continue - case "${changed_path}" in - scripts/apk/*|.gitea/workflows/build.yml|mise.toml) - mark_all_packages "shared build logic changed" - return - ;; - packaging/alpine/local/*) - package="${changed_path#packaging/alpine/local/}" - selected["${package%%/*}"]=1 - ;; - esac - done <<< "${changed_paths}" - - for package in "${!selected[@]}"; do - printf '%s\n' "${package}" - done | sort -} - download_registry_metadata() { - local arch url + local arch url key_dir require_registry_config INSTANCE_URL "${instance_url}" require_registry_config PACKAGE_OWNER "${owner}" @@ -99,19 +41,35 @@ download_registry_metadata() { --output "${registry_cache}/${arch}.APKINDEX.tar.gz" \ "${url}/${arch}/APKINDEX.tar.gz" done + key_dir="${registry_cache}/keys" + mkdir -p "${key_dir}" + find "${key_dir}" -maxdepth 1 -type f -name '*.rsa.pub' -delete curl --fail --silent --show-error --location \ - --output "${registry_cache}/registry.rsa.pub" \ + --remote-name --remote-header-name --output-dir "${key_dir}" \ "$(registry_key_url)" + registry_key_path="$(find "${key_dir}" -maxdepth 1 -type f -name '*.rsa.pub' -print -quit)" + if [[ -z "${registry_key_path}" ]]; then + printf 'registry signing key was not downloaded: %s\n' "$(registry_key_url)" >&2 + exit 1 + fi } load_registry_index() { - local arch package_name package_version - declare -gA registry_packages=() + local arch package_name package_version registry_key comparison + declare -gA registry_latest_versions=() for arch in ${arches}; do while IFS=$'\t' read -r package_name package_version; do [[ -n "${package_name}" && -n "${package_version}" ]] || continue - registry_packages["${arch}|${package_name}|${package_version}"]=1 + registry_key="${arch}|${package_name}" + if [[ -z "${registry_latest_versions[${registry_key}]:-}" ]]; then + registry_latest_versions["${registry_key}"]="${package_version}" + continue + fi + comparison="$(apk version -t "${package_version}" "${registry_latest_versions[${registry_key}]}")" + if [[ "${comparison}" == ">" ]]; then + registry_latest_versions["${registry_key}"]="${package_version}" + fi done < <( tar -xOzf "${registry_cache}/${arch}.APKINDEX.tar.gz" APKINDEX | awk ' /^P:/ { package = substr($0, 3); next } @@ -121,25 +79,34 @@ load_registry_index() { done } -registry_missing_packages() { - local package_dir package_name arch spec spec_name spec_version missing +registry_upgrade_packages() { + local package_dir package_name arch candidate_spec candidate_version published_version comparison selected while IFS= read -r package_dir; do apk_validate_package_dir "${package_dir}" package_name="$(apk_package_name "${package_dir}")" - missing=0 + selected=0 for arch in ${arches}; do apk_package_supports_arch "${package_dir}" "${arch}" || continue - while IFS= read -r spec; do - spec_name="${spec%%=*}" - spec_version="${spec#*=}" - if [[ -z "${registry_packages[${arch}|${spec_name}|${spec_version}]:-}" ]]; then - missing=1 - printf 'Registry miss: %s (%s, %s)\n' "${package_name}" "${arch}" "${spec}" >&2 - fi - done < <(apk_package_versioned_names "${package_dir}" "${arch}") + IFS= read -r candidate_spec < <(apk_package_versioned_names "${package_dir}" "${arch}") + candidate_version="${candidate_spec#*=}" + published_version="${registry_latest_versions[${arch}|${package_name}]:-}" + if [[ -z "${published_version}" ]]; then + selected=1 + printf 'Registry miss: %s (%s, %s)\n' "${package_name}" "${arch}" "${candidate_spec}" >&2 + continue + fi + comparison="$(apk version -t "${candidate_version}" "${published_version}")" + if [[ "${comparison}" == ">" ]]; then + selected=1 + printf 'Registry upgrade: %s (%s, %s -> %s)\n' \ + "${package_name}" "${arch}" "${published_version}" "${candidate_version}" >&2 + else + printf 'Registry current: %s (%s, source %s; published %s)\n' \ + "${package_name}" "${arch}" "${candidate_version}" "${published_version}" >&2 + fi done - [[ "${missing}" == 1 ]] && printf '%s\n' "${package_name}" + [[ "${selected}" == 1 ]] && printf '%s\n' "${package_name}" done < <(apk_all_package_dirs "${repo_root}") } @@ -151,10 +118,7 @@ prepare() { load_registry_index while IFS= read -r selected; do [[ -n "${selected}" ]] && selected_map["${selected}"]=1 - done < <(changed_packages) - while IFS= read -r selected; do - [[ -n "${selected}" ]] && selected_map["${selected}"]=1 - done < <(registry_missing_packages) + done < <(registry_upgrade_packages) ALPINE_PACKAGES="$(for selected in "${!selected_map[@]}"; do printf '%s\n' "${selected}"; done | sort | xargs)" if [[ -n "${ALPINE_PACKAGES}" ]]; then @@ -162,7 +126,7 @@ prepare() { printf 'Selected packages for rebuild: %s\n' "${ALPINE_PACKAGES}" else ALPINE_SKIP_BUILD=1 - printf 'No packages selected for rebuild: registry satisfies source metadata\n' + printf 'No packages selected for rebuild: registry has equal or newer package versions\n' fi if [[ -n "${GITHUB_ENV:-}" ]]; then @@ -170,7 +134,7 @@ prepare() { printf 'ALPINE_PACKAGES=%s\n' "${ALPINE_PACKAGES}" printf 'ALPINE_SKIP_BUILD=%s\n' "${ALPINE_SKIP_BUILD}" printf 'ALPINE_REGISTRY_URL=%s\n' "$(registry_url)" - printf 'ALPINE_REGISTRY_KEY=%s\n' "${registry_cache}/registry.rsa.pub" + printf 'ALPINE_REGISTRY_KEY=%s\n' "${registry_key_path}" } >> "${GITHUB_ENV}" fi if [[ -n "${GITHUB_OUTPUT:-}" ]]; then diff --git a/scripts/apk/ci-smoke.sh b/scripts/apk/ci-smoke.sh index 1c95c4c..4aa4f29 100755 --- a/scripts/apk/ci-smoke.sh +++ b/scripts/apk/ci-smoke.sh @@ -11,30 +11,27 @@ repo_dir="${repo_root}/packages/${repo_name}/${arch}" registry_url="${ALPINE_REGISTRY_URL:-}" registry_key="${ALPINE_REGISTRY_KEY:-}" read -r -a package_list <<< "${packages}" -declare -A versioned_packages=() +declare -A source_packages=() while IFS= read -r package_dir; do while IFS= read -r spec; do - versioned_packages["${spec%%=*}"]="${spec}" + source_packages["${spec%%=*}"]=1 done < <(apk_package_versioned_names "${package_dir}" "${arch}") done < <(apk_all_package_dirs "${repo_root}") for package in "${package_list[@]}"; do - if [[ -z "${versioned_packages[${package}]:-}" ]]; then + if [[ -z "${source_packages[${package}]:-}" ]]; then printf 'smoke package is not defined by a source APKBUILD: %s\n' "${package}" >&2 exit 2 fi - package="${versioned_packages[${package}]}" done if [[ -d "${repo_dir}" ]]; then cp "${repo_dir}"/*.rsa.pub /etc/apk/keys/ echo "${repo_root}/packages/${repo_name}" >> /etc/apk/repositories -else - if [[ -z "${registry_url}" ]]; then - printf 'missing local repository and no registry fallback: packages/%s/%s\n' "${repo_name}" "${arch}" >&2 - exit 1 - fi +fi + +if [[ -n "${registry_url}" ]]; then if [[ ! -f "${registry_key}" ]]; then printf 'missing Alpine registry signing key: %s\n' "${registry_key:-}" >&2 exit 1 @@ -43,9 +40,11 @@ else echo "${registry_url}" >> /etc/apk/repositories fi -for index in "${!package_list[@]}"; do - package_list[index]="${versioned_packages[${package_list[index]}]}" -done +if [[ ! -d "${repo_dir}" && -z "${registry_url}" ]]; then + printf 'missing local repository and no registry fallback: packages/%s/%s\n' "${repo_name}" "${arch}" >&2 + exit 1 +fi + apk update apk add "${package_list[@]}" if command -v weed >/dev/null 2>&1; then