Optimize APK update checks and CI builds
Build Alpine Packages / build-and-publish (v3.24, 3.24) (push) Failing after 2h20m30s
Build Alpine Packages / build-and-publish (v3.24, 3.24) (push) Failing after 2h20m30s
This commit is contained in:
Joachim Schlöffel
+94
-150
@@ -5,11 +5,29 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
# shellcheck source=scripts/apk/package-lib.sh
|
||||
. "${repo_root}/scripts/apk/package-lib.sh"
|
||||
|
||||
repo_name="${ALPINE_REPO_NAME:-local}"
|
||||
arches="${ALPINE_ARCHES:-x86_64 aarch64}"
|
||||
cache_root="${ALPINE_APK_REPO_CACHE:-/cache/public-alpine-packages/${ALPINE_REGISTRY_BRANCH:-local}/apk-repo}"
|
||||
repo_dest="${repo_root}/packages/${repo_name}"
|
||||
branch="${ALPINE_REGISTRY_BRANCH:-v${ALPINE_VERSION:-3.24}}"
|
||||
repository="${ALPINE_REGISTRY_REPOSITORY:-${PACKAGE_NAME:-main}}"
|
||||
instance_url="${INSTANCE_URL:-}"
|
||||
owner="${PACKAGE_OWNER:-}"
|
||||
command_name="${1:-prepare}"
|
||||
registry_cache="${CI_REGISTRY_CACHE_DIR:-${repo_root}/.cache/ci-registry}"
|
||||
|
||||
require_registry_config() {
|
||||
local name="$1" value="$2"
|
||||
[[ -n "${value}" ]] && return
|
||||
printf 'missing required CI registry setting: %s\n' "${name}" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
registry_url() {
|
||||
printf '%s/api/packages/%s/alpine/%s/%s\n' \
|
||||
"${instance_url%/}" "${owner}" "${branch}" "${repository}"
|
||||
}
|
||||
|
||||
registry_key_url() {
|
||||
printf '%s/api/packages/%s/alpine/key\n' "${instance_url%/}" "${owner}"
|
||||
}
|
||||
|
||||
all_packages() {
|
||||
while IFS= read -r package_dir; do
|
||||
@@ -18,9 +36,7 @@ all_packages() {
|
||||
}
|
||||
|
||||
mark_all_packages() {
|
||||
local reason="$1"
|
||||
|
||||
printf 'Rebuilding all packages: %s\n' "${reason}" >&2
|
||||
printf 'Rebuilding all packages: %s\n' "$1" >&2
|
||||
all_packages
|
||||
}
|
||||
|
||||
@@ -30,34 +46,23 @@ diff_base() {
|
||||
|
||||
if [[ -n "${base_ref}" ]] && git rev-parse --verify "origin/${base_ref}" >/dev/null 2>&1; then
|
||||
printf '%s...HEAD\n' "origin/${base_ref}"
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ -n "${before}" && ! "${before}" =~ ^0+$ ]] && git rev-parse --verify "${before}^{commit}" >/dev/null 2>&1; then
|
||||
elif [[ -n "${before}" && ! "${before}" =~ ^0+$ ]] && git rev-parse --verify "${before}^{commit}" >/dev/null 2>&1; then
|
||||
printf '%s..HEAD\n' "${before}"
|
||||
return
|
||||
fi
|
||||
|
||||
if git rev-parse --verify "HEAD^" >/dev/null 2>&1; then
|
||||
elif git rev-parse --verify HEAD^ >/dev/null 2>&1; then
|
||||
printf 'HEAD^..HEAD\n'
|
||||
return
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
changed_packages() {
|
||||
local base
|
||||
local changed_paths
|
||||
local changed_path
|
||||
local package
|
||||
local base changed_paths changed_path package
|
||||
declare -A selected=()
|
||||
|
||||
if ! base="$(diff_base)"; then
|
||||
mark_all_packages "no usable git diff base"
|
||||
return
|
||||
fi
|
||||
|
||||
if ! changed_paths="$(git diff --name-only "${base}")"; then
|
||||
mark_all_packages "git diff failed for ${base}"
|
||||
return
|
||||
@@ -72,8 +77,7 @@ changed_packages() {
|
||||
;;
|
||||
packaging/alpine/local/*)
|
||||
package="${changed_path#packaging/alpine/local/}"
|
||||
package="${package%%/*}"
|
||||
selected["${package}"]=1
|
||||
selected["${package%%/*}"]=1
|
||||
;;
|
||||
esac
|
||||
done <<< "${changed_paths}"
|
||||
@@ -83,167 +87,107 @@ changed_packages() {
|
||||
done | sort
|
||||
}
|
||||
|
||||
restore_cache() {
|
||||
if [[ -d "${cache_root}" ]]; then
|
||||
mkdir -p "${repo_dest}"
|
||||
cp -a "${cache_root}/." "${repo_dest}/"
|
||||
printf 'Restored Alpine package repository cache from %s\n' "${cache_root}"
|
||||
else
|
||||
printf 'No Alpine package repository cache found at %s\n' "${cache_root}"
|
||||
fi
|
||||
download_registry_metadata() {
|
||||
local arch url
|
||||
|
||||
require_registry_config INSTANCE_URL "${instance_url}"
|
||||
require_registry_config PACKAGE_OWNER "${owner}"
|
||||
mkdir -p "${registry_cache}"
|
||||
url="$(registry_url)"
|
||||
for arch in ${arches}; do
|
||||
curl --fail --silent --show-error --location \
|
||||
--output "${registry_cache}/${arch}.APKINDEX.tar.gz" \
|
||||
"${url}/${arch}/APKINDEX.tar.gz"
|
||||
done
|
||||
curl --fail --silent --show-error --location \
|
||||
--output "${registry_cache}/registry.rsa.pub" \
|
||||
"$(registry_key_url)"
|
||||
}
|
||||
|
||||
package_apk_names() {
|
||||
local package_dir="$1"
|
||||
local arch="$2"
|
||||
|
||||
apk_package_apk_names "${package_dir}" "${arch}"
|
||||
}
|
||||
|
||||
package_cache_complete() {
|
||||
local package_dir="$1"
|
||||
local package_name
|
||||
local arch
|
||||
local apk_name
|
||||
|
||||
package_name="$(apk_package_name "${package_dir}")"
|
||||
load_registry_index() {
|
||||
local arch package_name package_version
|
||||
declare -gA registry_packages=()
|
||||
|
||||
for arch in ${arches}; do
|
||||
case "${arch}" in
|
||||
x86_64|aarch64) ;;
|
||||
*) printf 'unsupported Alpine architecture: %s\n' "${arch}" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
if ! apk_package_supports_arch "${package_dir}" "${arch}"; then
|
||||
continue
|
||||
fi
|
||||
|
||||
while IFS= read -r apk_name; do
|
||||
if [[ ! -f "${repo_dest}/${arch}/${apk_name}" ]]; then
|
||||
printf 'Cache miss for %s: missing %s/%s\n' "${package_name}" "${arch}" "${apk_name}" >&2
|
||||
return 1
|
||||
fi
|
||||
done < <(package_apk_names "${package_dir}" "${arch}")
|
||||
while IFS=$'\t' read -r package_name package_version; do
|
||||
[[ -n "${package_name}" && -n "${package_version}" ]] || continue
|
||||
registry_packages["${arch}|${package_name}|${package_version}"]=1
|
||||
done < <(
|
||||
tar -xOzf "${registry_cache}/${arch}.APKINDEX.tar.gz" APKINDEX | awk '
|
||||
/^P:/ { package = substr($0, 3); next }
|
||||
/^V:/ { if (package != "") print package "\t" substr($0, 3); package = "" }
|
||||
'
|
||||
)
|
||||
done
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
select_missing_packages() {
|
||||
local package_dir
|
||||
local package_name
|
||||
registry_missing_packages() {
|
||||
local package_dir package_name arch spec spec_name spec_version missing
|
||||
|
||||
while IFS= read -r package_dir; do
|
||||
apk_validate_package_dir "${package_dir}"
|
||||
package_name="$(apk_package_name "${package_dir}")"
|
||||
if ! package_cache_complete "${package_dir}"; then
|
||||
printf '%s\n' "${package_name}"
|
||||
fi
|
||||
done < <(apk_all_package_dirs "${repo_root}")
|
||||
}
|
||||
|
||||
prune_stale_apks() {
|
||||
local package_dir
|
||||
local arch
|
||||
local apk_name
|
||||
local current_apk
|
||||
declare -A expected_apks=()
|
||||
|
||||
if [[ ! -d "${repo_dest}" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
while IFS= read -r package_dir; do
|
||||
apk_validate_package_dir "${package_dir}"
|
||||
missing=0
|
||||
for arch in ${arches}; do
|
||||
case "${arch}" in
|
||||
x86_64|aarch64) ;;
|
||||
*) printf 'unsupported Alpine architecture: %s\n' "${arch}" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
if ! apk_package_supports_arch "${package_dir}" "${arch}"; then
|
||||
continue
|
||||
fi
|
||||
|
||||
while IFS= read -r apk_name; do
|
||||
expected_apks["${repo_dest}/${arch}/${apk_name}"]=1
|
||||
done < <(package_apk_names "${package_dir}" "${arch}")
|
||||
apk_package_supports_arch "${package_dir}" "${arch}" || continue
|
||||
while IFS= read -r spec; do
|
||||
spec_name="${spec%%=*}"
|
||||
spec_version="${spec#*=}"
|
||||
if [[ -z "${registry_packages[${arch}|${spec_name}|${spec_version}]:-}" ]]; then
|
||||
missing=1
|
||||
printf 'Registry miss: %s (%s, %s)\n' "${package_name}" "${arch}" "${spec}" >&2
|
||||
fi
|
||||
done < <(apk_package_versioned_names "${package_dir}" "${arch}")
|
||||
done
|
||||
[[ "${missing}" == 1 ]] && printf '%s\n' "${package_name}"
|
||||
done < <(apk_all_package_dirs "${repo_root}")
|
||||
|
||||
shopt -s nullglob
|
||||
for current_apk in "${repo_dest}"/*/*.apk; do
|
||||
if [[ -z "${expected_apks[${current_apk}]:-}" ]]; then
|
||||
rm -f "${current_apk}"
|
||||
printf 'Pruned stale cached APK: %s\n' "${current_apk#"${repo_root}/"}"
|
||||
fi
|
||||
done
|
||||
shopt -u nullglob
|
||||
}
|
||||
|
||||
prepare() {
|
||||
local selected_packages
|
||||
local selected
|
||||
declare -A selected_map=()
|
||||
|
||||
restore_cache
|
||||
prune_stale_apks
|
||||
|
||||
download_registry_metadata
|
||||
load_registry_index
|
||||
while IFS= read -r selected; do
|
||||
[[ -n "${selected}" ]] || continue
|
||||
selected_map["${selected}"]=1
|
||||
[[ -n "${selected}" ]] && selected_map["${selected}"]=1
|
||||
done < <(changed_packages)
|
||||
|
||||
while IFS= read -r selected; do
|
||||
[[ -n "${selected}" ]] || continue
|
||||
selected_map["${selected}"]=1
|
||||
done < <(select_missing_packages)
|
||||
[[ -n "${selected}" ]] && selected_map["${selected}"]=1
|
||||
done < <(registry_missing_packages)
|
||||
|
||||
selected_packages="$(
|
||||
for selected in "${!selected_map[@]}"; do
|
||||
printf '%s\n' "${selected}"
|
||||
done | sort | xargs
|
||||
)"
|
||||
|
||||
if [[ -n "${selected_packages}" ]]; then
|
||||
printf 'Selected packages for rebuild: %s\n' "${selected_packages}"
|
||||
export ALPINE_PACKAGES="${selected_packages}"
|
||||
export ALPINE_SKIP_BUILD=0
|
||||
ALPINE_PACKAGES="$(for selected in "${!selected_map[@]}"; do printf '%s\n' "${selected}"; done | sort | xargs)"
|
||||
if [[ -n "${ALPINE_PACKAGES}" ]]; then
|
||||
ALPINE_SKIP_BUILD=0
|
||||
printf 'Selected packages for rebuild: %s\n' "${ALPINE_PACKAGES}"
|
||||
else
|
||||
printf 'No packages selected for rebuild\n'
|
||||
export ALPINE_PACKAGES=""
|
||||
export ALPINE_SKIP_BUILD=1
|
||||
ALPINE_SKIP_BUILD=1
|
||||
printf 'No packages selected for rebuild: registry satisfies source metadata\n'
|
||||
fi
|
||||
|
||||
if [[ -n "${GITHUB_ENV:-}" ]]; then
|
||||
{
|
||||
printf 'ALPINE_PACKAGES=%s\n' "${ALPINE_PACKAGES}"
|
||||
printf 'ALPINE_SKIP_BUILD=%s\n' "${ALPINE_SKIP_BUILD}"
|
||||
printf 'ALPINE_REGISTRY_URL=%s\n' "$(registry_url)"
|
||||
printf 'ALPINE_REGISTRY_KEY=%s\n' "${registry_cache}/registry.rsa.pub"
|
||||
} >> "${GITHUB_ENV}"
|
||||
fi
|
||||
}
|
||||
|
||||
sync_cache() {
|
||||
if [[ ! -d "${repo_dest}" ]]; then
|
||||
printf 'missing package repository: %s\n' "${repo_dest}" >&2
|
||||
exit 1
|
||||
if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
|
||||
if [[ -n "${ALPINE_PACKAGES}" ]]; then
|
||||
printf 'packages_selected=true\n' >> "${GITHUB_OUTPUT}"
|
||||
else
|
||||
printf 'packages_selected=false\n' >> "${GITHUB_OUTPUT}"
|
||||
fi
|
||||
if [[ " ${ALPINE_PACKAGES} " == *" greptimedb "* ]]; then
|
||||
printf 'greptimedb_selected=true\n' >> "${GITHUB_OUTPUT}"
|
||||
else
|
||||
printf 'greptimedb_selected=false\n' >> "${GITHUB_OUTPUT}"
|
||||
fi
|
||||
fi
|
||||
|
||||
prune_stale_apks
|
||||
mkdir -p "${cache_root}"
|
||||
rsync -a --delete "${repo_dest}/" "${cache_root}/"
|
||||
printf 'Updated Alpine package repository cache at %s\n' "${cache_root}"
|
||||
}
|
||||
|
||||
case "${command_name}" in
|
||||
prepare)
|
||||
prepare
|
||||
;;
|
||||
sync)
|
||||
sync_cache
|
||||
;;
|
||||
*)
|
||||
printf 'usage: %s [prepare|sync]\n' "$0" >&2
|
||||
exit 2
|
||||
;;
|
||||
prepare) prepare ;;
|
||||
*) printf 'usage: %s prepare\n' "$0" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
Reference in New Issue
Block a user