From ee5cdeb7b83f42e7ff392f38fa418781959f0eba Mon Sep 17 00:00:00 2001 From: termux-pacman-bot Date: Tue, 27 Jan 2026 01:34:55 +0000 Subject: [PATCH] Update repo --- scripts/bin/update-packages | 374 +++++++++++------- .../updates/internal/termux_github_graphql.sh | 73 +++- .../utils/termux_pkg_upgrade_version.sh | 46 +-- 3 files changed, 302 insertions(+), 191 deletions(-) diff --git a/scripts/bin/update-packages b/scripts/bin/update-packages index a9aba9602b..1b18b53baa 100755 --- a/scripts/bin/update-packages +++ b/scripts/bin/update-packages @@ -15,11 +15,13 @@ set -u : "${TERMUX_ARCH:="aarch64"}" export TERMUX_PKG_UPDATE_METHOD="" # Which method to use for updating? (repology, github or gitlab) -export TERMUX_PKG_UPDATE_TAG_TYPE="" # Whether to use latest-release-tag or newest-tag. +export TERMUX_PKG_UPDATE_TAG_TYPE="" # Whether to use latest-release-tag, latest-regex or newest-tag. export TERMUX_PKG_AUTO_UPDATE=false # Whether to auto-update or not. Disabled by default. export TERMUX_PKG_UPDATE_VERSION_REGEXP="" # Regexp to extract version with `grep -oP`. export TERMUX_PKG_UPDATE_VERSION_SED_REGEXP="" # Regexp to extract version with `sed`. export TERMUX_REPOLOGY_DATA_FILE +# Clean up any old repology data files +rm "${TMPDIR:-/tmp}/termux-repology".* &> /dev/null TERMUX_REPOLOGY_DATA_FILE="$(mktemp -t termux-repology.XXXXXX)" # File to store repology data. export TERMUX_SCRIPTDIR @@ -84,16 +86,18 @@ TERMUX_PACKAGES_DIRECTORIES=$(jq --raw-output 'del(.pkg_format) | keys | .[]' "$ # Converts milliseconds to human-readable format. # Example: `ms_to_human_readable 123456789` => 34h 17m 36s 789ms ms_to_human_readable() { - echo "$(($1/3600000))h $(($1%3600000/60000))m $(($1%60000/1000))s $(($1%1000))ms" | sed 's/0h //;s/0m //;s/0s //' + local t="$1" hrs=0 mins=0 secs=0 ms=0 + (( ms = t%1000, t /= 1000, secs = t%60, t /= 60, mins = t%60, hrs = t/60 )) + echo "${hrs}h ${mins}m ${secs}s ${ms}ms" | sed 's/0h //;s/0m //;s/0s //' } # Runs a command without displaying its output in the case if trace is disabled and displays output in the case if it is enabled. # Needed only for debugging quiet() { - if [[ "$-" =~ x ]]; then + if [[ "$-" == *"x"* ]]; then "$@" else - &>/dev/null "$@" + "$@" &> /dev/null fi return $? } @@ -107,8 +111,7 @@ _update() { IFS="," read -r -a EXCLUDED_ARCH <<< "${TERMUX_PKG_EXCLUDED_ARCHES:-}" export TERMUX_ARCH="" # Arch to test updates. for arch in aarch64 arm i686 x86_64; do - # shellcheck disable=SC2076 - if [[ ! " ${EXCLUDED_ARCH[*]} " =~ " ${arch} " ]]; then + if [[ " ${EXCLUDED_ARCH[*]} " != *" ${arch} "* ]]; then TERMUX_ARCH="${arch}" break fi @@ -123,7 +126,10 @@ _update() { echo # Newline. echo "INFO: Updating ${TERMUX_PKG_NAME} [Current version: ${TERMUX_PKG_VERSION}]" - termux_pkg_auto_update + # Throw FD 3 into the FIFO for reporting + exec {IPC_FIFO_FD}<>"$IPC_FIFO" + termux_pkg_auto_update 3>& "${IPC_FIFO_FD}" + exec {IPC_FIFO_FD}>&- } declare -A _LATEST_TAGS=() @@ -165,30 +171,29 @@ _fetch_and_cache_tags() { # Ignore non-constant sources # shellcheck disable=SC1091 for pkg_dir in "${__GITHUB_PACKAGES[@]}"; do - local PKG_SRCURL TAG_TYPE OWNER REPO - read -r PKG_SRCURL TAG_TYPE < <( - set +u + local PKG_SRCURL="" TAG_TYPE="" REGEXP="" + read -r PKG_SRCURL TAG_TYPE REGEXP < <( + set +eu source "${pkg_dir}/build.sh" - echo "${TERMUX_PKG_SRCURL} ${TERMUX_PKG_UPDATE_TAG_TYPE}" + echo "${TERMUX_PKG_SRCURL}" "${TERMUX_PKG_UPDATE_TAG_TYPE:-"none"}" "${TERMUX_PKG_UPDATE_VERSION_REGEXP:-}" ) - IFS=/ read -r OWNER REPO _ <<< "${PKG_SRCURL#*://github.com/}" + local user="" repo="" + IFS=/ read -r user repo _ <<< "${PKG_SRCURL#*://github.com/}" - if [[ -z "${TAG_TYPE}" ]]; then # If not set, then decide on the basis of url. + if [[ "${TAG_TYPE}" == "none" ]]; then # If not set, then decide on the basis of url. if [[ "${PKG_SRCURL:0:4}" == "git+" ]]; then - # Get newest tag. - TAG_TYPE="newest-tag" + TAG_TYPE="newest-tag" # Get newest tag. + elif [[ -n "$REGEXP" ]]; then + TAG_TYPE="latest-regex" # Get the latest release tag. else - # Get the latest release tag. - TAG_TYPE="latest-release-tag" + TAG_TYPE="latest-release-tag" # Get the latest release tag. fi fi # We prepare the query snippets for `termux_github_graphql` here # since we already have the needed information available. - GITHUB_GRAPHQL_QUERIES+=( "_$((COUNTER++)): repository(owner: \\\"${OWNER}\\\", name: \\\"${REPO}\\\") { ..._${TAG_TYPE//-/_} }" ) - - unset PKG_SRCURL TAG_TYPE OWNER REPO + GITHUB_GRAPHQL_QUERIES+=( "_$((COUNTER++)): repository(owner: \\\"${user}\\\", name: \\\"${repo}\\\") { ..._${TAG_TYPE//-/_} }" ) done # This is called indirectly in subshell @@ -210,7 +215,7 @@ _fetch_and_cache_tags() { termux_github_api_get_tag() { local ver="${TERMUX_PKG_VERSION#*:}" - echo "LOCAL|${TERMUX_PKG_NAME}|${ver#v}" + echo "LOCAL|${TERMUX_PKG_NAME}|${ver}" exit 0 } termux_pkg_upgrade_version() { @@ -234,114 +239,174 @@ _fetch_and_cache_tags() { } echo "$([[ "${CI-false}" == "true" ]] && echo "::group::" || :)INFO: Fetching GitHub packages via GraphQL API" - LATEST_TAGS_GITHUB="$( - termux_github_graphql "${GITHUB_GRAPHQL_QUERIES[@]}" - )" + + local line + local -a LATEST_TAGS=() LATEST_TAGS_GITHUB=() + + while IFS='' read -r line; do + case "$line" in + # The entries can be single or multi-line + # but each new entry starts with 'GIT|' + 'GIT|'*) LATEST_TAGS_GITHUB+=("$line");; + # If the line didn't start a new entry, append it to the previous entry. + *) LATEST_TAGS_GITHUB[-1]+=$'\n'"$line";; + esac + done < <(termux_github_graphql "${GITHUB_GRAPHQL_QUERIES[@]}") + [[ "${CI-false}" == "true" ]] && echo "::endgroup::" || : echo "INFO: Fetching non-GitHub packages" - local LATEST_TAGS='' - LATEST_TAGS="$( + while read -r line; do + case "$line" in + # The entries can be single or multi-line + # but each new entry starts with 'GIT|' + 'LOCAL|'*|'SKIP|'*|'PKG|'*) LATEST_TAGS+=("$line");; + # If the line didn't start a new entry, append it to the previous entry. + *) LATEST_TAGS[-1]+=$'\n'"$line";; + esac + # Provide a running tally for local runs. + # Don't do this in the CI since that would spam the logs + [[ -z "${CI:-}" ]] && printf '\rINFO: Caching tags - %s' "${#LATEST_TAGS[*]}" + done < <( export -f __main__ export TERMUX_SCRIPTDIR GITHUB_TOKEN TERMUX_REPOLOGY_DATA_FILE printf '%s\0' "${__PACKAGES[@]}" | xargs -0 -n1 -P"$(nproc)" bash -c '__main__ "$@"' _ |& grep -E '^(LOCAL|PKG|SKIP)\|' - )" + ) + # make sure the running tally gets a proper newline on local runs + [[ -z "${CI:-}" ]] && echo + unset -f __main__ - declare -A __GITHUB_CURRENT_TAGS=() + local -A __GITHUB_CURRENT_TAGS=() [[ "${CI-false}" == "true" ]] && echo "::group::INFO: Skipping the following up-to-date packages" || : - while IFS='|' read -r type pkg version; do - if [[ "${type}" == "LOCAL" ]]; then # Current git-originated package version - __GITHUB_CURRENT_TAGS["${pkg:-_}"]="$version" - continue - fi - _LATEST_TAGS["${pkg:-_}"]="$version" - if [[ "${type}" == "SKIP" ]] || [[ "${type}" == "GIT" && "${__GITHUB_CURRENT_TAGS["${pkg:-}"]:-}" == "${version}" ]]; then - echo "INFO: Skipping ${pkg}: already at version ${version}" - : "${_ALREADY_SEEN["${pkg}"]:=""}" - fi - done < <(printf '%s\n' "$LATEST_TAGS" "$LATEST_TAGS_GITHUB") + local pkg pkg_type pkg_name pkg_version + for pkg in "${LATEST_TAGS[@]}" "${LATEST_TAGS_GITHUB[@]}"; do + # local - + # PS4='+$0 \[\e[32m\]${FUNCNAME[0]:-}${FUNCNAME[*]:+()}:$LINENO\[\e[0m\] '; set -x + + pkg_type="${pkg%%|*}" + pkg_name="${pkg#*|}"; pkg_name="${pkg_name%|*}" + pkg_version="${pkg##*|}" + case "${pkg_type}" in + "LOCAL") # Local package version for a GitHub GraphQL request. + __GITHUB_CURRENT_TAGS["${pkg_name:-_}"]="$pkg_version" + continue + ;; + "GIT") # GitHub GraphQL results, can be skipped if it matches the local version. + if [[ "${__GITHUB_CURRENT_TAGS["${pkg_name:-}"]:-}" == "${pkg_version}" ]]; then + echo "INFO: Skipping ${pkg_name}: already at version ${pkg_version}" + : "${_ALREADY_SEEN["${pkg_name}"]:=""}" + fi + ;; + "SKIP") # SKIP means we know package is up to date and we can safely skip checks. + : "${_ALREADY_SEEN["${pkg_name}"]:=""}" + echo "INFO: Skipping ${pkg_name}: already at version ${pkg_version}" + ;; + esac + _LATEST_TAGS["${pkg_name:-_}"]="$pkg_version" + done + [[ "${CI-false}" == "true" ]] && echo "::endgroup::" || : - unset __GITHUB_CURRENT_TAGS + + # Sum up results + echo "INFO: Fetched ${#LATEST_TAGS[*]} tags." + echo "INFO: GitHub - ${#LATEST_TAGS_GITHUB[*]}" + echo "INFO: Other - $(( ${#LATEST_TAGS[*]} - ${#LATEST_TAGS_GITHUB[*]} ))" + echo "INFO: Cached - ${#_ALREADY_SEEN[*]}" } _check_updated() { - if [[ -n "${_LATEST_TAGS[${1##*/}]:-}" ]]; then - ( - set +eu - quiet source "${1}/build.sh" - set -eu - export TERMUX_PKG_UPGRADE_VERSION_DRY_RUN=1 - if quiet termux_pkg_upgrade_version "${_LATEST_TAGS[${1##*/}]}"; then - echo "INFO: Skipping ${1##*/}: already at version ${TERMUX_PKG_VERSION#*:}" - return 0 - fi - return 1 - ) - local _ANSWER=$? - if (( _ANSWER == 0 )) ; then - : "${_ALREADY_SEEN["$(basename "${1}")"]:=""}" - fi - return $_ANSWER + local latest_tag="${_LATEST_TAGS[${1##*/}]:-}" + + if [[ -z "${latest_tag}" ]]; then + return 1 + fi + + ( + set +eu + quiet source "${1}/build.sh" + TERMUX_PKG_NAME="${1##*/}" + export TERMUX_PKG_UPGRADE_VERSION_DRY_RUN=1 + # The call to `termux_pkg_upgrade_version` is delibrately + # not wrapped by `quiet` to let warnings on stderr bubble up. + if termux_pkg_upgrade_version "${latest_tag}" > /dev/null; then + echo "INFO: Skipping ${1##*/}: already at version ${TERMUX_PKG_VERSION}" + : "${_ALREADY_SEEN["${TERMUX_PKG_NAME}"]:=""}" + return 0 fi return 1 + ) } -_run_update() { - # PS4='+$0 \[\e[32m\]${FUNCNAME[0]:-}${FUNCNAME[*]:+()}:$LINENO\[\e[0m\] ' - # set -x - local pkg_dir="$1" pkg_name - pkg_name="$(basename "${pkg_dir}")" - # Run each package update in separate process since we include their environment variables. - local output="" _CURRENT_VERSION _NEWEST_VERSION - : > "$LATEST_VERSION_TEMP_FILE" # flush the sidechannel - # shellcheck disable=SC1091 # Ignore shellcheck's inability to follow the source - _CURRENT_VERSION="$(set +eu; . "$(realpath "$pkg_dir")/build.sh"; echo "${TERMUX_PKG_VERSION#*:}")" - - if output="$( - set -euo pipefail - # Pass cached tag we obtained earlier - [[ -n "${_LATEST_TAGS[${pkg_dir##*/}]:-}" ]] && export __CACHED_TAG="${_LATEST_TAGS[${pkg_dir##*/}]}" || : - exec > >(tee /dev/fd/2) 2>&1 # output everything on stderr to stdout as well. - _update "${pkg_dir}" - )"; then - _NEWEST_VERSION="$(< "$LATEST_VERSION_TEMP_FILE")" - : "${_NEWEST_VERSION:="$_CURRENT_VERSION"}" - if dpkg --compare-versions "${_CURRENT_VERSION}" lt "${_NEWEST_VERSION}"; then - _ALREADY_SEEN["$pkg_name"]="${pkg_name} ${_CURRENT_VERSION} => ${_NEWEST_VERSION}" - elif dpkg --compare-versions "${_CURRENT_VERSION}" gt "${_NEWEST_VERSION}"; then - _ALREADY_SEEN["$pkg_name"]="${pkg_name} ${_CURRENT_VERSION} => ${_NEWEST_VERSION} (Error? current version is larger than latest fetched version)" - else - _ALREADY_SEEN["$pkg_name"]="" - fi - else - _FAILED_UPDATES["$pkg_name"]="${output}" - fi -} - -declare -a _CACHED_ISSUE_TITLES=() +declare -A _CACHED_ISSUES=() # Check if an issue with same title already exists and is open. _gh_check_issue_exists() { - local pkg_name="$1" + local pkg_name="$1" number title if [[ -z "${_CACHED_ISSUE_TITLES[*]}" ]]; then - while read -r title; do - _CACHED_ISSUE_TITLES+=("'${title}'") # An extra quote ('') is added to avoid false positive matches. + while read -r number title; do + _CACHED_ISSUES["$number"]="'${title}'" # An extra quote ('') is added to avoid false positive matches. done < <( gh issue list \ --limit 10000 \ --label "auto update failing" --label "bot" \ --state open \ --search "Auto update failing for in:title type:issue" \ - --json title | jq -r '.[] | .title' | sort | uniq + --json number,title | jq -r '.[] | "\(.number) \(.title)"' | sort | uniq ) fi - # shellcheck disable=SC2076 # We want literal match here, not regex based. - if [[ "${_CACHED_ISSUE_TITLES[*]}" =~ "'Auto update failing for ${pkg_name}'" ]]; then - return 0 - fi + + for number in "${!_CACHED_ISSUES[@]}"; do + # shellcheck disable=SC2076 # We want literal match here, not regex based. + if [[ "${_CACHED_ISSUES[$number]}" =~ "'Auto update failing for ${pkg_name}'" ]]; then + echo "$number" + return 0 + fi + done return 1 } +_run_update() { + local pkg_dir="$1" pkg_name + pkg_name="$(basename "${pkg_dir}")" + # Run each package update in separate process since we include their environment variables. + local output="" _CURRENT_VERSION _NEWEST_VERSION _ISSUE + # shellcheck disable=SC1091 # Ignore shellcheck's inability to follow the source + _CURRENT_VERSION="$(set +eu; . "$(realpath "$pkg_dir")/build.sh"; echo "${TERMUX_PKG_VERSION#*:}")" + + if output="$( + set -euo pipefail + # Pass cached tag we obtained earlier + [[ -n "${_LATEST_TAGS[${pkg_name}]:-}" ]] && export __CACHED_TAG="${_LATEST_TAGS[${pkg_name}]}" || : + exec > >(tee /dev/fd/2) 2>&1 # output everything on stderr to stdout as well. + _update "${pkg_dir}" + )"; then + # For whatever reason reads from the FIFO are not instant even if there is data. + # So put a 1ms timeout on it that way it doesn't wait infinitely if there's nothing in the FIFO, + # since it only receives new data if `termux_pkg_upgrade_version()` ran. + read -rt0.001 _NEWEST_VERSION < "$IPC_FIFO" + # If we didn't get a version back treat it as current. + # This mainly happens when Repology doesn't report a newer version for a package. + # Which is returned as `null` in the API response, and which we treat as a skip. + # See: scripts/updates/internal/termux_repology_auto_update.sh + : "${_NEWEST_VERSION:="${_CURRENT_VERSION}"}" + + # Check associated issues if able. + if [[ -n "$GITHUB_TOKEN" ]] && command -v gh &> /dev/null; then + _ISSUE="$(_gh_check_issue_exists "${pkg_name}")" + fi + + if termux_pkg_is_update_needed "${_CURRENT_VERSION}" "${_NEWEST_VERSION}"; then + _ALREADY_SEEN["$pkg_name"]="${pkg_name} ${_CURRENT_VERSION} => ${_NEWEST_VERSION}${_ISSUE:+" - Open issue: #${_ISSUE}"}" + elif termux_pkg_is_update_needed "${_NEWEST_VERSION}" "${_CURRENT_VERSION}"; then + _ALREADY_SEEN["$pkg_name"]="${pkg_name} ${_CURRENT_VERSION} => ${_NEWEST_VERSION}${_ISSUE:+" - Open issue: #${_ISSUE}"} (Error? current version is larger than latest fetched version)" + else + _ALREADY_SEEN["$pkg_name"]="" + fi + else + _FAILED_UPDATES["$pkg_name"]="${output}" + fi +} + _should_update() { local pkg_dir="$1" @@ -357,12 +422,13 @@ _should_update() { *) [[ $'\n'"$(<"$pkg_dir/build.sh")"$'\n' == *$'\n'TERMUX_PKG_AUTO_UPDATE=true$'\n'* ]] || return 1 ;; # Skip. esac - # shellcheck disable=SC2076 - if [[ " ${!_ALREADY_SEEN[*]} ${!_FAILED_UPDATES[*]} " =~ " $(basename "${pkg_dir}") " ]]; then + if [[ " ${!_ALREADY_SEEN[*]} ${!_FAILED_UPDATES[*]} " == *" ${pkg_dir##*/} "* ]]; then return 1 # Skip. fi - if [[ "${BUILD_PACKAGES}${GITHUB_ACTIONS:-}" == "truetrue" ]] && _gh_check_issue_exists "$(basename "${pkg_dir}")"; then - echo "INFO: Skipping '$(basename "${pkg_dir}")', an update issue for it hasn't been resolved yet." + + local _ISSUE + if [[ "${BUILD_PACKAGES}${GITHUB_ACTIONS:-}" == "truetrue" ]] && _ISSUE="$(_gh_check_issue_exists "${pkg_dir##*/}")"; then + echo "INFO: Skipping '${pkg_dir##*/}', an update issue (#${_ISSUE}) for it hasn't been resolved yet." return 1 fi @@ -379,75 +445,87 @@ _update_dependencies() { fi # shellcheck disable=SC2086 # Allow splitting of TERMUX_PACKAGES_DIRECTORIES. while read -r dep dep_dir; do - if [[ -z $dep ]]; then + if [[ -z "${dep}" ]]; then continue elif [[ "${dep}" == "ERROR" ]]; then - termux_error_exit "Obtaining update order failed for $(basename "${pkg_dir}")" + termux_error_exit "Obtaining update order failed for ${pkg_dir##*/}" fi - _should_update "${dep_dir}" && ! _check_updated "${dep_dir}" && _run_update "${dep_dir}" + _should_update "${dep_dir}" && ! _check_updated "${dep_dir}" && _run_update "${dep_dir}" done < <("${TERMUX_SCRIPTDIR}"/scripts/buildorder.py "${pkg_dir}" $TERMUX_PACKAGES_DIRECTORIES || echo "ERROR") } echo "INFO: Running update for: $*" _start_time="$(date +%10s%3N)" -# Remove any left over temp files that may not have been cleaned up. -rm "${TMPDIR:-/tmp}"/termux_ipc_*.fifo &> /dev/null -# Set up a side channel for reporting parsed latest versions -export LATEST_VERSION_TEMP_FILE="${TMPDIR:-/tmp}/termux_ipc_$$.fifo" + +# Remove any leftover FIFOs that may not have been cleaned up. +rm "${TMPDIR:-/tmp}/termux_ipc_"*.fifo &> /dev/null +# Set up a side channel FIFO for reporting parsed latest versions +export IPC_FIFO="${TMPDIR:-/tmp}/termux_ipc_$$.fifo" +mkfifo "$IPC_FIFO" +exec {IPC_FIFO_FD}<>"$IPC_FIFO" if [[ "$1" == "@all" ]]; then + declare -a PACKAGE_LIST=() + # To conserve API limits and speed up update checking, + # pre-cache the latest tags and eliminate packages without updates + # when it is possible to determine that from the API response alone. _fetch_and_cache_tags - for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do - for pkg_dir in "${repo_dir}"/*; do - _unix_millis="$(date +%10s%3N)" - ! _should_update "${pkg_dir}" && continue # Skip if not needed. - _check_updated "${pkg_dir}" && continue # Skip if already up-to-date. - _update_dependencies "${pkg_dir}" # Update all its dependencies first. - # NOTE: We are not checking whether dependencies were updated successfully or not. - # There is no way to know whether this package will build with current - # available versions of its dependencies or needs new ones. - # So, whatever the case may be. We just need packages to be updated in order - # and not care about anything else in between. If something fails to update, - # it will be reported by failure handling code, so no worries. - _run_update "${pkg_dir}" - echo "termux - took $(ms_to_human_readable $(( $(date +%10s%3N) - _unix_millis )))" + for channel in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do + for pkg_dir in "${channel}"/*; do + # Check if we should be putting this package into the list. + # Skip it if: + # - it doesn't have auto-updates enabled + # - it was already cached by _fetch_and_cache_tags + # - or there is an open auto-update issue for it + _should_update "${pkg_dir}" || continue + PACKAGE_LIST+=("$pkg_dir") done done -else - for pkg in "$@"; do - _unix_millis="$(date +%10s%3N)" - if [ ! -d "${pkg}" ]; then # If only package name is given, try to find it's directory. - for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do - if [ -d "${repo_dir}/${pkg}" ]; then - pkg="${repo_dir}/${pkg}" - break - fi - done - fi - # Here `pkg` is a directory. - ! _should_update "${pkg}" && continue - _update_dependencies "${pkg}" - _run_update "${pkg}" - echo "termux - took $(ms_to_human_readable $(( $(date +%10s%3N) - _unix_millis )))" - done + set -- "${PACKAGE_LIST[@]}" + unset PACKAGE_LIST fi -# Remove temp file -rm "${TMPDIR:-/tmp}"/termux_ipc_*.fifo +for pkg_dir in "$@"; do + _unix_millis="$(date +%10s%3N)" + if [[ ! -d "${pkg_dir}" ]]; then # If only package name is given, try to find it's directory. + for channel in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do + if [[ -d "${channel}/${pkg_dir}" ]]; then + pkg_dir="${channel}/${pkg_dir}" + break + fi + done + fi + + ! _should_update "${pkg_dir}" && continue + _update_dependencies "${pkg_dir}" + _run_update "${pkg_dir}" + echo "termux - took $(ms_to_human_readable $(( $(date +%10s%3N) - _unix_millis )))" +done + +exec {IPC_FIFO_FD}<&- # close the FIFO +# Clean up our FIFO and repology data file +rm "$IPC_FIFO" "$TERMUX_REPOLOGY_DATA_FILE" #################################################REPORT CHANGES################################################## +total="${#_ALREADY_SEEN[*]}" +direct="$#" +indirect="$(( total - direct ))" + echo "" -echo "INFO: Checked ${#_ALREADY_SEEN[*]} packages - took $(ms_to_human_readable $(( $(date +%10s%3N) - _start_time ))) in total." -for _pkg in "${_ALREADY_SEEN[@]}"; do - [[ -n "$_pkg" ]] && echo "$_pkg" -done +echo "SUMMARY: Checked ${total} packages - took $(ms_to_human_readable $(( $(date +%10s%3N) - _start_time ))) in total." +echo "SUMMARY: ${direct} (Directly passed), ${indirect} (Dependencies)" echo "" -echo "INFO: ${#_FAILED_UPDATES[*]} failed updates." +echo "==================================================" +echo "" +echo "SUMMARY: Detected updates" +printf '%s\n' "${_ALREADY_SEEN[@]}" | grep -ve '^$' | sort +echo "" +echo "SUMMARY: ${#_FAILED_UPDATES[*]} failed updates." echo "${!_FAILED_UPDATES[@]}" -unset LATEST_VERSION_TEMP_FILE _unix_millis _start_time _pkg +unset IPC_FIFO IPC_FIFO_FD _unix_millis _start_time _pkg total direct indirect ################################################FAILURE HANDLING################################################# diff --git a/scripts/updates/internal/termux_github_graphql.sh b/scripts/updates/internal/termux_github_graphql.sh index 3215284ea9..833127c26c 100644 --- a/scripts/updates/internal/termux_github_graphql.sh +++ b/scripts/updates/internal/termux_github_graphql.sh @@ -5,6 +5,8 @@ termux_github_graphql() { # Batch size for fetching tags, 100 seems to work consistently. local BATCH BATCH_SIZE=100 + # echo "# vim: ft=graphql" > /tmp/query-12345 # Uncomment for debugging GraphQL queries + # echo "# $(date -Iseconds)" >> /tmp/query-12345 for (( BATCH = 0; ${#GITHUB_GRAPHQL_QUERIES[@]} >= BATCH_SIZE * BATCH ; BATCH++ )); do echo "Starting batch $BATCH at: ${GITHUB_GRAPHQL_QUERIES[$BATCH * $BATCH_SIZE]//\\/}" >&2 @@ -14,7 +16,9 @@ termux_github_graphql() { local QUERY # Start the GraphQL query with our two fragments for getting the latest tag from a release, and from refs/tags - # These are defined only if needed, so this one is for '_latest_release_tag' + # These are defined only if needed. + + # _latest_release_tag returns latestRelease.tagName from the repo its querying grep -q '_latest_release_tag' <<< "${GITHUB_GRAPHQL_QUERIES[@]:$BATCH * $BATCH_SIZE:$BATCH_SIZE}" && { QUERY+="$(printf '%s\n' \ 'fragment _latest_release_tag on Repository {' \ @@ -22,6 +26,17 @@ termux_github_graphql() { '}')" } + # _latest_regex returns the (20) latest tags by commit date + grep -q '_latest_regex' <<< "${GITHUB_GRAPHQL_QUERIES[@]:$BATCH * $BATCH_SIZE:$BATCH_SIZE}" && { + QUERY+="$(printf '%s\n' \ + 'fragment _latest_regex on Repository {' \ + ' refs( refPrefix: \"refs/tags/\" first: 20 orderBy: {field: TAG_COMMIT_DATE, direction: DESC}) {' \ + ' nodes { name }' \ + ' }' \ + '}')" + } + + # _newest_tag returns the (1) newest tag by commit date grep -q '_newest_tag' <<< "${GITHUB_GRAPHQL_QUERIES[@]:$BATCH * $BATCH_SIZE:$BATCH_SIZE}" && { QUERY+="$(printf '%s\n' \ 'fragment _newest_tag on Repository {' \ @@ -41,38 +56,56 @@ termux_github_graphql() { 'ratelimit: rateLimit { cost limit remaining used resetAt }' \ '}' \ - # echo "// Batch: $BATCH" >> /tmp/query-12345 # Uncomment for debugging GraphQL queries + # echo "# Batch: $BATCH" >> /tmp/query-12345 # Uncomment for debugging GraphQL queries # printf '%s' "${QUERY}" >> /tmp/query-12345 # Uncomment for debugging GraphQL queries # We use graphql intensively so we should slowdown our requests to avoid hitting github ratelimits. sleep 5 local response - response="$(printf '{ "query": "%s" }' "${QUERY//$'\n'/ }" | curl -fL \ + # Try up to 3 times to fetch the batch, GitHub's GraphQL API can be a bit unreliable at times. + if ! response="$(printf '{ "query": "%s" }' "${QUERY//$'\n'/ }" | curl -fL \ + --retry 3 --retry-delay 5 \ --no-progress-meter \ -H "Authorization: token ${GITHUB_TOKEN}" \ -H 'Accept: application/vnd.github.v3+json' \ -H 'Content-Type: application/json' \ -X POST \ --data @- \ - https://api.github.com/graphql 2>&1 - )" || termux_error_exit "ERR - termux_github_graphql: $response" + https://api.github.com/graphql)"; then + { + printf '\t%s\n' \ + "Did not receive a clean API response." \ + "Need to run a manual sanity check on the response." + if ! jq <<< "$response"; then + printf '\t%s\n' "Doesn't seem to be valid JSON, skipping batch." + continue + fi + printf '\t%s\n' "Seems to be valid JSON, let's try parsing it." + } >&2 + fi unset QUERY - jq -r --argjson pkgs "$pkg_json" ' - .data # From the data: table - | del(.ratelimit) # Remove the ratelimit: table - | to_entries[] # Convert the remaining entries to an array - | .key as $alias # Save key to variable - | ($alias | ltrimstr("_") | tonumber) as $idx # Extract iterator from bash array - | .value | ( # For each .value - .latestRelease?.tagName # Print out the tag name of the latest release - // .refs.nodes[0]?.name # or of the latest tag - // empty # If neither exists print nothing - ) | sub("^v"; "") as $tag # Strip leading `v` from tag name - | select($tag != "") # Filter out empty strings - | ($pkgs[$idx] | split("/")[-1]) as $pkgName # Get package name from bash array - | "GIT|\($pkgName)|\($tag)" # Print results - ' <<<"$response" + ret="$(jq -r --argjson pkgs "$pkg_json" ' + .data # From the data: table + | del(.ratelimit) # Remove the ratelimit: table + | to_entries[] # Convert the remaining entries to an array + | .key as $alias # Save key to variable + | ($alias | ltrimstr("_") | tonumber) as $idx # Extract iterator from bash array + | .value | ( # For each .value + .latestRelease?.tagName # Print out the tag name of the latest release + // (.refs.nodes | map(.name) | join("\n")) # or of the tags + // empty # If neither exists print nothing + ) as $tag # Save to variable + | select($tag != "") # Filter out empty strings + | ($pkgs[$idx] | split("/")[-1]) as $pkgName # Get package name from bash array + | "GIT|\($pkgName)|\($tag)" # Print results + ' <<< "$response" 2>/dev/null)" || { + echo "something ain't right with this response" + } + # # Uncomment for debugging GraphQL queries + # jq '.' <<< "$response" >> /tmp/query-12345 + # echo "$ret" >> /tmp/query-12345 + echo "$ret" done } diff --git a/scripts/updates/utils/termux_pkg_upgrade_version.sh b/scripts/updates/utils/termux_pkg_upgrade_version.sh index 9938e63061..567d96ab13 100755 --- a/scripts/updates/utils/termux_pkg_upgrade_version.sh +++ b/scripts/updates/utils/termux_pkg_upgrade_version.sh @@ -16,11 +16,10 @@ _termux_should_cleanup() { } termux_pkg_upgrade_version() { - if [[ "$#" -lt 1 ]]; then + if (( $# < 1 )); then termux_error_exit <<-EndUsage Usage: ${FUNCNAME[0]} LATEST_VERSION [--skip-version-check] - Also reports the fully parsed LATEST_VERSION - to \$LATEST_VERSION_TEMP_FILE if provided. + Also reports the fully parsed LATEST_VERSION on file descriptor 3 EndUsage fi @@ -38,34 +37,35 @@ termux_pkg_upgrade_version() { # If needed, filter version numbers using grep regexp. if [[ -n "${TERMUX_PKG_UPDATE_VERSION_REGEXP:-}" ]]; then # Extract version numbers. - local OLD_LATEST_VERSION="${LATEST_VERSION}" - LATEST_VERSION="$(grep --max-count=1 -oP "${TERMUX_PKG_UPDATE_VERSION_REGEXP}" <<< "${LATEST_VERSION}" || true)" + local ORIGINAL_LATEST_VERSION="${LATEST_VERSION}" + LATEST_VERSION="$(grep --max-count=1 -oP "${TERMUX_PKG_UPDATE_VERSION_REGEXP}" <<< "${LATEST_VERSION}" || :)" if [[ -z "${LATEST_VERSION:-}" ]]; then termux_error_exit <<-EndOfError - ERROR: failed to filter version numbers using regexp '${TERMUX_PKG_UPDATE_VERSION_REGEXP}'. - Ensure that it works correctly with ${OLD_LATEST_VERSION}. + ERROR: Failed to filter version numbers for '${TERMUX_PKG_NAME}'. + Ensure that '${TERMUX_PKG_UPDATE_VERSION_REGEXP}' works correctly to match '${ORIGINAL_LATEST_VERSION}'. EndOfError fi - unset OLD_LATEST_VERSION - else # Otherwise remove any leading non-digits as that would not be a valid version. - # shellcheck disable=SC2001 # This is something parameter expansion can't handle well, so we use sed. - LATEST_VERSION="$(sed -e "s/^[^0-9]*//" <<< "$LATEST_VERSION")" + unset ORIGINAL_LATEST_VERSION fi # If needed, filter version numbers using sed regexp. if [[ -n "${TERMUX_PKG_UPDATE_VERSION_SED_REGEXP:-}" ]]; then # Extract version numbers. - local OLD_LATEST_VERSION="${LATEST_VERSION}" - LATEST_VERSION="$(sed -E "${TERMUX_PKG_UPDATE_VERSION_SED_REGEXP}" <<< "${LATEST_VERSION}" || true)" + local ORIGINAL_LATEST_VERSION="${LATEST_VERSION}" + LATEST_VERSION="$(sed -E "${TERMUX_PKG_UPDATE_VERSION_SED_REGEXP}" <<< "${LATEST_VERSION}" || :)" if [[ -z "${LATEST_VERSION:-}" ]]; then termux_error_exit <<-EndOfError - ERROR: failed to filter version numbers using regexp '${TERMUX_PKG_UPDATE_VERSION_SED_REGEXP}'. - Ensure that it works correctly with ${OLD_LATEST_VERSION}. + ERROR: Failed to filter version numbers for '${TERMUX_PKG_NAME}'. + Ensure that '${TERMUX_PKG_UPDATE_VERSION_SED_REGEXP}' works correctly to match '${ORIGINAL_LATEST_VERSION}'. EndOfError fi - unset OLD_LATEST_VERSION + unset ORIGINAL_LATEST_VERSION fi + # Remove any leading non-digits as that would not be a valid version. + # shellcheck disable=SC2001 # This is something parameter expansion can't handle well, so we use sed. + LATEST_VERSION="$(sed -e "s/^[^0-9]*//" <<< "$LATEST_VERSION")" + # Translate "_" into ".": some packages use underscores to separate # version numbers, but we require them to be separated by dots. LATEST_VERSION="${LATEST_VERSION//_/.}" @@ -76,9 +76,10 @@ termux_pkg_upgrade_version() { LATEST_VERSION="$(sed -E "s/[-.]?(${suffix}[0-9]*)/~\1/ig" <<< "$LATEST_VERSION")" done - # Report back the fully parsed $LATEST_VERSION for the summary. - # Or discard it straight into /dev/null if no tempfile was provided. - echo "$LATEST_VERSION" > "${LATEST_VERSION_TEMP_FILE:-/dev/null}" + # If FD 3 is open, use it for reporting the fully parsed $LATEST_VERSION + # If it's not open use the brace group to be able to + # discard the `3: Bad file descriptor` error silently. + { echo "$LATEST_VERSION" >&3; } 2> /dev/null if [[ "${SKIP_VERSION_CHECK}" != "--skip-version-check" ]]; then if ! termux_pkg_is_update_needed \ @@ -117,8 +118,8 @@ termux_pkg_upgrade_version() { for repo_path in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do _buildsh_path="${TERMUX_SCRIPTDIR}/${repo_path}/${TERMUX_PKG_NAME}/build.sh" - repo=$(jq --raw-output ".\"${repo_path}\".name" "${TERMUX_SCRIPTDIR}/repo.json") - repo=${repo#"termux-"} + repo="$(jq --raw-output ".\"${repo_path}\".name" "${TERMUX_SCRIPTDIR}/repo.json")" + repo="${repo#"termux-"}" if [[ -f "${_buildsh_path}" ]]; then echo "INFO: Package ${TERMUX_PKG_NAME} exists in ${repo} repo." @@ -127,8 +128,7 @@ termux_pkg_upgrade_version() { fi done - local force_cleanup="false" - + # check cleanup conditions local big_package=false while IFS= read -r p; do if [[ "${p}" == "${TERMUX_PKG_NAME}" ]]; then