diff --git a/scripts/bin/apt-compare-versions b/scripts/bin/apt-compare-versions deleted file mode 100755 index b1225448f2..0000000000 --- a/scripts/bin/apt-compare-versions +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 -# apt-compare-versions: Simple script which takes two arguments and compares -# their version according to apt rules. This can be used to verify the ordering -# between two versions. -# -# Note that ~ (tilde) construct, which allows for beta and preview releases. -# A version with ~ is considered earlier than one without, so 1.6~beta1 is -# considered earlier than 1.6. If both versions contains ~ the version comparison -# is made first on the part preceding the tilde, then the part coming after, -# so 1.6~beta1 comes before 1.6~beta2. - -import apt_pkg, sys -apt_pkg.init_system() - -if len(sys.argv) != 3: - sys.exit('usage: apt-compare-versions ') - -version1 = sys.argv[1] -version2 = sys.argv[2] - -comparison_result = apt_pkg.version_compare(version1, version2) -if comparison_result > 0: - operator = ' > ' -elif comparison_result < 0: - operator = ' < ' -elif comparison_result == 0: - operator = ' == ' - -print(version1 + operator + version2) diff --git a/scripts/bin/apt-compare-versions b/scripts/bin/apt-compare-versions new file mode 120000 index 0000000000..00f20e6cf3 --- /dev/null +++ b/scripts/bin/apt-compare-versions @@ -0,0 +1 @@ +../updates/utils/termux_pkg_is_update_needed.sh \ No newline at end of file diff --git a/scripts/bin/update-packages b/scripts/bin/update-packages index 417498a00d..c059db191f 100755 --- a/scripts/bin/update-packages +++ b/scripts/bin/update-packages @@ -1,8 +1,8 @@ #!/usr/bin/env bash -set -e -u -BASEDIR=$(dirname "$(realpath "$0")") +# shellcheck source-path=/data/data/com.termux/files/home/termux-packages +set -u -# These variables should be set in environment outside of this script. +# Following variables should be set in environment outside of this script. # Build updated packages. : "${BUILD_PACKAGES:=false}" # Commit changes to Git. @@ -10,110 +10,184 @@ BASEDIR=$(dirname "$(realpath "$0")") # Push changes to remote. : "${GIT_PUSH_PACKAGES:=false}" -if [ -z "${GITHUB_API_TOKEN-}" ]; then - echo "Error: you need a Github Personal Access Token be set in variable GITHUB_API_TOKEN." - exit 1 -fi +export TERMUX_PKG_UPDATE_METHOD="" # Which method to use for updating? (repology, github or gitlab) +export TERMUX_PKG_UPDATE_TAG_TYPE="" # Whether to use latest-release-tag or newest-tag. +export TERMUX_GITLAB_API_HOST="gitlab.com" # Default host for gitlab-ci. +export TERMUX_PKG_AUTO_UPDATE=false # Whether to auto-update or not. Disabled by default. +export TERMUX_PKG_UPDATE_VERSION_REGEXP="" # Regexp to extract version. +export TERMUX_REPOLOGY_DATA_FILE +TERMUX_REPOLOGY_DATA_FILE="$(mktemp -t termux-repology.XXXXXX)" # File to store repology data. -for pkg_dir in "${BASEDIR}"/../../packages/*; do - if [ -f "${pkg_dir}/build.sh" ]; then - package=$(basename "$pkg_dir") - else - # Fail if detected a non-package directory. - echo "Error: directory '${pkg_dir}' is not a package." - exit 1 - fi +export TERMUX_SCRIPTDIR +TERMUX_SCRIPTDIR="$(realpath "$(dirname "$0")/../..")" # Script directory. - # Extract the package auto-update configuration. - build_vars=$( - set +e +u - . "${BASEDIR}/../../packages/${package}/build.sh" 2>/dev/null - echo "auto_update_flag=${TERMUX_PKG_AUTO_UPDATE};" - echo "termux_version=\"${TERMUX_PKG_VERSION}\";" - echo "srcurl=\"${TERMUX_PKG_SRCURL}\";" - echo "version_regexp=\"${TERMUX_PKG_AUTO_UPDATE_TAG_REGEXP//\\/\\\\}\";" - ) - auto_update_flag=""; termux_version=""; srcurl=""; version_regexp=""; - eval "$build_vars" +# Define few more variables used by scripts. +# shellcheck source=scripts/properties.sh +. "${TERMUX_SCRIPTDIR}/scripts/properties.sh" - # Ignore packages that have auto-update disabled. - if [ "${auto_update_flag}" != "true" ]; then - continue - fi +# Utility function to write error message to stderr. +# shellcheck source=scripts/updates/utils/termux_error_exit.sh +. "${TERMUX_SCRIPTDIR}"/scripts/updates/utils/termux_error_exit.sh - # Extract github project from TERMUX_PKG_SRCURL - project="$(echo "${srcurl}" | grep github.com | cut -d / -f4-5)" - if [ -z "${project}" ]; then - echo "Error: package '${package}' doesn't use GitHub archive source URL but has been configured for automatic updates." - exit 1 - fi +# Utility function to write updated version to build.sh. +# shellcheck source=scripts/updates/utils/termux_pkg_upgrade_version.sh +. "${TERMUX_SCRIPTDIR}"/scripts/updates/utils/termux_pkg_upgrade_version.sh - # Our local version of package. - termux_epoch="$(echo "$termux_version" | cut -d: -f1)" - termux_version=$(echo "$termux_version" | cut -d: -f2-) - if [ "$termux_version" == "$termux_epoch" ]; then - # No epoch set. - termux_epoch="" - else - termux_epoch+=":" - fi +# Utility function to check if package needs to be updated, based on version comparison. +# shellcheck source=scripts/updates/utils/termux_pkg_is_update_needed.sh +. "${TERMUX_SCRIPTDIR}"/scripts/updates/utils/termux_pkg_is_update_needed.sh - # Get the latest release tag. - latest_tag=$(curl --silent --location -H "Authorization: token ${GITHUB_API_TOKEN}" "https://api.github.com/repos/${project}/releases/latest" | jq -r .tag_name) +# Wrapper around github api to get latest release or newest tag. +# shellcheck source=scripts/updates/api/termux_github_api_get_tag.sh +. "${TERMUX_SCRIPTDIR}"/scripts/updates/api/termux_github_api_get_tag.sh - # If the github api returns error - if [ -z "$latest_tag" ] || [ "${latest_tag}" = "null" ]; then - echo "Error: failed to get the latest release tag for '${package}'. GitHub API returned 'null' which indicates that no releases available." - exit 1 - fi +# Wrapper around gitlab api to get latest release or newest tag. +# shellcheck source=scripts/updates/api/termux_gitlab_api_get_tag.sh +. "${TERMUX_SCRIPTDIR}"/scripts/updates/api/termux_gitlab_api_get_tag.sh - # Remove leading 'v' which is common in version tag. - latest_version=${latest_tag#v} +# Function to get latest version of a package as per repology. +# shellcheck source=scripts/updates/api/termux_repology_api_get_latest_version.sh +. "${TERMUX_SCRIPTDIR}"/scripts/updates/api/termux_repology_api_get_latest_version.sh - # If needed, filter version numbers from tag by using regexp. - if [ -n "$version_regexp" ]; then - latest_version=$(grep -oP "$version_regexp" <<< "$latest_version" || true) - fi - if [ -z "$latest_version" ]; then - echo "Error: failed to get latest version for '${package}'. Check whether the TERMUX_PKG_AUTO_UPDATE_TAG_REGEXP='${version_regexp}' is work right with latest_release='${latest_tag}'." - exit 1 - fi +# Default auto update script for packages hosted on github.com. Should not be overrided by build.sh. +# To use custom algorithm, one should override termux_pkg_auto_update(). +# shellcheck source=scripts/updates/internal/termux_github_auto_update.sh +. "${TERMUX_SCRIPTDIR}"/scripts/updates/internal/termux_github_auto_update.sh - # Translate "_" into ".": some packages use underscores to seperate - # version numbers, but we require them to be separated by dots. - latest_version=${latest_version//_/.} +# Default auto update script for packages hosted on hosts using gitlab-ci. Should not be overrided by build.sh. +# To use custom algorithm, one should override termux_pkg_auto_update(). +# shellcheck source=scripts/updates/internal/termux_gitlab_auto_update.sh +. "${TERMUX_SCRIPTDIR}"/scripts/updates/internal/termux_gitlab_auto_update.sh - # We have no better choice for comparing versions. - if [ "$(echo -e "${termux_version}\n${latest_version}" | sort -V | head -n 1)" != "$latest_version" ] ;then - if [ "$BUILD_PACKAGES" = "false" ]; then - echo "Package '${package}' needs update to '${latest_version}'." - else - echo "Updating '${package}' to '${latest_version}'." - sed -i "s/^\(TERMUX_PKG_VERSION=\)\(.*\)\$/\1${termux_epoch}${latest_version}/g" "${BASEDIR}/../../packages/${package}/build.sh" - sed -i "/TERMUX_PKG_REVISION=/d" "${BASEDIR}/../../packages/${package}/build.sh" - echo n | "${BASEDIR}/../bin/update-checksum" "$package" || { - echo "Warning: failed to update checksum for '${package}', skipping..." - git checkout -- "${BASEDIR}/../../packages/${package}" - git pull --rebase - continue - } +# Default auto update script for rest packages. Should not be overrided by build.sh. +# To use custom algorithm, one should override termux_pkg_auto_update(). +# shellcheck source=scripts/updates/internal/termux_repology_auto_update.sh +. "${TERMUX_SCRIPTDIR}"/scripts/updates/internal/termux_repology_auto_update.sh - echo "Trying to build package '${package}'." - if "${BASEDIR}/../run-docker.sh" ./build-package.sh -a aarch64 -I "$package" && \ - "${BASEDIR}/../run-docker.sh" ./build-package.sh -a arm -I "$package"; then - if [ "$GIT_COMMIT_PACKAGES" = "true" ]; then - git add "${BASEDIR}/../../packages/${package}" - git commit -m "$(echo -e "${package}: update to ${latest_version}\n\nThis commit has been automatically submitted by Github Actions.")" - fi +# Main script to: +# - by default, decide which update method to use, +# - but can be overrided by build.sh to use custom update method. +# - For example: see neovim-nightly's build.sh. +# shellcheck source=scripts/updates/termux_pkg_auto_update.sh +. "${TERMUX_SCRIPTDIR}"/scripts/updates/termux_pkg_auto_update.sh - if [ "$GIT_PUSH_PACKAGES" = "true" ]; then - git pull --rebase - git push - fi - else - echo "Warning: failed to build '${package}'." - git checkout -- "${BASEDIR}/../../packages/${package}" - fi +_update() { + export TERMUX_PKG_NAME + TERMUX_PKG_NAME="$(basename "$1")" + # Avoid: + # - ending on errors such as $(which prog), where prog is not installed. + # - error on unbound variable. + # + # Variables used by auto update script should be covered by above variables and properties.sh. + set +e +u + # shellcheck source=/dev/null + . "${pkg_dir}"/build.sh 2>/dev/null + set -e -u + + IFS="," read -r -a BLACKLISTED_ARCH <<<"${TERMUX_PKG_BLACKLISTED_ARCHES:-}" + export TERMUX_ARCH="" # Arch to test updates. + for arch in aarch64 arm i686 x86_64; do + # shellcheck disable=SC2076 + if [[ ! " ${BLACKLISTED_ARCH[*]} " =~ " ${arch} " ]]; then + TERMUX_ARCH="${arch}" + break fi + done + + echo # Newline. + echo "INFO: Updating ${TERMUX_PKG_NAME}..." + # Only update if auto update is enabled. + if [[ "${TERMUX_PKG_AUTO_UPDATE}" == "true" ]]; then + echo "INFO: Current version: ${TERMUX_PKG_VERSION}" + termux_pkg_auto_update + echo # Newline. + else + echo "INFO: Skipping update. Auto update is disabled." fi -done +} + +_test_pkg_build_file() { + local pkg_dir="$1" + if [[ ! -f "${pkg_dir}/build.sh" ]]; then + # Fail if detected a non-package directory. + termux_error_exit "ERROR: directory '${pkg_dir}' is not a package." + fi +} + +declare -a _FAILED_UPDATES=() + +_run_update() { + local pkg_dir="$1" + _test_pkg_build_file "${pkg_dir}" + # Run each package update in separate process since we include their environment variables. + ( + set -euo pipefail + _update "${pkg_dir}" + ) + # shellcheck disable=SC2181 + if [[ $? -ne 0 ]]; then + _FAILED_UPDATES+=("$(basename "${pkg_dir}")") + fi +} + +_get_unique_packages() { + local -a unique_packages=() + + while read -r pkg; do + unique_packages+=("${pkg}") + done < <(curl --silent --location --retry 5 --retry-delay 5 --retry-max-time 60 \ + "https://repology.org/api/v1/projects/?inrepo=termux&&repos=1" | + jq -r keys) + + echo "${unique_packages[@]}" +} + +declare -a _UNIQUE_PACKAGES +read -r -a _UNIQUE_PACKAGES <<<"$(_get_unique_packages)" + +_unique_to_termux() { + local pkg_dir="$1" + # shellcheck disable=2076 # We want literal match not regex. + if [[ "${_UNIQUE_PACKAGES[*]}" =~ "$(basename "${pkg_dir}")" ]]; then + return 0 + else + return 1 + fi +} + +main() { + echo "INFO: Running update for: $*" + + if [[ "$1" == "@all" ]]; then + for pkg_dir in "${TERMUX_SCRIPTDIR}"/packages/*; do + # Skip update if package is unique to Termux. + if _unique_to_termux "${pkg_dir}"; then + echo # Newline. + echo "INFO: Skipping update for unique to Termux package: $(basename "${pkg_dir}")" + continue + fi + _run_update "${pkg_dir}" + done + else + for pkg in "$@"; do + # Skip update if package is unique to Termux. + if _unique_to_termux "${TERMUX_SCRIPTDIR}"/packages/"${pkg}"; then + echo # Newline. + echo "INFO: Skipping update for unique to Termux package: ${pkg}" + continue + fi + _run_update "${TERMUX_SCRIPTDIR}/packages/${pkg}" + done + fi + + if ((${#_FAILED_UPDATES[@]} > 0)); then + echo # Newline. + echo "===========================Failed updates===========================" + for failed_update in "${_FAILED_UPDATES[@]}"; do + echo "==> ${failed_update}" + done + exit 1 # Exit with error code, so that we know that some/all updates failed. + fi +} + +main "$@" diff --git a/scripts/updates/api/dump-repology-data b/scripts/updates/api/dump-repology-data new file mode 100755 index 0000000000..bd17957604 --- /dev/null +++ b/scripts/updates/api/dump-repology-data @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 + +# The MIT License (MIT) + +# Copyright (c) 2022 Aditya Alok (aka. @MrAdityaAlok) + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +from requests import get as requests_get + + +def get_repology_data(last_project): + repology_data = requests_get( + f"https://repology.org/api/v1/projects/{last_project}?inrepo=termux&outdated=True&families_newest=2-" + ).json() # NOTE: We are using 2- so that api will return a package as outdated if it is so in 2 or more + # repo family. This helps us avoid false positives. + + return repology_data + + +def get_outdated_packages(): + termux_outdated_packages = {} + last_project = "" + + while True: + repology_data = get_repology_data(last_project) + last_project = sorted(repology_data.keys())[ + -1 + ] # This used to query repology for next set of packages. + # Quoting repology documentation: "You may iterate through + # all projects by using the last project name in the next request" + # For more info, visit https://repology.org/api + # NOTE: next response to request will include the last_project given. + if len(repology_data) <= 1: + # Break the loop now. Since api returned only one package, it + # must be the last_project, which was already included in previous iteration. + break + + for package_name, package_data in repology_data.items(): + if package_name in termux_outdated_packages: + # Skip if package is already in the dict. + continue + newest_stable = None + newest_devel = None + for repo_data in package_data: + if repo_data.get("status", "") == "newest": + newest_stable = repo_data["version"] + # If we found stable version, break the loop. + break + elif repo_data.get("status", "") == "devel": + # Do not break the loop if we found devel version as there may be stable version later. + newest_devel = repo_data["version"] + + if newest_stable: + termux_outdated_packages[package_name] = newest_stable + elif newest_devel: + termux_outdated_packages[package_name] = newest_devel + else: + # If we don't find any version, skip the package. + continue + + return termux_outdated_packages + + +if __name__ == "__main__": + import json + import sys + + try: + output_file = sys.argv[1] + except IndexError: + sys.exit("Please provide an output file") + + with open(output_file, "w") as f: + json.dump(get_outdated_packages(), f) diff --git a/scripts/updates/api/termux_github_api_get_tag.sh b/scripts/updates/api/termux_github_api_get_tag.sh new file mode 100644 index 0000000000..1c4dc4c0c0 --- /dev/null +++ b/scripts/updates/api/termux_github_api_get_tag.sh @@ -0,0 +1,136 @@ +# shellcheck shell=bash +termux_github_api_get_tag() { + if [[ -z "$1" ]]; then + termux_error_exit <<-EndOfUsage + Usage: ${FUNCNAME[0]} PKG_SRCURL [TAG_TYPE] + Returns the latest tag of the given package. + EndOfUsage + fi + + if [[ -z "${GITHUB_TOKEN:-}" ]]; then + # Needed to use graphql API. + termux_error_exit "ERROR: GITHUB_TOKEN environment variable not set." + fi + + local PKG_SRCURL="$1" + local TAG_TYPE="${2:-}" + + local project + project="$(echo "${PKG_SRCURL}" | cut -d'/' -f4-5)" + project="${project%.git}" + + if [[ -z "${TAG_TYPE}" ]]; then # If not set, then decide on the basis of url. + if [[ "${PKG_SRCURL: -4}" == ".git" ]]; then + # Get newest tag. + TAG_TYPE="newest-tag" + else + # Get the latest release tag. + TAG_TYPE="latest-release-tag" + fi + fi + + local jq_filter + local api_url="https://api.github.com" + local -a extra_curl_opts + + if [[ "${TAG_TYPE}" == "newest-tag" ]]; then + api_url="${api_url}/graphql" + jq_filter='.data.repository.refs.edges[0].node.name' + extra_curl_opts=( + "-X POST" + "-d $( + cat <<-EOF | tr '\n' ' ' + { + "query": "query { + repository(owner: \"${project%/*}\", name: \"${project##*/}\") { + refs(refPrefix: \"refs/tags/\", first: 1, orderBy: { + field: TAG_COMMIT_DATE, direction: DESC + }) + { + edges { + node { + name + } + } + } + } + }" + } + EOF + )" + ) + + elif [[ "${TAG_TYPE}" == "latest-release-tag" ]]; then + api_url="${api_url}/repos/${project}/releases/latest" + jq_filter=".tag_name" + else + termux_error_exit <<-EndOfError + ERROR: Invalid TAG_TYPE: '${TAG_TYPE}'. + Allowed values: 'newest-tag', 'latest-release-tag'. + EndOfError + fi + + local response + # shellcheck disable=SC2086 # we need expansion of ${extra_curl_opts[0]} + response="$( + curl --silent --location --retry 10 --retry-delay 1 \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "Accept: application/vnd.github.v3+json" \ + --write-out '|%{http_code}' \ + ${extra_curl_opts[0]:-} \ + "${extra_curl_opts[1]:-}" \ + "${api_url}" + )" + + local http_code + http_code="${response##*|}" + # Why printf "%s\n"? Because echo interpolates control characters, which jq does not like. + response="$(printf "%s\n" "${response%|*}")" + + local tag_name + if [[ "${http_code}" == "200" ]]; then + if jq --exit-status --raw-output "${jq_filter}" <<<"${response}" >/dev/null; then + tag_name="$(jq --exit-status --raw-output "${jq_filter}" <<<"${response}")" + else + termux_error_exit "ERROR: Failed to parse tag name from: '${response}'" + fi + elif [[ "${http_code}" == "404" ]]; then + if jq --exit-status "has(\"message\") and .message == \"Not Found\"" <<<"${response}"; then + termux_error_exit <<-EndOfError + ERROR: No '${TAG_TYPE}' found (${api_url}). + Try using '$( + if [ ${TAG_TYPE} = "newest-tag" ]; then + echo "latest-release-tag" + else + echo "newest-tag" + fi + )'. + EndOfError + else + termux_error_exit <<-EndOfError + ERROR: Failed to get '${TAG_TYPE}'(${api_url})'. + Response: + ${response} + EndOfError + fi + else + termux_error_exit <<-EndOfError + ERROR: Failed to get '${TAG_TYPE}'(${api_url})'. + HTTP code: ${http_code} + Response: + ${response} + EndOfError + fi + + # If program control reached here and still no tag_name, then something went wrong. + if [[ -z "${tag_name:-}" ]] || [[ "${tag_name}" == "null" ]]; then + termux_error_exit <<-EndOfError + ERROR: JQ could not find '${TAG_TYPE}'(${api_url})'. + Response: + ${response} + Please report this as bug. + EndOfError + fi + + echo "${tag_name#v}" # Remove leading 'v' which is common in version tag. +} diff --git a/scripts/updates/api/termux_gitlab_api_get_tag.sh b/scripts/updates/api/termux_gitlab_api_get_tag.sh new file mode 100644 index 0000000000..6a4b5c24b6 --- /dev/null +++ b/scripts/updates/api/termux_gitlab_api_get_tag.sh @@ -0,0 +1,112 @@ +# shellcheck shell=bash +termux_gitlab_api_get_tag() { + if [[ -z "$1" ]]; then + termux_error_exit <<-EndOfUsage + Usage: ${FUNCNAME[0]} PKG_SRCURL [TAG_TYPE] [API_HOST] + Returns the latest tag of the given package. + EndOfUsage + + fi + local PKG_SRCURL="$1" + local TAG_TYPE="${2:-}" + local API_HOST="${3:-gitlab.com}" + + local project + project="$(echo "${PKG_SRCURL}" | cut -d'/' -f4-5)" + project="${project%.git}" + + if [[ -z "${TAG_TYPE}" ]]; then # If not set, then decide on the basis of url. + if [[ "${PKG_SRCURL: -4}" == ".git" ]]; then + # Get newest tag. + TAG_TYPE="newest-tag" + else + # Get the latest release tag. + TAG_TYPE="latest-release-tag" + fi + fi + + local jq_filter + local api_path + + case "${TAG_TYPE}" in + latest-release-tag) + api_path="/releases" + jq_filter=".[0].tag_name" + ;; + newest-tag) + api_path="/repository/tags" + jq_filter=".[0].name" + ;; + *) + termux_error_exit <<-EndOfError + ERROR: Invalid TAG_TYPE: '${TAG_TYPE}'. + Allowed values: 'newest-tag', 'latest-release-tag'. + EndOfError + ;; + esac + + # Replace slash '/' with '%2F' in project name. It is required for Gitlab API. + local api_url="https://${API_HOST}/api/v4/projects/${project//\//%2F}${api_path}" + # Api can be accessed without authentication if the repository is publicly accessible. + # Default rate limit for gitlab.com is 300 requests per minute for unauthenticated users + # and non-protected paths which should be enough for most use cases. + # see: https://docs.gitlab.com/ee/user/gitlab_com/index.html#gitlabcom-specific-rate-limits + local response + response="$( + curl --silent --location --retry 10 --retry-delay 1 \ + --write-out '|%{http_code}' \ + "${api_url}" + )" + + local http_code + http_code="${response##*|}" + # Why printf "%s\n"? Because echo interpolates control characters, which jq does not like. + response="$(printf "%s\n" "${response%|*}")" + + local tag_name + if [[ "${http_code}" == "200" ]]; then + if jq --exit-status --raw-output "${jq_filter}" <<<"${response}" >/dev/null; then + tag_name="$(jq --exit-status --raw-output "${jq_filter}" <<<"${response}")" + else + termux_error_exit "ERROR: Failed to parse tag name from: '${response}'" + fi + elif [[ "${http_code}" == "404" ]]; then + if jq --exit-status "has(\"message\") and .message == \"Not Found\"" <<<"${response}"; then + termux_error_exit <<-EndOfError + ERROR: No '${TAG_TYPE}' found. (${api_url}) + Try using '$( + if [ ${TAG_TYPE} = "newest-tag" ]; then + echo "latest-release-tag" + else + echo "newest-tag" + fi + )'. + EndOfError + else + termux_error_exit <<-EndOfError + ERROR: Failed to get '${TAG_TYPE}' (${api_url}). + Response: + ${response} + EndOfError + fi + else + termux_error_exit <<-EndOfError + ERROR: Failed to get '${TAG_TYPE}' (${api_url}). + HTTP code: ${http_code} + Response: + ${response} + EndOfError + fi + + # If program control reached here and still no tag_name, then something is not right. + if [[ -z "${tag_name:-}" ]] || [[ "${tag_name}" == "null" ]]; then + termux_error_exit <<-EndOfError + ERROR: JQ could not find '${TAG_TYPE}' (${api_url}). + Response: + ${response} + Please report this as bug. + EndOfError + fi + + echo "${tag_name#v}" # Strip leading 'v'. +} diff --git a/scripts/updates/api/termux_repology_api_get_latest_version.sh b/scripts/updates/api/termux_repology_api_get_latest_version.sh new file mode 100644 index 0000000000..46ac698fcf --- /dev/null +++ b/scripts/updates/api/termux_repology_api_get_latest_version.sh @@ -0,0 +1,18 @@ +# shellcheck shell=bash +termux_repology_api_get_latest_version() { + if [[ -z "$1" ]]; then + termux_error_exit "Usage: ${FUNCNAME[0]} PKG_NAME" + fi + + if [[ ! -s "${TERMUX_REPOLOGY_DATA_FILE}" ]]; then + pip3 install bs4 requests >/dev/null # Install python dependencies. + python3 "${TERMUX_SCRIPTDIR}"/scripts/updates/api/dump-repology-data \ + "${TERMUX_REPOLOGY_DATA_FILE}" >/dev/null + fi + + local PKG_NAME="$1" + local version + # Why `--arg`? See: https://stackoverflow.com/a/54674832/15086226 + version="$(jq -r --arg packageName "$PKG_NAME" '.[$packageName]' <"${TERMUX_REPOLOGY_DATA_FILE}")" + echo "${version#v}" +} diff --git a/scripts/updates/internal/termux_github_auto_update.sh b/scripts/updates/internal/termux_github_auto_update.sh new file mode 100644 index 0000000000..3ff1dd3952 --- /dev/null +++ b/scripts/updates/internal/termux_github_auto_update.sh @@ -0,0 +1,25 @@ +# shellcheck shell=bash +# Default algorithm to use for packages hosted on github.com +termux_github_auto_update() { + local pkg_version + pkg_version="$(echo "${TERMUX_PKG_VERSION}" | cut -d: -f2-)" + local pkg_epoch + pkg_epoch="$(echo "${TERMUX_PKG_VERSION}" | cut -d: -f1)" + + if [[ "${pkg_version}" == "${pkg_epoch}" ]]; then + # No epoch set. + pkg_epoch="" + else + pkg_epoch+=":" + fi + + local latest_tag + latest_tag="$( + termux_github_api_get_tag "${TERMUX_PKG_SRCURL}" "${TERMUX_PKG_UPDATE_TAG_TYPE}" + )" + + if [[ -z "${latest_tag}" ]]; then + termux_error_exit "ERROR: Unable to get tag from ${TERMUX_PKG_SRCURL}" + fi + termux_pkg_upgrade_version "${pkg_epoch}${latest_tag}" +} diff --git a/scripts/updates/internal/termux_gitlab_auto_update.sh b/scripts/updates/internal/termux_gitlab_auto_update.sh new file mode 100644 index 0000000000..535ad80bf9 --- /dev/null +++ b/scripts/updates/internal/termux_gitlab_auto_update.sh @@ -0,0 +1,28 @@ +# shellcheck shell=bash +# Default algorithm to use for packages hosted on hosts using gitlab-ci. +termux_gitlab_auto_update() { + # Our local version of package. + local pkg_version + pkg_version="$(echo "${TERMUX_PKG_VERSION}" | cut -d: -f2-)" + local pkg_epoch + pkg_epoch="$(echo "${TERMUX_PKG_VERSION}" | cut -d: -f1)" + + if [[ "${pkg_version}" == "${pkg_epoch}" ]]; then + # No epoch set. + pkg_epoch="" + else + pkg_epoch+=":" + fi + + local latest_tag + latest_tag="$( + termux_gitlab_api_get_tag \ + "${TERMUX_PKG_SRCURL}" "${TERMUX_PKG_UPDATE_TAG_TYPE}" "${TERMUX_GITLAB_API_HOST}" + )" + # No need to check for return code `2`, since gitlab api does not implement cache control. + + if [[ -z "${latest_tag}" ]]; then + termux_error_exit "ERROR: Unable to get tag from ${TERMUX_PKG_SRCURL}" + fi + termux_pkg_upgrade_version "${pkg_epoch}${latest_tag}" +} diff --git a/scripts/updates/internal/termux_repology_auto_update.sh b/scripts/updates/internal/termux_repology_auto_update.sh new file mode 100644 index 0000000000..ef927fc864 --- /dev/null +++ b/scripts/updates/internal/termux_repology_auto_update.sh @@ -0,0 +1,26 @@ +# shellcheck shell=bash +termux_repology_auto_update() { + # Our local version of package. + local pkg_version + pkg_version="$(echo "${TERMUX_PKG_VERSION}" | cut -d: -f2-)" + local pkg_epoch + pkg_epoch="$(echo "${TERMUX_PKG_VERSION}" | cut -d: -f1)" + + if [[ "${pkg_version}" == "${pkg_epoch}" ]]; then + # No epoch set. + pkg_epoch="" + else + pkg_epoch+=":" + fi + + local latest_version + latest_version="$(termux_repology_api_get_latest_version "${TERMUX_PKG_NAME}")" + + # Repology api returns null if package is not tracked by repology or is already upto date. + if [[ "${latest_version}" == "null" ]]; then + echo "INFO: Already up to date." # Since we exclude unique to termux packages, this package + # should be tracked by repology and be already up to date. + return 0 + fi + termux_pkg_upgrade_version "${pkg_epoch}${latest_version}" +} diff --git a/scripts/updates/termux_pkg_auto_update.sh b/scripts/updates/termux_pkg_auto_update.sh new file mode 100644 index 0000000000..835da85871 --- /dev/null +++ b/scripts/updates/termux_pkg_auto_update.sh @@ -0,0 +1,44 @@ +# shellcheck shell=bash +termux_pkg_auto_update() { + local project_host + project_host="$(echo "${TERMUX_PKG_SRCURL}" | cut -d"/" -f3)" + + if [[ -z "${TERMUX_PKG_UPDATE_METHOD}" ]]; then + if [[ "${project_host}" == "github.com" ]]; then + TERMUX_PKG_UPDATE_METHOD="github" + elif [[ "${project_host}" == "gitlab.com" ]]; then + TERMUX_PKG_UPDATE_METHOD="gitlab" + else + TERMUX_PKG_UPDATE_METHOD="repology" + fi + fi + + local _err_msg="ERROR: source url's hostname is not ${TERMUX_PKG_UPDATE_METHOD}.com, but has been +configured to use ${TERMUX_PKG_UPDATE_METHOD}'s method." + + case "${TERMUX_PKG_UPDATE_METHOD}" in + github) + if [[ "${project_host}" != "${TERMUX_PKG_UPDATE_METHOD}.com" ]]; then + termux_error_exit "${_err_msg}" + else + termux_github_auto_update + fi + ;; + gitlab) + if [[ "${project_host}" != "${TERMUX_PKG_UPDATE_METHOD}.com" ]]; then + termux_error_exit "${_err_msg}" + else + termux_gitlab_auto_update + fi + ;; + repology) + termux_repology_auto_update + ;; + *) + termux_error_exit <<-EndOfError + ERROR: wrong value '${TERMUX_PKG_UPDATE_METHOD}' for TERMUX_PKG_UPDATE_METHOD. + Can be 'github', 'gitlab' or 'repology' + EndOfError + ;; + esac +} diff --git a/scripts/updates/utils/termux_error_exit.sh b/scripts/updates/utils/termux_error_exit.sh new file mode 100644 index 0000000000..28425b6cc2 --- /dev/null +++ b/scripts/updates/utils/termux_error_exit.sh @@ -0,0 +1,10 @@ +# shellcheck shell=bash +termux_error_exit() { + if [ "$#" -eq 0 ]; then + # Read from stdin. + printf '%s\n' "$(cat)" >&2 + else + printf '%s\n' "$*" >&2 + fi + exit 1 +} diff --git a/scripts/updates/utils/termux_pkg_is_update_needed.sh b/scripts/updates/utils/termux_pkg_is_update_needed.sh new file mode 100755 index 0000000000..250725e523 --- /dev/null +++ b/scripts/updates/utils/termux_pkg_is_update_needed.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# +# NOTE: This function returns true even when CURRENT_VERSION = "1.0" and LATEST_VERSION = "1.0-1". +# This is logically correct, but repology sometimes returns "1.0-1" as the latest version even +# if "1.0" is latest. This happens when any of the repositories tracked by repology has specified +# "1.0-1" as the latest. +# +# For example: +# latest lua:lpeg version (as of 2021-11-20T12:21:31) is "1.0.2" but MacPorts specifies as "1.0.2-1". +# Hence repology returns "1.0.2-1" as the latest. +# +# But hopefully, all this can be avoided if TERMUX_PKG_AUTO_UPDATE_TAG_REGEXP is set. +# +termux_pkg_is_update_needed() { + # USAGE: termux_pkg_is_update_needed [regexp] + + if [[ -z "$1" ]] || [[ -z "$2" ]]; then + termux_error_exit "${BASH_SOURCE[0]}: at least 2 arguments expected" + fi + + local CURRENT_VERSION="$1" + local LATEST_VERSION="$2" + local VERSION_REGEX="${3:-}" + + # If needed, filter version numbers from tag by using regexp. + if [[ -n "${VERSION_REGEX}" ]]; then + LATEST_VERSION="$(grep -oP "${VERSION_REGEX}" <<<"${LATEST_VERSION}" || true)" + + if [[ -z "${LATEST_VERSION}" ]]; then + termux_error_exit <<-EndOfError + ERROR: failed to compare versions. Ensure whether the version regex '${VERSION_REGEX}' + works correctly with given versions. + EndOfError + fi + fi + + # Translate "_" into ".": some packages use underscores to seperate + # version numbers, but we require them to be separated by dots. + LATEST_VERSION="${LATEST_VERSION//_/.}" + + # Compare versions. + # shellcheck disable=SC2091 + if $( + cat <<-EOF | python3 - + import sys + + from pkg_resources import parse_version + + if parse_version("${CURRENT_VERSION}") < parse_version("${LATEST_VERSION}"): + sys.exit(0) + else: + sys.exit(1) + EOF + ); then + return 0 # true. Update needed. + fi + return 1 # false. Update not needed. +} + +show_help() { + echo "Usage: ${BASH_SOURCE[0]} [--help] ] [version-regex]" + echo "--help - show this help message and exit" + echo " - first version to compare" + echo " - second version to compare" + echo " [version-regex] - optional regular expression to filter version numbers" + exit 0 +} + +# Make script sourceable as well as executable. +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + declare -f termux_error_exit >/dev/null || + . "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/termux_error_exit.sh" # realpath is used to resolve symlinks. + + if [[ "${1}" == "--help" ]]; then + show_help + fi + + # Print in human readable format. + first_version="$1" + second_version="$2" + version_regexp="${3:-}" + if termux_pkg_is_update_needed "${first_version}" "${second_version}" "${version_regexp}"; then + echo "${first_version} < ${second_version}" + else + echo "${first_version} >= ${second_version}" + fi +fi diff --git a/scripts/updates/utils/termux_pkg_upgrade_version.sh b/scripts/updates/utils/termux_pkg_upgrade_version.sh new file mode 100755 index 0000000000..fc41556134 --- /dev/null +++ b/scripts/updates/utils/termux_pkg_upgrade_version.sh @@ -0,0 +1,79 @@ +# shellcheck shell=bash +termux_pkg_upgrade_version() { + if [[ "$#" -lt 1 ]]; then + # Show usage. + termux_error_exit <<-EndUsage + Usage: ${FUNCNAME[0]} LATEST_VERSION [--skip-version-check] + Version should be passed with epoch, if any. + EndUsage + fi + + local LATEST_VERSION="$1" + local SKIP_VERSION_CHECK="${2:-}" + local PKG_DIR + PKG_DIR="${TERMUX_SCRIPTDIR}/packages/${TERMUX_PKG_NAME}" + + if [[ "${SKIP_VERSION_CHECK}" != "--skip-version-check" ]]; then + if ! termux_pkg_is_update_needed \ + "${TERMUX_PKG_VERSION}" "${LATEST_VERSION}" "${TERMUX_PKG_UPDATE_VERSION_REGEXP}"; then + echo "INFO: No update needed. Already at version '${TERMUX_PKG_VERSION}'." + return 0 + fi + fi + + if [[ "${BUILD_PACKAGES}" == "false" ]]; then + echo "INFO: package needs to be updated to $(echo "${LATEST_VERSION}" | cut -d':' -f2)." + else + echo "INFO: package being updated to $(echo "${LATEST_VERSION}" | cut -d':' -f2)." + + sed -i \ + "s/^\(TERMUX_PKG_VERSION=\)\(.*\)\$/\1\"${LATEST_VERSION}\"/g" \ + "${PKG_DIR}/build.sh" + sed -i \ + "/TERMUX_PKG_REVISION=/d" \ + "${PKG_DIR}/build.sh" + + # Update checksum + if [[ "${TERMUX_PKG_SHA256[*]}" != "SKIP_CHECKSUM" ]] && [[ "${TERMUX_PKG_SRCURL: -4}" != ".git" ]]; then + echo n | "${TERMUX_SCRIPTDIR}/scripts/bin/update-checksum" "${TERMUX_PKG_NAME}" || { + git checkout -- "${PKG_DIR}" + git pull --rebase + termux_error_exit "ERROR: failed to update checksum." + } + fi + + echo "INFO: Trying to build package." + if "${TERMUX_SCRIPTDIR}/scripts/run-docker.sh" ./build-package.sh -a "${TERMUX_ARCH}" -I "${TERMUX_PKG_NAME}"; then + if [[ "${GIT_COMMIT_PACKAGES}" == "true" ]]; then + echo "INFO: Committing package." + stderr="$( + git add "${PKG_DIR}" 2>&1 >/dev/null + git commit -m "${TERMUX_PKG_NAME}: update to $(echo "${LATEST_VERSION}" | cut -d':' -f2)" \ + -m "This commit has been automatically submitted by Github Actions." 2>&1 >/dev/null + )" || { + termux_error_exit <<-EndOfError + ERROR: git commit failed. See below for details. + ${stderr} + EndOfError + } + fi + + if [[ "${GIT_PUSH_PACKAGES}" == "true" ]]; then + echo "INFO: Pushing package." + stderr="$( + git pull --rebase 2>&1 >/dev/null + git push 2>&1 >/dev/null + )" || { + termux_error_exit <<-EndOfError + ERROR: git push failed. See below for details. + ${stderr} + EndOfError + } + fi + else + git checkout -- "${PKG_DIR}" + termux_error_exit "ERROR: failed to build." + fi + + fi +}