From c7bef4b5adea834d63d2655df1d7395cd81d325b Mon Sep 17 00:00:00 2001 From: Maxython Date: Tue, 2 Nov 2021 15:00:20 +0300 Subject: [PATCH] Compiler update --- .github/workflows/package_updates.yml | 27 +++++ .github/workflows/packages.yml | 102 ++++++++++++------- scripts/build/setup/termux_setup_protobuf.sh | 2 +- scripts/build/termux_extract_dep_info.sh | 1 + scripts/build/termux_get_repo_files.sh | 78 +++++++------- scripts/build/termux_step_massage.sh | 13 +-- scripts/build/termux_step_setup_toolchain.sh | 5 +- scripts/build/termux_step_setup_variables.sh | 1 + 8 files changed, 147 insertions(+), 82 deletions(-) create mode 100644 .github/workflows/package_updates.yml diff --git a/.github/workflows/package_updates.yml b/.github/workflows/package_updates.yml new file mode 100644 index 0000000000..0a048f9aad --- /dev/null +++ b/.github/workflows/package_updates.yml @@ -0,0 +1,27 @@ +name: Package updates + +on: + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +jobs: + update-packages: + if: github.repository == 'termux/termux-packages' + runs-on: ubuntu-latest + steps: + - name: Clone repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + token: ${{ secrets.GH_API_KEY }} + - name: Process package updates + env: + GITHUB_API_TOKEN: ${{ secrets.GH_API_KEY }} + BUILD_PACKAGES: "true" + GIT_COMMIT_PACKAGES: "true" + GIT_PUSH_PACKAGES: "true" + run: | + git config --global user.name "Termux Github Actions" + git config --global user.email "contact@termux.com" + bash ./scripts/bin/update-packages diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 3ee4f38249..f35348c874 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -9,6 +9,11 @@ on: pull_request: paths: - 'packages/**' + workflow_dispatch: + inputs: + packages: + description: "A space-separated names of packages selected for rebuilding" + required: true jobs: build: @@ -27,24 +32,26 @@ jobs: fetch-depth: 1000 - name: Build run: | - BASE_COMMIT=$(jq --raw-output .pull_request.base.sha "$GITHUB_EVENT_PATH") - OLD_COMMIT=$(jq --raw-output .commits[0].id "$GITHUB_EVENT_PATH") - HEAD_COMMIT=$(jq --raw-output .commits[-1].id "$GITHUB_EVENT_PATH") - if [ "$BASE_COMMIT" = "null" ]; then - if [ "$OLD_COMMIT" = "$HEAD_COMMIT" ]; then - # Single-commit push. - echo "Processing commit: ${HEAD_COMMIT}" - CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${HEAD_COMMIT}") + if [ "${{ github.event_name }}" != "workflow_dispatch" ]; then + BASE_COMMIT=$(jq --raw-output .pull_request.base.sha "$GITHUB_EVENT_PATH") + OLD_COMMIT=$(jq --raw-output .commits[0].id "$GITHUB_EVENT_PATH") + HEAD_COMMIT=$(jq --raw-output .commits[-1].id "$GITHUB_EVENT_PATH") + if [ "$BASE_COMMIT" = "null" ]; then + if [ "$OLD_COMMIT" = "$HEAD_COMMIT" ]; then + # Single-commit push. + echo "Processing commit: ${HEAD_COMMIT}" + CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${HEAD_COMMIT}") + else + # Multi-commit push. + OLD_COMMIT="${OLD_COMMIT}~1" + echo "Processing commit range: ${OLD_COMMIT}..${HEAD_COMMIT}" + CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${OLD_COMMIT}" "${HEAD_COMMIT}") + fi else - # Multi-commit push. - OLD_COMMIT="${OLD_COMMIT}~1" - echo "Processing commit range: ${OLD_COMMIT}..${HEAD_COMMIT}" - CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${OLD_COMMIT}" "${HEAD_COMMIT}") + # Pull requests. + echo "Processing pull request #$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH"): ${BASE_COMMIT}..HEAD" + CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${BASE_COMMIT}" "HEAD") fi - else - # Pull requests. - echo "Processing pull request #$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH"): ${BASE_COMMIT}..HEAD" - CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${BASE_COMMIT}" "HEAD") fi mkdir -p ./artifacts ./output touch ./output/.placeholder @@ -71,28 +78,49 @@ jobs: # This file does not belong to a package, so ignore it continue fi - if [[ $file =~ ^packages/([a-z0-9+-]*)/([a-z0-9+-]*).subpackage.sh$ ]]; then - # A subpackage was modified, check if it was deleted or just updated - pkg=${BASH_REMATCH[1]} - subpkg=${BASH_REMATCH[2]} - if [ ! -f "packages/${pkg}/${subpkg}.subpackage.sh" ]; then - echo "$subpkg" >> ./deleted_packages.txt - fi - elif [[ $file =~ ^packages/([a-z0-9+-]*)/.*$ ]]; then - # package, check if it was deleted or updated - pkg=${BASH_REMATCH[1]} - if [ ! -d "packages/${pkg}" ]; then - echo "$pkg" >> ./deleted_packages.txt - else - echo "$pkg" >> ./built_packages.txt - # If there are subpackages we want to create a list of those - # as well - for file in $(find "packages/${pkg}/" -maxdepth 1 -type f -name \*.subpackage.sh | sort); do - echo "$(basename "${file%%.subpackage.sh}")" >> ./built_subpackages.txt - done - fi + # Build local Docker image if setup scripts were changed. + # Useful for pull requests submitting changes for both build environment and packages. + if grep -qP '^scripts/(Dockerfile|setup-ubuntu\.sh)$' <<< "$CHANGED_FILES"; then + echo "Detected changes for environment setup scripts. Building custom Docker image now." + cd ./scripts + docker build -t termux/package-builder:latest . + cd .. fi - done<<<${CHANGED_FILES} + # Parse changed files and identify new packages and deleted packages. + # Create lists of those packages that will be passed to upload job for + # further processing. + while read -r file; do + if ! [[ $file == packages/* ]]; then + # This file does not belong to a package, so ignore it + continue + fi + if [[ $file =~ ^packages/([a-z0-9+-]*)/([a-z0-9+-]*).subpackage.sh$ ]]; then + # A subpackage was modified, check if it was deleted or just updated + pkg=${BASH_REMATCH[1]} + subpkg=${BASH_REMATCH[2]} + if [ ! -f "packages/${pkg}/${subpkg}.subpackage.sh" ]; then + echo "$subpkg" >> ./deleted_packages.txt + fi + elif [[ $file =~ ^packages/([a-z0-9+-]*)/.*$ ]]; then + # package, check if it was deleted or updated + pkg=${BASH_REMATCH[1]} + if [ ! -d "packages/${pkg}" ]; then + echo "$pkg" >> ./deleted_packages.txt + else + echo "$pkg" >> ./built_packages.txt + # If there are subpackages we want to create a list of those + # as well + for file in $(find "packages/${pkg}/" -maxdepth 1 -type f -name \*.subpackage.sh | sort); do + echo "$(basename "${file%%.subpackage.sh}")" >> ./built_subpackages.txt + done + fi + fi + done<<<${CHANGED_FILES} + else + for i in ${{ github.event.inputs.packages }}; do + echo "$i" >> ./built_packages.txt + done + fi # Fix so that lists do not contain duplicates if [ -f ./built_packages.txt ]; then diff --git a/scripts/build/setup/termux_setup_protobuf.sh b/scripts/build/setup/termux_setup_protobuf.sh index 3092b4dfce..1ac4ed6126 100644 --- a/scripts/build/setup/termux_setup_protobuf.sh +++ b/scripts/build/setup/termux_setup_protobuf.sh @@ -14,7 +14,7 @@ termux_setup_protobuf() { termux_download \ https://github.com/protocolbuffers/protobuf/releases/download/v$_PROTOBUF_VERSION/$_PROTOBUF_ZIP \ $TERMUX_PKG_TMPDIR/$_PROTOBUF_ZIP \ - 2994b7256f7416b90ad831dbf76a27c0934386deb514587109f39141f2636f37 + 4b18a69b3093432ee0531bc9bf3c4114f81bde1670ade2875f694180ac8bd7f6 rm -Rf "$TERMUX_PKG_TMPDIR/protoc-$_PROTOBUF_VERSION-linux-x86_64" unzip $TERMUX_PKG_TMPDIR/$_PROTOBUF_ZIP -d $TERMUX_PKG_TMPDIR/protobuf-$_PROTOBUF_VERSION diff --git a/scripts/build/termux_extract_dep_info.sh b/scripts/build/termux_extract_dep_info.sh index 9d4fbdbda4..7d6e5be138 100755 --- a/scripts/build/termux_extract_dep_info.sh +++ b/scripts/build/termux_extract_dep_info.sh @@ -23,6 +23,7 @@ termux_extract_dep_info() { ) else TERMUX_ARCH=$( + TERMUX_PKG_PLATFORM_INDEPENDENT="false" source ${PKG_DIR}/build.sh if [ "$TERMUX_PKG_PLATFORM_INDEPENDENT" = "true" ]; then echo all diff --git a/scripts/build/termux_get_repo_files.sh b/scripts/build/termux_get_repo_files.sh index 8386acf432..a5bd8b3d18 100644 --- a/scripts/build/termux_get_repo_files.sh +++ b/scripts/build/termux_get_repo_files.sh @@ -1,51 +1,55 @@ termux_get_repo_files() { - # Not needed for on-device builds. - [ "$TERMUX_ON_DEVICE_BUILD" = "true" ] && return + # Not needed for on-device builds or when building + # dependencies. + if [ "$TERMUX_ON_DEVICE_BUILD" = "true" ] || [ "$TERMUX_INSTALL_DEPS" = "false" ]; then + return + fi - if [ "$TERMUX_INSTALL_DEPS" = true ]; then - for idx in $(seq ${#TERMUX_REPO_URL[@]}); do - local TERMUX_REPO_NAME=$(echo ${TERMUX_REPO_URL[$idx-1]} | sed -e 's%https://%%g' -e 's%http://%%g' -e 's%/%-%g') - local RELEASE_FILE=${TERMUX_COMMON_CACHEDIR}/${TERMUX_REPO_NAME}-${TERMUX_REPO_DISTRIBUTION[$idx-1]}-Release + for idx in $(seq ${#TERMUX_REPO_URL[@]}); do + local TERMUX_REPO_NAME=$(echo ${TERMUX_REPO_URL[$idx-1]} | sed -e 's%https://%%g' -e 's%http://%%g' -e 's%/%-%g') + local RELEASE_FILE=${TERMUX_COMMON_CACHEDIR}/${TERMUX_REPO_NAME}-${TERMUX_REPO_DISTRIBUTION[$idx-1]}-Release + local repo_base="${TERMUX_REPO_URL[$idx-1]}/dists/${TERMUX_REPO_DISTRIBUTION[$idx-1]}" + local dl_prefix="${TERMUX_REPO_NAME}-${TERMUX_REPO_DISTRIBUTION[$idx-1]}-${TERMUX_REPO_COMPONENT[$idx-1]}" - local download_attempts=6 - while ((download_attempts > 0)); do - if termux_download "${TERMUX_REPO_URL[$idx-1]}/dists/${TERMUX_REPO_DISTRIBUTION[$idx-1]}/Release" \ - "$RELEASE_FILE" SKIP_CHECKSUM && \ - termux_download "${TERMUX_REPO_URL[$idx-1]}/dists/${TERMUX_REPO_DISTRIBUTION[$idx-1]}/Release.gpg" \ - "${RELEASE_FILE}.gpg" SKIP_CHECKSUM; then + local download_attempts=6 + while ((download_attempts > 0)); do + if termux_download "${repo_base}/Release" \ + "$RELEASE_FILE" SKIP_CHECKSUM && \ + termux_download "${repo_base}/Release.gpg" \ + "${RELEASE_FILE}.gpg" SKIP_CHECKSUM; then - if gpg --verify "${RELEASE_FILE}.gpg" "$RELEASE_FILE"; then - local failed=false + if ! gpg --verify "${RELEASE_FILE}.gpg" "$RELEASE_FILE"; then + termux_error_exit "failed to verify gpg signature of $RELEASE_FILE" + fi - for arch in all $TERMUX_ARCH; do - local PACKAGES_HASH=$(./scripts/get_hash_from_file.py ${RELEASE_FILE} $arch ${TERMUX_REPO_COMPONENT[$idx-1]}) + local failed=false + for arch in all $TERMUX_ARCH; do + local PACKAGES_HASH=$(./scripts/get_hash_from_file.py ${RELEASE_FILE} $arch ${TERMUX_REPO_COMPONENT[$idx-1]}) - # If packages_hash = "" then the repo probably doesn't contain debs for $arch - if [ -n "$PACKAGES_HASH" ]; then - if ! termux_download "${TERMUX_REPO_URL[$idx-1]}/dists/${TERMUX_REPO_DISTRIBUTION[$idx-1]}/${TERMUX_REPO_COMPONENT[$idx-1]}/binary-$arch/Packages" \ - "${TERMUX_COMMON_CACHEDIR}-$arch/${TERMUX_REPO_NAME}-${TERMUX_REPO_DISTRIBUTION[$idx-1]}-${TERMUX_REPO_COMPONENT[$idx-1]}-Packages" \ - $PACKAGES_HASH; then - failed=true - break - fi - fi - done - - if ! $failed; then + # If packages_hash = "" then the repo probably doesn't contain debs for $arch + if [ -n "$PACKAGES_HASH" ]; then + if ! termux_download "${repo_base}/${TERMUX_REPO_COMPONENT[$idx-1]}/binary-$arch/Packages" \ + "${TERMUX_COMMON_CACHEDIR}-$arch/${dl_prefix}-Packages" \ + $PACKAGES_HASH; then + failed=true break fi fi + done + + if ! $failed; then + break fi + fi - download_attempts=$((download_attempts - 1)) - if ((download_attempts < 1)); then - termux_error_exit "Failed to download package repository metadata. Try to build without -i/-I option." - fi - - echo "Retrying download in 30 seconds (${download_attempts} attempts left)..." >&2 - sleep 30 - done + download_attempts=$((download_attempts - 1)) + if ((download_attempts < 1)); then + termux_error_exit "Failed to download package repository metadata. Try to build without -i/-I option." + fi + echo "Retrying download in 30 seconds (${download_attempts} attempts left)..." >&2 + sleep 30 done - fi + + done } diff --git a/scripts/build/termux_step_massage.sh b/scripts/build/termux_step_massage.sh index 422e17f383..78e6044c0e 100644 --- a/scripts/build/termux_step_massage.sh +++ b/scripts/build/termux_step_massage.sh @@ -32,12 +32,13 @@ termux_step_massage() { find . \( -path "./bin/*" -o -path "./lib/*" -o -path "./libexec/*" -o -path "./opt/*" \) -type f -print0 | xargs -r -0 "$TERMUX_ELF_CLEANER" fi - # Fix shebang paths: - while IFS= read -r -d '' file - do - head -c 100 "$file" | grep -E "^#\!.*\\/bin\\/.*" | grep -q -E -v "^#\! ?\\/system" && \ - sed --follow-symlinks -i -E "1 s@^#\!(.*)/bin/(.*)@#\!$TERMUX_PREFIX/bin/\2@" "$file" - done < <(find -L . -type f -print0) + if [ "$TERMUX_PKG_NO_SHEBANG_FIX" != "true" ]; then + # Fix shebang paths: + while IFS= read -r -d '' file; do + head -c 100 "$file" | grep -E "^#\!.*\\/bin\\/.*" | grep -q -E -v "^#\! ?\\/system" && \ + sed --follow-symlinks -i -E "1 s@^#\!(.*)/bin/(.*)@#\!$TERMUX_PREFIX/bin/\2@" "$file" + done < <(find -L . -type f -print0) + fi # Delete the info directory file. rm -rf ./share/info/dir diff --git a/scripts/build/termux_step_setup_toolchain.sh b/scripts/build/termux_step_setup_toolchain.sh index 2d511fa957..ecbd9c283e 100644 --- a/scripts/build/termux_step_setup_toolchain.sh +++ b/scripts/build/termux_step_setup_toolchain.sh @@ -8,19 +8,21 @@ termux_step_setup_toolchain() { export AS=$TERMUX_HOST_PLATFORM-clang export CC=$TERMUX_HOST_PLATFORM-clang export CXX=$TERMUX_HOST_PLATFORM-clang++ - export AR=llvm-ar export CPP=$TERMUX_HOST_PLATFORM-cpp export LD=ld.lld + export AR=llvm-ar export OBJCOPY=llvm-objcopy export OBJDUMP=llvm-objdump export RANLIB=llvm-ranlib export READELF=llvm-readelf export STRIP=llvm-strip + export NM=llvm-nm if [ "$TERMUX_ON_DEVICE_BUILD" = "false" ]; then export PATH=$TERMUX_STANDALONE_TOOLCHAIN/bin:$PATH export CC_FOR_BUILD=gcc export PKG_CONFIG=$TERMUX_STANDALONE_TOOLCHAIN/bin/pkg-config + export PKGCONFIG=$PKG_CONFIG export CCTERMUX_HOST_PLATFORM=$TERMUX_HOST_PLATFORM$TERMUX_PKG_API_LEVEL if [ $TERMUX_ARCH = arm ]; then CCTERMUX_HOST_PLATFORM=armv7a-linux-androideabi$TERMUX_PKG_API_LEVEL @@ -97,6 +99,7 @@ termux_step_setup_toolchain() { export RUSTFLAGS="-C link-arg=-Wl,-rpath=$TERMUX_PREFIX/lib -C link-arg=-Wl,--enable-new-dtags" export ac_cv_func_getpwent=no + export ac_cv_func_endpwent=yes export ac_cv_func_getpwnam=no export ac_cv_func_getpwuid=no export ac_cv_func_sigsetmask=no diff --git a/scripts/build/termux_step_setup_variables.sh b/scripts/build/termux_step_setup_variables.sh index 368386a4b3..1e6a1bbea3 100644 --- a/scripts/build/termux_step_setup_variables.sh +++ b/scripts/build/termux_step_setup_variables.sh @@ -135,6 +135,7 @@ termux_step_setup_variables() { TERMUX_PKG_TMPDIR=$TERMUX_TOPDIR/$TERMUX_PKG_NAME/tmp TERMUX_PKG_SERVICE_SCRIPT=() # Fill with entries like: ("daemon name" 'script to execute'). Script is echoed with -e so can contain \n for multiple lines TERMUX_PKG_GROUPS="" # https://wiki.archlinux.org/title/Pacman#Installing_package_groups + TERMUX_PKG_NO_SHEBANG_FIX=false # if true, skip fixing shebang accordingly to TERMUX_PREFIX unset CFLAGS CPPFLAGS LDFLAGS CXXFLAGS }