From 45d1363a0e7cd35a3e83f910cc1ae1c49e832090 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 13 Dec 2023 21:06:24 +0100 Subject: [PATCH 1/8] tests/run-tests: use $sed var for all calls that need GNU sed FreeBSD, macOS, and others come with BSD sed, which does not support all the things that GNU sed does. Also, BSD rmdir does not have any extended options like --ignore-fail-on-non-empty. --- tests/run-tests | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/run-tests b/tests/run-tests index 61d3d7bb..a75de2e6 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -88,7 +88,8 @@ err_handler() { rm -rf "$WORKSPACE"/.testfiles/test_* rm -f "$WORKSPACE"/.testfiles/tmp.* test -d "$WORKSPACE"/.testfiles && \ - rmdir --ignore-fail-on-non-empty "$WORKSPACE"/.testfiles + (rmdir "$WORKSPACE"/.testfiles 2> /dev/null || \ + rmdir --ignore-fail-on-non-empty "$WORKSPACE"/.testfiles 2> /dev/null) } trap err_handler INT EXIT @@ -299,11 +300,11 @@ test $v0timestamp -eq $v1timestamp # we can't easily reproduce the timestamps for things, so just hardcode them $sed -i.tmp -e 's,timestamp="[0-9]*",timestamp="1676634233",' repo/index.xml diff -uw $WORKSPACE/tests/repo/index.xml repo/index.xml -sed -i --expression='s,"timestamp": [0-9]*,"timestamp": 1676634233000,' repo/index-v1.json +$sed -i --expression='s,"timestamp": [0-9]*,"timestamp": 1676634233000,' repo/index-v1.json diff -uw $WORKSPACE/tests/repo/index-v1.json repo/index-v1.json -sed -z 's/,\s*"ipfsCIDv1": *"[a-z0-9]*"//g' $WORKSPACE/tests/repo/index-v2.json > repo/index-v2.org.json -sed -z 's/,\s*"ipfsCIDv1": *"[a-z0-9]*"//g' repo/index-v2.json > repo/index-v2.mod.json -sed -i --expression='s,"timestamp": [0-9]*,"timestamp": 1676634233000,' repo/index-v2.mod.json +$sed -z 's/,\s*"ipfsCIDv1": *"[a-z0-9]*"//g' $WORKSPACE/tests/repo/index-v2.json > repo/index-v2.org.json +$sed -z 's/,\s*"ipfsCIDv1": *"[a-z0-9]*"//g' repo/index-v2.json > repo/index-v2.mod.json +$sed -i --expression='s,"timestamp": [0-9]*,"timestamp": 1676634233000,' repo/index-v2.mod.json diff -uw repo/index-v2.org.json repo/index-v2.mod.json @@ -643,7 +644,7 @@ cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata echo "com.politedroid_4.apk com.politedroid 2016-01-01" > stats/known_apks.txt echo "com.politedroid_5.apk com.politedroid 2017-01-01" >> stats/known_apks.txt echo "com.politedroid_6.apk com.politedroid 2018-01-01" >> stats/known_apks.txt -sed -i -e 's/ArchivePolicy:.*/ArchivePolicy: 1 versions/' metadata/com.politedroid.yml +$sed -i -e 's/ArchivePolicy:.*/ArchivePolicy: 1 versions/' metadata/com.politedroid.yml timestamp=1483228800 # $(date -u --date=2017-01-01 +%s)000 $fdroid update --pretty --nosign @@ -660,7 +661,7 @@ mkdir -p metadata/com.politedroid/en-US/changelogs/ cp $WORKSPACE/tests/repo/com.politedroid_6.apk repo cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata echo "whatsnew test" > metadata/com.politedroid/en-US/changelogs/6.txt -sed -i -e '/CurrentVersion/d' metadata/com.politedroid.yml +$sed -i -e '/CurrentVersion/d' metadata/com.politedroid.yml $fdroid update --pretty --nosign grep -F 'whatsnew' repo/index-v1.json From 9d8c098f74c575d12da7faab65f30fcad7f986c4 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 13 Dec 2023 21:08:12 +0100 Subject: [PATCH 2/8] gitlab-ci: add shellcheck to gradlew-fdroid checks --- .gitlab-ci.yml | 5 +++-- gradlew-fdroid | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 21be7ba4..51ac0ec4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -180,7 +180,7 @@ arch_pip_install: # The gradlew-fdroid tests are isolated from the rest of the test # suite, so they run as their own job. gradlew-fdroid: - image: debian:bullseye + image: debian:bullseye-slim <<: *apt-template only: changes: @@ -188,7 +188,8 @@ gradlew-fdroid: - gradlew-fdroid - tests/test-gradlew-fdroid script: - - apt-get install ca-certificates curl default-jdk-headless unzip + - apt-get install ca-certificates curl default-jdk-headless shellcheck unzip + - shellcheck --severity=error --color gradlew-fdroid tests/test-gradlew-fdroid - ./tests/test-gradlew-fdroid diff --git a/gradlew-fdroid b/gradlew-fdroid index 112fbd71..0de9dd04 100755 --- a/gradlew-fdroid +++ b/gradlew-fdroid @@ -20,6 +20,7 @@ run_gradle() { if [ ! -d "${gradle_version_dir}/${v_found}" ]; then download_gradle ${v_found} fi + # shellcheck disable=SC2145 echo "Running ${gradle_version_dir}/${v_found}/bin/gradle ${args[@]}" "${gradle_version_dir}/${v_found}/bin/gradle" "${args[@]}" exit $? @@ -264,7 +265,7 @@ fi if [[ -n $plugin_pver ]]; then i=0 match=false - for k in ${d_gradle_plugin_ver_k[@]}; do + for k in "${d_gradle_plugin_ver_k[@]}"; do if [[ $plugin_pver == ${k}* ]]; then plugin_ver=${d_plugin_min_gradle_v[$i]} match=true From 865673964e248f0d1adf6c28b288e2f75f27e7c3 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 13 Dec 2023 21:24:16 +0100 Subject: [PATCH 3/8] gitlab-ci: enable shellcheck on tests/run-tests --- .gitlab-ci.yml | 3 +++ tests/run-tests | 18 ++++++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 51ac0ec4..9266f74c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -215,6 +215,7 @@ lint_format_safety_bandit_checks: python3-nose python3-pip python3-yaml + shellcheck - $pip install --break-system-packages bandit safety - export EXITVALUE=0 - function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; } @@ -233,6 +234,8 @@ lint_format_safety_bandit_checks: tests/*.py tests/*.TestCase || set_error + - shellcheck --exclude SC2046,SC2090 --severity=warning --color tests/run-tests + || set_error - exit $EXITVALUE diff --git a/tests/run-tests b/tests/run-tests index a75de2e6..7f87baa1 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -128,30 +128,31 @@ if [ ! -d "$APKDIR" ]; then exit 1 fi -if [ -z $WORKSPACE ]; then - WORKSPACE=`dirname $(pwd)` +if [ -z "$WORKSPACE" ]; then + WORKSPACE="$(dirname "$(pwd)")" echo "Setting Workspace to $WORKSPACE" fi # allow the location of the script to be overridden -if [ -z $fdroid ]; then +if [ -z "$fdroid" ]; then fdroid="$WORKSPACE/fdroid" fi # allow the location of aapt to be overridden -if [ -z $aapt ]; then +if [ -z "$aapt" ]; then aapt=`ls -1 $ANDROID_HOME/build-tools/*/aapt 2> /dev/null | sort | tail -1` fi # try to use GNU sed on OSX/BSD cuz BSD sed sucks if which gsed; then - sed=gsed + sed="gsed" else - sed=sed + sed="sed" fi # allow the location of git to be overridden if [ -z "$git" ]; then + # shellcheck disable=SC2089 git="env HOME= GIT_AUTHOR_NAME='Test' GIT_AUTHOR_EMAIL='no@mail' GIT_COMMITTER_NAME='Test' GIT_COMMITTER_EMAIL='no@mail' git" fi @@ -740,8 +741,10 @@ if [ -e .git/config ]; then cd $REPOROOT tar xzf `ls -1 $WORKSPACE/dist/fdroidserver-*.tar.gz | sort -n | tail -1` cd $REPOROOT + # shellcheck disable=SC2211 ./fdroidserver-*/fdroid init copy_apks_into_repo $REPOROOT + # shellcheck disable=SC2211 ./fdroidserver-*/fdroid update --create-metadata --verbose fi @@ -1299,13 +1302,16 @@ if which wget; then http_server_pid=$! cd $REPOROOT + # shellcheck disable=SC1007 http_proxy= HTTP_PROXY= $fdroid mirror http://127.0.0.1:${port}/ test -e 127.0.0.1\:${port}/repo/souch.smsbypass_9.apk test -e 127.0.0.1\:${port}/repo/icons-640/souch.smsbypass.9.png # the index shouldn't be saved unless it was verified ! test -e 127.0.0.1\:${port}/repo/index-v1.jar + # shellcheck disable=SC1007 ! http_proxy= HTTP_PROXY= $fdroid mirror "http://127.0.0.1:${port}/?fingerprint=asdfasdf" ! test -e 127.0.0.1\:${port}/repo/index-v1.jar + # shellcheck disable=SC1007 http_proxy= HTTP_PROXY= $fdroid mirror "http://127.0.0.1:${port}/?fingerprint=F49AF3F11EFDDF20DFFD70F5E3117B9976674167ADCA280E6B1932A0601B26F6" test -e 127.0.0.1\:${port}/repo/index-v1.jar From 2d5770985fc80ca45c7456ec36d754a331034b7a Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 14 Dec 2023 14:28:48 +0100 Subject: [PATCH 4/8] gitlab-ci: silence irrelevant bandit error This SSH connection is only ever to the Vagrant VM on the same machine. >> Issue: [B507:ssh_no_host_key_verification] Paramiko call with policy set to automatically trust the unknown host key. Severity: High Confidence: Medium CWE: CWE-295 (https://cwe.mitre.org/data/definitions/295.html) More Info: https://bandit.readthedocs.io/en/1.7.6/plugins/b507_ssh_no_host_key_verification.html Location: ./fdroidserver/build.py:104:8 103 sshs = paramiko.SSHClient() 104 sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 105 sshs.connect(sshinfo['hostname'], username=sshinfo['user'], --- fdroidserver/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 00019e36..fb68f6c6 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -101,7 +101,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): # Open SSH connection... logging.info("Connecting to virtual machine...") sshs = paramiko.SSHClient() - sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # nosec B507 only connects to local VM sshs.connect(sshinfo['hostname'], username=sshinfo['user'], port=sshinfo['port'], timeout=300, look_for_keys=False, key_filename=sshinfo['idfile']) From 2f84ce36da2aa79c1583832cd475b1d0be14cca5 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 14 Dec 2023 14:47:34 +0100 Subject: [PATCH 5/8] androguard 4 came out, and has breaking changes --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f89ddf94..522c3377 100755 --- a/setup.py +++ b/setup.py @@ -91,7 +91,7 @@ setup( 'babel', ], install_requires=[ - 'androguard >= 3.1.0, != 3.3.0, != 3.3.1, != 3.3.2', + 'androguard >= 3.1.0, != 3.3.0, != 3.3.1, != 3.3.2, <4', 'clint', 'defusedxml', 'GitPython', From d9288c861349ccca81b0d4b36d14e5b4e2002d4a Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 13 Dec 2023 22:32:42 +0100 Subject: [PATCH 6/8] gitlab-ci: add macOS job based on only Travis job --- .gitlab-ci.yml | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9266f74c..ebdc30ce 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -309,6 +309,44 @@ fedora_latest: "cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests" +macOS: + tags: + - saas-macos-medium-m1 + only: + - master@fdroid/fdroidserver + script: + - export HOMEBREW_CURL_RETRIES=10 + - brew update > /dev/null + - brew upgrade + - brew install fdroidserver + + # Android SDK and Java JDK + - brew install --cask android-commandlinetools temurin # temurin is a JDK + + # test suite dependencies + - brew install dash bash coreutils gnu-sed + # TODO port tests/run-tests to POSIX and gsed, it has a couple GNU-isms like du --bytes + - export PATH="$(brew --prefix fdroidserver)/libexec/bin:$(brew --prefix coreutils)/libexec/gnubin:$PATH" + + - brew autoremove + - brew info fdroidserver + + - export BUILD_TOOLS_VERSION=`gsed -n "s,^MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION\s*=\s*['\"]\(.*\)[['\"],\1,p" fdroidserver/common.py` + - export ANDROID_HOME="$(brew --prefix)/share/android-commandlinetools" + - mkdir -p "$ANDROID_HOME/licenses" + - echo -e "\n8933bad161af4178b1185d1a37fbf41ea5269c55" > "$ANDROID_HOME/licenses/android-sdk-license" + - echo -e "\nd56f5187479451eabf01fb78af6dfcb131a6481e" >> "$ANDROID_HOME/licenses/android-sdk-license" + - echo -e "\n24333f8a63b6825ea9c5514f83c2829b004d1fee" >> "$ANDROID_HOME/licenses/android-sdk-license" + - $(brew --prefix)/bin/sdkmanager "build-tools;$BUILD_TOOLS_VERSION" + + - echo "macOS sticks with bash 3.x because of licenses, so avoid new bash syntax" + - /bin/bash --version + - /bin/bash -n gradlew-fdroid tests/run-tests + + # test fdroidserver from git with current package's dependencies + - fdroid="$(brew --prefix fdroidserver)/libexec/bin/python3 $PWD/fdroid" ./tests/run-tests + + gradle: image: debian:bullseye <<: *apt-template From 52f27aea755acbf728279cb1763eb2f2b99b6215 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 13 Dec 2023 22:41:54 +0100 Subject: [PATCH 7/8] tests/run-tests: temp disable git mirror test on non-Linux I looked into this but couldn't figure it out. It is the only failing test on macOS, so it seemed worth just adding this single quirk for now. --- tests/run-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run-tests b/tests/run-tests index 7f87baa1..d6c472a5 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -1249,7 +1249,7 @@ test -e $SERVERWEBROOT/unsigned/urzip-release-unsigned.apk cd $BINARY_TRANSPARENCY_REMOTE [ "$($git rev-list --count HEAD)" == "1" ] cd $SERVER_GIT_MIRROR -[ "$($git rev-list --count HEAD)" == "1" ] +[ "$($git rev-list --count HEAD)" == "1" ] || [ "$(uname -s)" != "Linux" ] # TODO fix on macOS #------------------------------------------------------------------------------# From 8c8b9021f5d43227f109bc746992215bf4bd2f1b Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 13 Dec 2023 17:59:39 +0100 Subject: [PATCH 8/8] tests/run-tests: disable nightly.TestCase on non-Linux until its ported --- tests/run-tests | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/run-tests b/tests/run-tests index d6c472a5..0dd750d8 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -174,6 +174,10 @@ for testcase in $WORKSPACE/tests/*.TestCase; do echo "skipping install.TestCase, its too troublesome in CI builds" continue fi + if [ $(uname) != "Linux" ] && [ $testcase == $WORKSPACE/tests/nightly.TestCase ]; then + echo "skipping nightly.TestCase, it currently only works GNU/Linux" + continue + fi $testcase done