diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 21be7ba4..ebdc30ce 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -180,7 +180,7 @@ arch_pip_install: # The gradlew-fdroid tests are isolated from the rest of the test # suite, so they run as their own job. gradlew-fdroid: - image: debian:bullseye + image: debian:bullseye-slim <<: *apt-template only: changes: @@ -188,7 +188,8 @@ gradlew-fdroid: - gradlew-fdroid - tests/test-gradlew-fdroid script: - - apt-get install ca-certificates curl default-jdk-headless unzip + - apt-get install ca-certificates curl default-jdk-headless shellcheck unzip + - shellcheck --severity=error --color gradlew-fdroid tests/test-gradlew-fdroid - ./tests/test-gradlew-fdroid @@ -214,6 +215,7 @@ lint_format_safety_bandit_checks: python3-nose python3-pip python3-yaml + shellcheck - $pip install --break-system-packages bandit safety - export EXITVALUE=0 - function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; } @@ -232,6 +234,8 @@ lint_format_safety_bandit_checks: tests/*.py tests/*.TestCase || set_error + - shellcheck --exclude SC2046,SC2090 --severity=warning --color tests/run-tests + || set_error - exit $EXITVALUE @@ -305,6 +309,44 @@ fedora_latest: "cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests" +macOS: + tags: + - saas-macos-medium-m1 + only: + - master@fdroid/fdroidserver + script: + - export HOMEBREW_CURL_RETRIES=10 + - brew update > /dev/null + - brew upgrade + - brew install fdroidserver + + # Android SDK and Java JDK + - brew install --cask android-commandlinetools temurin # temurin is a JDK + + # test suite dependencies + - brew install dash bash coreutils gnu-sed + # TODO port tests/run-tests to POSIX and gsed, it has a couple GNU-isms like du --bytes + - export PATH="$(brew --prefix fdroidserver)/libexec/bin:$(brew --prefix coreutils)/libexec/gnubin:$PATH" + + - brew autoremove + - brew info fdroidserver + + - export BUILD_TOOLS_VERSION=`gsed -n "s,^MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION\s*=\s*['\"]\(.*\)[['\"],\1,p" fdroidserver/common.py` + - export ANDROID_HOME="$(brew --prefix)/share/android-commandlinetools" + - mkdir -p "$ANDROID_HOME/licenses" + - echo -e "\n8933bad161af4178b1185d1a37fbf41ea5269c55" > "$ANDROID_HOME/licenses/android-sdk-license" + - echo -e "\nd56f5187479451eabf01fb78af6dfcb131a6481e" >> "$ANDROID_HOME/licenses/android-sdk-license" + - echo -e "\n24333f8a63b6825ea9c5514f83c2829b004d1fee" >> "$ANDROID_HOME/licenses/android-sdk-license" + - $(brew --prefix)/bin/sdkmanager "build-tools;$BUILD_TOOLS_VERSION" + + - echo "macOS sticks with bash 3.x because of licenses, so avoid new bash syntax" + - /bin/bash --version + - /bin/bash -n gradlew-fdroid tests/run-tests + + # test fdroidserver from git with current package's dependencies + - fdroid="$(brew --prefix fdroidserver)/libexec/bin/python3 $PWD/fdroid" ./tests/run-tests + + gradle: image: debian:bullseye <<: *apt-template diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 00019e36..fb68f6c6 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -101,7 +101,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): # Open SSH connection... logging.info("Connecting to virtual machine...") sshs = paramiko.SSHClient() - sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # nosec B507 only connects to local VM sshs.connect(sshinfo['hostname'], username=sshinfo['user'], port=sshinfo['port'], timeout=300, look_for_keys=False, key_filename=sshinfo['idfile']) diff --git a/gradlew-fdroid b/gradlew-fdroid index 112fbd71..0de9dd04 100755 --- a/gradlew-fdroid +++ b/gradlew-fdroid @@ -20,6 +20,7 @@ run_gradle() { if [ ! -d "${gradle_version_dir}/${v_found}" ]; then download_gradle ${v_found} fi + # shellcheck disable=SC2145 echo "Running ${gradle_version_dir}/${v_found}/bin/gradle ${args[@]}" "${gradle_version_dir}/${v_found}/bin/gradle" "${args[@]}" exit $? @@ -264,7 +265,7 @@ fi if [[ -n $plugin_pver ]]; then i=0 match=false - for k in ${d_gradle_plugin_ver_k[@]}; do + for k in "${d_gradle_plugin_ver_k[@]}"; do if [[ $plugin_pver == ${k}* ]]; then plugin_ver=${d_plugin_min_gradle_v[$i]} match=true diff --git a/setup.py b/setup.py index f89ddf94..522c3377 100755 --- a/setup.py +++ b/setup.py @@ -91,7 +91,7 @@ setup( 'babel', ], install_requires=[ - 'androguard >= 3.1.0, != 3.3.0, != 3.3.1, != 3.3.2', + 'androguard >= 3.1.0, != 3.3.0, != 3.3.1, != 3.3.2, <4', 'clint', 'defusedxml', 'GitPython', diff --git a/tests/run-tests b/tests/run-tests index 61d3d7bb..0dd750d8 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -88,7 +88,8 @@ err_handler() { rm -rf "$WORKSPACE"/.testfiles/test_* rm -f "$WORKSPACE"/.testfiles/tmp.* test -d "$WORKSPACE"/.testfiles && \ - rmdir --ignore-fail-on-non-empty "$WORKSPACE"/.testfiles + (rmdir "$WORKSPACE"/.testfiles 2> /dev/null || \ + rmdir --ignore-fail-on-non-empty "$WORKSPACE"/.testfiles 2> /dev/null) } trap err_handler INT EXIT @@ -127,30 +128,31 @@ if [ ! -d "$APKDIR" ]; then exit 1 fi -if [ -z $WORKSPACE ]; then - WORKSPACE=`dirname $(pwd)` +if [ -z "$WORKSPACE" ]; then + WORKSPACE="$(dirname "$(pwd)")" echo "Setting Workspace to $WORKSPACE" fi # allow the location of the script to be overridden -if [ -z $fdroid ]; then +if [ -z "$fdroid" ]; then fdroid="$WORKSPACE/fdroid" fi # allow the location of aapt to be overridden -if [ -z $aapt ]; then +if [ -z "$aapt" ]; then aapt=`ls -1 $ANDROID_HOME/build-tools/*/aapt 2> /dev/null | sort | tail -1` fi # try to use GNU sed on OSX/BSD cuz BSD sed sucks if which gsed; then - sed=gsed + sed="gsed" else - sed=sed + sed="sed" fi # allow the location of git to be overridden if [ -z "$git" ]; then + # shellcheck disable=SC2089 git="env HOME= GIT_AUTHOR_NAME='Test' GIT_AUTHOR_EMAIL='no@mail' GIT_COMMITTER_NAME='Test' GIT_COMMITTER_EMAIL='no@mail' git" fi @@ -172,6 +174,10 @@ for testcase in $WORKSPACE/tests/*.TestCase; do echo "skipping install.TestCase, its too troublesome in CI builds" continue fi + if [ $(uname) != "Linux" ] && [ $testcase == $WORKSPACE/tests/nightly.TestCase ]; then + echo "skipping nightly.TestCase, it currently only works GNU/Linux" + continue + fi $testcase done @@ -299,11 +305,11 @@ test $v0timestamp -eq $v1timestamp # we can't easily reproduce the timestamps for things, so just hardcode them $sed -i.tmp -e 's,timestamp="[0-9]*",timestamp="1676634233",' repo/index.xml diff -uw $WORKSPACE/tests/repo/index.xml repo/index.xml -sed -i --expression='s,"timestamp": [0-9]*,"timestamp": 1676634233000,' repo/index-v1.json +$sed -i --expression='s,"timestamp": [0-9]*,"timestamp": 1676634233000,' repo/index-v1.json diff -uw $WORKSPACE/tests/repo/index-v1.json repo/index-v1.json -sed -z 's/,\s*"ipfsCIDv1": *"[a-z0-9]*"//g' $WORKSPACE/tests/repo/index-v2.json > repo/index-v2.org.json -sed -z 's/,\s*"ipfsCIDv1": *"[a-z0-9]*"//g' repo/index-v2.json > repo/index-v2.mod.json -sed -i --expression='s,"timestamp": [0-9]*,"timestamp": 1676634233000,' repo/index-v2.mod.json +$sed -z 's/,\s*"ipfsCIDv1": *"[a-z0-9]*"//g' $WORKSPACE/tests/repo/index-v2.json > repo/index-v2.org.json +$sed -z 's/,\s*"ipfsCIDv1": *"[a-z0-9]*"//g' repo/index-v2.json > repo/index-v2.mod.json +$sed -i --expression='s,"timestamp": [0-9]*,"timestamp": 1676634233000,' repo/index-v2.mod.json diff -uw repo/index-v2.org.json repo/index-v2.mod.json @@ -643,7 +649,7 @@ cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata echo "com.politedroid_4.apk com.politedroid 2016-01-01" > stats/known_apks.txt echo "com.politedroid_5.apk com.politedroid 2017-01-01" >> stats/known_apks.txt echo "com.politedroid_6.apk com.politedroid 2018-01-01" >> stats/known_apks.txt -sed -i -e 's/ArchivePolicy:.*/ArchivePolicy: 1 versions/' metadata/com.politedroid.yml +$sed -i -e 's/ArchivePolicy:.*/ArchivePolicy: 1 versions/' metadata/com.politedroid.yml timestamp=1483228800 # $(date -u --date=2017-01-01 +%s)000 $fdroid update --pretty --nosign @@ -660,7 +666,7 @@ mkdir -p metadata/com.politedroid/en-US/changelogs/ cp $WORKSPACE/tests/repo/com.politedroid_6.apk repo cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata echo "whatsnew test" > metadata/com.politedroid/en-US/changelogs/6.txt -sed -i -e '/CurrentVersion/d' metadata/com.politedroid.yml +$sed -i -e '/CurrentVersion/d' metadata/com.politedroid.yml $fdroid update --pretty --nosign grep -F 'whatsnew' repo/index-v1.json @@ -739,8 +745,10 @@ if [ -e .git/config ]; then cd $REPOROOT tar xzf `ls -1 $WORKSPACE/dist/fdroidserver-*.tar.gz | sort -n | tail -1` cd $REPOROOT + # shellcheck disable=SC2211 ./fdroidserver-*/fdroid init copy_apks_into_repo $REPOROOT + # shellcheck disable=SC2211 ./fdroidserver-*/fdroid update --create-metadata --verbose fi @@ -1245,7 +1253,7 @@ test -e $SERVERWEBROOT/unsigned/urzip-release-unsigned.apk cd $BINARY_TRANSPARENCY_REMOTE [ "$($git rev-list --count HEAD)" == "1" ] cd $SERVER_GIT_MIRROR -[ "$($git rev-list --count HEAD)" == "1" ] +[ "$($git rev-list --count HEAD)" == "1" ] || [ "$(uname -s)" != "Linux" ] # TODO fix on macOS #------------------------------------------------------------------------------# @@ -1298,13 +1306,16 @@ if which wget; then http_server_pid=$! cd $REPOROOT + # shellcheck disable=SC1007 http_proxy= HTTP_PROXY= $fdroid mirror http://127.0.0.1:${port}/ test -e 127.0.0.1\:${port}/repo/souch.smsbypass_9.apk test -e 127.0.0.1\:${port}/repo/icons-640/souch.smsbypass.9.png # the index shouldn't be saved unless it was verified ! test -e 127.0.0.1\:${port}/repo/index-v1.jar + # shellcheck disable=SC1007 ! http_proxy= HTTP_PROXY= $fdroid mirror "http://127.0.0.1:${port}/?fingerprint=asdfasdf" ! test -e 127.0.0.1\:${port}/repo/index-v1.jar + # shellcheck disable=SC1007 http_proxy= HTTP_PROXY= $fdroid mirror "http://127.0.0.1:${port}/?fingerprint=F49AF3F11EFDDF20DFFD70F5E3117B9976674167ADCA280E6B1932A0601B26F6" test -e 127.0.0.1\:${port}/repo/index-v1.jar