Merge branch 'macOS' into 'master'

GitLab CI: add shellcheck, androguard <4, silence irrelevant bandit error, enable macOS job

See merge request fdroid/fdroidserver!1421

.gitlab-ci.yml

@@ -180,7 +180,7 @@ arch_pip_install:
 # The gradlew-fdroid tests are isolated from the rest of the test
 # suite, so they run as their own job.
 gradlew-fdroid:
-  image: debian:bullseye
+  image: debian:bullseye-slim
   <<: *apt-template
   only:
     changes:
@@ -188,7 +188,8 @@ gradlew-fdroid:
       - gradlew-fdroid
       - tests/test-gradlew-fdroid
   script:
-    - apt-get install ca-certificates curl default-jdk-headless unzip
+    - apt-get install ca-certificates curl default-jdk-headless shellcheck unzip
+    - shellcheck --severity=error --color gradlew-fdroid tests/test-gradlew-fdroid
     - ./tests/test-gradlew-fdroid
 
 
@@ -214,6 +215,7 @@ lint_format_safety_bandit_checks:
           python3-nose
           python3-pip
           python3-yaml
+          shellcheck
     - $pip install --break-system-packages bandit safety
     - export EXITVALUE=0
     - function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; }
@@ -232,6 +234,8 @@ lint_format_safety_bandit_checks:
             tests/*.py
             tests/*.TestCase
         || set_error
+    - shellcheck --exclude SC2046,SC2090 --severity=warning --color tests/run-tests
+        || set_error
     - exit $EXITVALUE
 
 
@@ -305,6 +309,44 @@ fedora_latest:
         "cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests"
 
 
+macOS:
+  tags:
+    - saas-macos-medium-m1
+  only:
+    - master@fdroid/fdroidserver
+  script:
+    - export HOMEBREW_CURL_RETRIES=10
+    - brew update > /dev/null
+    - brew upgrade
+    - brew install fdroidserver
+
+    # Android SDK and Java JDK
+    - brew install --cask android-commandlinetools temurin  # temurin is a JDK
+
+    # test suite dependencies
+    - brew install dash bash coreutils gnu-sed
+    # TODO port tests/run-tests to POSIX and gsed, it has a couple GNU-isms like du --bytes
+    - export PATH="$(brew --prefix fdroidserver)/libexec/bin:$(brew --prefix coreutils)/libexec/gnubin:$PATH"
+
+    - brew autoremove
+    - brew info fdroidserver
+
+    - export BUILD_TOOLS_VERSION=`gsed -n "s,^MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION\s*=\s*['\"]\(.*\)[['\"],\1,p" fdroidserver/common.py`
+    - export ANDROID_HOME="$(brew --prefix)/share/android-commandlinetools"
+    - mkdir -p "$ANDROID_HOME/licenses"
+    - echo -e "\n8933bad161af4178b1185d1a37fbf41ea5269c55" > "$ANDROID_HOME/licenses/android-sdk-license"
+    - echo -e "\nd56f5187479451eabf01fb78af6dfcb131a6481e" >> "$ANDROID_HOME/licenses/android-sdk-license"
+    - echo -e "\n24333f8a63b6825ea9c5514f83c2829b004d1fee" >> "$ANDROID_HOME/licenses/android-sdk-license"
+    - $(brew --prefix)/bin/sdkmanager "build-tools;$BUILD_TOOLS_VERSION"
+
+    - echo "macOS sticks with bash 3.x because of licenses, so avoid new bash syntax"
+    - /bin/bash --version
+    - /bin/bash -n gradlew-fdroid tests/run-tests
+
+    # test fdroidserver from git with current package's dependencies
+    - fdroid="$(brew --prefix fdroidserver)/libexec/bin/python3 $PWD/fdroid" ./tests/run-tests
+
+
 gradle:
   image: debian:bullseye
   <<: *apt-template

fdroidserver/build.py

@@ -101,7 +101,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
         # Open SSH connection...
         logging.info("Connecting to virtual machine...")
         sshs = paramiko.SSHClient()
-        sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy())  # nosec B507 only connects to local VM
         sshs.connect(sshinfo['hostname'], username=sshinfo['user'],
                      port=sshinfo['port'], timeout=300,
                      look_for_keys=False, key_filename=sshinfo['idfile'])

gradlew-fdroid

@@ -20,6 +20,7 @@ run_gradle() {
     if [ ! -d "${gradle_version_dir}/${v_found}" ]; then
         download_gradle ${v_found}
     fi
+    # shellcheck disable=SC2145
     echo "Running ${gradle_version_dir}/${v_found}/bin/gradle ${args[@]}"
     "${gradle_version_dir}/${v_found}/bin/gradle" "${args[@]}"
     exit $?
@@ -264,7 +265,7 @@ fi
 if [[ -n $plugin_pver ]]; then
     i=0
     match=false
-    for k in ${d_gradle_plugin_ver_k[@]}; do
+    for k in "${d_gradle_plugin_ver_k[@]}"; do
         if [[ $plugin_pver == ${k}* ]]; then
             plugin_ver=${d_plugin_min_gradle_v[$i]}
             match=true

setup.py

@@ -91,7 +91,7 @@ setup(
         'babel',
     ],
     install_requires=[
-        'androguard >= 3.1.0, != 3.3.0, != 3.3.1, != 3.3.2',
+        'androguard >= 3.1.0, != 3.3.0, != 3.3.1, != 3.3.2, <4',
         'clint',
         'defusedxml',
         'GitPython',

tests/run-tests

@@ -88,7 +88,8 @@ err_handler() {
     rm -rf "$WORKSPACE"/.testfiles/test_*
     rm -f "$WORKSPACE"/.testfiles/tmp.*
     test -d "$WORKSPACE"/.testfiles && \
-	rmdir --ignore-fail-on-non-empty "$WORKSPACE"/.testfiles
+	(rmdir "$WORKSPACE"/.testfiles 2> /dev/null || \
+             rmdir --ignore-fail-on-non-empty "$WORKSPACE"/.testfiles 2> /dev/null)
 }
 
 trap err_handler INT EXIT
@@ -127,30 +128,31 @@ if [ ! -d "$APKDIR" ]; then
     exit 1
 fi
 
-if [ -z $WORKSPACE ]; then
+if [ -z "$WORKSPACE" ]; then
-    WORKSPACE=`dirname $(pwd)`
+    WORKSPACE="$(dirname "$(pwd)")"
     echo "Setting Workspace to $WORKSPACE"
 fi
 
 # allow the location of the script to be overridden
-if [ -z $fdroid ]; then
+if [ -z "$fdroid" ]; then
     fdroid="$WORKSPACE/fdroid"
 fi
 
 # allow the location of aapt to be overridden
-if [ -z $aapt ]; then
+if [ -z "$aapt" ]; then
     aapt=`ls -1 $ANDROID_HOME/build-tools/*/aapt 2> /dev/null | sort | tail -1`
 fi
 
 # try to use GNU sed on OSX/BSD cuz BSD sed sucks
 if which gsed; then
-    sed=gsed
+    sed="gsed"
 else
-    sed=sed
+    sed="sed"
 fi
 
 # allow the location of git to be overridden
 if [ -z "$git" ]; then
+    # shellcheck disable=SC2089
     git="env HOME= GIT_AUTHOR_NAME='Test' GIT_AUTHOR_EMAIL='no@mail' GIT_COMMITTER_NAME='Test' GIT_COMMITTER_EMAIL='no@mail' git"
 fi
 
@@ -172,6 +174,10 @@ for testcase in $WORKSPACE/tests/*.TestCase; do
         echo "skipping install.TestCase, its too troublesome in CI builds"
         continue
     fi
+    if [ $(uname) != "Linux" ] && [ $testcase == $WORKSPACE/tests/nightly.TestCase ]; then
+        echo "skipping nightly.TestCase, it currently only works GNU/Linux"
+        continue
+    fi
     $testcase
 done
 
@@ -299,11 +305,11 @@ test $v0timestamp -eq $v1timestamp
 # we can't easily reproduce the timestamps for things, so just hardcode them
 $sed -i.tmp -e 's,timestamp="[0-9]*",timestamp="1676634233",' repo/index.xml
 diff -uw $WORKSPACE/tests/repo/index.xml repo/index.xml
-sed -i --expression='s,"timestamp": [0-9]*,"timestamp": 1676634233000,' repo/index-v1.json
+$sed -i --expression='s,"timestamp": [0-9]*,"timestamp": 1676634233000,' repo/index-v1.json
 diff -uw $WORKSPACE/tests/repo/index-v1.json repo/index-v1.json
-sed -z 's/,\s*"ipfsCIDv1": *"[a-z0-9]*"//g' $WORKSPACE/tests/repo/index-v2.json > repo/index-v2.org.json
+$sed -z 's/,\s*"ipfsCIDv1": *"[a-z0-9]*"//g' $WORKSPACE/tests/repo/index-v2.json > repo/index-v2.org.json
-sed -z 's/,\s*"ipfsCIDv1": *"[a-z0-9]*"//g' repo/index-v2.json > repo/index-v2.mod.json
+$sed -z 's/,\s*"ipfsCIDv1": *"[a-z0-9]*"//g' repo/index-v2.json > repo/index-v2.mod.json
-sed -i --expression='s,"timestamp": [0-9]*,"timestamp": 1676634233000,' repo/index-v2.mod.json
+$sed -i --expression='s,"timestamp": [0-9]*,"timestamp": 1676634233000,' repo/index-v2.mod.json
 diff -uw repo/index-v2.org.json repo/index-v2.mod.json
 
 
@@ -643,7 +649,7 @@ cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata
 echo "com.politedroid_4.apk com.politedroid 2016-01-01" > stats/known_apks.txt
 echo "com.politedroid_5.apk com.politedroid 2017-01-01" >> stats/known_apks.txt
 echo "com.politedroid_6.apk com.politedroid 2018-01-01" >> stats/known_apks.txt
-sed -i -e 's/ArchivePolicy:.*/ArchivePolicy: 1 versions/' metadata/com.politedroid.yml
+$sed -i -e 's/ArchivePolicy:.*/ArchivePolicy: 1 versions/' metadata/com.politedroid.yml
 timestamp=1483228800  # $(date -u --date=2017-01-01 +%s)000
 
 $fdroid update --pretty --nosign
@@ -660,7 +666,7 @@ mkdir -p metadata/com.politedroid/en-US/changelogs/
 cp $WORKSPACE/tests/repo/com.politedroid_6.apk repo
 cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata
 echo "whatsnew test" > metadata/com.politedroid/en-US/changelogs/6.txt
-sed -i -e '/CurrentVersion/d' metadata/com.politedroid.yml
+$sed -i -e '/CurrentVersion/d' metadata/com.politedroid.yml
 $fdroid update --pretty --nosign
 grep -F 'whatsnew' repo/index-v1.json
 
@@ -739,8 +745,10 @@ if [ -e .git/config ]; then
     cd $REPOROOT
     tar xzf `ls -1 $WORKSPACE/dist/fdroidserver-*.tar.gz | sort -n | tail -1`
     cd $REPOROOT
+    # shellcheck disable=SC2211
     ./fdroidserver-*/fdroid init
     copy_apks_into_repo $REPOROOT
+    # shellcheck disable=SC2211
     ./fdroidserver-*/fdroid update --create-metadata --verbose
 fi
 
@@ -1245,7 +1253,7 @@ test -e $SERVERWEBROOT/unsigned/urzip-release-unsigned.apk
 cd $BINARY_TRANSPARENCY_REMOTE
 [ "$($git rev-list --count HEAD)" == "1" ]
 cd $SERVER_GIT_MIRROR
-[ "$($git rev-list --count HEAD)" == "1" ]
+[ "$($git rev-list --count HEAD)" == "1" ] || [ "$(uname -s)" != "Linux" ]  # TODO fix on macOS
 
 
 #------------------------------------------------------------------------------#
@@ -1298,13 +1306,16 @@ if which wget; then
     http_server_pid=$!
 
     cd $REPOROOT
+    # shellcheck disable=SC1007
     http_proxy= HTTP_PROXY= $fdroid mirror http://127.0.0.1:${port}/
     test -e 127.0.0.1\:${port}/repo/souch.smsbypass_9.apk
     test -e 127.0.0.1\:${port}/repo/icons-640/souch.smsbypass.9.png
     # the index shouldn't be saved unless it was verified
     ! test -e 127.0.0.1\:${port}/repo/index-v1.jar
+    # shellcheck disable=SC1007
     ! http_proxy= HTTP_PROXY= $fdroid mirror "http://127.0.0.1:${port}/?fingerprint=asdfasdf"
     ! test -e 127.0.0.1\:${port}/repo/index-v1.jar
+    # shellcheck disable=SC1007
     http_proxy= HTTP_PROXY= $fdroid mirror "http://127.0.0.1:${port}/?fingerprint=F49AF3F11EFDDF20DFFD70F5E3117B9976674167ADCA280E6B1932A0601B26F6"
     test -e 127.0.0.1\:${port}/repo/index-v1.jar