Merge branch 'master' into 'master'

allow running without aapt, support Java versions newer than 9

Closes #474

See merge request fdroid/fdroidserver!497

.gitlab-ci.yml

@@ -62,13 +62,18 @@ ubuntu_lts:
   image: ubuntu:latest
   only:
     - master@fdroid/fdroidserver
+  variables:
+    DEBIAN_FRONTEND: noninteractive
   script:
+    - echo Etc/UTC > /etc/timezone
+    - apt-get -qy update
+    - apt-get -qy install gnupg
     - while ! apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv-key 9AAC253193B65D4DF1D0A13EEC4632C79C5E0151; do sleep 15; done
     - export RELEASE=`sed -n 's,^deb [^ ][^ ]* \([a-z]*\).*,\1,p' /etc/apt/sources.list | head -1`
     - echo "deb http://ppa.launchpad.net/fdroid/fdroidserver/ubuntu $RELEASE main" >> /etc/apt/sources.list
     - apt-get -qy update
     - apt-get -qy dist-upgrade
-    - apt-get -qy install fdroidserver git python3-setuptools
+    - apt-get -qy install --install-recommends fdroidserver git python3-setuptools
     - export ANDROID_HOME=/usr/lib/android-sdk
     - export LANG=C.UTF-8
     - cd tests

.travis.yml

@@ -7,7 +7,7 @@ matrix:
     - os: linux
       language: android
     - os: osx
-      osx_image: xcode9.3beta
+      osx_image: xcode9.3
       env: ANDROID_SDK_ROOT=/usr/local/share/android-sdk
       env: ANDROID_HOME=/usr/local/share/android-sdk
     - os: osx
@@ -15,11 +15,7 @@ matrix:
       env: ANDROID_SDK_ROOT=/usr/local/share/android-sdk
       env: ANDROID_HOME=/usr/local/share/android-sdk
     - os: osx
-      osx_image: xcode7.3
+      osx_image: xcode8.3
-      env: ANDROID_SDK_ROOT=/usr/local/share/android-sdk
-      env: ANDROID_HOME=/usr/local/share/android-sdk
-    - os: osx
-      osx_image: xcode6.4
       env: ANDROID_SDK_ROOT=/usr/local/share/android-sdk
       env: ANDROID_HOME=/usr/local/share/android-sdk
 

fdroidserver/common.py

@@ -160,18 +160,18 @@ def _add_java_paths_to_config(pathlist, thisconfig):
         j = os.path.basename(d)
         # the last one found will be the canonical one, so order appropriately
         for regex in [
-                r'^1\.([6-9])\.0\.jdk$',  # OSX
+                r'^1\.([16-9][0-9]?)\.0\.jdk$',  # OSX
-                r'^jdk1\.([6-9])\.0_[0-9]+.jdk$',  # OSX and Oracle tarball
+                r'^jdk1\.([16-9][0-9]?)\.0_[0-9]+.jdk$',  # OSX and Oracle tarball
-                r'^jdk1\.([6-9])\.0_[0-9]+$',  # Oracle Windows
+                r'^jdk1\.([16-9][0-9]?)\.0_[0-9]+$',  # Oracle Windows
-                r'^jdk([6-9])-openjdk$',  # Arch
+                r'^jdk([16-9][0-9]?)-openjdk$',  # Arch
-                r'^java-([6-9])-openjdk$',  # Arch
+                r'^java-([16-9][0-9]?)-openjdk$',  # Arch
-                r'^java-([6-9])-jdk$',  # Arch (oracle)
+                r'^java-([16-9][0-9]?)-jdk$',  # Arch (oracle)
-                r'^java-1\.([6-9])\.0-.*$',  # RedHat
+                r'^java-1\.([16-9][0-9]?)\.0-.*$',  # RedHat
-                r'^java-([6-9])-oracle$',  # Debian WebUpd8
+                r'^java-([16-9][0-9]?)-oracle$',  # Debian WebUpd8
-                r'^jdk-([6-9])-oracle-.*$',  # Debian make-jpkg
+                r'^jdk-([16-9][0-9]?)-oracle-.*$',  # Debian make-jpkg
-                r'^java-([6-9])-openjdk-[^c][^o][^m].*$',  # Debian
+                r'^java-([16-9][0-9]?)-openjdk-[^c][^o][^m].*$',  # Debian
-                r'^oracle-jdk-bin-1\.([7-9]).*$',  # Gentoo (oracle)
+                r'^oracle-jdk-bin-1\.([17-9][0-9]?).*$',  # Gentoo (oracle)
-                r'^icedtea-bin-([7-9]).*$',  # Gentoo (openjdk)
+                r'^icedtea-bin-([17-9][0-9]?).*$',  # Gentoo (openjdk)
                 ]:
             m = re.match(regex, j)
             if not m:
@@ -208,19 +208,19 @@ def fill_config_defaults(thisconfig):
     if thisconfig['java_paths'] is None:
         thisconfig['java_paths'] = dict()
         pathlist = []
-        pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
+        pathlist += glob.glob('/usr/lib/jvm/j*[16-9]*')
-        pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
+        pathlist += glob.glob('/usr/java/jdk1.[16-9]*')
-        pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
+        pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[16-9][0-9]?.0.jdk')
-        pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
+        pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[0-9]*')
-        pathlist += glob.glob('/opt/oracle-jdk-*1.[7-9]*')
+        pathlist += glob.glob('/opt/oracle-jdk-*1.[0-9]*')
-        pathlist += glob.glob('/opt/icedtea-*[7-9]*')
+        pathlist += glob.glob('/opt/icedtea-*[0-9]*')
         if os.getenv('JAVA_HOME') is not None:
             pathlist.append(os.getenv('JAVA_HOME'))
         if os.getenv('PROGRAMFILES') is not None:
-            pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
+            pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[16-9][0-9]?.*'))
         _add_java_paths_to_config(pathlist, thisconfig)
 
-    for java_version in ('7', '8', '9'):
+    for java_version in ('14', '13', '12', '11', '10', '9', '8', '7'):
         if java_version not in thisconfig['java_paths']:
             continue
         java_home = thisconfig['java_paths'][java_version]
@@ -228,7 +228,7 @@ def fill_config_defaults(thisconfig):
         if os.path.exists(jarsigner):
             thisconfig['jarsigner'] = jarsigner
             thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
-            break  # Java7 is preferred, so quit if found
+            break
 
     for k in ['ndk_paths', 'java_paths']:
         d = thisconfig[k]

fdroidserver/init.py

@@ -75,6 +75,8 @@ def main():
     # in ANDROID_HOME if that exists, otherwise None
     if options.android_home is not None:
         test_config['sdk_path'] = options.android_home
+    elif common.use_androguard():
+        pass
     elif not common.test_sdk_exists(test_config):
         if os.path.isfile('/usr/bin/aapt'):
             # remove sdk_path and build_tools, they are not required
@@ -110,7 +112,8 @@ def main():
                     test_config['sdk_path'] = s
                 if common.test_sdk_exists(test_config):
                     break
-    if not common.test_sdk_exists(test_config):
+    if (options.android_home is not None or not common.use_androguard()) \
+       and not common.test_sdk_exists(test_config):
         raise FDroidException("Android SDK not found.")
 
     if not os.path.exists('config.py'):

tests/run-tests

@@ -6,11 +6,19 @@ echo_header() {
     { echo -e "==============================================================================\n$1"; } 2>/dev/null
 }
 
+get_fdroid_apk_filename() {
+    if [ -z $aapt ]; then
+        python3 -c "from androguard.core.bytecodes.apk import APK; a=APK('$1'); print(a.package+'_'+a.get_androidversion_code()+'.apk')"
+    else
+        $aapt dump badging "$1" | sed -n "s,^package: name='\(.*\)' versionCode='\([0-9][0-9]*\)' .*,\1_\2.apk,p"
+    fi
+}
+
 copy_apks_into_repo() {
     set +x
     find $APKDIR -type f -name '*.apk' -print0 | while IFS= read -r -d '' f; do
         echo $f | grep -F -v -e unaligned -e unsigned -e badsig -e badcert -e bad-unicode -e janus.apk || continue
-        apk=`$aapt dump badging "$f" | sed -n "s,^package: name='\(.*\)' versionCode='\([0-9][0-9]*\)' .*,\1_\2.apk,p"`
+        apk=`get_fdroid_apk_filename "$f"`
         test "$f" -nt repo/$apk && rm -f repo/$apk  # delete existing if $f is newer
         if [ ! -e repo/$apk ] && [ ! -e archive/$apk ]; then
             echo "$f --> repo/$apk"
@@ -72,10 +80,15 @@ if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
 fi
 
 if [ -z "$ANDROID_HOME" ]; then
-    echo "ANDROID_HOME must be set with the path to the Android SDK, i.e.: "
+    if python3 -c "import androguard"; then
-    echo "  export ANDROID_HOME=/opt/android-sdk"
+        echo "ANDROID_HOME is not set, using androguard"
+    else
+        echo "ERROR: ANDROID_HOME is not set, androguard is not available!"
         exit 1
     fi
+else
+    echo "Using ANDROID_HOME=$ANDROID_HOME"
+fi
 
 if [ -d tests ]; then
     cd tests
@@ -715,8 +728,8 @@ $fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME
 #------------------------------------------------------------------------------#
 echo_header "check that 'fdroid init' fails when build-tools cannot be found"
 
-if [ -e /usr/bin/aapt ]; then
+if [ -e /usr/bin/aapt ] || python3 -c 'import androguard'; then
-    echo "/usr/bin/aapt exists, not running test"
+    echo "/usr/bin/aapt or androguard installed, not running test"
 else
     REPOROOT=`create_test_dir`
     FAKE_ANDROID_HOME=`create_test_dir`
@@ -755,6 +768,9 @@ set -e
 # by --android-home over the one in ANDROID_HOME, therefore if it uses the one
 # in ANDROID_HOME, it won't work because it is a fake one.  Only
 # --android-home provides a working one.
+if [ -z "$ANDROID_HOME" ]; then
+    echo_header "SKIP setup a new repo from scratch with keystore and android-home set on cmd line"
+else
     echo_header "setup a new repo from scratch with keystore and android-home set on cmd line"
 
     REPOROOT=`create_test_dir`
@@ -777,7 +793,7 @@ test -e repo/index-v1.jar
     test -e tmp/apkcache
     ! test -z tmp/apkcache
     export ANDROID_HOME=$STORED_ANDROID_HOME
-
+fi
 
 #------------------------------------------------------------------------------#
 echo_header "check duplicate files are properly handled by fdroid update"