Merge branch 'purge-aapt' into 'master'

purge aapt requirements and elevate apksigner

Closes #627

See merge request fdroid/fdroidserver!821

fdroidserver/common.py

@@ -114,7 +114,7 @@ default_config = {
         'r16b': None,
     },
     'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'),
-    'build_tools': MINIMUM_AAPT_BUILD_TOOLS_VERSION,
+    'build_tools': MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION,
     'java_paths': None,
     'scan_binary': False,
     'ant': "ant",
@@ -423,7 +423,8 @@ def find_apksigner():
     Returns the best version of apksigner following this algorithm:
     * use config['apksigner'] if set
     * try to find apksigner in path
-    * find apksigner in build-tools starting from newest installed going down to MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION
+    * find apksigner in build-tools starting from newest installed
+      going down to MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION
     :return: path to apksigner or None if no version is found
     """
     if set_command_in_config('apksigner'):
@@ -434,7 +435,7 @@ def find_apksigner():
     for f in sorted(os.listdir(build_tools_path), reverse=True):
         if not os.path.isdir(os.path.join(build_tools_path, f)):
             continue
-        if LooseVersion(f) < LooseVersion(MINIMUM_AAPT_BUILD_TOOLS_VERSION):
+        if LooseVersion(f) < LooseVersion(MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION):
             return None
         if os.path.exists(os.path.join(build_tools_path, f, 'apksigner')):
             apksigner = os.path.join(build_tools_path, f, 'apksigner')
@@ -507,6 +508,7 @@ def test_aapt_version(aapt):
 
 def test_sdk_exists(thisconfig):
     if 'sdk_path' not in thisconfig:
+        # TODO convert this to apksigner once it is required
         if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
             test_aapt_version(thisconfig['aapt'])
             return True
@@ -2388,19 +2390,15 @@ def ensure_final_value(packageName, arsc, value):
     return ''
 
 
-def is_apk_and_debuggable_aapt(apkfile):
-    p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
-                      output=False)
-    if p.returncode != 0:
-        raise FDroidException(_("Failed to get APK manifest information"))
-    for line in p.output.splitlines():
-        if 'android:debuggable' in line and not line.endswith('0x0'):
-            return True
-    return False
+def is_apk_and_debuggable(apkfile):
+    """Returns True if the given file is an APK and is debuggable
 
+    Parse only <application android:debuggable=""> from the APK.
 
-def is_apk_and_debuggable_androguard(apkfile):
-    """Parse only <application android:debuggable=""> from the APK"""
+    :param apkfile: full path to the apk to check"""
+
+    if get_file_extension(apkfile) != 'apk':
+        return False
     from androguard.core.bytecodes.axml import AXMLParser, format_value, START_TAG
     with ZipFile(apkfile) as apk:
         with apk.open('AndroidManifest.xml') as manifest:
@@ -2422,20 +2420,6 @@ def is_apk_and_debuggable_androguard(apkfile):
     return False
 
 
-def is_apk_and_debuggable(apkfile):
-    """Returns True if the given file is an APK and is debuggable
-
-    :param apkfile: full path to the apk to check"""
-
-    if get_file_extension(apkfile) != 'apk':
-        return False
-
-    if use_androguard():
-        return is_apk_and_debuggable_androguard(apkfile)
-    else:
-        return is_apk_and_debuggable_aapt(apkfile)
-
-
 def get_apk_id(apkfile):
     """Extract identification information from APK.
 
@@ -2448,15 +2432,12 @@ def get_apk_id(apkfile):
     :returns: triplet (appid, version code, version name)
 
     """
-    if use_androguard():
-        try:
-            return get_apk_id_androguard(apkfile)
-        except zipfile.BadZipFile as e:
-            logging.error(apkfile + ': ' + str(e))
-            if 'aapt' in config:
-                return get_apk_id_aapt(apkfile)
-    else:
-        return get_apk_id_aapt(apkfile)
+    try:
+        return get_apk_id_androguard(apkfile)
+    except zipfile.BadZipFile as e:
+        logging.error(apkfile + ': ' + str(e))
+        if 'aapt' in config:
+            return get_apk_id_aapt(apkfile)
 
 
 def get_apk_id_androguard(apkfile):

fdroidserver/init.py

@@ -75,46 +75,44 @@ def main():
     # in ANDROID_HOME if that exists, otherwise None
     if options.android_home is not None:
         test_config['sdk_path'] = options.android_home
-    elif common.use_androguard():
-        pass
     elif not common.test_sdk_exists(test_config):
-        if os.path.isfile('/usr/bin/aapt'):
-            # remove sdk_path and build_tools, they are not required
-            test_config.pop('sdk_path', None)
-            test_config.pop('build_tools', None)
-            # make sure at least aapt is found, since this can't do anything without it
-            test_config['aapt'] = common.find_sdk_tools_cmd('aapt')
+        # if neither --android-home nor the default sdk_path
+        # exist, prompt the user using platform-specific default
+        # and if the user leaves it blank, ignore and move on.
+        default_sdk_path = ''
+        if sys.platform == 'win32' or sys.platform == 'cygwin':
+            p = os.path.join(os.getenv('USERPROFILE'),
+                             'AppData', 'Local', 'Android', 'android-sdk')
+        elif sys.platform == 'darwin':
+            # on OSX, Homebrew is common and has an easy path to detect
+            p = '/usr/local/opt/android-sdk'
+        elif os.path.isdir('/usr/lib/android-sdk'):
+            # if the Debian packages are installed, suggest them
+            p = '/usr/lib/android-sdk'
         else:
-            # if neither --android-home nor the default sdk_path
-            # exist, prompt the user using platform-specific default
-            default_sdk_path = '/opt/android-sdk'
-            if sys.platform == 'win32' or sys.platform == 'cygwin':
-                p = os.path.join(os.getenv('USERPROFILE'),
-                                 'AppData', 'Local', 'Android', 'android-sdk')
-            elif sys.platform == 'darwin':
-                # on OSX, Homebrew is common and has an easy path to detect
-                p = '/usr/local/opt/android-sdk'
-            else:
-                # if the Debian packages are installed, suggest them
-                p = '/usr/lib/android-sdk'
-            if os.path.exists(p):
-                default_sdk_path = p
+            p = '/opt/android-sdk'
+        if os.path.exists(p):
+            default_sdk_path = p
 
-            while not options.no_prompt:
-                try:
-                    s = input(_('Enter the path to the Android SDK (%s) here:\n> ') % default_sdk_path)
-                except KeyboardInterrupt:
-                    print('')
-                    sys.exit(1)
-                if re.match(r'^\s*$', s) is not None:
-                    test_config['sdk_path'] = default_sdk_path
-                else:
-                    test_config['sdk_path'] = s
-                if common.test_sdk_exists(test_config):
-                    break
-    if (options.android_home is not None or not common.use_androguard()) \
-       and not common.test_sdk_exists(test_config):
-        raise FDroidException("Android SDK not found.")
+        while not options.no_prompt:
+            try:
+                s = input(_('Enter the path to the Android SDK (%s) here:\n> ') % default_sdk_path)
+            except KeyboardInterrupt:
+                print('')
+                sys.exit(1)
+            if re.match(r'^\s*$', s) is not None:
+                test_config['sdk_path'] = default_sdk_path
+            else:
+                test_config['sdk_path'] = s
+            if not default_sdk_path:
+                del(test_config['sdk_path'])
+                break
+            if common.test_sdk_exists(test_config):
+                break
+
+    if test_config.get('sdk_path') and not common.test_sdk_exists(test_config):
+        raise FDroidException(_("Android SDK not found at {path}!")
+                              .format(path=test_config['sdk_path']))
 
     if not os.path.exists('config.py'):
         # 'metadata' and 'tmp' are created in fdroid
@@ -134,34 +132,13 @@ def main():
         logging.info('Try running `fdroid init` in an empty directory.')
         raise FDroidException('Repository already exists.')
 
-    if common.use_androguard():
-        pass
-    elif 'aapt' not in test_config or not os.path.isfile(test_config['aapt']):
-        # try to find a working aapt, in all the recent possible paths
-        build_tools = os.path.join(test_config['sdk_path'], 'build-tools')
-        aaptdirs = []
-        aaptdirs.append(os.path.join(build_tools, test_config['build_tools']))
-        aaptdirs.append(build_tools)
-        for f in os.listdir(build_tools):
-            if os.path.isdir(os.path.join(build_tools, f)):
-                aaptdirs.append(os.path.join(build_tools, f))
-        for d in sorted(aaptdirs, reverse=True):
-            if os.path.isfile(os.path.join(d, 'aapt')):
-                aapt = os.path.join(d, 'aapt')
-                break
-        if aapt and os.path.isfile(aapt):
-            dirname = os.path.basename(os.path.dirname(aapt))
-            if dirname == 'build-tools':
-                # this is the old layout, before versioned build-tools
-                test_config['build_tools'] = ''
-            else:
-                test_config['build_tools'] = dirname
-            common.write_to_config(test_config, 'build_tools')
-        common.ensure_build_tools_exists(test_config)
-
     # now that we have a local config.py, read configuration...
     config = common.read_config(options)
 
+    # enable apksigner by default so v2/v3 APK signatures validate
+    if common.find_apksigner() is not None:
+        test_config['apksigner'] = common.find_apksigner()
+
     # the NDK is optional and there may be multiple versions of it, so it's
     # left for the user to configure
 

fdroidserver/update.py

@@ -49,7 +49,6 @@ from . import _
 from . import common
 from . import index
 from . import metadata
-from .common import SdkToolsPopen
 from .exception import BuildException, FDroidException
 
 from PIL import Image, PngImagePlugin
@@ -1350,10 +1349,7 @@ def scan_apk(apk_file):
         'antiFeatures': set(),
     }
 
-    if common.use_androguard():
-        scan_apk_androguard(apk, apk_file)
-    else:
-        scan_apk_aapt(apk, apk_file)
+    scan_apk_androguard(apk, apk_file)
 
     if not common.is_valid_package_name(apk['packageName']):
         raise BuildException(_("{appid} from {path} is not a valid Java Package Name!")
@@ -1412,83 +1408,6 @@ def _get_apk_icons_src(apkfile, icon_name):
     return icons_src
 
 
-def scan_apk_aapt(apk, apkfile):
-    p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
-    if p.returncode != 0:
-        if options.delete_unknown:
-            if os.path.exists(apkfile):
-                logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
-                os.remove(apkfile)
-            else:
-                logging.error("Could not find {0} to remove it".format(apkfile))
-        else:
-            logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
-        raise BuildException(_("Invalid APK"))
-    icon_name = None
-    for line in p.output.splitlines():
-        if line.startswith("package:"):
-            try:
-                apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
-                apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
-                apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
-            except Exception as e:
-                raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
-        elif line.startswith("application:"):
-            m = re.match(APK_LABEL_ICON_PAT, line)
-            if m:
-                apk['name'] = m.group(1)
-                icon_name = os.path.splitext(os.path.basename(m.group(2)))[0]
-        elif not apk.get('name') and line.startswith("launchable-activity:"):
-            # Only use launchable-activity as fallback to application
-            apk['name'] = re.match(APK_LABEL_ICON_PAT, line).group(1)
-        elif line.startswith("sdkVersion:"):
-            m = re.match(APK_SDK_VERSION_PAT, line)
-            if m is None:
-                logging.error(line.replace('sdkVersion:', '')
-                              + ' is not a valid minSdkVersion!')
-            else:
-                apk['minSdkVersion'] = int(m.group(1))
-        elif line.startswith("targetSdkVersion:"):
-            m = re.match(APK_SDK_VERSION_PAT, line)
-            if m is None:
-                logging.error(line.replace('targetSdkVersion:', '')
-                              + ' is not a valid targetSdkVersion!')
-            else:
-                apk['targetSdkVersion'] = int(m.group(1))
-        elif line.startswith("maxSdkVersion:"):
-            apk['maxSdkVersion'] = int(re.match(APK_SDK_VERSION_PAT, line).group(1))
-        elif line.startswith("native-code:"):
-            apk['nativecode'] = []
-            for arch in line[13:].split(' '):
-                apk['nativecode'].append(arch[1:-1])
-        elif line.startswith('uses-permission:'):
-            perm_match = re.match(APK_PERMISSION_PAT, line).groups()
-            permission = UsesPermission(
-                perm_match[0],  # name
-                None if perm_match[1] is None else int(perm_match[1]),  # maxSdkVersion
-            )
-            apk['uses-permission'].append(permission)
-
-        elif line.startswith('uses-permission-sdk-23:'):
-            perm_match = re.match(APK_PERMISSION_PAT, line).groups()
-            permission_sdk_23 = UsesPermissionSdk23(
-                perm_match[0],  # name
-                None if perm_match[1] is None else int(perm_match[1]),  # maxSdkVersion
-            )
-            apk['uses-permission-sdk-23'].append(permission_sdk_23)
-
-        elif line.startswith('uses-feature:'):
-            feature = re.match(APK_FEATURE_PAT, line).group(1)
-            # Filter out this, it's only added with the latest SDK tools and
-            # causes problems for lots of apps.
-            if feature != "android.hardware.screen.portrait" \
-                    and feature != "android.hardware.screen.landscape":
-                if feature.startswith("android.feature."):
-                    feature = feature[16:]
-                apk['features'].add(feature)
-    apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name)
-
-
 def _sanitize_sdk_version(value):
     """Sanitize the raw values from androguard to handle bad values
 
@@ -1529,8 +1448,6 @@ def scan_apk_androguard(apk, apkfile):
                 logging.error(_("Failed to get apk information, skipping {path}")
                               .format(path=apkfile))
             raise BuildException(_("Invalid APK"))
-    except ImportError:
-        raise FDroidException("androguard library is not installed and aapt not present")
     except FileNotFoundError:
         logging.error(_("Could not open apk file for analysis"))
         raise BuildException(_("Invalid APK"))