From f11b2e8d450e0ef530f1ce260e4c5419e4567da9 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 20 Sep 2018 11:35:07 +0200 Subject: [PATCH 1/4] point out the easy way to generate the locale files fdroid/fdroidserver!560 fdroid/fdroidserver#546 --- locale/Makefile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/locale/Makefile b/locale/Makefile index 4f68f627..a8b62eb6 100644 --- a/locale/Makefile +++ b/locale/Makefile @@ -13,11 +13,17 @@ TEMPLATE = fdroidserver.pot VERSION = $(shell git describe) +default: + @printf "Build the translation files using: ./setup.py compile_catalog\n\n" + +message: + @printf "\nYou probably want to use this instead: ./setup.py compile_catalog\n\n" + # refresh everything from the source code update: $(POFILES) # generate .mo files from the .po files -compile: $(MOFILES) +compile: message $(MOFILES) clean: -rm -f -- $(MOFILES) From fa09337b4b177f8e666043ef861215b46dc22ad5 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 18 Sep 2018 15:20:37 +0200 Subject: [PATCH 2/4] APK_ID_TRIPLET_REGEX only matches first line of aapt output Stop expensive aapt parsing after the first line when looking with APK_ID_TRIPLET_REGEX. As is seen with the `aapt dump badging` output files in tests/build-tools/, the first line is the only line that will ever match. #557 --- fdroidserver/common.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index cedc8c5a..7b8842a1 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -2120,10 +2120,9 @@ def get_apk_id_androguard(apkfile): def get_apk_id_aapt(apkfile): p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False) - for line in p.output.splitlines(): - m = APK_ID_TRIPLET_REGEX.match(line) - if m: - return m.group(1), m.group(2), m.group(3) + m = APK_ID_TRIPLET_REGEX.match(p.output[0:p.output.index('\n')]) + if m: + return m.group(1), m.group(2), m.group(3) raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'") .format(apkfilename=apkfile)) From a3cecc16a37cc2eec02b5bcf2a382d28885270c9 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 18 Sep 2018 15:29:24 +0200 Subject: [PATCH 3/4] use partial androguard binary XML parsing to speed up APK ID lookup Normally, androguard parses the entire APK before it is possible to get any values from it. This uses androguard primitives to only attempt to parse the AndroidManifest.xml, then to quit as soon as it gets what it needs. This greatly speeds up the parsing (1 minute vs 60 minutes). fdroid/fdroidserver#557 --- fdroidserver/common.py | 67 ++++++++++++++++++++++++++++++++++++++---- tests/common.TestCase | 14 ++++----- 2 files changed, 69 insertions(+), 12 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 7b8842a1..53c4d41a 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -2096,26 +2096,83 @@ def is_apk_and_debuggable(apkfile): def get_apk_id(apkfile): - """Extract identification information from APK using aapt. + """Extract identification information from APK. + + Androguard is preferred since it is more reliable and a lot + faster. Occasionally, when androguard can't get the info from the + APK, aapt still can. So aapt is also used as the final fallback + method. :param apkfile: path to an APK file. :returns: triplet (appid, version code, version name) + """ if use_androguard(): - return get_apk_id_androguard(apkfile) + try: + return get_apk_id_androguard(apkfile) + except zipfile.BadZipFile as e: + logging.error(apkfile + ': ' + str(e)) + if 'aapt' in config: + return get_apk_id_aapt(apkfile) else: return get_apk_id_aapt(apkfile) def get_apk_id_androguard(apkfile): + """Read (appid, versionCode, versionName) from an APK + + This first tries to do quick binary XML parsing to just get the + values that are needed. It will fallback to full androguard + parsing, which is slow, if it can't find the versionName value or + versionName is set to a Android String Resource (e.g. an integer + hex value that starts with @). + + """ if not os.path.exists(apkfile): raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'") .format(apkfilename=apkfile)) - a = _get_androguard_APK(apkfile) - versionName = ensure_final_value(a.package, a.get_android_resources(), a.get_androidversion_name()) + + from androguard.core.bytecodes.axml import AXMLParser, format_value, START_TAG, END_TAG, TEXT, END_DOCUMENT + + appid = None + versionCode = None + versionName = None + with zipfile.ZipFile(apkfile) as apk: + with apk.open('AndroidManifest.xml') as manifest: + axml = AXMLParser(manifest.read()) + count = 0 + while axml.is_valid(): + _type = next(axml) + count += 1 + if _type == START_TAG: + for i in range(0, axml.getAttributeCount()): + name = axml.getAttributeName(i) + _type = axml.getAttributeValueType(i) + _data = axml.getAttributeValueData(i) + value = format_value(_type, _data, lambda _: axml.getAttributeValue(i)) + if appid is None and name == 'package': + appid = value + elif versionCode is None and name == 'versionCode': + if value.startswith('0x'): + versionCode = str(int(value, 16)) + else: + versionCode = value + elif versionName is None and name == 'versionName': + versionName = value + + if axml.getName() == 'manifest': + break + elif _type == END_TAG or _type == TEXT or _type == END_DOCUMENT: + raise RuntimeError('{path}: must be the first element in AndroidManifest.xml' + .format(path=apkfile)) + + if not versionName or versionName[0] == '@': + a = _get_androguard_APK(apkfile) + versionName = ensure_final_value(a.package, a.get_android_resources(), a.get_androidversion_name()) if not versionName: versionName = '' # versionName is expected to always be a str - return a.package, a.get_androidversion_code(), versionName + + return appid, versionCode, versionName.strip('\0') def get_apk_id_aapt(apkfile): diff --git a/tests/common.TestCase b/tests/common.TestCase index f1449392..841da524 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -611,14 +611,14 @@ class CommonTest(unittest.TestCase): for apkfilename, appid, versionCode, versionName in testcases: if 'aapt' in config: a, vc, vn = fdroidserver.common.get_apk_id_aapt(apkfilename) - self.assertEqual(appid, a) - self.assertEqual(versionCode, vc) - self.assertEqual(versionName, vn) + self.assertEqual(appid, a, 'aapt appid parsing failed for ' + apkfilename) + self.assertEqual(versionCode, vc, 'aapt versionCode parsing failed for ' + apkfilename) + self.assertEqual(versionName, vn, 'aapt versionName parsing failed for ' + apkfilename) if fdroidserver.common.use_androguard(): - a, vc, vn = fdroidserver.common.get_apk_id_androguard(apkfilename) - self.assertEqual(appid, a) - self.assertEqual(versionCode, vc) - self.assertEqual(versionName, vn) + a, vc, vn = fdroidserver.common.get_apk_id(apkfilename) + self.assertEqual(appid, a, 'androguard appid parsing failed for ' + apkfilename) + self.assertEqual(versionName, vn, 'androguard versionName parsing failed for ' + apkfilename) + self.assertEqual(versionCode, vc, 'androguard versionCode parsing failed for ' + apkfilename) with self.assertRaises(FDroidException): fdroidserver.common.get_apk_id('nope') From 11d46072ab4495b53e49059a9285779bd76ff0b9 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 18 Sep 2018 15:59:49 +0200 Subject: [PATCH 4/4] use androguard primitives to speed up finding debuggable flag androguard parses the whole APK before handing the instance back, this uses the primitives to just find the value, then stop parsing. #557 --- fdroidserver/common.py | 24 +++++++++++++++++++----- tests/common.TestCase | 15 ++++++++++++++- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 53c4d41a..084f982f 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -2073,11 +2073,25 @@ def is_apk_and_debuggable_aapt(apkfile): def is_apk_and_debuggable_androguard(apkfile): - apkobject = _get_androguard_APK(apkfile) - if apkobject.is_valid_APK(): - debuggable = apkobject.get_element("application", "debuggable") - if debuggable == 'true': - return True + """Parse only from the APK""" + from androguard.core.bytecodes.axml import AXMLParser, format_value, START_TAG + with ZipFile(apkfile) as apk: + with apk.open('AndroidManifest.xml') as manifest: + axml = AXMLParser(manifest.read()) + while axml.is_valid(): + _type = next(axml) + if _type == START_TAG and axml.getName() == 'application': + for i in range(0, axml.getAttributeCount()): + name = axml.getAttributeName(i) + if name == 'debuggable': + _type = axml.getAttributeValueType(i) + _data = axml.getAttributeValueData(i) + value = format_value(_type, _data, lambda _: axml.getAttributeValue(i)) + if value == 'true': + return True + else: + return False + break return False diff --git a/tests/common.TestCase b/tests/common.TestCase index 841da524..125f0961 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -131,7 +131,7 @@ class CommonTest(unittest.TestCase): fdroidserver.common._add_java_paths_to_config(pathlist, config) self.assertEqual(config['java_paths']['8'], choice[1:]) - def testIsApkDebuggable(self): + def test_is_apk_and_debuggable(self): config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config @@ -150,6 +150,13 @@ class CommonTest(unittest.TestCase): debuggable = fdroidserver.common.is_apk_and_debuggable(apkfile) self.assertTrue(debuggable, "debuggable APK state was not properly parsed!") + if 'aapt' in config: + self.assertTrue(fdroidserver.common.is_apk_and_debuggable_aapt(apkfile), + 'aapt parsing missed !') + if fdroidserver.common.use_androguard(): + self.assertTrue(fdroidserver.common.is_apk_and_debuggable_androguard(apkfile), + 'androguard missed !') + # these are set NOT debuggable testfiles = [] testfiles.append(os.path.join(self.basedir, 'urzip-release.apk')) @@ -158,6 +165,12 @@ class CommonTest(unittest.TestCase): debuggable = fdroidserver.common.is_apk_and_debuggable(apkfile) self.assertFalse(debuggable, "debuggable APK state was not properly parsed!") + if 'aapt' in config: + self.assertFalse(fdroidserver.common.is_apk_and_debuggable_aapt(apkfile), + 'aapt parsing missed !') + if fdroidserver.common.use_androguard(): + self.assertFalse(fdroidserver.common.is_apk_and_debuggable_androguard(apkfile), + 'androguard missed !') def test_is_valid_package_name(self): for name in ["cafebabe",