From 56a5d868f1c7075d6123ba670525556962bd5e40 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 13 Jul 2018 11:04:37 +0200 Subject: [PATCH 1/8] update: purge --editor and --interactive, they were removed in 2013 48296df5b0bd2a93d37f72ffe10cf6f955f21767 --- completion/bash-completion | 2 +- fdroidserver/update.py | 5 ----- locale/fdroidserver.pot | 17 ----------------- 3 files changed, 1 insertion(+), 23 deletions(-) diff --git a/completion/bash-completion b/completion/bash-completion index af9acf00..6018c19d 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -146,7 +146,7 @@ __complete_install() { __complete_update() { opts="-c -v -q -b -i -I -e -w" lopts="--create-metadata --verbose --quiet --buildreport - --interactive --icons --editor --wiki --pretty --clean --delete-unknown + --icons --wiki --pretty --clean --delete-unknown --nosign --rename-apks --use-date-from-apk" case "${prev}" in -e|--editor) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 968ee050..15b055e0 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1851,13 +1851,8 @@ def main(): help=_("Delete APKs and/or OBBs without metadata from the repo")) parser.add_argument("-b", "--buildreport", action="store_true", default=False, help=_("Report on build data status")) - parser.add_argument("-i", "--interactive", default=False, action="store_true", - help=_("Interactively ask about things that need updating.")) parser.add_argument("-I", "--icons", action="store_true", default=False, help=_("Resize all the icons exceeding the max pixel size and exit")) - parser.add_argument("-e", "--editor", default="/etc/alternatives/editor", - help=_("Specify editor to use in interactive mode. Default is {path}") - .format(path='/etc/alternatives/editor')) parser.add_argument("-w", "--wiki", default=False, action="store_true", help=_("Update the wiki")) parser.add_argument("--pretty", action="store_true", default=False, diff --git a/locale/fdroidserver.pot b/locale/fdroidserver.pot index 5c5323c3..f09b9b61 100644 --- a/locale/fdroidserver.pot +++ b/locale/fdroidserver.pot @@ -784,10 +784,6 @@ msgstr "" msgid "Interact with the repo HTTP server" msgstr "" -#: ../fdroidserver/update.py -msgid "Interactively ask about things that need updating." -msgstr "" - #: ../fdroidserver/update.py msgid "Invalid APK" msgstr "" @@ -1325,19 +1321,6 @@ msgstr "" msgid "Specify an identity file to provide to SSH for rsyncing" msgstr "" -#: ../fdroidserver/update.py -msgid "Specify editor to use in interactive mode. Default " -msgstr "" - -#, c-format -msgid "Specify editor to use in interactive mode. Default %s" -msgstr "" - -#: ../fdroidserver/update.py -#, python-brace-format -msgid "Specify editor to use in interactive mode. Default is {path}" -msgstr "" - #: ../fdroidserver/build.py msgid "Specify that we're running on the build server" msgstr "" From a9cc7f63588be030378677dcbf6b737725d0d2ed Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 13 Jul 2018 11:05:56 +0200 Subject: [PATCH 2/8] update translation template with latest strings --- locale/fdroidserver.pot | 88 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 3 deletions(-) diff --git a/locale/fdroidserver.pot b/locale/fdroidserver.pot index f09b9b61..64ff855c 100644 --- a/locale/fdroidserver.pot +++ b/locale/fdroidserver.pot @@ -5,9 +5,9 @@ #, fuzzy msgid "" msgstr "" -"Project-Id-Version: fdroidserver 1.0.0-95-gd7af22b\n" +"Project-Id-Version: fdroidserver 1.0.6-70-g54bc858\n" "Report-Msgid-Bugs-To: https://gitlab.com/fdroid/fdroidserver/issues\n" -"POT-Creation-Date: 2018-02-13 09:01+0100\n" +"POT-Creation-Date: 2018-07-13 11:05+0200\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -211,7 +211,7 @@ msgstr "" msgid "Also warn about formatting issues, like rewritemeta -l" msgstr "" -#: ../fdroidserver/common.py +#: ../fdroidserver/common.py ../fdroidserver/build.py #, python-brace-format msgid "Android SDK '{path}' does not have '{dirname}' installed!" msgstr "" @@ -283,6 +283,10 @@ msgstr "" msgid "Build generated by `fdroid import` - remove disable line once ready" msgstr "" +#: ../fdroidserver/checkupdates.py +msgid "Build metadata git repo has uncommited changes!" +msgstr "" + #: ../fdroidserver/build.py msgid "Build only the latest version of each package" msgstr "" @@ -487,6 +491,10 @@ msgstr "" msgid "Description of length {length} is over the {limit} char limit" msgstr "" +#: ../fdroidserver/nightly.py +msgid "Do not deploy the new files to the repo" +msgstr "" + #: ../fdroidserver/mirror.py #, python-brace-format msgid "Do not include \"{path}\" in URL!" @@ -496,6 +504,10 @@ msgstr "" msgid "Do not prompt for Android SDK path, just fail" msgstr "" +#: ../fdroidserver/nightly.py +msgid "Do not remove the private keys generated from the keystore" +msgstr "" + #: ../fdroidserver/build.py msgid "Don't create a source tarball, useful when testing a build" msgstr "" @@ -788,6 +800,11 @@ msgstr "" msgid "Invalid APK" msgstr "" +#: ../fdroidserver/lint.py ../fdroidserver/checkupdates.py +#, python-brace-format +msgid "Invalid VercodeOperation: {field}" +msgstr "" + #: ../fdroidserver/metadata.py #, python-format msgid "Invalid boolean '%s'" @@ -835,6 +852,11 @@ msgstr "" msgid "Invalid package name {0}" msgstr "" +#: ../fdroidserver/common.py +#, python-brace-format +msgid "Invalid redirect to non-HTTPS: {before} -> {after} " +msgstr "" + #: ../fdroidserver/metadata.py #, python-brace-format msgid "Invalid versionCode: \"{versionCode}\" is not an integer!" @@ -1011,6 +1033,10 @@ msgstr "" msgid "Old APK signature failed to verify: {path}" msgstr "" +#: ../fdroid +msgid "Old, deprecated name for fdroid deploy" +msgstr "" + #: ../fdroidserver/update.py #, python-brace-format msgid "Only PNG and JPEG are supported for graphics, found: {path}" @@ -1036,6 +1062,11 @@ msgstr "" msgid "Override path for repo APKs (default: ./repo)" msgstr "" +#: ../fdroidserver/index.py +#, python-brace-format +msgid "Overriding blank versionName in {apkfilename} from metadata: {version}" +msgstr "" + #: ../fdroidserver/common.py #, python-brace-format msgid "Parsing manifest at '{path}'" @@ -1210,6 +1241,10 @@ msgstr "" msgid "Rewriting '{appid}' to '{path}'" msgstr "" +#: ../fdroidserver/checkupdates.py +msgid "Run on git repo that has uncommitted changes" +msgstr "" + #: ../fdroidserver/lint.py msgid "Run rewritemeta to fix formatting" msgstr "" @@ -1535,6 +1570,21 @@ msgstr "" msgid "Update the wiki" msgstr "" +#: ../fdroidserver/checkupdates.py +#, python-brace-format +msgid "UpdateCheckData has invalid URL: {url}" +msgstr "" + +#: ../fdroidserver/lint.py +#, python-brace-format +msgid "UpdateCheckData must use HTTPS URL: {url}" +msgstr "" + +#: ../fdroidserver/lint.py +#, python-brace-format +msgid "UpdateCheckData not a valid URL: {url}" +msgstr "" + #: /usr/lib/python3.5/optparse.py /usr/lib/python3.6/optparse.py msgid "Usage" msgstr "" @@ -1572,6 +1622,11 @@ msgstr "" msgid "Using Java's jarsigner, not recommended for verifying APKs! Use apksigner" msgstr "" +#: ../fdroidserver/common.py +#, python-brace-format +msgid "Using androguard from \"{path}\"" +msgstr "" + #: ../fdroidserver/init.py #, python-brace-format msgid "Using existing keystore \"{path}\"" @@ -1586,6 +1641,10 @@ msgstr "" msgid "Valid commands are:" msgstr "" +#: ../fdroidserver/verify.py +msgid "Verify against locally cached copy rather than redownloading." +msgstr "" + #: ../fdroid msgid "Verify the integrity of downloaded packages" msgstr "" @@ -1710,6 +1769,11 @@ msgstr "" msgid "deleting: repo/{apkfilename}" msgstr "" +#: ../fdroidserver/common.py +#, python-brace-format +msgid "deployeded build logs to '{path}'" +msgstr "" + #: /usr/lib/python3.5/argparse.py /usr/lib/python3.6/argparse.py #, python-format msgid "dest= is required for options like %r" @@ -1734,6 +1798,11 @@ msgstr "" msgid "expected one argument" msgstr "" +#: ../fdroidserver/common.py +#, python-brace-format +msgid "failed deploying build logs to '{path}'" +msgstr "" + #: ../fdroid msgid "fdroid [-h|--help|--version] []" msgstr "" @@ -1931,6 +2000,14 @@ msgstr "" msgid "signed APK, either a file-path or HTTPS URL." msgstr "" +#: ../fdroidserver/common.py +msgid "skip deploying full build logs: log content is empty" +msgstr "" + +#: ../fdroidserver/common.py +msgid "skip deploying full build logs: not enabled in config" +msgstr "" + #: ../fdroidserver/update.py #, python-brace-format msgid "skipping source tarball: {path}" @@ -1940,6 +2017,11 @@ msgstr "" msgid "srclibs missing name and/or @" msgstr "" +#: ../fdroidserver/common.py +#, python-brace-format +msgid "supplied timestamp value '{timestamp}' is not a unix timestamp" +msgstr "" + #: /usr/lib/python3.5/argparse.py /usr/lib/python3.6/argparse.py #, python-format msgid "the following arguments are required: %s" From c8e61aa02dc2fb3afa6d818f2e14dbaaf11f7cdd Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 13 Jul 2018 11:28:11 +0200 Subject: [PATCH 3/8] update: handle random drawable folder names when parsing icons The app com.android.acehk.aceapp37423 had this invalid stuff in it: 6165 2014-03-28 12:52 res/drawable-320dpi/ic_launcher.png --- fdroidserver/update.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 15b055e0..e1e095fc 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1081,6 +1081,11 @@ def scan_apk(apk_file): def _get_apk_icons_src(apkfile, icon_name): """Extract the paths to the app icon in all available densities + The folder name is normally generated by the Android Tools, but + there is nothing that prevents people from using whatever DPI + names they make up. Android will just ignore them, so we should + too. + """ icons_src = dict() density_re = re.compile(r'^res/(.*)/{}\.(png|xml)$'.format(icon_name)) @@ -1089,9 +1094,9 @@ def _get_apk_icons_src(apkfile, icon_name): m = density_re.match(filename) if m: folder = m.group(1).split('-') - if len(folder) > 1 and folder[1].endswith('dpi'): + try: density = screen_resolutions[folder[1]] - else: + except Exception: density = '160' icons_src[density] = m.group(0) if icons_src.get('-1') is None and '160' in icons_src: From 75e9a9e176cbbf23bc9b17ce2141d834c9152713 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 13 Jul 2018 12:01:34 +0200 Subject: [PATCH 4/8] update: support versionCode values in hex --- fdroidserver/update.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index e1e095fc..8c1fb6ea 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1231,7 +1231,11 @@ def scan_apk_androguard(apk, apkfile): raise BuildException(_("Invalid APK")) apk['packageName'] = apkobject.get_package() - apk['versionCode'] = int(apkobject.get_androidversion_code()) + vcstr = apkobject.get_androidversion_code() + if vcstr.startswith('0x'): + apk['versionCode'] = int(vcstr, 16) + else: + apk['versionCode'] = int(vcstr) apk['name'] = apkobject.get_app_name() apk['versionName'] = common.ensure_final_value(apk['packageName'], arsc, From 0e0a5019ab76ba94e7425b6ee3c2e1d5dd63603d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 13 Jul 2018 15:38:26 +0200 Subject: [PATCH 5/8] update: little trick to avoid some cases of XML app icons refs #344 --- fdroidserver/update.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 8c1fb6ea..6d1ddcfa 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1260,7 +1260,8 @@ def scan_apk_androguard(apk, apkfile): if resource_id: icon_name = arsc.get_id(apk['packageName'], icon_id)[1] else: - icon_name = os.path.splitext(os.path.basename(apkobject.get_app_icon()))[0] + # don't use 'anydpi' aka 0xFFFE aka 65534 since it is XML + icon_name = os.path.splitext(os.path.basename(apkobject.get_app_icon(max_dpi=65534 - 1)))[0] apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name) arch_re = re.compile("^lib/(.*)/.*$") From 4352aa2792deaddedd88410f1e54565952a4ae2d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 13 Jul 2018 15:43:20 +0200 Subject: [PATCH 6/8] update: pass resourceIds through if they can't be deferenced People do odd tricks with `aapt --rename-manifest-package` and androguard doesn't handle them all. * https://github.com/androguard/androguard/issues/503 --- fdroidserver/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 1212afca..1b3bf8a3 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -2009,7 +2009,7 @@ def ensure_final_value(packageName, arsc, value): res_id = int('0x' + value[1:], 16) res_id = arsc.get_id(packageName, res_id)[1] returnValue = arsc.get_string(packageName, res_id)[1] - except ValueError: + except (ValueError, TypeError): pass return returnValue From 84dfd6b06af1799d776a72af13d5c32642bda84e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 17 Jul 2018 12:32:18 +0200 Subject: [PATCH 7/8] update: more reliable handling of XMLNS with androguard Sometimes androguard returns the XMLNS as entirely empty, which would make it an invalid APK since normally the 'android' name is mapped to the 'http://schemas.amazon.com/apk/res/android' value. Occasionally, a different key is used. closes fdroid/fdroidserver#515 --- fdroidserver/update.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 6d1ddcfa..bf848061 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1231,7 +1231,19 @@ def scan_apk_androguard(apk, apkfile): raise BuildException(_("Invalid APK")) apk['packageName'] = apkobject.get_package() - vcstr = apkobject.get_androidversion_code() + + xml = apkobject.get_android_manifest_xml() + androidmanifest_xml = apkobject.xml['AndroidManifest.xml'] + if len(xml.nsmap) > 0: + # one of them surely will be the Android one, or its corrupt + xmlns = '{http://schemas.android.com/apk/res/android}' + else: + # strange but sometimes the namespace is blank. This seems to + # only happen with the Bromite/Chromium APKs + xmlns = '{}' + + vcstr = androidmanifest_xml.get(xmlns + 'versionCode') + if vcstr.startswith('0x'): apk['versionCode'] = int(vcstr, 16) else: @@ -1239,7 +1251,7 @@ def scan_apk_androguard(apk, apkfile): apk['name'] = apkobject.get_app_name() apk['versionName'] = common.ensure_final_value(apk['packageName'], arsc, - apkobject.get_androidversion_name()) + androidmanifest_xml.get(xmlns + 'versionName')) minSdkVersion = _sanitize_sdk_version(apkobject.get_min_sdk_version()) if minSdkVersion is not None: @@ -1270,14 +1282,9 @@ def scan_apk_androguard(apk, apkfile): apk['nativecode'] = [] apk['nativecode'].extend(sorted(list(arch))) - xml = apkobject.get_android_manifest_xml() - xmlns = xml.nsmap.get('android') - if not xmlns: - xmlns = 'http://schemas.android.com/apk/res/android' - for item in xml.findall('uses-permission'): - name = str(item.attrib['{' + xmlns + '}name']) - maxSdkVersion = item.attrib.get('{' + xmlns + '}maxSdkVersion') + name = str(item.attrib[xmlns + 'name']) + maxSdkVersion = item.attrib.get(xmlns + 'maxSdkVersion') maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None permission = UsesPermission( name, @@ -1292,8 +1299,8 @@ def scan_apk_androguard(apk, apkfile): apk['uses-permission'].append(permission) for item in xml.findall('uses-permission-sdk-23'): - name = str(item.attrib['{' + xmlns + '}name']) - maxSdkVersion = item.attrib.get('{' + xmlns + '}maxSdkVersion') + name = str(item.attrib[xmlns + 'name']) + maxSdkVersion = item.attrib.get(xmlns + 'maxSdkVersion') maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None permission_sdk_23 = UsesPermissionSdk23( name, @@ -1302,7 +1309,7 @@ def scan_apk_androguard(apk, apkfile): apk['uses-permission-sdk-23'].append(permission_sdk_23) for item in xml.findall('uses-feature'): - key = '{' + xmlns + '}name' + key = xmlns + 'name' if key not in item.attrib: continue feature = str(item.attrib[key]) @@ -1310,7 +1317,7 @@ def scan_apk_androguard(apk, apkfile): and feature != "android.hardware.screen.landscape": if feature.startswith("android.feature."): feature = feature[16:] - required = item.attrib.get('{' + xmlns + '}required') + required = item.attrib.get(xmlns + 'required') if required is None or required == 'true': apk['features'].append(feature) From 60ee69b8bddb3e8e0e53fd5b6d0835b785fe8146 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 18 Jul 2018 17:11:21 +0200 Subject: [PATCH 8/8] gitlab-ci: pylint<2.0 workaround to avoid typed-ast's gcc requirement To keep those tests light and small, no gcc please! * https://gitlab.com/eighthave/fdroidserver/-/jobs/82274815 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 36021b42..4e1d6947 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -126,7 +126,7 @@ lint_format_safety_checks: script: - apk add --no-cache bash dash ca-certificates python3 - python3 -m ensurepip - - pip3 install pycodestyle pyflakes pylint safety + - pip3 install pycodestyle pyflakes 'pylint<2.0' safety - export EXITVALUE=0 - ./hooks/pre-commit || export EXITVALUE=1 - safety check --full-report || export EXITVALUE=1