From 7f9f47496f8f380ea0dc22730a1e349c010c504f Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 28 Jun 2017 23:12:04 +0200 Subject: [PATCH 1/6] gitlab-ci: fix metadata_v0 test to run on the right commits Checking out master will often mean its testing the wrong commit, since merge requests rarely are in master. --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 259e25f2..571253af 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,6 +16,7 @@ metadata_v0: script: - cd tests - cp dump_internal_metadata_format.py dump.py # since this isn't in old commits + - export GITCOMMIT=`git describe` - git checkout 0.7.0 # or any old commit of your choosing - cd .. - sed -i "s/'Author Email',/'Author Email',\n'Author Web Site',/" fdroidserver/metadata.py @@ -24,7 +25,7 @@ metadata_v0: - ../tests/dump.py - cd .. - git reset --hard - - git checkout master + - git checkout $GITCOMMIT - cd fdroiddata - ../tests/dump.py - "sed -i -e '/AuthorWebSite/d' From 5fd014a8524b38f183e3ee6918645b4aaac2fea9 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 15 Jun 2017 17:02:46 +0200 Subject: [PATCH 2/6] update: move btlog import since btlog.py requires python3-git To keep the dependencies limited to where they are needed. --- fdroidserver/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index b7119ab7..9d7c92ee 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -37,7 +37,6 @@ from binascii import hexlify from PIL import Image import logging -from . import btlog from . import common from . import index from . import metadata @@ -1695,6 +1694,7 @@ def main(): git_remote = config.get('binary_transparency_remote') if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')): + from . import btlog btlog.make_binary_transparency_log(repodirs) if config['update_stats']: From 9886e539d3b1b0a033ca5d9dbe58d82caf87b0f4 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 21 Jun 2017 14:01:01 +0200 Subject: [PATCH 3/6] scan APKs for signs of "Master Key" exploit This exploit is old, and was fixed in 4.4. But it was easy to exploit, so it is still worth scanning for it. It is also easy to scan for, since valid APKs should not have files with duplicate names. In theory, this could look for duplicate file names for any file, but this limits the false positives by only checking names of files related to executing code. fdroidclient#40 --- fdroidserver/update.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 9d7c92ee..1f322837 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -470,13 +470,24 @@ def sha256sum(filename): return sha.hexdigest() -def has_old_openssl(filename): - '''checks for known vulnerable openssl versions in the APK''' +def has_known_vulnerability(filename): + """checks for known vulnerabilities in the APK + + Checks OpenSSL .so files in the APK to see if they are a known vulnerable + version. Google also enforces this: + https://support.google.com/faqs/answer/6376725?hl=en + + Checks whether there are more than one classes.dex or AndroidManifest.xml + files, which is invalid and an essential part of the "Master Key" attack. + + http://www.saurik.com/id/17 + """ # statically load this pattern - if not hasattr(has_old_openssl, "pattern"): - has_old_openssl.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)') + if not hasattr(has_known_vulnerability, "pattern"): + has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)') + files_in_apk = set() with zipfile.ZipFile(filename) as zf: for name in zf.namelist(): if name.endswith('libcrypto.so') or name.endswith('libssl.so'): @@ -485,7 +496,7 @@ def has_old_openssl(filename): chunk = lib.read(4096) if chunk == b'': break - m = has_old_openssl.pattern.search(chunk) + m = has_known_vulnerability.pattern.search(chunk) if m: version = m.group(1).decode('ascii') if version.startswith('1.0.1') and version[5] >= 'r' \ @@ -495,6 +506,11 @@ def has_old_openssl(filename): logging.warning('"%s" contains outdated %s (%s)', filename, name, version) return True break + elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'): + if name in files_in_apk: + return True + files_in_apk.add(name) + return False @@ -1172,7 +1188,7 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk): if not common.verify_apk_signature(apkfile): return True, None, False - if has_old_openssl(apkfile): + if has_known_vulnerability(apkfile): apk['antiFeatures'].add('KnownVuln') apkzip = zipfile.ZipFile(apkfile, 'r') From 4b99a505b3278f67e32a1cf4f1e060ccee2c0c9e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 23 Jun 2017 16:38:40 +0200 Subject: [PATCH 4/6] locale: fix broken source string --- locale/fdroidserver.pot | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/locale/fdroidserver.pot b/locale/fdroidserver.pot index 02face5f..f0219aa2 100644 --- a/locale/fdroidserver.pot +++ b/locale/fdroidserver.pot @@ -327,7 +327,9 @@ msgid "Download logs we don't have" msgstr "" #: ../fdroidserver/stats.py:66 -msgid "Recalculate aggregate stats - use when changes " +msgid "" +"Recalculate aggregate stats - use when changes " +"have been made that would invalidate old cached data." msgstr "" #: ../fdroidserver/stats.py:69 From 214c9f7a2bc1a8c52aad405e4c304ca0f1fcaca1 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 23 Jun 2017 21:58:46 +0200 Subject: [PATCH 5/6] lint: check file extension for metadata This helps keep fdroiddata clean, on @krt's request. closes #222 --- fdroidserver/lint.py | 26 +++++++++++++++++++- tests/lint.TestCase | 56 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) create mode 100755 tests/lint.TestCase diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index 3ffdf5e3..200bddf6 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -17,6 +17,7 @@ # along with this program. If not, see . from argparse import ArgumentParser +import glob import os import re import sys @@ -378,6 +379,29 @@ def check_extlib_dir(apps): yield "Unused extlib at %s" % os.path.join(dir_path, path) +def check_for_unsupported_metadata_files(basedir=""): + """Checks whether any non-metadata files are in metadata/""" + + global config + + return_value = False + formats = config['accepted_formats'] + for f in glob.glob(basedir + 'metadata/*') + glob.glob(basedir + 'metadata/.*'): + if os.path.isdir(f): + exists = False + for t in formats: + exists = exists or os.path.exists(f + '.' + t) + if not exists: + print('"' + f + '/" has no matching metadata file!') + return_value = True + elif not os.path.splitext(f)[1][1:] in formats: + print('"' + f.replace(basedir, '') + + '" is not a supported file format: (' + ','.join(formats) + ')') + return_value = True + + return return_value + + def main(): global config, options @@ -398,7 +422,7 @@ def main(): allapps = metadata.read_metadata(xref=True) apps = common.read_app_args(options.appid, allapps, False) - anywarns = False + anywarns = check_for_unsupported_metadata_files() apps_check_funcs = [] if len(options.appid) == 0: diff --git a/tests/lint.TestCase b/tests/lint.TestCase new file mode 100755 index 00000000..158c5cb1 --- /dev/null +++ b/tests/lint.TestCase @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +# http://www.drdobbs.com/testing/unit-testing-with-python/240165163 + +import inspect +import optparse +import os +import shutil +import sys +import tempfile +import unittest + +localmodule = os.path.realpath( + os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) +print('localmodule: ' + localmodule) +if localmodule not in sys.path: + sys.path.insert(0, localmodule) + +import fdroidserver.common +import fdroidserver.lint + + +class LintTest(unittest.TestCase): + '''fdroidserver/lint.py''' + + def test_check_for_unsupported_metadata_files(self): + config = dict() + fdroidserver.common.fill_config_defaults(config) + config['accepted_formats'] = ('txt', 'yml') + fdroidserver.common.config = config + fdroidserver.lint.config = config + self.assertTrue(fdroidserver.lint.check_for_unsupported_metadata_files()) + + tmpdir = os.path.join(localmodule, '.testfiles') + tmptestsdir = tempfile.mkdtemp(prefix='test_check_for_unsupported_metadata_files-', + dir=tmpdir) + self.assertFalse(fdroidserver.lint.check_for_unsupported_metadata_files(tmptestsdir + '/')) + shutil.copytree(os.path.join(localmodule, 'tests', 'metadata'), + os.path.join(tmptestsdir, 'metadata'), + ignore=shutil.ignore_patterns('apk', 'dump', '*.json')) + self.assertFalse(fdroidserver.lint.check_for_unsupported_metadata_files(tmptestsdir + '/')) + shutil.copy(os.path.join(localmodule, 'tests', 'metadata', 'org.adaway.json'), + os.path.join(tmptestsdir, 'metadata')) + self.assertTrue(fdroidserver.lint.check_for_unsupported_metadata_files(tmptestsdir + '/')) + + +if __name__ == "__main__": + parser = optparse.OptionParser() + parser.add_option("-v", "--verbose", action="store_true", default=False, + help="Spew out even more information than normal") + (fdroidserver.lint.options, args) = parser.parse_args(['--verbose']) + fdroidserver.common.options = fdroidserver.lint.options + + newSuite = unittest.TestSuite() + newSuite.addTest(unittest.makeSuite(LintTest)) + unittest.main() From 7de6e4dfed90df8fe4a1d56850f4e14a2ff184b5 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 28 Jun 2017 22:23:04 +0200 Subject: [PATCH 6/6] init: update docs links for next steps --- fdroidserver/init.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdroidserver/init.py b/fdroidserver/init.py index e6f400cb..5a895464 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -246,6 +246,6 @@ then run "fdroid update -c; fdroid update". You might also want to edit "config.py" to set the URL, repo name, and more. You should also set up a signing key (a temporary one might have been automatically generated). -For more info: https://f-droid.org/manual/fdroid.html#Simple-Binary-Repository -and https://f-droid.org/manual/fdroid.html#Signing +For more info: https://f-droid.org/docs/Setup_an_F-Droid_App_Repo +and https://f-droid.org/docs/Signing_Process ''')