diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 259e25f2..571253af 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,6 +16,7 @@ metadata_v0: script: - cd tests - cp dump_internal_metadata_format.py dump.py # since this isn't in old commits + - export GITCOMMIT=`git describe` - git checkout 0.7.0 # or any old commit of your choosing - cd .. - sed -i "s/'Author Email',/'Author Email',\n'Author Web Site',/" fdroidserver/metadata.py @@ -24,7 +25,7 @@ metadata_v0: - ../tests/dump.py - cd .. - git reset --hard - - git checkout master + - git checkout $GITCOMMIT - cd fdroiddata - ../tests/dump.py - "sed -i -e '/AuthorWebSite/d' diff --git a/fdroidserver/init.py b/fdroidserver/init.py index e6f400cb..5a895464 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -246,6 +246,6 @@ then run "fdroid update -c; fdroid update". You might also want to edit "config.py" to set the URL, repo name, and more. You should also set up a signing key (a temporary one might have been automatically generated). -For more info: https://f-droid.org/manual/fdroid.html#Simple-Binary-Repository -and https://f-droid.org/manual/fdroid.html#Signing +For more info: https://f-droid.org/docs/Setup_an_F-Droid_App_Repo +and https://f-droid.org/docs/Signing_Process ''') diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index 3ffdf5e3..200bddf6 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -17,6 +17,7 @@ # along with this program. If not, see . from argparse import ArgumentParser +import glob import os import re import sys @@ -378,6 +379,29 @@ def check_extlib_dir(apps): yield "Unused extlib at %s" % os.path.join(dir_path, path) +def check_for_unsupported_metadata_files(basedir=""): + """Checks whether any non-metadata files are in metadata/""" + + global config + + return_value = False + formats = config['accepted_formats'] + for f in glob.glob(basedir + 'metadata/*') + glob.glob(basedir + 'metadata/.*'): + if os.path.isdir(f): + exists = False + for t in formats: + exists = exists or os.path.exists(f + '.' + t) + if not exists: + print('"' + f + '/" has no matching metadata file!') + return_value = True + elif not os.path.splitext(f)[1][1:] in formats: + print('"' + f.replace(basedir, '') + + '" is not a supported file format: (' + ','.join(formats) + ')') + return_value = True + + return return_value + + def main(): global config, options @@ -398,7 +422,7 @@ def main(): allapps = metadata.read_metadata(xref=True) apps = common.read_app_args(options.appid, allapps, False) - anywarns = False + anywarns = check_for_unsupported_metadata_files() apps_check_funcs = [] if len(options.appid) == 0: diff --git a/fdroidserver/update.py b/fdroidserver/update.py index b7119ab7..1f322837 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -37,7 +37,6 @@ from binascii import hexlify from PIL import Image import logging -from . import btlog from . import common from . import index from . import metadata @@ -471,13 +470,24 @@ def sha256sum(filename): return sha.hexdigest() -def has_old_openssl(filename): - '''checks for known vulnerable openssl versions in the APK''' +def has_known_vulnerability(filename): + """checks for known vulnerabilities in the APK + + Checks OpenSSL .so files in the APK to see if they are a known vulnerable + version. Google also enforces this: + https://support.google.com/faqs/answer/6376725?hl=en + + Checks whether there are more than one classes.dex or AndroidManifest.xml + files, which is invalid and an essential part of the "Master Key" attack. + + http://www.saurik.com/id/17 + """ # statically load this pattern - if not hasattr(has_old_openssl, "pattern"): - has_old_openssl.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)') + if not hasattr(has_known_vulnerability, "pattern"): + has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)') + files_in_apk = set() with zipfile.ZipFile(filename) as zf: for name in zf.namelist(): if name.endswith('libcrypto.so') or name.endswith('libssl.so'): @@ -486,7 +496,7 @@ def has_old_openssl(filename): chunk = lib.read(4096) if chunk == b'': break - m = has_old_openssl.pattern.search(chunk) + m = has_known_vulnerability.pattern.search(chunk) if m: version = m.group(1).decode('ascii') if version.startswith('1.0.1') and version[5] >= 'r' \ @@ -496,6 +506,11 @@ def has_old_openssl(filename): logging.warning('"%s" contains outdated %s (%s)', filename, name, version) return True break + elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'): + if name in files_in_apk: + return True + files_in_apk.add(name) + return False @@ -1173,7 +1188,7 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk): if not common.verify_apk_signature(apkfile): return True, None, False - if has_old_openssl(apkfile): + if has_known_vulnerability(apkfile): apk['antiFeatures'].add('KnownVuln') apkzip = zipfile.ZipFile(apkfile, 'r') @@ -1695,6 +1710,7 @@ def main(): git_remote = config.get('binary_transparency_remote') if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')): + from . import btlog btlog.make_binary_transparency_log(repodirs) if config['update_stats']: diff --git a/locale/fdroidserver.pot b/locale/fdroidserver.pot index 02face5f..f0219aa2 100644 --- a/locale/fdroidserver.pot +++ b/locale/fdroidserver.pot @@ -327,7 +327,9 @@ msgid "Download logs we don't have" msgstr "" #: ../fdroidserver/stats.py:66 -msgid "Recalculate aggregate stats - use when changes " +msgid "" +"Recalculate aggregate stats - use when changes " +"have been made that would invalidate old cached data." msgstr "" #: ../fdroidserver/stats.py:69 diff --git a/tests/lint.TestCase b/tests/lint.TestCase new file mode 100755 index 00000000..158c5cb1 --- /dev/null +++ b/tests/lint.TestCase @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +# http://www.drdobbs.com/testing/unit-testing-with-python/240165163 + +import inspect +import optparse +import os +import shutil +import sys +import tempfile +import unittest + +localmodule = os.path.realpath( + os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) +print('localmodule: ' + localmodule) +if localmodule not in sys.path: + sys.path.insert(0, localmodule) + +import fdroidserver.common +import fdroidserver.lint + + +class LintTest(unittest.TestCase): + '''fdroidserver/lint.py''' + + def test_check_for_unsupported_metadata_files(self): + config = dict() + fdroidserver.common.fill_config_defaults(config) + config['accepted_formats'] = ('txt', 'yml') + fdroidserver.common.config = config + fdroidserver.lint.config = config + self.assertTrue(fdroidserver.lint.check_for_unsupported_metadata_files()) + + tmpdir = os.path.join(localmodule, '.testfiles') + tmptestsdir = tempfile.mkdtemp(prefix='test_check_for_unsupported_metadata_files-', + dir=tmpdir) + self.assertFalse(fdroidserver.lint.check_for_unsupported_metadata_files(tmptestsdir + '/')) + shutil.copytree(os.path.join(localmodule, 'tests', 'metadata'), + os.path.join(tmptestsdir, 'metadata'), + ignore=shutil.ignore_patterns('apk', 'dump', '*.json')) + self.assertFalse(fdroidserver.lint.check_for_unsupported_metadata_files(tmptestsdir + '/')) + shutil.copy(os.path.join(localmodule, 'tests', 'metadata', 'org.adaway.json'), + os.path.join(tmptestsdir, 'metadata')) + self.assertTrue(fdroidserver.lint.check_for_unsupported_metadata_files(tmptestsdir + '/')) + + +if __name__ == "__main__": + parser = optparse.OptionParser() + parser.add_option("-v", "--verbose", action="store_true", default=False, + help="Spew out even more information than normal") + (fdroidserver.lint.options, args) = parser.parse_args(['--verbose']) + fdroidserver.common.options = fdroidserver.lint.options + + newSuite = unittest.TestSuite() + newSuite.addTest(unittest.makeSuite(LintTest)) + unittest.main()