From f56b1f3012c6dc6576f07c1deb6865c794890e0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 22 Sep 2022 01:33:23 +0200 Subject: [PATCH 01/21] basic downloading for scan_binary signatures --- fdroidserver/common.py | 1 + fdroidserver/exception.py | 7 ++ fdroidserver/scanner.py | 161 +++++++++++++++++++++++--- tests/scanner.TestCase | 235 ++++++++++++++++++++++++++++++-------- tests/testcommon.py | 9 +- 5 files changed, 351 insertions(+), 62 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 8ed76fb7..f4a1fe77 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -125,6 +125,7 @@ default_config = { 'sdk_path': "$ANDROID_HOME", 'ndk_paths': {}, 'cachedir': str(Path.home() / '.cache/fdroidserver'), + 'cachedir_scanner': str(Path.home() / '.cache/fdroidserver/scanner'), 'java_paths': None, 'scan_binary': False, 'ant': "ant", diff --git a/fdroidserver/exception.py b/fdroidserver/exception.py index 097f4f68..a598b368 100644 --- a/fdroidserver/exception.py +++ b/fdroidserver/exception.py @@ -45,3 +45,10 @@ class BuildException(FDroidException): class VerificationException(FDroidException): pass + + +class ConfigurationException(FDroidException): + def __init__(self, value=None, detail=None): + super().__init__() + self.value = value + self.detail = detail diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 710d0c23..6321af8b 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -23,19 +23,23 @@ import re import sys import traceback import zipfile +import yaml from argparse import ArgumentParser from collections import namedtuple from copy import deepcopy from tempfile import TemporaryDirectory +from pathlib import Path import logging import itertools +import urllib.request +from datetime import datetime, timedelta import requests from . import _ from . import common from . import metadata -from .exception import BuildException, VCSException +from .exception import BuildException, VCSException, ConfigurationException from . import scanner config = None @@ -47,17 +51,6 @@ json_per_build = deepcopy(DEFAULT_JSON_PER_BUILD) MAVEN_URL_REGEX = re.compile(r"""\smaven\s*(?:{.*?(?:setUrl|url)|\((?:url)?)\s*=?\s*(?:uri)?\(?\s*["']?([^\s"']+)["']?[^})]*[)}]""", re.DOTALL) -CODE_SIGNATURES = { - exp: re.compile(r'.*' + exp, re.IGNORECASE) for exp in [ - r'com/google/firebase', - r'com/google/android/gms', - r'com/google/android/play/core', - r'com/google/tagmanager', - r'com/google/analytics', - r'com/android/billing', - ] -} - # Common known non-free blobs (always lower case): NON_FREE_GRADLE_LINES = { exp: re.compile(r'.*' + exp, re.IGNORECASE) for exp in [ @@ -109,6 +102,9 @@ NON_FREE_GRADLE_LINES = { } +SCANNER_CACHE_VERSION = 1 + + def get_gradle_compile_commands(build): compileCommands = ['compile', 'provided', @@ -188,6 +184,145 @@ def _exodus_compile_signatures(signatures): return compiled_tracker_signature +def _datetime_now(): + """ + simple warpper for datetime.now to allow mocking it for testing + """ + return datetime.now().astimezone() + + +def _scanner_cachedir(): + """ + get `Path` to local cache dir + """ + if not common.config or "cachedir_scanner" not in common.config: + raise ConfigurationException("could not load 'cachedir_scanner' config") + cachedir = Path(config["cachedir_scanner"]) + cachedir.mkdir(exist_ok=True, parents=True) + return cachedir + + +class SignatureCacheMalformedException(Exception): + pass + + +class SignatureCacheOutdatedException(Exception): + pass + + +class SignatureDataController: + def __init__(self, name, filename): + self.name = name + self.filename = filename + self.cache_outdated_interval = timedelta(days=7) + self.data = {} + + def check_data_version(self): + if self.data.get("version") != SCANNER_CACHE_VERSION: + raise SignatureCacheMalformedException() + + def check_last_updated(self): + timestamp = self.data.get("timestamp") + if not timestamp: + raise SignatureCacheMalformedException() + try: + timestamp = datetime.fromisoformat(timestamp) + except ValueError as e: + raise SignatureCacheMalformedException() from e + except TypeError as e: + raise SignatureCacheMalformedException() from e + if (timestamp + self.cache_outdated_interval) < scanner._datetime_now(): + raise SignatureCacheOutdatedException() + + def load_from_defaults(self): + sig_file = Path(__file__).absolute().parent / 'scanner_signatures' / self.file_name + with open(sig_file) as f: + self.data = yaml.safe_load(f) + + def load_from_cache(self): + sig_file = scanner._scanner_cachedir() / self.filename + if not sig_file.exists(): + raise SignatureCacheMalformedException() + with open(sig_file) as f: + self.data = yaml.safe_load(f) + + def write_to_cache(self): + sig_file = scanner._scanner_cachedir() / self.filename + with open(sig_file, "w", encoding="utf-8") as f: + yaml.safe_dump(self.data, f) + logging.debug("write '{}' to cache".format(self.filename)) + + def verify_data(self): + valid_keys = ['timestamp', 'version', 'signatures'] + for k in [x for x in self.data.keys() if x not in valid_keys]: + del self.data[k] + + # def scan + + +class ExodusSignatureDataController(SignatureDataController): + def __init__(self): + super().__init__('Exodus signatures', 'exodus.yml') + + def fetch_signatures_from_web(): + pass + # TODO + # exodus_url = "https://reports.exodus-privacy.eu.org/api/trackers" + # sigs = { + # "signatures": [], + # "timestamp": scanner._datetime_now().isoformat(), + # "version": SCANNER_CACHE_VERSION, + # } + + # with urllib.request.urlopen(exodus_url) as f: + # data = json.load(f) + # for tracker in data["trackers"].values(): + # sigs["signatures"].append({ + # "name": tracker["name"], + # "binary_signature": tracker["code_signature"], + # "network_signature": tracker["network_signature"], + # "types": ["tracker", "non-free"] # right now we assume all trackers in exodus are non-free + # }) + + +class ScannerSignatureDataController(SignatureDataController): + def __init__(self): + super().__init__('Scanner signatures', 'scanner.yml') + + def fetch_signatures_from_web(self): + url = "https://uniqx.gitlab.io/fdroid-scanner-signatures/sigs.json" + with urllib.request.urlopen(url) as f: + data = yaml.safe_load(f) + # TODO: validate parsed data + # TODO: error message 'please update fdroidserver/report' when fetching failed due to changes in the data strucutre + self.data = data + + +class SignatureTool(): + def __init__(self): + self.sdcs = [ScannerSignatureDataController()] + for sdc in self.sdcs: + sdc.fetch_signatures_from_web() + # TODO: use cache + # if not sdc.check_cache(): + # sdc.load_from_defaults() + self.compile_regexes() + + def compile_regexes(self): + self.regex = {'code_signatures': {}} + for sdc in self.sdcs: + for lname, ldef in sdc.data.get('signatures', []).items(): + self.regex['code_signatures'].update({(x, re.compile(x)) for x in ldef.get('code_signatures', [])}) + + def binary_signatures(self): + for sdc in self.sdcs: + for sig in sdc.binary_signatures(): + yield sig + + +SIGNATURE_TOOL = SignatureTool() + + # taken from exodus_core def load_exodus_trackers_signatures(): """ @@ -215,7 +350,7 @@ def scan_binary(apkfile, extract_signatures=None): result = get_embedded_classes(apkfile) problems = 0 for classname in result: - for suspect, regexp in CODE_SIGNATURES.items(): + for suspect, regexp in SIGNATURE_TOOL.regex['code_signatures'].items(): if regexp.match(classname): logging.debug("Found class '%s'" % classname) problems += 1 diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index 991cd9f0..01076a6e 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -4,6 +4,7 @@ import glob import inspect import logging import optparse +import io import os import re import shutil @@ -17,6 +18,7 @@ import zipfile import collections import pathlib from unittest import mock +from datetime import datetime, timedelta localmodule = os.path.realpath( os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..') @@ -446,21 +448,19 @@ class Test_scan_binary(unittest.TestCase): fdroidserver.common.config = config fdroidserver.common.options = mock.Mock() + fdroidserver.scanner.SIGNATURE_TOOL = mock.Mock() + fdroidserver.scanner.SIGNATURE_TOOL.regex = {} + fdroidserver.scanner.SIGNATURE_TOOL.regex['code_signatures'] = { + "java/lang/Object": re.compile(r'.*java/lang/Object', re.IGNORECASE | re.UNICODE) + } + def test_code_signature_match(self): apkfile = os.path.join(self.basedir, 'no_targetsdk_minsdk1_unsigned.apk') - mock_code_signatures = { - "java/lang/Object": re.compile( - r'.*java/lang/Object', re.IGNORECASE | re.UNICODE - ) - } - with mock.patch("fdroidserver.scanner.CODE_SIGNATURES", mock_code_signatures): - self.assertEqual( - 1, - fdroidserver.scanner.scan_binary(apkfile), - "Did not find expected code signature '{}' in binary '{}'".format( - fdroidserver.scanner.CODE_SIGNATURES.values(), apkfile - ), - ) + self.assertEqual( + 1, + fdroidserver.scanner.scan_binary(apkfile), + "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner.SIGNATURE_TOOL.regex['code_signatures'].values(), apkfile), + ) @unittest.skipIf( sys.version_info < (3, 9), @@ -470,44 +470,85 @@ class Test_scan_binary(unittest.TestCase): ) def test_bottom_level_embedded_apk_code_signature(self): apkfile = os.path.join(self.basedir, 'apk.embedded_1.apk') - mock_code_signatures = { + fdroidserver.scanner.SIGNATURE_TOOL.regex['code_signatures'] = { "org/bitbucket/tickytacky/mirrormirror/MainActivity": re.compile( - r'.*org/bitbucket/tickytacky/mirrormirror/MainActivity', - re.IGNORECASE | re.UNICODE, + r'.*org/bitbucket/tickytacky/mirrormirror/MainActivity', re.IGNORECASE | re.UNICODE ) } - with mock.patch("fdroidserver.scanner.CODE_SIGNATURES", mock_code_signatures): - self.assertEqual( - 1, - fdroidserver.scanner.scan_binary(apkfile), - "Did not find expected code signature '{}' in binary '{}'".format( - fdroidserver.scanner.CODE_SIGNATURES.values(), apkfile - ), - ) + + self.assertEqual( + 1, + fdroidserver.scanner.scan_binary(apkfile), + "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner.SIGNATURE_TOOL.regex['code_signatures'].values(), apkfile), + ) def test_top_level_signature_embedded_apk_present(self): apkfile = os.path.join(self.basedir, 'apk.embedded_1.apk') - mock_code_signatures = { + fdroidserver.scanner.SIGNATURE_TOOL.regex['code_signatures'] = { "org/fdroid/ci/BuildConfig": re.compile( r'.*org/fdroid/ci/BuildConfig', re.IGNORECASE | re.UNICODE ) } - with mock.patch("fdroidserver.scanner.CODE_SIGNATURES", mock_code_signatures): - self.assertEqual( - 1, - fdroidserver.scanner.scan_binary(apkfile), - "Did not find expected code signature '{}' in binary '{}'".format( - fdroidserver.scanner.CODE_SIGNATURES.values(), apkfile - ), - ) - - def test_no_match(self): - apkfile = os.path.join(self.basedir, 'no_targetsdk_minsdk1_unsigned.apk') - result = fdroidserver.scanner.scan_binary(apkfile) self.assertEqual( - 0, result, "Found false positives in binary '{}'".format(apkfile) + 1, + fdroidserver.scanner.scan_binary(apkfile), + "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner.SIGNATURE_TOOL.regex['code_signatures'].values(), apkfile), ) + # TODO: re-enable once allow-listing migrated to more complex regexes + # def test_no_match(self): + # apkfile = os.path.join(self.basedir, 'no_targetsdk_minsdk1_unsigned.apk') + # result = fdroidserver.scanner.scan_binary(apkfile) + # self.assertEqual(0, result, "Found false positives in binary '{}'".format(apkfile)) + + +# class Test__fetch_exodus_signatures_to_cache(unittest.TestCase): +# def setUp(self): +# self.web_req_func = mock.Mock(return_value=io.StringIO(json.dumps({ +# "trackers": { +# "1": { +# "id": 1, +# "name": "Steyer Puch 1", +# "description": "blah blah blah", +# "creation_date": "1956-01-01", +# "code_signature": "com.puch.|com.steyer.", +# "network_signature": "pst\\.com", +# "website": "https://pst.com", +# "categories": ["tracker"], +# "documentation": [], +# }, +# "2": { +# "id": 2, +# "name": "Steyer Puch 2", +# "description": "blah blah blah", +# "creation_date": "1956-01-01", +# "code_signature": "com.puch.|com.steyer.", +# "network_signature": "pst\\.com", +# "website": "https://pst.com", +# "categories": ["tracker"], +# "documentation": [], +# } +# }, +# }))) +# self.open_func = mock.mock_open() +# self.cachedir_func = mock.Mock(return_value=pathlib.Path("mocked/path")) +# +# def test_ok(self): +# with mock.patch("urllib.request.urlopen", self.web_req_func), mock.patch( +# "builtins.open", self.open_func +# ) as outfilemock, mock.patch( +# "fdroidserver.scanner._scanner_cachedir", self.cachedir_func +# ), mock.patch("fdroidserver.scanner._datetime_now", unittest.mock.Mock(return_value=datetime(1999, 12, 31, 23, 59, 59))): +# fdroidserver.scanner.fetch_exodus_signatures_to_cache() +# +# self.cachedir_func.assert_called_once() +# self.web_req_func.assert_called_once_with("https://reports.exodus-privacy.eu.org/api/trackers") +# self.open_func.assert_called_once_with(pathlib.Path("mocked/path/exodus.json"), "w", encoding="utf-8") +# self.assertEqual( +# mock_open_to_str(self.open_func), +# """{"signatures": {"exodus-1": {"name": "Steyer Puch 1", "code_signature": "com.puch.|com.steyer.", "network_signature": "pst\\\\.com", "types": ["tracker", "non-free"]}, "exodus-2": {"name": "Steyer Puch 2", "code_signature": "com.puch.|com.steyer.", "network_signature": "pst\\\\.com", "types": ["tracker", "non-free"]}}, "timestamp": "1999-12-31T23:59:59"}""" +# ) + class Test__exodus_compile_signatures(unittest.TestCase): def setUp(self): @@ -581,6 +622,106 @@ class Test_load_exodus_trackers_signatures(unittest.TestCase): self.assertEqual(regex, "mocked return value") +class Test_SignatureDataController(unittest.TestCase): + # __init__ + def test_init(self): + sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') + self.assertEqual(sdc.name, 'nnn') + self.assertEqual(sdc.filename, 'fff.yml') + self.assertEqual(sdc.cache_outdated_interval, timedelta(days=7)) + self.assertDictEqual(sdc.data, {}) + + # check_last_updated + def test_check_last_updated_ok(self): + sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') + sdc.data['timestamp'] = datetime.now().astimezone().isoformat() + sdc.check_last_updated() + + def test_check_last_updated_exception_cache_outdated(self): + sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') + sdc.data['timestamp'] = (datetime.now().astimezone() - timedelta(days=30)).isoformat() + with self.assertRaises(fdroidserver.scanner.SignatureCacheOutdatedException): + sdc.check_last_updated() + + def test_check_last_updated_exception_missing_timestamp_value(self): + sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') + with self.assertRaises(fdroidserver.scanner.SignatureCacheMalformedException): + sdc.check_last_updated() + + def test_check_last_updated_exception_not_string(self): + sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') + sdc.data['timestamp'] = 12345 + with self.assertRaises(fdroidserver.scanner.SignatureCacheMalformedException): + sdc.check_last_updated() + + def test_check_last_updated_exception_not_iso_formatted_string(self): + sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') + sdc.data['timestamp'] = '01/09/2002 10:11' + with self.assertRaises(fdroidserver.scanner.SignatureCacheMalformedException): + sdc.check_last_updated() + + # check_data_version + def test_check_data_version_ok(self): + sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') + sdc.data['version'] = fdroidserver.scanner.SCANNER_CACHE_VERSION + sdc.check_data_version() + + def test_check_data_version_exception(self): + sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') + with self.assertRaises(fdroidserver.scanner.SignatureCacheMalformedException): + sdc.check_data_version() + + +class Test_ScannerSignatureDataController_fetch_signatures_from_web(unittest.TestCase): + def setUp(self): + self.uo_func = mock.Mock(return_value=io.StringIO(textwrap.dedent('''\ + version: 999 + timestamp: "1999-12-31T23:59:59.999999+00:00" + signatures: + - binary_signature: com/google/firebase + name: Google Firebase + types: + - tracker + - non-free + - gradle_signature: com/google/android/gms + name: Google Mobile Services + types: + - non-free + - network_signature: doubleclick\\.net + name: Another thing to test. + types: + - ads + '''))) + + def test_fetch_signatures_from_web(self): + sdc = fdroidserver.scanner.ScannerSignatureDataController() + with unittest.mock.patch('urllib.request.urlopen', self.uo_func): + sdc.fetch_signatures_from_web() + self.assertEqual(sdc.data.get('version'), 999) + self.assertEqual(sdc.data.get('timestamp'), "1999-12-31T23:59:59.999999+00:00") + self.assertListEqual( + sdc.data.get('signatures'), + [ + { + 'binary_signature': 'com/google/firebase', + 'name': 'Google Firebase', + 'types': ['tracker', 'non-free'], + }, + { + 'gradle_signature': 'com/google/android/gms', + 'name': 'Google Mobile Services', + 'types': ['non-free'], + }, + { + 'network_signature': 'doubleclick\\.net', + 'name': 'Another thing to test.', + 'types': ['ads'], + }, + ] + ) + self.assertEqual(len(sdc.data), 3) + + class Test_main(unittest.TestCase): def setUp(self): self.args = ["com.example.app", "local/additional.apk", "another.apk"] @@ -644,13 +785,13 @@ if __name__ == "__main__": (fdroidserver.common.options, args) = parser.parse_args(['--verbose']) newSuite = unittest.TestSuite() - newSuite.addTests( - [ - unittest.makeSuite(ScannerTest), - unittest.makeSuite(Test_scan_binary), - unittest.makeSuite(Test__exodus_compile_signatures), - unittest.makeSuite(Test_load_exodus_trackers_signatures), - unittest.makeSuite(Test_main), - ] - ) + newSuite.addTests([ + unittest.makeSuite(ScannerTest), + unittest.makeSuite(Test_scan_binary), + unittest.makeSuite(Test__exodus_compile_signatures), + unittest.makeSuite(Test_load_exodus_trackers_signatures), + unittest.makeSuite(Test_SignatureDataController), + unittest.makeSuite(Test_ScannerSignatureDataController_fetch_signatures_from_web), + unittest.makeSuite(Test_main), + ]) unittest.main(failfast=False) diff --git a/tests/testcommon.py b/tests/testcommon.py index 2557bd61..128ead00 100644 --- a/tests/testcommon.py +++ b/tests/testcommon.py @@ -36,8 +36,7 @@ class TmpCwd(): class TmpPyPath(): - """Context-manager for temporarily changing the current working - directory. + """Context-manager for temporarily adding a direcory to python path """ def __init__(self, additional_path): @@ -48,3 +47,9 @@ class TmpPyPath(): def __exit__(self, a, b, c): sys.path.remove(self.additional_path) + + +def mock_open_to_str(mock): + return "".join([ + x.args[0] for x in mock.mock_calls if str(x).startswith("call().write(") + ]) From 9c5b35742df52a55f3c21f23c3e3292ae6c75079 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 22 Sep 2022 14:43:16 +0200 Subject: [PATCH 02/21] use ScannerTool for basic cacheing and scanning --- fdroidserver/data/scanner/scanner.json | 42 ++++++++++++++++++++++ fdroidserver/scanner.py | 48 ++++++++++++++++---------- 2 files changed, 71 insertions(+), 19 deletions(-) create mode 100644 fdroidserver/data/scanner/scanner.json diff --git a/fdroidserver/data/scanner/scanner.json b/fdroidserver/data/scanner/scanner.json new file mode 100644 index 00000000..19eb1564 --- /dev/null +++ b/fdroidserver/data/scanner/scanner.json @@ -0,0 +1,42 @@ +{ + "signatures": { + "com.android.billing": { + "code_signatures": [ + "com/android/billing" + ], + "license": "NonFree" + }, + "com.google.analytics": { + "code_signatures": [ + "com/google/analytics" + ], + "license": "NonFree" + }, + "com.google.android.gms": { + "code_signatures": [ + "com/google/android/gms" + ], + "license": "NonFree" + }, + "com.google.android.play.core": { + "code_signatures": [ + "com/google/android/play/core" + ], + "license": "NonFree" + }, + "com.google.firebase": { + "code_signatures": [ + "com/google/firebase" + ], + "license": "NonFree" + }, + "com.google.tagmanager": { + "code_signatures": [ + "com/google/tagmanager" + ], + "license": "NonFree" + } + }, + "timestamp": "2022-09-15T12:43:06.508370+00:00", + "version": 1 +} diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 6321af8b..6848e1e8 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -234,22 +234,30 @@ class SignatureDataController: if (timestamp + self.cache_outdated_interval) < scanner._datetime_now(): raise SignatureCacheOutdatedException() + def load(self): + try: + self.load_from_cache() + self.verify_data() + except SignatureCacheMalformedException as e: + self.load_from_defaults() + self.write_to_cache() + def load_from_defaults(self): - sig_file = Path(__file__).absolute().parent / 'scanner_signatures' / self.file_name + sig_file = Path(__file__).absolute().parent / 'data' / 'scanner' / self.filename with open(sig_file) as f: - self.data = yaml.safe_load(f) + self.data = json.load(f) def load_from_cache(self): sig_file = scanner._scanner_cachedir() / self.filename if not sig_file.exists(): raise SignatureCacheMalformedException() with open(sig_file) as f: - self.data = yaml.safe_load(f) + self.data = json.load(f) def write_to_cache(self): sig_file = scanner._scanner_cachedir() / self.filename with open(sig_file, "w", encoding="utf-8") as f: - yaml.safe_dump(self.data, f) + json.dump(self.data, f, indent=2) logging.debug("write '{}' to cache".format(self.filename)) def verify_data(self): @@ -287,7 +295,7 @@ class ExodusSignatureDataController(SignatureDataController): class ScannerSignatureDataController(SignatureDataController): def __init__(self): - super().__init__('Scanner signatures', 'scanner.yml') + super().__init__('Scanner signatures', 'scanner.json') def fetch_signatures_from_web(self): url = "https://uniqx.gitlab.io/fdroid-scanner-signatures/sigs.json" @@ -298,29 +306,31 @@ class ScannerSignatureDataController(SignatureDataController): self.data = data -class SignatureTool(): +class ScannerTool(): def __init__(self): self.sdcs = [ScannerSignatureDataController()] for sdc in self.sdcs: - sdc.fetch_signatures_from_web() - # TODO: use cache - # if not sdc.check_cache(): - # sdc.load_from_defaults() + sdc.load() self.compile_regexes() def compile_regexes(self): self.regex = {'code_signatures': {}} for sdc in self.sdcs: - for lname, ldef in sdc.data.get('signatures', []).items(): - self.regex['code_signatures'].update({(x, re.compile(x)) for x in ldef.get('code_signatures', [])}) - - def binary_signatures(self): - for sdc in self.sdcs: - for sig in sdc.binary_signatures(): - yield sig + print(']]]', sdc.data) + for signame, sigdef in sdc.data.get('signatures', {}).items(): + for sig in sigdef['code_signatures']: + self.regex['code_signatures'][sig] = re.compile(sig, re.IGNORECASE) + print(')))', self.regex['code_signatures']) -SIGNATURE_TOOL = SignatureTool() +# TODO: change this from global instance to dependency injection +SCANNER_TOOL = None + + +def _get_tool(): + if not scanner.SCANNER_TOOL: + scanner.SCANNER_TOOL = ScannerTool() + return scanner.SCANNER_TOOL # taken from exodus_core @@ -350,7 +360,7 @@ def scan_binary(apkfile, extract_signatures=None): result = get_embedded_classes(apkfile) problems = 0 for classname in result: - for suspect, regexp in SIGNATURE_TOOL.regex['code_signatures'].items(): + for suspect, regexp in _get_tool().regex['code_signatures'].items(): if regexp.match(classname): logging.debug("Found class '%s'" % classname) problems += 1 From e4b54fe4a7cb31afb799beea75674edfbdc3828d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 22 Sep 2022 15:34:25 +0200 Subject: [PATCH 03/21] move NON_FREE_GRADLE_LINES to signature definition files --- fdroidserver/data/scanner/scanner.json | 253 ++++++++++++++++++++++++- fdroidserver/scanner.py | 62 +----- 2 files changed, 257 insertions(+), 58 deletions(-) diff --git a/fdroidserver/data/scanner/scanner.json b/fdroidserver/data/scanner/scanner.json index 19eb1564..f2479f22 100644 --- a/fdroidserver/data/scanner/scanner.json +++ b/fdroidserver/data/scanner/scanner.json @@ -1,23 +1,116 @@ { "signatures": { + "admob": { + "gradle_signatures": [ + "admob.*sdk.*android" + ], + "license": "NonFree" + }, + "androidx": { + "gradle_signatures": [ + "androidx.navigation:navigation-dynamic-features", + "androidx.work:work-gcm" + ], + "license": "NonFree" + }, + "appcenter-push": { + "gradle_signatures": [ + "appcenter-push" + ], + "license": "NonFree" + }, + "bugsense": { + "gradle_signatures": [ + "bugsense" + ], + "license": "NonFree" + }, + "cloudrail": { + "gradle_signatures": [ + "cloudrail" + ], + "license": "NonFree" + }, "com.android.billing": { "code_signatures": [ "com/android/billing" ], "license": "NonFree" }, + "com.android.billingclient": { + "gradle_signatures": [ + "com.android.billingclient" + ], + "license": "NonFree" + }, + "com.anjlab.android.iab.v3": { + "gradle_signatures": [ + "com.anjlab.android.iab.v3:library" + ], + "license": "NonFree" + }, + "com.cloudinary": { + "gradle_signatures": [ + "com.cloudinary:cloudinary-android" + ], + "license": "NonFree" + }, + "com.evernote": { + "gradle_signatures": [ + "com.evernote:android-job" + ], + "license": "NonFree" + }, + "com.facebook": { + "gradle_signatures": [ + "[\"']com.facebook.android['\":]" + ], + "license": "NonFree" + }, + "com.github.junrar": { + "gradle_signatures": [ + "com.github.junrar:junrar" + ], + "license": "NonFree" + }, + "com.github.penn5": { + "gradle_signatures": [ + "com.github.penn5:donations" + ], + "license": "NonFree" + }, "com.google.analytics": { "code_signatures": [ "com/google/analytics" ], "license": "NonFree" }, + "com.google.android.exoplayer": { + "gradle_signatures": [ + "com.google.android.exoplayer:extension-cast", + "com.google.android.exoplayer:extension-cronet" + ], + "license": "NonFree" + }, "com.google.android.gms": { "code_signatures": [ "com/google/android/gms" ], "license": "NonFree" }, + "com.google.android.libraries.places": { + "gradle_signatures": [ + "com.google.android.libraries.places:places" + ], + "license": "NonFree" + }, + "com.google.android.play": { + "gradle_signatures": [ + "com.google.android.play:app-update", + "com.google.android.play:core.*" + ], + "license": "NonFree" + }, "com.google.android.play.core": { "code_signatures": [ "com/google/android/play/core" @@ -30,13 +123,169 @@ ], "license": "NonFree" }, + "com.google.mlkit": { + "gradle_signatures": [ + "com.google.mlkit" + ], + "license": "NonFree" + }, "com.google.tagmanager": { "code_signatures": [ "com/google/tagmanager" ], "license": "NonFree" + }, + "com.hypertrack": { + "gradle_signatures": [ + "com.hypertrack" + ], + "license": "NonFree" + }, + "com.mapbox": { + "gradle_signatures": [ + "com.mapbox" + ], + "license": "NonFree" + }, + "com.onesignal": { + "gradle_signatures": [ + "com.onesignal:OneSignal" + ], + "license": "NonFree" + }, + "com.tencent.bugly": { + "gradle_signatures": [ + "com.tencent.bugly" + ], + "license": "NonFree" + }, + "com.umeng.umsdk": { + "gradle_signatures": [ + "com.umeng.umsdk" + ], + "license": "NonFree" + }, + "com.yandex.android": { + "gradle_signatures": [ + "com.yandex.android" + ], + "license": "NonFree" + }, + "com.yayandroid": { + "gradle_signatures": [ + "com.yayandroid:LocationManager" + ], + "license": "NonFree" + }, + "crashlytics": { + "gradle_signatures": [ + "crashlytics" + ], + "license": "NonFree" + }, + "crittercism": { + "gradle_signatures": [ + "crittercism" + ], + "license": "NonFree" + }, + "firebase": { + "gradle_signatures": [ + "firebase" + ], + "license": "NonFree" + }, + "flurryagent": { + "gradle_signatures": [ + "flurryagent" + ], + "license": "NonFree" + }, + "google-ad": { + "gradle_signatures": [ + "google.*ad.*view" + ], + "license": "NonFree" + }, + "google.admob": { + "gradle_signatures": [ + "google.*admob" + ], + "license": "NonFree" + }, + "google.play.services": { + "gradle_signatures": [ + "google.*play.*services" + ], + "license": "NonFree" + }, + "heyzap": { + "gradle_signatures": [ + "heyzap" + ], + "license": "NonFree" + }, + "io.github.sinaweibosdk": { + "gradle_signatures": [ + "io.github.sinaweibosdk" + ], + "license": "NonFree" + }, + "io.objectbox": { + "gradle_signatures": [ + "io.objectbox:objectbox-gradle-plugin" + ], + "license": "NonFree" + }, + "jpct": { + "gradle_signatures": [ + "jpct.*ae" + ], + "license": "NonFree" + }, + "libspen23": { + "gradle_signatures": [ + "libspen23" + ], + "license": "NonFree" + }, + "me.pushy": { + "gradle_signatures": [ + "me.pushy:sdk" + ], + "license": "NonFree" + }, + "org.jetbrains.kotlinx": { + "gradle_signatures": [ + "org.jetbrains.kotlinx:kotlinx-coroutines-play-services" + ], + "license": "NonFree" + }, + "ouya": { + "gradle_signatures": [ + "ouya.*sdk" + ], + "license": "NonFree" + }, + "paypal": { + "gradle_signatures": [ + "paypal.*mpl" + ], + "license": "NonFree" + }, + "xyz.belvi.mobilevision": { + "gradle_signatures": [ + "xyz.belvi.mobilevision:barcodescanner" + ], + "license": "NonFree" + }, + "youtube": { + "gradle_signatures": [ + "youtube.*android.*player.*api" + ], + "license": "NonFree" } }, - "timestamp": "2022-09-15T12:43:06.508370+00:00", + "timestamp": "2022-09-22T13:16:29.313769+00:00", "version": 1 -} +} \ No newline at end of file diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 6848e1e8..4b949723 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -51,56 +51,6 @@ json_per_build = deepcopy(DEFAULT_JSON_PER_BUILD) MAVEN_URL_REGEX = re.compile(r"""\smaven\s*(?:{.*?(?:setUrl|url)|\((?:url)?)\s*=?\s*(?:uri)?\(?\s*["']?([^\s"']+)["']?[^})]*[)}]""", re.DOTALL) -# Common known non-free blobs (always lower case): -NON_FREE_GRADLE_LINES = { - exp: re.compile(r'.*' + exp, re.IGNORECASE) for exp in [ - r'flurryagent', - r'paypal.*mpl', - r'admob.*sdk.*android', - r'google.*ad.*view', - r'google.*admob', - r'google.*play.*services', - r'com.google.android.play:core.*', - r'com.google.android.play:app-update', - r'com.google.android.libraries.places:places', - r'com.google.mlkit', - r'com.android.billingclient', - r'androidx.work:work-gcm', - r'crittercism', - r'heyzap', - r'jpct.*ae', - r'youtube.*android.*player.*api', - r'bugsense', - r'crashlytics', - r'ouya.*sdk', - r'libspen23', - r'firebase', - r'''["']com.facebook.android['":]''', - r'cloudrail', - r'com.tencent.bugly', - r'appcenter-push', - r'com.github.junrar:junrar', - r'androidx.navigation:navigation-dynamic-features', - r'xyz.belvi.mobilevision:barcodescanner', - r'org.jetbrains.kotlinx:kotlinx-coroutines-play-services', - r'me.pushy:sdk', - r'io.github.sinaweibosdk', - r'com.umeng.umsdk', - r'com.google.android.exoplayer:extension-cast', - r'io.objectbox:objectbox-gradle-plugin', - r'com.evernote:android-job', - r'com.yayandroid:LocationManager', - r'com.onesignal:OneSignal', - r'com.cloudinary:cloudinary-android', - r'com.google.android.exoplayer:extension-cronet', - r'com.anjlab.android.iab.v3:library', - r'com.github.penn5:donations', - r'com.mapbox', - r'com.yandex.android', - r'com.hypertrack', - ] -} - SCANNER_CACHE_VERSION = 1 @@ -314,13 +264,13 @@ class ScannerTool(): self.compile_regexes() def compile_regexes(self): - self.regex = {'code_signatures': {}} + self.regex = {'code_signatures': {}, 'gradle_signatures': {}} for sdc in self.sdcs: - print(']]]', sdc.data) for signame, sigdef in sdc.data.get('signatures', {}).items(): - for sig in sigdef['code_signatures']: - self.regex['code_signatures'][sig] = re.compile(sig, re.IGNORECASE) - print(')))', self.regex['code_signatures']) + for sig in sigdef.get('code_signatures', []): + self.regex['code_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE) + for sig in sigdef.get('gradle_signatures', []): + self.regex['gradle_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE) # TODO: change this from global instance to dependency injection @@ -412,7 +362,7 @@ def scan_source(build_dir, build=metadata.Build()): return any(al in s for al in allowlisted) def suspects_found(s): - for n, r in NON_FREE_GRADLE_LINES.items(): + for n, r in _get_tool().regex['gradle_signatures'].items(): if r.match(s) and not is_allowlisted(s): yield n From d5ef1b2e95fc7ffe09ba6b5487b34d0b50942aaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Fri, 23 Sep 2022 18:33:02 +0200 Subject: [PATCH 04/21] add --clear-cache option to scanner --- fdroidserver/scanner.py | 101 ++++++++++++++++++++++++++++------------ tests/scanner.TestCase | 26 +++++------ 2 files changed, 83 insertions(+), 44 deletions(-) diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 4b949723..8dcb512a 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -16,26 +16,26 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import imghdr -import json import os import re import sys -import traceback -import zipfile +import json import yaml +import imghdr +import shutil +import logging +import zipfile +import requests +import itertools +import traceback +import urllib.request from argparse import ArgumentParser from collections import namedtuple from copy import deepcopy from tempfile import TemporaryDirectory from pathlib import Path -import logging -import itertools -import urllib.request from datetime import datetime, timedelta -import requests - from . import _ from . import common from . import metadata @@ -145,18 +145,24 @@ def _scanner_cachedir(): """ get `Path` to local cache dir """ - if not common.config or "cachedir_scanner" not in common.config: - raise ConfigurationException("could not load 'cachedir_scanner' config") - cachedir = Path(config["cachedir_scanner"]) + if not common.config: + raise ConfigurationException('config not initialized') + if "cachedir_scanner" not in common.config: + raise ConfigurationException("could not load 'cachedir_scanner' from config") + cachedir = Path(common.config["cachedir_scanner"]) cachedir.mkdir(exist_ok=True, parents=True) return cachedir -class SignatureCacheMalformedException(Exception): +class SignatureDataMalformedException(Exception): pass -class SignatureCacheOutdatedException(Exception): +class SignatureDataOutdatedException(Exception): + pass + + +class SignatureDataVersionMismatchException(Exception): pass @@ -169,26 +175,37 @@ class SignatureDataController: def check_data_version(self): if self.data.get("version") != SCANNER_CACHE_VERSION: - raise SignatureCacheMalformedException() + raise SignatureDataVersionMismatchException() def check_last_updated(self): + ''' + NOTE: currently not in use + + Checks if the timestamp value is ok. Raises an exception if something + is not ok. + + :raises SignatureDataMalformedException: when timestamp value is + inaccessible or not parseable + :raises SignatureDataOutdatedException: when timestamp is older then + `self.cache_outdated_interval` + ''' timestamp = self.data.get("timestamp") if not timestamp: - raise SignatureCacheMalformedException() + raise SignatureDataMalformedException() try: timestamp = datetime.fromisoformat(timestamp) except ValueError as e: - raise SignatureCacheMalformedException() from e + raise SignatureDataMalformedException() from e except TypeError as e: - raise SignatureCacheMalformedException() from e + raise SignatureDataMalformedException() from e if (timestamp + self.cache_outdated_interval) < scanner._datetime_now(): - raise SignatureCacheOutdatedException() + raise SignatureDataOutdatedException() def load(self): try: self.load_from_cache() self.verify_data() - except SignatureCacheMalformedException as e: + except (SignatureDataMalformedException, SignatureDataVersionMismatchException): self.load_from_defaults() self.write_to_cache() @@ -200,7 +217,7 @@ class SignatureDataController: def load_from_cache(self): sig_file = scanner._scanner_cachedir() / self.filename if not sig_file.exists(): - raise SignatureCacheMalformedException() + raise SignatureDataMalformedException() with open(sig_file) as f: self.data = json.load(f) @@ -211,7 +228,12 @@ class SignatureDataController: logging.debug("write '{}' to cache".format(self.filename)) def verify_data(self): + ''' + cleans and validates and cleans `self.data` + ''' + self.check_data_version() valid_keys = ['timestamp', 'version', 'signatures'] + for k in [x for x in self.data.keys() if x not in valid_keys]: del self.data[k] @@ -264,23 +286,35 @@ class ScannerTool(): self.compile_regexes() def compile_regexes(self): - self.regex = {'code_signatures': {}, 'gradle_signatures': {}} + self.err_regex = {'code_signatures': {}, 'gradle_signatures': {}} for sdc in self.sdcs: for signame, sigdef in sdc.data.get('signatures', {}).items(): for sig in sigdef.get('code_signatures', []): - self.regex['code_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE) + self.err_regex['code_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE) for sig in sigdef.get('gradle_signatures', []): - self.regex['gradle_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE) + self.err_regex['gradle_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE) + + def clear_cache(self): + # delete cache folder and all its contents + shutil.rmtree(scanner._scanner_cachedir(), ignore_errors=True) + # re-initialize, this will re-populate the cache from default values + self.__init__() -# TODO: change this from global instance to dependency injection -SCANNER_TOOL = None +# TODO: change this from singleton instance to dependency injection +# use `_get_tool()` instead of accessing this directly +_SCANNER_TOOL = None def _get_tool(): - if not scanner.SCANNER_TOOL: - scanner.SCANNER_TOOL = ScannerTool() - return scanner.SCANNER_TOOL + ''' + lazy loading factory for ScannerTool singleton + + ScannerTool initialization need to access `common.config` values. Those are only available after initialization through `common.read_config()` So this factory assumes config was called at an erlier point in time + ''' + if not scanner._SCANNER_TOOL: + scanner._SCANNER_TOOL = ScannerTool() + return scanner._SCANNER_TOOL # taken from exodus_core @@ -310,7 +344,7 @@ def scan_binary(apkfile, extract_signatures=None): result = get_embedded_classes(apkfile) problems = 0 for classname in result: - for suspect, regexp in _get_tool().regex['code_signatures'].items(): + for suspect, regexp in _get_tool().err_regex['code_signatures'].items(): if regexp.match(classname): logging.debug("Found class '%s'" % classname) problems += 1 @@ -362,7 +396,7 @@ def scan_source(build_dir, build=metadata.Build()): return any(al in s for al in allowlisted) def suspects_found(s): - for n, r in _get_tool().regex['gradle_signatures'].items(): + for n, r in _get_tool().err_regex['gradle_signatures'].items(): if r.match(s) and not is_allowlisted(s): yield n @@ -652,6 +686,8 @@ def main(): help=_("Force scan of disabled apps and builds.")) parser.add_argument("--json", action="store_true", default=False, help=_("Output JSON to stdout.")) + parser.add_argument("--clear-cache", action="store_true", default=False, + help=_("purge local scanner definitions cache")) metadata.add_metadata_arguments(parser) options = parser.parse_args() metadata.warnings_action = options.W @@ -665,6 +701,9 @@ def main(): config = common.read_config(options) + if options.clear_cache: + scanner._get_tool().clear_cache() + probcount = 0 exodus = [] diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index 01076a6e..bb595832 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -448,9 +448,9 @@ class Test_scan_binary(unittest.TestCase): fdroidserver.common.config = config fdroidserver.common.options = mock.Mock() - fdroidserver.scanner.SIGNATURE_TOOL = mock.Mock() - fdroidserver.scanner.SIGNATURE_TOOL.regex = {} - fdroidserver.scanner.SIGNATURE_TOOL.regex['code_signatures'] = { + fdroidserver.scanner._SCANNER_TOOL = mock.Mock() + fdroidserver.scanner._SCANNER_TOOL.err_regex = {} + fdroidserver.scanner._SCANNER_TOOL.err_regex['code_signatures'] = { "java/lang/Object": re.compile(r'.*java/lang/Object', re.IGNORECASE | re.UNICODE) } @@ -459,7 +459,7 @@ class Test_scan_binary(unittest.TestCase): self.assertEqual( 1, fdroidserver.scanner.scan_binary(apkfile), - "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner.SIGNATURE_TOOL.regex['code_signatures'].values(), apkfile), + "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner._SCANNER_TOOL.err_regex['code_signatures'].values(), apkfile), ) @unittest.skipIf( @@ -470,7 +470,7 @@ class Test_scan_binary(unittest.TestCase): ) def test_bottom_level_embedded_apk_code_signature(self): apkfile = os.path.join(self.basedir, 'apk.embedded_1.apk') - fdroidserver.scanner.SIGNATURE_TOOL.regex['code_signatures'] = { + fdroidserver.scanner._SCANNER_TOOL.err_regex['code_signatures'] = { "org/bitbucket/tickytacky/mirrormirror/MainActivity": re.compile( r'.*org/bitbucket/tickytacky/mirrormirror/MainActivity', re.IGNORECASE | re.UNICODE ) @@ -479,12 +479,12 @@ class Test_scan_binary(unittest.TestCase): self.assertEqual( 1, fdroidserver.scanner.scan_binary(apkfile), - "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner.SIGNATURE_TOOL.regex['code_signatures'].values(), apkfile), + "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner._SCANNER_TOOL.err_regex['code_signatures'].values(), apkfile), ) def test_top_level_signature_embedded_apk_present(self): apkfile = os.path.join(self.basedir, 'apk.embedded_1.apk') - fdroidserver.scanner.SIGNATURE_TOOL.regex['code_signatures'] = { + fdroidserver.scanner._SCANNER_TOOL.err_regex['code_signatures'] = { "org/fdroid/ci/BuildConfig": re.compile( r'.*org/fdroid/ci/BuildConfig', re.IGNORECASE | re.UNICODE ) @@ -492,7 +492,7 @@ class Test_scan_binary(unittest.TestCase): self.assertEqual( 1, fdroidserver.scanner.scan_binary(apkfile), - "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner.SIGNATURE_TOOL.regex['code_signatures'].values(), apkfile), + "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner._SCANNER_TOOL.err_regex['code_signatures'].values(), apkfile), ) # TODO: re-enable once allow-listing migrated to more complex regexes @@ -640,24 +640,24 @@ class Test_SignatureDataController(unittest.TestCase): def test_check_last_updated_exception_cache_outdated(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') sdc.data['timestamp'] = (datetime.now().astimezone() - timedelta(days=30)).isoformat() - with self.assertRaises(fdroidserver.scanner.SignatureCacheOutdatedException): + with self.assertRaises(fdroidserver.scanner.SignatureDataOutdatedException): sdc.check_last_updated() def test_check_last_updated_exception_missing_timestamp_value(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') - with self.assertRaises(fdroidserver.scanner.SignatureCacheMalformedException): + with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException): sdc.check_last_updated() def test_check_last_updated_exception_not_string(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') sdc.data['timestamp'] = 12345 - with self.assertRaises(fdroidserver.scanner.SignatureCacheMalformedException): + with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException): sdc.check_last_updated() def test_check_last_updated_exception_not_iso_formatted_string(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') sdc.data['timestamp'] = '01/09/2002 10:11' - with self.assertRaises(fdroidserver.scanner.SignatureCacheMalformedException): + with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException): sdc.check_last_updated() # check_data_version @@ -668,7 +668,7 @@ class Test_SignatureDataController(unittest.TestCase): def test_check_data_version_exception(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') - with self.assertRaises(fdroidserver.scanner.SignatureCacheMalformedException): + with self.assertRaises(fdroidserver.scanner.SignatureDataVersionMismatchException): sdc.check_data_version() From c10633eac594c1a5613afcebe5b4ddec081778ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 28 Sep 2022 17:35:31 +0200 Subject: [PATCH 05/21] convert fdroid scanner --exodus to SignatureDataController --- fdroidserver/common.py | 18 ++++- fdroidserver/scanner.py | 169 ++++++++++++++++++++++------------------ tests/build.TestCase | 2 +- tests/scanner.TestCase | 64 +++++++-------- tests/testcommon.py | 5 ++ 5 files changed, 142 insertions(+), 116 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index f4a1fe77..0da602ad 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -76,7 +76,7 @@ from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesEx BuildException, VerificationException, MetaDataException from .asynchronousfilereader import AsynchronousFileReader -from . import apksigcopier +from . import apksigcopier, common # The path to this fdroidserver distribution @@ -321,6 +321,22 @@ def fill_config_defaults(thisconfig): break +def get_config(options=None): + """ + helper function for getting access to commons.config while safely + initializing if it wasn't initialized yet. + """ + global config + + if config is not None: + return config + + config = {} + common.fill_config_defaults(config) + common.read_config(options) + return config + + def regsub_file(pattern, repl, path): with open(path, 'rb') as f: text = f.read() diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 8dcb512a..31344288 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -20,7 +20,6 @@ import os import re import sys import json -import yaml import imghdr import shutil import logging @@ -42,7 +41,6 @@ from . import metadata from .exception import BuildException, VCSException, ConfigurationException from . import scanner -config = None options = None DEFAULT_JSON_PER_BUILD = {'errors': [], 'warnings': [], 'infos': []} # type: ignore @@ -136,7 +134,7 @@ def _exodus_compile_signatures(signatures): def _datetime_now(): """ - simple warpper for datetime.now to allow mocking it for testing + simple wrapper for datetime.now to allow mocking it for testing """ return datetime.now().astimezone() @@ -145,11 +143,12 @@ def _scanner_cachedir(): """ get `Path` to local cache dir """ - if not common.config: + cfg = common.get_config() + if not cfg: raise ConfigurationException('config not initialized') - if "cachedir_scanner" not in common.config: + if "cachedir_scanner" not in cfg: raise ConfigurationException("could not load 'cachedir_scanner' from config") - cachedir = Path(common.config["cachedir_scanner"]) + cachedir = Path(cfg["cachedir_scanner"]) cachedir.mkdir(exist_ok=True, parents=True) return cachedir @@ -170,7 +169,7 @@ class SignatureDataController: def __init__(self, name, filename): self.name = name self.filename = filename - self.cache_outdated_interval = timedelta(days=7) + self.cache_outdated_interval = None self.data = {} def check_data_version(self): @@ -198,8 +197,27 @@ class SignatureDataController: raise SignatureDataMalformedException() from e except TypeError as e: raise SignatureDataMalformedException() from e - if (timestamp + self.cache_outdated_interval) < scanner._datetime_now(): - raise SignatureDataOutdatedException() + if self.cache_outdated_interval: + if (timestamp + self.cache_outdated_interval) < scanner._datetime_now(): + raise SignatureDataOutdatedException() + + def fetch(self): + try: + self.load_from_cache() + self.verify_data() + self.check_last_updated() + except ( + SignatureDataMalformedException, + SignatureDataVersionMismatchException, + SignatureDataOutdatedException + ): + try: + self.fetch_signatures_from_web() + except AttributeError: + # just load from defaults if fetch_signatures_from_web is not + # implemented + self.load_from_defaults() + self.write_to_cache() def load(self): try: @@ -237,62 +255,70 @@ class SignatureDataController: for k in [x for x in self.data.keys() if x not in valid_keys]: del self.data[k] - # def scan - class ExodusSignatureDataController(SignatureDataController): def __init__(self): super().__init__('Exodus signatures', 'exodus.yml') + self.cache_outdated_interval = timedelta(days=1) # refresh exodus cache after one day - def fetch_signatures_from_web(): - pass - # TODO - # exodus_url = "https://reports.exodus-privacy.eu.org/api/trackers" - # sigs = { - # "signatures": [], - # "timestamp": scanner._datetime_now().isoformat(), - # "version": SCANNER_CACHE_VERSION, - # } + def fetch_signatures_from_web(self): + exodus_url = "https://reports.exodus-privacy.eu.org/api/trackers" + self.data = { + "signatures": {}, + "timestamp": scanner._datetime_now().isoformat(), + "version": SCANNER_CACHE_VERSION, + } - # with urllib.request.urlopen(exodus_url) as f: - # data = json.load(f) - # for tracker in data["trackers"].values(): - # sigs["signatures"].append({ - # "name": tracker["name"], - # "binary_signature": tracker["code_signature"], - # "network_signature": tracker["network_signature"], - # "types": ["tracker", "non-free"] # right now we assume all trackers in exodus are non-free - # }) + with urllib.request.urlopen(exodus_url) as f: + d = json.load(f) + for tracker in d["trackers"].values(): + if tracker.get('code_signature'): + self.data["signatures"][tracker["name"]] = { + "name": tracker["name"], + "warn_code_signatures": [tracker["code_signature"]], + # exodus also provides network signatures, unused atm. + # "network_signatures": [tracker["network_signature"]], + "AntiFeatures": ["Tracking"], + "license": "NonFree" # We assume all trackers in exodus + # are non-free, alought free + # trackers like piwik, acra, + # etc. might be listed by exodus + # too. + } class ScannerSignatureDataController(SignatureDataController): def __init__(self): super().__init__('Scanner signatures', 'scanner.json') - def fetch_signatures_from_web(self): - url = "https://uniqx.gitlab.io/fdroid-scanner-signatures/sigs.json" - with urllib.request.urlopen(url) as f: - data = yaml.safe_load(f) - # TODO: validate parsed data - # TODO: error message 'please update fdroidserver/report' when fetching failed due to changes in the data strucutre - self.data = data - class ScannerTool(): def __init__(self): self.sdcs = [ScannerSignatureDataController()] - for sdc in self.sdcs: - sdc.load() + self.load() self.compile_regexes() + def load(self): + for sdc in self.sdcs: + sdc.load() + def compile_regexes(self): - self.err_regex = {'code_signatures': {}, 'gradle_signatures': {}} + self.regexs = { + 'err_code_signatures': {}, + 'err_gradle_signatures': {}, + 'warn_code_signatures': {}, + 'warn_gradle_signatures': {}, + } for sdc in self.sdcs: for signame, sigdef in sdc.data.get('signatures', {}).items(): for sig in sigdef.get('code_signatures', []): - self.err_regex['code_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE) + self.regexs['err_code_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE) for sig in sigdef.get('gradle_signatures', []): - self.err_regex['gradle_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE) + self.regexs['err_gradle_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE) + for sig in sigdef.get('warn_code_signatures', []): + self.regexs['warn_code_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE) + for sig in sigdef.get('warn_gradle_signatures', []): + self.regexs['warn_gradle_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE) def clear_cache(self): # delete cache folder and all its contents @@ -300,6 +326,10 @@ class ScannerTool(): # re-initialize, this will re-populate the cache from default values self.__init__() + def add(self, new_controller: SignatureDataController): + self.sdcs.append(new_controller) + self.compile_regexes() + # TODO: change this from singleton instance to dependency injection # use `_get_tool()` instead of accessing this directly @@ -342,36 +372,20 @@ def scan_binary(apkfile, extract_signatures=None): """Scan output of dexdump for known non-free classes.""" logging.info(_('Scanning APK with dexdump for known non-free classes.')) result = get_embedded_classes(apkfile) - problems = 0 + problems, warnings = 0, 0 for classname in result: - for suspect, regexp in _get_tool().err_regex['code_signatures'].items(): + for suspect, regexp in _get_tool().regexs['warn_code_signatures'].items(): if regexp.match(classname): - logging.debug("Found class '%s'" % classname) + logging.debug("Warning: found class '%s'" % classname) + warnings += 1 + for suspect, regexp in _get_tool().regexs['err_code_signatures'].items(): + if regexp.match(classname): + logging.debug("Problem: found class '%s'" % classname) problems += 1 - - if extract_signatures: - - def _detect_tracker(sig, tracker, class_list): - for clazz in class_list: - if sig.search(clazz): - logging.debug("Found tracker, class {} matching {}".format(clazz, tracker.code_signature)) - return tracker - return None - - results = [] - args = [(extract_signatures[1][index], tracker, result) - for (index, tracker) in enumerate(extract_signatures[0]) if - len(tracker.code_signature) > 3] - - for res in itertools.starmap(_detect_tracker, args): - if res: - results.append(res) - - trackers = [t for t in results if t is not None] - problems += len(trackers) - + if warnings: + logging.warning(_("Found {count} warnings in {filename}").format(count=warnings, filename=apkfile)) if problems: - logging.critical("Found problems in %s" % apkfile) + logging.critical(_("Found {count} problems in {filename}").format(count=problems, filename=apkfile)) return problems @@ -396,7 +410,7 @@ def scan_source(build_dir, build=metadata.Build()): return any(al in s for al in allowlisted) def suspects_found(s): - for n, r in _get_tool().err_regex['gradle_signatures'].items(): + for n, r in _get_tool().regexs['err_gradle_signatures'].items(): if r.match(s) and not is_allowlisted(s): yield n @@ -669,7 +683,7 @@ def scan_source(build_dir, build=metadata.Build()): def main(): - global config, options, json_per_build + global options, json_per_build # Parse command line... parser = ArgumentParser( @@ -699,24 +713,25 @@ def main(): else: logging.getLogger().setLevel(logging.ERROR) - config = common.read_config(options) + # initialize/load configuration values + common.get_config(options) if options.clear_cache: scanner._get_tool().clear_cache() + if options.exodus: + c = ExodusSignatureDataController() + c.fetch() + scanner._get_tool().add(c) probcount = 0 - exodus = [] - if options.exodus: - exodus = load_exodus_trackers_signatures() - appids = [] for apk in options.appid: if os.path.isfile(apk): - count = scanner.scan_binary(apk, exodus) + count = scanner.scan_binary(apk) if count > 0: logging.warning( - _('Scanner found {count} problems in {apk}:').format( + _('Scanner found {count} problems in {apk}').format( count=count, apk=apk ) ) diff --git a/tests/build.TestCase b/tests/build.TestCase index 13378060..cca5fa0c 100755 --- a/tests/build.TestCase +++ b/tests/build.TestCase @@ -402,7 +402,7 @@ class BuildTest(unittest.TestCase): os.chdir(testdir) os.mkdir("build") - config = dict() + config = fdroidserver.common.get_config() config['sdk_path'] = os.getenv('ANDROID_HOME') config['ndk_paths'] = {'r10d': os.getenv('ANDROID_NDK_HOME')} fdroidserver.common.config = config diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index bb595832..ade8bd0a 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -31,7 +31,7 @@ import fdroidserver.build import fdroidserver.common import fdroidserver.metadata import fdroidserver.scanner -from testcommon import TmpCwd +from testcommon import TmpCwd, mock_open_to_str class ScannerTest(unittest.TestCase): @@ -449,17 +449,18 @@ class Test_scan_binary(unittest.TestCase): fdroidserver.common.options = mock.Mock() fdroidserver.scanner._SCANNER_TOOL = mock.Mock() - fdroidserver.scanner._SCANNER_TOOL.err_regex = {} - fdroidserver.scanner._SCANNER_TOOL.err_regex['code_signatures'] = { + fdroidserver.scanner._SCANNER_TOOL.regexs = {} + fdroidserver.scanner._SCANNER_TOOL.regexs['err_code_signatures'] = { "java/lang/Object": re.compile(r'.*java/lang/Object', re.IGNORECASE | re.UNICODE) } + fdroidserver.scanner._SCANNER_TOOL.regexs['warn_code_signatures'] = {} def test_code_signature_match(self): apkfile = os.path.join(self.basedir, 'no_targetsdk_minsdk1_unsigned.apk') self.assertEqual( 1, fdroidserver.scanner.scan_binary(apkfile), - "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner._SCANNER_TOOL.err_regex['code_signatures'].values(), apkfile), + "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner._SCANNER_TOOL.regexs['err_code_signatures'].values(), apkfile), ) @unittest.skipIf( @@ -470,7 +471,7 @@ class Test_scan_binary(unittest.TestCase): ) def test_bottom_level_embedded_apk_code_signature(self): apkfile = os.path.join(self.basedir, 'apk.embedded_1.apk') - fdroidserver.scanner._SCANNER_TOOL.err_regex['code_signatures'] = { + fdroidserver.scanner._SCANNER_TOOL.regexs['err_code_signatures'] = { "org/bitbucket/tickytacky/mirrormirror/MainActivity": re.compile( r'.*org/bitbucket/tickytacky/mirrormirror/MainActivity', re.IGNORECASE | re.UNICODE ) @@ -479,12 +480,12 @@ class Test_scan_binary(unittest.TestCase): self.assertEqual( 1, fdroidserver.scanner.scan_binary(apkfile), - "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner._SCANNER_TOOL.err_regex['code_signatures'].values(), apkfile), + "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner._SCANNER_TOOL.regexs['err_code_signatures'].values(), apkfile), ) def test_top_level_signature_embedded_apk_present(self): apkfile = os.path.join(self.basedir, 'apk.embedded_1.apk') - fdroidserver.scanner._SCANNER_TOOL.err_regex['code_signatures'] = { + fdroidserver.scanner._SCANNER_TOOL.regexs['err_code_signatures'] = { "org/fdroid/ci/BuildConfig": re.compile( r'.*org/fdroid/ci/BuildConfig', re.IGNORECASE | re.UNICODE ) @@ -492,7 +493,7 @@ class Test_scan_binary(unittest.TestCase): self.assertEqual( 1, fdroidserver.scanner.scan_binary(apkfile), - "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner._SCANNER_TOOL.err_regex['code_signatures'].values(), apkfile), + "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner._SCANNER_TOOL.regexs['err_code_signatures'].values(), apkfile), ) # TODO: re-enable once allow-listing migrated to more complex regexes @@ -671,6 +672,23 @@ class Test_SignatureDataController(unittest.TestCase): with self.assertRaises(fdroidserver.scanner.SignatureDataVersionMismatchException): sdc.check_data_version() + def test_write_to_cache(self): + open_func = mock.mock_open() + sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') + sdc.data = {"mocked": "data"} + + with mock.patch("builtins.open", open_func), mock.patch( + "fdroidserver.scanner._scanner_cachedir", + return_value=pathlib.Path('.'), + ): + sdc.write_to_cache() + + open_func.assert_called_with(pathlib.Path('fff.yml'), 'w', encoding="utf-8") + self.assertEqual( + mock_open_to_str(open_func), + """{\n "mocked": "data"\n}""" + ) + class Test_ScannerSignatureDataController_fetch_signatures_from_web(unittest.TestCase): def setUp(self): @@ -693,34 +711,6 @@ class Test_ScannerSignatureDataController_fetch_signatures_from_web(unittest.Tes - ads '''))) - def test_fetch_signatures_from_web(self): - sdc = fdroidserver.scanner.ScannerSignatureDataController() - with unittest.mock.patch('urllib.request.urlopen', self.uo_func): - sdc.fetch_signatures_from_web() - self.assertEqual(sdc.data.get('version'), 999) - self.assertEqual(sdc.data.get('timestamp'), "1999-12-31T23:59:59.999999+00:00") - self.assertListEqual( - sdc.data.get('signatures'), - [ - { - 'binary_signature': 'com/google/firebase', - 'name': 'Google Firebase', - 'types': ['tracker', 'non-free'], - }, - { - 'gradle_signature': 'com/google/android/gms', - 'name': 'Google Mobile Services', - 'types': ['non-free'], - }, - { - 'network_signature': 'doubleclick\\.net', - 'name': 'Another thing to test.', - 'types': ['ads'], - }, - ] - ) - self.assertEqual(len(sdc.data), 3) - class Test_main(unittest.TestCase): def setUp(self): @@ -768,7 +758,7 @@ class Test_main(unittest.TestCase): self.exit_func.assert_not_called() self.read_app_args_func.assert_not_called() - self.scan_binary_func.assert_called_once_with('local.application.apk', []) + self.scan_binary_func.assert_called_once_with('local.application.apk') if __name__ == "__main__": diff --git a/tests/testcommon.py b/tests/testcommon.py index 128ead00..29b3f9b6 100644 --- a/tests/testcommon.py +++ b/tests/testcommon.py @@ -50,6 +50,11 @@ class TmpPyPath(): def mock_open_to_str(mock): + """ + helper function for accessing all data written into a + unittest.mock.mock_open() instance as a string. + """ + return "".join([ x.args[0] for x in mock.mock_calls if str(x).startswith("call().write(") ]) From c9b59b525d486662d6e007711a3814ef35b2f0a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 28 Sep 2022 18:18:19 +0200 Subject: [PATCH 06/21] fix timestamp check; remove dead code --- fdroidserver/scanner.py | 40 ------------ tests/scanner.TestCase | 131 +--------------------------------------- 2 files changed, 2 insertions(+), 169 deletions(-) diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 31344288..b4d46a4d 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -24,12 +24,10 @@ import imghdr import shutil import logging import zipfile -import requests import itertools import traceback import urllib.request from argparse import ArgumentParser -from collections import namedtuple from copy import deepcopy from tempfile import TemporaryDirectory from pathlib import Path @@ -115,23 +113,6 @@ def get_embedded_classes(apkfile, depth=0): return classes -# taken from exodus_core -def _exodus_compile_signatures(signatures): - """ - Compiles the regex associated to each signature, in order to speed up the trackers detection. - - :return: A compiled list of signatures. - """ - compiled_tracker_signature = [] - try: - compiled_tracker_signature = [ - re.compile(track.code_signature) for track in signatures - ] - except TypeError: - print("signatures is not iterable") - return compiled_tracker_signature - - def _datetime_now(): """ simple wrapper for datetime.now to allow mocking it for testing @@ -347,27 +328,6 @@ def _get_tool(): return scanner._SCANNER_TOOL -# taken from exodus_core -def load_exodus_trackers_signatures(): - """ - Load trackers signatures from the official Exodus database. - - :return: a dictionary containing signatures. - """ - signatures = [] - exodus_url = "https://reports.exodus-privacy.eu.org/api/trackers" - r = requests.get(exodus_url, timeout=300) - data = r.json() - for e in data['trackers']: - signatures.append( - namedtuple('tracker', data['trackers'][e].keys())( - *data['trackers'][e].values() - ) - ) - logging.debug('{} trackers signatures loaded'.format(len(signatures))) - return signatures, scanner._exodus_compile_signatures(signatures) - - def scan_binary(apkfile, extract_signatures=None): """Scan output of dexdump for known non-free classes.""" logging.info(_('Scanning APK with dexdump for known non-free classes.')) diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index ade8bd0a..4922fdf3 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -496,132 +496,6 @@ class Test_scan_binary(unittest.TestCase): "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner._SCANNER_TOOL.regexs['err_code_signatures'].values(), apkfile), ) - # TODO: re-enable once allow-listing migrated to more complex regexes - # def test_no_match(self): - # apkfile = os.path.join(self.basedir, 'no_targetsdk_minsdk1_unsigned.apk') - # result = fdroidserver.scanner.scan_binary(apkfile) - # self.assertEqual(0, result, "Found false positives in binary '{}'".format(apkfile)) - - -# class Test__fetch_exodus_signatures_to_cache(unittest.TestCase): -# def setUp(self): -# self.web_req_func = mock.Mock(return_value=io.StringIO(json.dumps({ -# "trackers": { -# "1": { -# "id": 1, -# "name": "Steyer Puch 1", -# "description": "blah blah blah", -# "creation_date": "1956-01-01", -# "code_signature": "com.puch.|com.steyer.", -# "network_signature": "pst\\.com", -# "website": "https://pst.com", -# "categories": ["tracker"], -# "documentation": [], -# }, -# "2": { -# "id": 2, -# "name": "Steyer Puch 2", -# "description": "blah blah blah", -# "creation_date": "1956-01-01", -# "code_signature": "com.puch.|com.steyer.", -# "network_signature": "pst\\.com", -# "website": "https://pst.com", -# "categories": ["tracker"], -# "documentation": [], -# } -# }, -# }))) -# self.open_func = mock.mock_open() -# self.cachedir_func = mock.Mock(return_value=pathlib.Path("mocked/path")) -# -# def test_ok(self): -# with mock.patch("urllib.request.urlopen", self.web_req_func), mock.patch( -# "builtins.open", self.open_func -# ) as outfilemock, mock.patch( -# "fdroidserver.scanner._scanner_cachedir", self.cachedir_func -# ), mock.patch("fdroidserver.scanner._datetime_now", unittest.mock.Mock(return_value=datetime(1999, 12, 31, 23, 59, 59))): -# fdroidserver.scanner.fetch_exodus_signatures_to_cache() -# -# self.cachedir_func.assert_called_once() -# self.web_req_func.assert_called_once_with("https://reports.exodus-privacy.eu.org/api/trackers") -# self.open_func.assert_called_once_with(pathlib.Path("mocked/path/exodus.json"), "w", encoding="utf-8") -# self.assertEqual( -# mock_open_to_str(self.open_func), -# """{"signatures": {"exodus-1": {"name": "Steyer Puch 1", "code_signature": "com.puch.|com.steyer.", "network_signature": "pst\\\\.com", "types": ["tracker", "non-free"]}, "exodus-2": {"name": "Steyer Puch 2", "code_signature": "com.puch.|com.steyer.", "network_signature": "pst\\\\.com", "types": ["tracker", "non-free"]}}, "timestamp": "1999-12-31T23:59:59"}""" -# ) - - -class Test__exodus_compile_signatures(unittest.TestCase): - def setUp(self): - self.m1 = mock.Mock() - self.m1.code_signature = r"^random\sregex$" - self.m2 = mock.Mock() - self.m2.code_signature = r"^another.+regex$" - self.mock_sigs = [self.m1, self.m2] - - def test_ok(self): - result = fdroidserver.scanner._exodus_compile_signatures(self.mock_sigs) - self.assertListEqual( - result, - [ - re.compile(self.m1.code_signature), - re.compile(self.m2.code_signature), - ], - ) - - def test_not_iterable(self): - result = fdroidserver.scanner._exodus_compile_signatures(123) - self.assertListEqual(result, []) - - -class Test_load_exodus_trackers_signatures(unittest.TestCase): - def setUp(self): - self.requests_ret = mock.Mock() - self.requests_ret.json = mock.Mock( - return_value={ - "trackers": { - "1": { - "id": 1, - "name": "Steyer Puch 1", - "description": "blah blah blah", - "creation_date": "1956-01-01", - "code_signature": "com.puch.|com.steyer.", - "network_signature": "pst\\.com", - "website": "https://pst.com", - "categories": ["tracker"], - "documentation": [], - }, - "2": { - "id": 2, - "name": "Steyer Puch 2", - "description": "blah blah blah", - "creation_date": "1956-01-01", - "code_signature": "com.puch.|com.steyer.", - "network_signature": "pst\\.com", - "website": "https://pst.com", - "categories": ["tracker"], - "documentation": [], - }, - }, - } - ) - self.requests_func = mock.Mock(return_value=self.requests_ret) - self.compilesig_func = mock.Mock(return_value="mocked return value") - - def test_ok(self): - with mock.patch("requests.get", self.requests_func), mock.patch( - "fdroidserver.scanner._exodus_compile_signatures", self.compilesig_func - ): - sigs, regex = fdroidserver.scanner.load_exodus_trackers_signatures() - self.requests_func.assert_called_once_with( - "https://reports.exodus-privacy.eu.org/api/trackers", timeout=300 - ) - self.assertEqual(len(sigs), 2) - self.assertListEqual([1, 2], sorted([x.id for x in sigs])) - - self.compilesig_func.assert_called_once_with(sigs) - self.assertEqual(regex, "mocked return value") - class Test_SignatureDataController(unittest.TestCase): # __init__ @@ -629,7 +503,7 @@ class Test_SignatureDataController(unittest.TestCase): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') self.assertEqual(sdc.name, 'nnn') self.assertEqual(sdc.filename, 'fff.yml') - self.assertEqual(sdc.cache_outdated_interval, timedelta(days=7)) + self.assertEqual(sdc.cache_outdated_interval, None) self.assertDictEqual(sdc.data, {}) # check_last_updated @@ -640,6 +514,7 @@ class Test_SignatureDataController(unittest.TestCase): def test_check_last_updated_exception_cache_outdated(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') + sdc.cache_outdated_interval = timedelta(days=7) sdc.data['timestamp'] = (datetime.now().astimezone() - timedelta(days=30)).isoformat() with self.assertRaises(fdroidserver.scanner.SignatureDataOutdatedException): sdc.check_last_updated() @@ -778,8 +653,6 @@ if __name__ == "__main__": newSuite.addTests([ unittest.makeSuite(ScannerTest), unittest.makeSuite(Test_scan_binary), - unittest.makeSuite(Test__exodus_compile_signatures), - unittest.makeSuite(Test_load_exodus_trackers_signatures), unittest.makeSuite(Test_SignatureDataController), unittest.makeSuite(Test_ScannerSignatureDataController_fetch_signatures_from_web), unittest.makeSuite(Test_main), From dc1a2db3f49f48c1bb99ba8e87648c48c5a49274 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 29 Sep 2022 10:33:01 +0200 Subject: [PATCH 07/21] scanner: use Path.resovle instead of undocumented absolute() --- fdroidserver/scanner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index b4d46a4d..c77caab1 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -209,7 +209,7 @@ class SignatureDataController: self.write_to_cache() def load_from_defaults(self): - sig_file = Path(__file__).absolute().parent / 'data' / 'scanner' / self.filename + sig_file = (Path(__file__).parent / 'data' / 'scanner' / self.filename).resolve() with open(sig_file) as f: self.data = json.load(f) From 1e6694112a09292827fda3993cfee9300c7601e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 29 Sep 2022 16:42:04 +0200 Subject: [PATCH 08/21] rename to suss --- fdroidserver/data/scanner/{scanner.json => suss.json} | 0 fdroidserver/scanner.py | 6 +++--- tests/scanner.TestCase | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) rename fdroidserver/data/scanner/{scanner.json => suss.json} (100%) diff --git a/fdroidserver/data/scanner/scanner.json b/fdroidserver/data/scanner/suss.json similarity index 100% rename from fdroidserver/data/scanner/scanner.json rename to fdroidserver/data/scanner/suss.json diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index c77caab1..6fe49dd1 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -268,14 +268,14 @@ class ExodusSignatureDataController(SignatureDataController): } -class ScannerSignatureDataController(SignatureDataController): +class SUSSDataController(SignatureDataController): def __init__(self): - super().__init__('Scanner signatures', 'scanner.json') + super().__init__('SUSS', 'suss.json') class ScannerTool(): def __init__(self): - self.sdcs = [ScannerSignatureDataController()] + self.sdcs = [SUSSDataController()] self.load() self.compile_regexes() diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index 4922fdf3..1e21e7b5 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -565,7 +565,7 @@ class Test_SignatureDataController(unittest.TestCase): ) -class Test_ScannerSignatureDataController_fetch_signatures_from_web(unittest.TestCase): +class Test_SUSSDataController_fetch_signatures_from_web(unittest.TestCase): def setUp(self): self.uo_func = mock.Mock(return_value=io.StringIO(textwrap.dedent('''\ version: 999 @@ -654,7 +654,7 @@ if __name__ == "__main__": unittest.makeSuite(ScannerTest), unittest.makeSuite(Test_scan_binary), unittest.makeSuite(Test_SignatureDataController), - unittest.makeSuite(Test_ScannerSignatureDataController_fetch_signatures_from_web), + unittest.makeSuite(Test_SUSSDataController_fetch_signatures_from_web), unittest.makeSuite(Test_main), ]) unittest.main(failfast=False) From 4a38908bd7b455805905389f4440d6ee362eca7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 29 Sep 2022 16:43:52 +0200 Subject: [PATCH 09/21] make cachedir subdirectories pick their place correctly based on parent folder --- fdroidserver/common.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 0da602ad..7e6bed29 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -125,13 +125,11 @@ default_config = { 'sdk_path': "$ANDROID_HOME", 'ndk_paths': {}, 'cachedir': str(Path.home() / '.cache/fdroidserver'), - 'cachedir_scanner': str(Path.home() / '.cache/fdroidserver/scanner'), 'java_paths': None, 'scan_binary': False, 'ant': "ant", 'mvn3': "mvn", 'gradle': os.path.join(FDROID_PATH, 'gradlew-fdroid'), - 'gradle_version_dir': str(Path.home() / '.cache/fdroidserver/gradle'), 'sync_from_local_copy_dir': False, 'allow_disabled_algorithms': False, 'per_app_repos': False, @@ -320,6 +318,11 @@ def fill_config_defaults(thisconfig): ndk_paths[ndkdict['release']] = ndk_paths.pop(k) break + if 'cachedir_scanner' not in thisconfig: + thisconfig['cachedir_scanner'] = str(Path(thisconfig['cachedir']) / 'scanner') + if 'gradle_version_dir' not in thisconfig: + thisconfig['gradle_version_dir'] = str(Path(thisconfig['cachedir']) / 'gradle') + def get_config(options=None): """ From bfcc30b85441ae736e22161e955f91f89ce57c5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 29 Sep 2022 19:55:27 +0200 Subject: [PATCH 10/21] add --refresh to scanner --- fdroidserver/scanner.py | 71 ++++++++++++++++++++++++----------------- tests/scanner.TestCase | 44 ++++++------------------- 2 files changed, 52 insertions(+), 63 deletions(-) diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 6fe49dd1..9fc5fcb2 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -21,7 +21,6 @@ import re import sys import json import imghdr -import shutil import logging import zipfile import itertools @@ -147,10 +146,12 @@ class SignatureDataVersionMismatchException(Exception): class SignatureDataController: - def __init__(self, name, filename): + def __init__(self, name, filename, url): self.name = name self.filename = filename - self.cache_outdated_interval = None + self.url = url + # by default we assume cache is valid indefinitely + self.cache_outdated_interval = timedelta(days=999999) self.data = {} def check_data_version(self): @@ -165,7 +166,7 @@ class SignatureDataController: is not ok. :raises SignatureDataMalformedException: when timestamp value is - inaccessible or not parseable + inaccessible or not parse-able :raises SignatureDataOutdatedException: when timestamp is older then `self.cache_outdated_interval` ''' @@ -178,9 +179,8 @@ class SignatureDataController: raise SignatureDataMalformedException() from e except TypeError as e: raise SignatureDataMalformedException() from e - if self.cache_outdated_interval: - if (timestamp + self.cache_outdated_interval) < scanner._datetime_now(): - raise SignatureDataOutdatedException() + if (timestamp + self.cache_outdated_interval) < scanner._datetime_now(): + raise SignatureDataOutdatedException() def fetch(self): try: @@ -233,24 +233,31 @@ class SignatureDataController: self.check_data_version() valid_keys = ['timestamp', 'version', 'signatures'] - for k in [x for x in self.data.keys() if x not in valid_keys]: - del self.data[k] + for k in list(self.data.keys()): + if k not in valid_keys: + del self.data[k] + + def fetch_signatures_from_web(self): + logging.debug(_("downloading '{}'").format(self.url)) + with urllib.request.urlopen(self.url) as f: + self.data = json.load(f) class ExodusSignatureDataController(SignatureDataController): def __init__(self): - super().__init__('Exodus signatures', 'exodus.yml') + super().__init__('Exodus signatures', 'exodus.yml', 'https://reports.exodus-privacy.eu.org/api/trackers') self.cache_outdated_interval = timedelta(days=1) # refresh exodus cache after one day def fetch_signatures_from_web(self): - exodus_url = "https://reports.exodus-privacy.eu.org/api/trackers" + logging.debug(_("downloading '{}'").format(self.url)) + self.data = { "signatures": {}, "timestamp": scanner._datetime_now().isoformat(), "version": SCANNER_CACHE_VERSION, } - with urllib.request.urlopen(exodus_url) as f: + with urllib.request.urlopen(self.url) as f: d = json.load(f) for tracker in d["trackers"].values(): if tracker.get('code_signature'): @@ -261,21 +268,26 @@ class ExodusSignatureDataController(SignatureDataController): # "network_signatures": [tracker["network_signature"]], "AntiFeatures": ["Tracking"], "license": "NonFree" # We assume all trackers in exodus - # are non-free, alought free + # are non-free, although free # trackers like piwik, acra, # etc. might be listed by exodus # too. } -class SUSSDataController(SignatureDataController): - def __init__(self): - super().__init__('SUSS', 'suss.json') - - class ScannerTool(): def __init__(self): - self.sdcs = [SUSSDataController()] + self.sdcs = [ + SignatureDataController( + 'SUSS', + 'suss.json', + 'https://fdroid.gitlab.io/fdroid-suss/suss.json' + ), + ] + + # we could add support for loading additional signature source + # definitions from config.yml here + self.load() self.compile_regexes() @@ -301,11 +313,9 @@ class ScannerTool(): for sig in sigdef.get('warn_gradle_signatures', []): self.regexs['warn_gradle_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE) - def clear_cache(self): - # delete cache folder and all its contents - shutil.rmtree(scanner._scanner_cachedir(), ignore_errors=True) - # re-initialize, this will re-populate the cache from default values - self.__init__() + def refresh(self): + for sdc in self.sdcs: + sdc.fetch_signatures_from_web() def add(self, new_controller: SignatureDataController): self.sdcs.append(new_controller) @@ -660,8 +670,8 @@ def main(): help=_("Force scan of disabled apps and builds.")) parser.add_argument("--json", action="store_true", default=False, help=_("Output JSON to stdout.")) - parser.add_argument("--clear-cache", action="store_true", default=False, - help=_("purge local scanner definitions cache")) + parser.add_argument("--refresh", "-r", action="store_true", default=False, + help=_("fetach the latest version of signatures from the web")) metadata.add_metadata_arguments(parser) options = parser.parse_args() metadata.warnings_action = options.W @@ -676,11 +686,14 @@ def main(): # initialize/load configuration values common.get_config(options) - if options.clear_cache: - scanner._get_tool().clear_cache() + if options.refresh: + scanner._get_tool().refresh() if options.exodus: c = ExodusSignatureDataController() - c.fetch() + if options.refresh: + c.fetch_signatures_from_web() + else: + c.fetch() scanner._get_tool().add(c) probcount = 0 diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index 1e21e7b5..65b70b60 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -4,7 +4,6 @@ import glob import inspect import logging import optparse -import io import os import re import shutil @@ -500,56 +499,56 @@ class Test_scan_binary(unittest.TestCase): class Test_SignatureDataController(unittest.TestCase): # __init__ def test_init(self): - sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') + sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') self.assertEqual(sdc.name, 'nnn') self.assertEqual(sdc.filename, 'fff.yml') - self.assertEqual(sdc.cache_outdated_interval, None) + self.assertEqual(sdc.cache_outdated_interval, timedelta(999999)) self.assertDictEqual(sdc.data, {}) # check_last_updated def test_check_last_updated_ok(self): - sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') + sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') sdc.data['timestamp'] = datetime.now().astimezone().isoformat() sdc.check_last_updated() def test_check_last_updated_exception_cache_outdated(self): - sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') + sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') sdc.cache_outdated_interval = timedelta(days=7) sdc.data['timestamp'] = (datetime.now().astimezone() - timedelta(days=30)).isoformat() with self.assertRaises(fdroidserver.scanner.SignatureDataOutdatedException): sdc.check_last_updated() def test_check_last_updated_exception_missing_timestamp_value(self): - sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') + sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException): sdc.check_last_updated() def test_check_last_updated_exception_not_string(self): - sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') + sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') sdc.data['timestamp'] = 12345 with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException): sdc.check_last_updated() def test_check_last_updated_exception_not_iso_formatted_string(self): - sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') + sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') sdc.data['timestamp'] = '01/09/2002 10:11' with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException): sdc.check_last_updated() # check_data_version def test_check_data_version_ok(self): - sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') + sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') sdc.data['version'] = fdroidserver.scanner.SCANNER_CACHE_VERSION sdc.check_data_version() def test_check_data_version_exception(self): - sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') + sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') with self.assertRaises(fdroidserver.scanner.SignatureDataVersionMismatchException): sdc.check_data_version() def test_write_to_cache(self): open_func = mock.mock_open() - sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') + sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') sdc.data = {"mocked": "data"} with mock.patch("builtins.open", open_func), mock.patch( @@ -565,28 +564,6 @@ class Test_SignatureDataController(unittest.TestCase): ) -class Test_SUSSDataController_fetch_signatures_from_web(unittest.TestCase): - def setUp(self): - self.uo_func = mock.Mock(return_value=io.StringIO(textwrap.dedent('''\ - version: 999 - timestamp: "1999-12-31T23:59:59.999999+00:00" - signatures: - - binary_signature: com/google/firebase - name: Google Firebase - types: - - tracker - - non-free - - gradle_signature: com/google/android/gms - name: Google Mobile Services - types: - - non-free - - network_signature: doubleclick\\.net - name: Another thing to test. - types: - - ads - '''))) - - class Test_main(unittest.TestCase): def setUp(self): self.args = ["com.example.app", "local/additional.apk", "another.apk"] @@ -654,7 +631,6 @@ if __name__ == "__main__": unittest.makeSuite(ScannerTest), unittest.makeSuite(Test_scan_binary), unittest.makeSuite(Test_SignatureDataController), - unittest.makeSuite(Test_SUSSDataController_fetch_signatures_from_web), unittest.makeSuite(Test_main), ]) unittest.main(failfast=False) From a8bcaa3d7081a7fb63f04564ac346c317f67d933 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Fri, 30 Sep 2022 04:44:14 +0200 Subject: [PATCH 11/21] scanner: implement caching rules for suss --- fdroidserver/build.py | 2 +- fdroidserver/common.py | 13 ++++-- fdroidserver/scanner.py | 99 ++++++++++++++++++++++++----------------- tests/scanner.TestCase | 21 +++++---- 4 files changed, 78 insertions(+), 57 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 32b3306f..ca37a4bb 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -987,7 +987,7 @@ def main(): if not options.appid and not options.all: parser.error("option %s: If you really want to build all the apps, use --all" % "all") - config = common.read_config(options) + config = common.read_config(opts=options) if config['build_server_always']: options.server = True diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 7e6bed29..7793c51f 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -324,19 +324,26 @@ def fill_config_defaults(thisconfig): thisconfig['gradle_version_dir'] = str(Path(thisconfig['cachedir']) / 'gradle') -def get_config(options=None): +def get_config(opts=None): """ helper function for getting access to commons.config while safely initializing if it wasn't initialized yet. """ - global config + global config, options if config is not None: return config config = {} common.fill_config_defaults(config) - common.read_config(options) + common.read_config(opts=opts) + + # make sure these values are available in common.py even if they didn't + # declare global in a scope + common.config = config + if opts is not None: + common.options = opts + return config diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 9fc5fcb2..e0852eea 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -141,6 +141,10 @@ class SignatureDataOutdatedException(Exception): pass +class SignatureDataCacheMissException(Exception): + pass + + class SignatureDataVersionMismatchException(Exception): pass @@ -151,7 +155,7 @@ class SignatureDataController: self.filename = filename self.url = url # by default we assume cache is valid indefinitely - self.cache_outdated_interval = timedelta(days=999999) + self.cache_duration = timedelta(days=999999) self.data = {} def check_data_version(self): @@ -162,63 +166,65 @@ class SignatureDataController: ''' NOTE: currently not in use - Checks if the timestamp value is ok. Raises an exception if something - is not ok. + Checks if the last_updated value is ok. Raises an exception if + it's expired or inaccessible. :raises SignatureDataMalformedException: when timestamp value is inaccessible or not parse-able :raises SignatureDataOutdatedException: when timestamp is older then - `self.cache_outdated_interval` + `self.cache_duration` ''' - timestamp = self.data.get("timestamp") - if not timestamp: - raise SignatureDataMalformedException() - try: - timestamp = datetime.fromisoformat(timestamp) - except ValueError as e: - raise SignatureDataMalformedException() from e - except TypeError as e: - raise SignatureDataMalformedException() from e - if (timestamp + self.cache_outdated_interval) < scanner._datetime_now(): - raise SignatureDataOutdatedException() + last_updated = self.data.get("last_updated", None) + if last_updated: + try: + last_updated = datetime.fromisoformat(last_updated) + except ValueError as e: + raise SignatureDataMalformedException() from e + except TypeError as e: + raise SignatureDataMalformedException() from e + delta = (last_updated + self.cache_duration) - scanner._datetime_now() + if delta > timedelta(seconds=0): + logging.debug(_('next {name} cache update due in {time}').format( + name=self.filename, time=delta + )) + else: + raise SignatureDataOutdatedException() def fetch(self): try: - self.load_from_cache() - self.verify_data() - self.check_last_updated() - except ( - SignatureDataMalformedException, - SignatureDataVersionMismatchException, - SignatureDataOutdatedException - ): - try: - self.fetch_signatures_from_web() - except AttributeError: - # just load from defaults if fetch_signatures_from_web is not - # implemented - self.load_from_defaults() + self.fetch_signatures_from_web() self.write_to_cache() + except Exception as e: + raise Exception(_("downloading scanner signatures from '{}' failed").format(self.url)) from e def load(self): try: - self.load_from_cache() - self.verify_data() - except (SignatureDataMalformedException, SignatureDataVersionMismatchException): - self.load_from_defaults() + try: + self.load_from_cache() + self.verify_data() + self.check_last_updated() + except SignatureDataCacheMissException: + self.load_from_defaults() + except SignatureDataOutdatedException: + self.fetch_signatures_from_web() self.write_to_cache() + except (SignatureDataMalformedException, SignatureDataVersionMismatchException) as e: + logging.critical(_("scanner cache is malformed! You can clear it with: '{clear}'").format( + clear='rm -r {}'.format(common.get_config()['cachedir_scanner']) + )) + raise e def load_from_defaults(self): sig_file = (Path(__file__).parent / 'data' / 'scanner' / self.filename).resolve() with open(sig_file) as f: - self.data = json.load(f) + self.set_data(json.load(f)) def load_from_cache(self): sig_file = scanner._scanner_cachedir() / self.filename if not sig_file.exists(): - raise SignatureDataMalformedException() + raise SignatureDataCacheMissException() with open(sig_file) as f: - self.data = json.load(f) + self.set_data(json.load(f)) def write_to_cache(self): sig_file = scanner._scanner_cachedir() / self.filename @@ -231,29 +237,36 @@ class SignatureDataController: cleans and validates and cleans `self.data` ''' self.check_data_version() - valid_keys = ['timestamp', 'version', 'signatures'] + valid_keys = ['timestamp', 'last_updated', 'version', 'signatures', 'cache_duration'] for k in list(self.data.keys()): if k not in valid_keys: del self.data[k] + def set_data(self, new_data): + self.data = new_data + if 'cache_duration' in new_data: + self.cache_duration = timedelta(seconds=new_data['cache_duration']) + def fetch_signatures_from_web(self): logging.debug(_("downloading '{}'").format(self.url)) with urllib.request.urlopen(self.url) as f: - self.data = json.load(f) + self.set_data(json.load(f)) + self.data['last_updated'] = scanner._datetime_now().isoformat() class ExodusSignatureDataController(SignatureDataController): def __init__(self): super().__init__('Exodus signatures', 'exodus.yml', 'https://reports.exodus-privacy.eu.org/api/trackers') - self.cache_outdated_interval = timedelta(days=1) # refresh exodus cache after one day + self.cache_duration = timedelta(days=1) # refresh exodus cache after one day def fetch_signatures_from_web(self): logging.debug(_("downloading '{}'").format(self.url)) - self.data = { + data = { "signatures": {}, "timestamp": scanner._datetime_now().isoformat(), + "last_updated": scanner._datetime_now().isoformat(), "version": SCANNER_CACHE_VERSION, } @@ -261,7 +274,7 @@ class ExodusSignatureDataController(SignatureDataController): d = json.load(f) for tracker in d["trackers"].values(): if tracker.get('code_signature'): - self.data["signatures"][tracker["name"]] = { + data["signatures"][tracker["name"]] = { "name": tracker["name"], "warn_code_signatures": [tracker["code_signature"]], # exodus also provides network signatures, unused atm. @@ -273,6 +286,7 @@ class ExodusSignatureDataController(SignatureDataController): # etc. might be listed by exodus # too. } + self.set_data(data) class ScannerTool(): @@ -316,6 +330,7 @@ class ScannerTool(): def refresh(self): for sdc in self.sdcs: sdc.fetch_signatures_from_web() + sdc.write_to_cache() def add(self, new_controller: SignatureDataController): self.sdcs.append(new_controller) @@ -684,7 +699,7 @@ def main(): logging.getLogger().setLevel(logging.ERROR) # initialize/load configuration values - common.get_config(options) + common.get_config(opts=options) if options.refresh: scanner._get_tool().refresh() diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index 65b70b60..32274396 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -502,39 +502,38 @@ class Test_SignatureDataController(unittest.TestCase): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') self.assertEqual(sdc.name, 'nnn') self.assertEqual(sdc.filename, 'fff.yml') - self.assertEqual(sdc.cache_outdated_interval, timedelta(999999)) + self.assertEqual(sdc.cache_duration, timedelta(999999)) self.assertDictEqual(sdc.data, {}) # check_last_updated def test_check_last_updated_ok(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') - sdc.data['timestamp'] = datetime.now().astimezone().isoformat() + sdc.data['last_updated'] = datetime.now().astimezone().isoformat() sdc.check_last_updated() def test_check_last_updated_exception_cache_outdated(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') - sdc.cache_outdated_interval = timedelta(days=7) - sdc.data['timestamp'] = (datetime.now().astimezone() - timedelta(days=30)).isoformat() + sdc.cache_duration = timedelta(days=7) + sdc.data['last_updated'] = (datetime.now().astimezone() - timedelta(days=30)).isoformat() with self.assertRaises(fdroidserver.scanner.SignatureDataOutdatedException): sdc.check_last_updated() - def test_check_last_updated_exception_missing_timestamp_value(self): - sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') - with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException): - sdc.check_last_updated() - def test_check_last_updated_exception_not_string(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') - sdc.data['timestamp'] = 12345 + sdc.data['last_updated'] = 12345 with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException): sdc.check_last_updated() def test_check_last_updated_exception_not_iso_formatted_string(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') - sdc.data['timestamp'] = '01/09/2002 10:11' + sdc.data['last_updated'] = '01/09/2002 10:11' with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException): sdc.check_last_updated() + def test_check_last_updated_no_exception_missing_when_last_updated_not_set(self): + sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') + sdc.check_last_updated() + # check_data_version def test_check_data_version_ok(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') From 6899c22f48d0003379c47f2f2424787e7f7abbfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Fri, 30 Sep 2022 05:32:18 +0200 Subject: [PATCH 12/21] scanner: remove allowlisted from scan_source --- fdroidserver/scanner.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index e0852eea..dd6272d0 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -383,20 +383,9 @@ def scan_source(build_dir, build=metadata.Build()): """ count = 0 - allowlisted = [ - 'firebase-jobdispatcher', # https://github.com/firebase/firebase-jobdispatcher-android/blob/master/LICENSE - 'com.firebaseui', # https://github.com/firebase/FirebaseUI-Android/blob/master/LICENSE - 'geofire-android', # https://github.com/firebase/geofire-java/blob/master/LICENSE - 'com.yandex.android:authsdk', # https://github.com/yandexmobile/yandex-login-sdk-android/blob/master/LICENSE.txt - 'com.hypertrack:hyperlog', # https://github.com/hypertrack/hyperlog-android#license - ] - - def is_allowlisted(s): - return any(al in s for al in allowlisted) - def suspects_found(s): for n, r in _get_tool().regexs['err_gradle_signatures'].items(): - if r.match(s) and not is_allowlisted(s): + if r.match(s): yield n allowed_repos = [re.compile(r'^https://' + re.escape(repo) + r'/*') for repo in [ From 0921863fa6dd791b2806a75f50c18976fad4ba97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Fri, 30 Sep 2022 05:36:36 +0200 Subject: [PATCH 13/21] scanner: update suss defaults after removal of allowlisted feature --- fdroidserver/data/scanner/suss.json | 15 +++++++++------ tests/scanner.TestCase | 4 ++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/fdroidserver/data/scanner/suss.json b/fdroidserver/data/scanner/suss.json index f2479f22..bc58db2a 100644 --- a/fdroidserver/data/scanner/suss.json +++ b/fdroidserver/data/scanner/suss.json @@ -1,4 +1,5 @@ { + "cache_duration": 86400, "signatures": { "admob": { "gradle_signatures": [ @@ -137,13 +138,14 @@ }, "com.hypertrack": { "gradle_signatures": [ - "com.hypertrack" + "com\\.hypertrack(?!:hyperlog)" ], "license": "NonFree" }, "com.mapbox": { + "MaintainerNotes": "com.mapbox.mapboxsdk:mapbox-sdk-services seems to be fully under this license:\nhttps://github.com/mapbox/mapbox-java/blob/main/LICENSE\n", "gradle_signatures": [ - "com.mapbox" + "com\\.mapbox(?!\\.mapboxsdk:mapbox-sdk-services)" ], "license": "NonFree" }, @@ -167,7 +169,7 @@ }, "com.yandex.android": { "gradle_signatures": [ - "com.yandex.android" + "com\\.yandex\\.android(?!:authsdk)" ], "license": "NonFree" }, @@ -191,7 +193,7 @@ }, "firebase": { "gradle_signatures": [ - "firebase" + "com(\\.google)?\\.firebase[.:](?!firebase-jobdispatcher|geofire-java)" ], "license": "NonFree" }, @@ -286,6 +288,7 @@ "license": "NonFree" } }, - "timestamp": "2022-09-22T13:16:29.313769+00:00", - "version": 1 + "timestamp": "2022-09-29T19:35:04.875586+00:00", + "version": 1, + "last_updated": "2022-09-30T05:13:18.411707+02:00" } \ No newline at end of file diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index 32274396..c3c70ffe 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -54,8 +54,8 @@ class ScannerTest(unittest.TestCase): 'com.integreight.onesheeld': 11, 'com.jens.automation2': 2, 'firebase-suspect': 1, - 'org.mozilla.rocket': 3, - 'org.tasks': 3, + 'org.mozilla.rocket': 1, + 'org.tasks': 2, 'realm': 1, 'se.manyver': 2, } From 036b788424474e9d65bcb2da50b1f7147d3823a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Fri, 30 Sep 2022 06:56:26 +0200 Subject: [PATCH 14/21] move suss defaults into scanner.py As discussed with _hc, we're going to keep the default values for SUSS inside the python code for now. To avoid compilcated packaging issues. Once fdroidserver does not have to support python verison older than 3.7 we can utilize to `importlib.resources` for shipping these defaults as separate file. --- fdroidserver/data/scanner/suss.json | 294 -------------------------- fdroidserver/scanner.py | 314 +++++++++++++++++++++++++++- 2 files changed, 309 insertions(+), 299 deletions(-) delete mode 100644 fdroidserver/data/scanner/suss.json diff --git a/fdroidserver/data/scanner/suss.json b/fdroidserver/data/scanner/suss.json deleted file mode 100644 index bc58db2a..00000000 --- a/fdroidserver/data/scanner/suss.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "cache_duration": 86400, - "signatures": { - "admob": { - "gradle_signatures": [ - "admob.*sdk.*android" - ], - "license": "NonFree" - }, - "androidx": { - "gradle_signatures": [ - "androidx.navigation:navigation-dynamic-features", - "androidx.work:work-gcm" - ], - "license": "NonFree" - }, - "appcenter-push": { - "gradle_signatures": [ - "appcenter-push" - ], - "license": "NonFree" - }, - "bugsense": { - "gradle_signatures": [ - "bugsense" - ], - "license": "NonFree" - }, - "cloudrail": { - "gradle_signatures": [ - "cloudrail" - ], - "license": "NonFree" - }, - "com.android.billing": { - "code_signatures": [ - "com/android/billing" - ], - "license": "NonFree" - }, - "com.android.billingclient": { - "gradle_signatures": [ - "com.android.billingclient" - ], - "license": "NonFree" - }, - "com.anjlab.android.iab.v3": { - "gradle_signatures": [ - "com.anjlab.android.iab.v3:library" - ], - "license": "NonFree" - }, - "com.cloudinary": { - "gradle_signatures": [ - "com.cloudinary:cloudinary-android" - ], - "license": "NonFree" - }, - "com.evernote": { - "gradle_signatures": [ - "com.evernote:android-job" - ], - "license": "NonFree" - }, - "com.facebook": { - "gradle_signatures": [ - "[\"']com.facebook.android['\":]" - ], - "license": "NonFree" - }, - "com.github.junrar": { - "gradle_signatures": [ - "com.github.junrar:junrar" - ], - "license": "NonFree" - }, - "com.github.penn5": { - "gradle_signatures": [ - "com.github.penn5:donations" - ], - "license": "NonFree" - }, - "com.google.analytics": { - "code_signatures": [ - "com/google/analytics" - ], - "license": "NonFree" - }, - "com.google.android.exoplayer": { - "gradle_signatures": [ - "com.google.android.exoplayer:extension-cast", - "com.google.android.exoplayer:extension-cronet" - ], - "license": "NonFree" - }, - "com.google.android.gms": { - "code_signatures": [ - "com/google/android/gms" - ], - "license": "NonFree" - }, - "com.google.android.libraries.places": { - "gradle_signatures": [ - "com.google.android.libraries.places:places" - ], - "license": "NonFree" - }, - "com.google.android.play": { - "gradle_signatures": [ - "com.google.android.play:app-update", - "com.google.android.play:core.*" - ], - "license": "NonFree" - }, - "com.google.android.play.core": { - "code_signatures": [ - "com/google/android/play/core" - ], - "license": "NonFree" - }, - "com.google.firebase": { - "code_signatures": [ - "com/google/firebase" - ], - "license": "NonFree" - }, - "com.google.mlkit": { - "gradle_signatures": [ - "com.google.mlkit" - ], - "license": "NonFree" - }, - "com.google.tagmanager": { - "code_signatures": [ - "com/google/tagmanager" - ], - "license": "NonFree" - }, - "com.hypertrack": { - "gradle_signatures": [ - "com\\.hypertrack(?!:hyperlog)" - ], - "license": "NonFree" - }, - "com.mapbox": { - "MaintainerNotes": "com.mapbox.mapboxsdk:mapbox-sdk-services seems to be fully under this license:\nhttps://github.com/mapbox/mapbox-java/blob/main/LICENSE\n", - "gradle_signatures": [ - "com\\.mapbox(?!\\.mapboxsdk:mapbox-sdk-services)" - ], - "license": "NonFree" - }, - "com.onesignal": { - "gradle_signatures": [ - "com.onesignal:OneSignal" - ], - "license": "NonFree" - }, - "com.tencent.bugly": { - "gradle_signatures": [ - "com.tencent.bugly" - ], - "license": "NonFree" - }, - "com.umeng.umsdk": { - "gradle_signatures": [ - "com.umeng.umsdk" - ], - "license": "NonFree" - }, - "com.yandex.android": { - "gradle_signatures": [ - "com\\.yandex\\.android(?!:authsdk)" - ], - "license": "NonFree" - }, - "com.yayandroid": { - "gradle_signatures": [ - "com.yayandroid:LocationManager" - ], - "license": "NonFree" - }, - "crashlytics": { - "gradle_signatures": [ - "crashlytics" - ], - "license": "NonFree" - }, - "crittercism": { - "gradle_signatures": [ - "crittercism" - ], - "license": "NonFree" - }, - "firebase": { - "gradle_signatures": [ - "com(\\.google)?\\.firebase[.:](?!firebase-jobdispatcher|geofire-java)" - ], - "license": "NonFree" - }, - "flurryagent": { - "gradle_signatures": [ - "flurryagent" - ], - "license": "NonFree" - }, - "google-ad": { - "gradle_signatures": [ - "google.*ad.*view" - ], - "license": "NonFree" - }, - "google.admob": { - "gradle_signatures": [ - "google.*admob" - ], - "license": "NonFree" - }, - "google.play.services": { - "gradle_signatures": [ - "google.*play.*services" - ], - "license": "NonFree" - }, - "heyzap": { - "gradle_signatures": [ - "heyzap" - ], - "license": "NonFree" - }, - "io.github.sinaweibosdk": { - "gradle_signatures": [ - "io.github.sinaweibosdk" - ], - "license": "NonFree" - }, - "io.objectbox": { - "gradle_signatures": [ - "io.objectbox:objectbox-gradle-plugin" - ], - "license": "NonFree" - }, - "jpct": { - "gradle_signatures": [ - "jpct.*ae" - ], - "license": "NonFree" - }, - "libspen23": { - "gradle_signatures": [ - "libspen23" - ], - "license": "NonFree" - }, - "me.pushy": { - "gradle_signatures": [ - "me.pushy:sdk" - ], - "license": "NonFree" - }, - "org.jetbrains.kotlinx": { - "gradle_signatures": [ - "org.jetbrains.kotlinx:kotlinx-coroutines-play-services" - ], - "license": "NonFree" - }, - "ouya": { - "gradle_signatures": [ - "ouya.*sdk" - ], - "license": "NonFree" - }, - "paypal": { - "gradle_signatures": [ - "paypal.*mpl" - ], - "license": "NonFree" - }, - "xyz.belvi.mobilevision": { - "gradle_signatures": [ - "xyz.belvi.mobilevision:barcodescanner" - ], - "license": "NonFree" - }, - "youtube": { - "gradle_signatures": [ - "youtube.*android.*player.*api" - ], - "license": "NonFree" - } - }, - "timestamp": "2022-09-29T19:35:04.875586+00:00", - "version": 1, - "last_updated": "2022-09-30T05:13:18.411707+02:00" -} \ No newline at end of file diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index dd6272d0..0fe97cb4 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -289,14 +289,22 @@ class ExodusSignatureDataController(SignatureDataController): self.set_data(data) +class SUSSDataController(SignatureDataController): + def __init__(self): + super().__init__( + 'SUSS', + 'suss.json', + 'https://fdroid.gitlab.io/fdroid-suss/suss.json' + ) + + def load_from_defaults(self): + self.set_data(json.loads(SUSS_DEFAULT)) + + class ScannerTool(): def __init__(self): self.sdcs = [ - SignatureDataController( - 'SUSS', - 'suss.json', - 'https://fdroid.gitlab.io/fdroid-suss/suss.json' - ), + SUSSDataController(), ] # we could add support for loading additional signature source @@ -809,3 +817,299 @@ def main(): if __name__ == "__main__": main() + + +SUSS_DEFAULT = '''{ + "cache_duration": 86400, + "signatures": { + "admob": { + "gradle_signatures": [ + "admob.*sdk.*android" + ], + "license": "NonFree" + }, + "androidx": { + "gradle_signatures": [ + "androidx.navigation:navigation-dynamic-features", + "androidx.work:work-gcm" + ], + "license": "NonFree" + }, + "appcenter-push": { + "gradle_signatures": [ + "appcenter-push" + ], + "license": "NonFree" + }, + "bugsense": { + "gradle_signatures": [ + "bugsense" + ], + "license": "NonFree" + }, + "cloudrail": { + "gradle_signatures": [ + "cloudrail" + ], + "license": "NonFree" + }, + "com.android.billing": { + "code_signatures": [ + "com/android/billing" + ], + "license": "NonFree" + }, + "com.android.billingclient": { + "gradle_signatures": [ + "com.android.billingclient" + ], + "license": "NonFree" + }, + "com.anjlab.android.iab.v3": { + "gradle_signatures": [ + "com.anjlab.android.iab.v3:library" + ], + "license": "NonFree" + }, + "com.cloudinary": { + "gradle_signatures": [ + "com.cloudinary:cloudinary-android" + ], + "license": "NonFree" + }, + "com.evernote": { + "gradle_signatures": [ + "com.evernote:android-job" + ], + "license": "NonFree" + }, + "com.facebook": { + "gradle_signatures": [ + "[\\"']com.facebook.android['\\":]" + ], + "license": "NonFree" + }, + "com.github.junrar": { + "gradle_signatures": [ + "com.github.junrar:junrar" + ], + "license": "NonFree" + }, + "com.github.penn5": { + "gradle_signatures": [ + "com.github.penn5:donations" + ], + "license": "NonFree" + }, + "com.google.analytics": { + "code_signatures": [ + "com/google/analytics" + ], + "license": "NonFree" + }, + "com.google.android.exoplayer": { + "gradle_signatures": [ + "com.google.android.exoplayer:extension-cast", + "com.google.android.exoplayer:extension-cronet" + ], + "license": "NonFree" + }, + "com.google.android.gms": { + "code_signatures": [ + "com/google/android/gms" + ], + "license": "NonFree" + }, + "com.google.android.libraries.places": { + "gradle_signatures": [ + "com.google.android.libraries.places:places" + ], + "license": "NonFree" + }, + "com.google.android.play": { + "gradle_signatures": [ + "com.google.android.play:app-update", + "com.google.android.play:core.*" + ], + "license": "NonFree" + }, + "com.google.android.play.core": { + "code_signatures": [ + "com/google/android/play/core" + ], + "license": "NonFree" + }, + "com.google.firebase": { + "code_signatures": [ + "com/google/firebase" + ], + "license": "NonFree" + }, + "com.google.mlkit": { + "gradle_signatures": [ + "com.google.mlkit" + ], + "license": "NonFree" + }, + "com.google.tagmanager": { + "code_signatures": [ + "com/google/tagmanager" + ], + "license": "NonFree" + }, + "com.hypertrack": { + "gradle_signatures": [ + "com\\\\.hypertrack(?!:hyperlog)" + ], + "license": "NonFree" + }, + "com.mapbox": { + "MaintainerNotes": "com.mapbox.mapboxsdk:mapbox-sdk-services seems to be fully under this license:\\nhttps://github.com/mapbox/mapbox-java/blob/main/LICENSE\\n", + "gradle_signatures": [ + "com\\\\.mapbox(?!\\\\.mapboxsdk:mapbox-sdk-services)" + ], + "license": "NonFree" + }, + "com.onesignal": { + "gradle_signatures": [ + "com.onesignal:OneSignal" + ], + "license": "NonFree" + }, + "com.tencent.bugly": { + "gradle_signatures": [ + "com.tencent.bugly" + ], + "license": "NonFree" + }, + "com.umeng.umsdk": { + "gradle_signatures": [ + "com.umeng.umsdk" + ], + "license": "NonFree" + }, + "com.yandex.android": { + "gradle_signatures": [ + "com\\\\.yandex\\\\.android(?!:authsdk)" + ], + "license": "NonFree" + }, + "com.yayandroid": { + "gradle_signatures": [ + "com.yayandroid:LocationManager" + ], + "license": "NonFree" + }, + "crashlytics": { + "gradle_signatures": [ + "crashlytics" + ], + "license": "NonFree" + }, + "crittercism": { + "gradle_signatures": [ + "crittercism" + ], + "license": "NonFree" + }, + "firebase": { + "gradle_signatures": [ + "com(\\\\.google)?\\\\.firebase[.:](?!firebase-jobdispatcher|geofire-java)" + ], + "license": "NonFree" + }, + "flurryagent": { + "gradle_signatures": [ + "flurryagent" + ], + "license": "NonFree" + }, + "google-ad": { + "gradle_signatures": [ + "google.*ad.*view" + ], + "license": "NonFree" + }, + "google.admob": { + "gradle_signatures": [ + "google.*admob" + ], + "license": "NonFree" + }, + "google.play.services": { + "gradle_signatures": [ + "google.*play.*services" + ], + "license": "NonFree" + }, + "heyzap": { + "gradle_signatures": [ + "heyzap" + ], + "license": "NonFree" + }, + "io.github.sinaweibosdk": { + "gradle_signatures": [ + "io.github.sinaweibosdk" + ], + "license": "NonFree" + }, + "io.objectbox": { + "gradle_signatures": [ + "io.objectbox:objectbox-gradle-plugin" + ], + "license": "NonFree" + }, + "jpct": { + "gradle_signatures": [ + "jpct.*ae" + ], + "license": "NonFree" + }, + "libspen23": { + "gradle_signatures": [ + "libspen23" + ], + "license": "NonFree" + }, + "me.pushy": { + "gradle_signatures": [ + "me.pushy:sdk" + ], + "license": "NonFree" + }, + "org.jetbrains.kotlinx": { + "gradle_signatures": [ + "org.jetbrains.kotlinx:kotlinx-coroutines-play-services" + ], + "license": "NonFree" + }, + "ouya": { + "gradle_signatures": [ + "ouya.*sdk" + ], + "license": "NonFree" + }, + "paypal": { + "gradle_signatures": [ + "paypal.*mpl" + ], + "license": "NonFree" + }, + "xyz.belvi.mobilevision": { + "gradle_signatures": [ + "xyz.belvi.mobilevision:barcodescanner" + ], + "license": "NonFree" + }, + "youtube": { + "gradle_signatures": [ + "youtube.*android.*player.*api" + ], + "license": "NonFree" + } + }, + "timestamp": "2022-09-29T19:35:04.875586+00:00", + "version": 1, + "last_updated": "2022-09-30T05:13:18.411707+02:00" +}''' From 7933623e9372efb38ec423d4e387bf0f445d7368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Fri, 30 Sep 2022 06:08:00 +0200 Subject: [PATCH 15/21] fix pydocstyle lint errors --- fdroidserver/common.py | 5 +---- fdroidserver/scanner.py | 31 +++++++++++++------------------ 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 7793c51f..0f001019 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -325,10 +325,7 @@ def fill_config_defaults(thisconfig): def get_config(opts=None): - """ - helper function for getting access to commons.config while safely - initializing if it wasn't initialized yet. - """ + """Get config instace. This function takes care of initaling config data before returning it.""" global config, options if config is not None: diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 0fe97cb4..81f4ce70 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -113,16 +113,12 @@ def get_embedded_classes(apkfile, depth=0): def _datetime_now(): - """ - simple wrapper for datetime.now to allow mocking it for testing - """ + """Get datetime.now(), using this funciton allows mocking it for testing.""" return datetime.now().astimezone() def _scanner_cachedir(): - """ - get `Path` to local cache dir - """ + """Get `Path` to fdroidserver cache dir.""" cfg = common.get_config() if not cfg: raise ConfigurationException('config not initialized') @@ -163,17 +159,14 @@ class SignatureDataController: raise SignatureDataVersionMismatchException() def check_last_updated(self): - ''' - NOTE: currently not in use - - Checks if the last_updated value is ok. Raises an exception if - it's expired or inaccessible. + """ + Check if the last_updated value is ok and raise an exception if expired or inaccessible. :raises SignatureDataMalformedException: when timestamp value is inaccessible or not parse-able :raises SignatureDataOutdatedException: when timestamp is older then `self.cache_duration` - ''' + """ last_updated = self.data.get("last_updated", None) if last_updated: try: @@ -233,9 +226,11 @@ class SignatureDataController: logging.debug("write '{}' to cache".format(self.filename)) def verify_data(self): - ''' - cleans and validates and cleans `self.data` - ''' + """ + Clean and validate `self.data`. + + Right now this function does just a basic key sanitation. + """ self.check_data_version() valid_keys = ['timestamp', 'last_updated', 'version', 'signatures', 'cache_duration'] @@ -351,11 +346,11 @@ _SCANNER_TOOL = None def _get_tool(): - ''' - lazy loading factory for ScannerTool singleton + """ + Lazy loading function for getting a ScannerTool instance. ScannerTool initialization need to access `common.config` values. Those are only available after initialization through `common.read_config()` So this factory assumes config was called at an erlier point in time - ''' + """ if not scanner._SCANNER_TOOL: scanner._SCANNER_TOOL = ScannerTool() return scanner._SCANNER_TOOL From 9560ed955c0d07d1e56588c8d373e29a13591bbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Fri, 30 Sep 2022 11:13:59 +0200 Subject: [PATCH 16/21] avoid running into native-date object issue Seem I ran into this issue: https://bugs.python.org/issue47228 This change tries to fix it by using utcnow insteas of astimezone. --- fdroidserver/scanner.py | 2 +- tests/scanner.TestCase | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 81f4ce70..797d694e 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -114,7 +114,7 @@ def get_embedded_classes(apkfile, depth=0): def _datetime_now(): """Get datetime.now(), using this funciton allows mocking it for testing.""" - return datetime.now().astimezone() + return datetime.utcnow() def _scanner_cachedir(): diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index c3c70ffe..a5ad3779 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -508,13 +508,13 @@ class Test_SignatureDataController(unittest.TestCase): # check_last_updated def test_check_last_updated_ok(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') - sdc.data['last_updated'] = datetime.now().astimezone().isoformat() + sdc.data['last_updated'] = datetime.utcnow().isoformat() sdc.check_last_updated() def test_check_last_updated_exception_cache_outdated(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') sdc.cache_duration = timedelta(days=7) - sdc.data['last_updated'] = (datetime.now().astimezone() - timedelta(days=30)).isoformat() + sdc.data['last_updated'] = (datetime.utcnow() - timedelta(days=30)).isoformat() with self.assertRaises(fdroidserver.scanner.SignatureDataOutdatedException): sdc.check_last_updated() From 59b1899d7957b1f0c7a7425a3d34934524f9898b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Fri, 30 Sep 2022 14:18:31 +0200 Subject: [PATCH 17/21] scanner: switch form iso8601 to unix timestamp for python3.5 support --- fdroidserver/scanner.py | 12 ++++++------ tests/scanner.TestCase | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 797d694e..8eb94a8c 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -170,7 +170,7 @@ class SignatureDataController: last_updated = self.data.get("last_updated", None) if last_updated: try: - last_updated = datetime.fromisoformat(last_updated) + last_updated = datetime.fromtimestamp(last_updated) except ValueError as e: raise SignatureDataMalformedException() from e except TypeError as e: @@ -247,7 +247,7 @@ class SignatureDataController: logging.debug(_("downloading '{}'").format(self.url)) with urllib.request.urlopen(self.url) as f: self.set_data(json.load(f)) - self.data['last_updated'] = scanner._datetime_now().isoformat() + self.data['last_updated'] = scanner._datetime_now().timestamp() class ExodusSignatureDataController(SignatureDataController): @@ -260,8 +260,8 @@ class ExodusSignatureDataController(SignatureDataController): data = { "signatures": {}, - "timestamp": scanner._datetime_now().isoformat(), - "last_updated": scanner._datetime_now().isoformat(), + "timestamp": scanner._datetime_now().timestamp(), + "last_updated": scanner._datetime_now().timestamp(), "version": SCANNER_CACHE_VERSION, } @@ -1104,7 +1104,7 @@ SUSS_DEFAULT = '''{ "license": "NonFree" } }, - "timestamp": "2022-09-29T19:35:04.875586+00:00", + "timestamp": 1664480104.875586, "version": 1, - "last_updated": "2022-09-30T05:13:18.411707+02:00" + "last_updated": 1664480104.875586 }''' diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index a5ad3779..e674197f 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -508,19 +508,19 @@ class Test_SignatureDataController(unittest.TestCase): # check_last_updated def test_check_last_updated_ok(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') - sdc.data['last_updated'] = datetime.utcnow().isoformat() + sdc.data['last_updated'] = datetime.utcnow().timestamp() sdc.check_last_updated() def test_check_last_updated_exception_cache_outdated(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') sdc.cache_duration = timedelta(days=7) - sdc.data['last_updated'] = (datetime.utcnow() - timedelta(days=30)).isoformat() + sdc.data['last_updated'] = (datetime.utcnow() - timedelta(days=30)).timestamp() with self.assertRaises(fdroidserver.scanner.SignatureDataOutdatedException): sdc.check_last_updated() def test_check_last_updated_exception_not_string(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') - sdc.data['last_updated'] = 12345 + sdc.data['last_updated'] = 'sepp' with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException): sdc.check_last_updated() From b7233dfb2e20184c4c11a00bc05e53f67ff45962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Fri, 30 Sep 2022 14:54:28 +0200 Subject: [PATCH 18/21] ignore cache write test case on older python versions --- tests/scanner.TestCase | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index e674197f..85c55cf2 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -545,6 +545,10 @@ class Test_SignatureDataController(unittest.TestCase): with self.assertRaises(fdroidserver.scanner.SignatureDataVersionMismatchException): sdc.check_data_version() + @unittest.skipIf( + sys.version_info < (3, 9, 0), + "mock_open doesn't allow easy access to written data in older python versions", + ) def test_write_to_cache(self): open_func = mock.mock_open() sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') From d815a64fdb3a7f214dd82b8ea023d4bbfeb555c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Fri, 30 Sep 2022 17:34:41 +0200 Subject: [PATCH 19/21] scanner: filter urlopen schemes, mute bandit --- fdroidserver/scanner.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 8eb94a8c..39bffd67 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -244,8 +244,10 @@ class SignatureDataController: self.cache_duration = timedelta(seconds=new_data['cache_duration']) def fetch_signatures_from_web(self): + if not self.url.startswith("https://"): + raise Exception(_("can't open non-https url: '{};".format(self.url))) logging.debug(_("downloading '{}'").format(self.url)) - with urllib.request.urlopen(self.url) as f: + with urllib.request.urlopen(self.url) as f: # nosec B310 scheme filtered above self.set_data(json.load(f)) self.data['last_updated'] = scanner._datetime_now().timestamp() @@ -265,7 +267,9 @@ class ExodusSignatureDataController(SignatureDataController): "version": SCANNER_CACHE_VERSION, } - with urllib.request.urlopen(self.url) as f: + if not self.url.startswith("https://"): + raise Exception(_("can't open non-https url: '{};".format(self.url))) + with urllib.request.urlopen(self.url) as f: # nosec B310 scheme filtered above d = json.load(f) for tracker in d["trackers"].values(): if tracker.get('code_signature'): @@ -356,7 +360,7 @@ def _get_tool(): return scanner._SCANNER_TOOL -def scan_binary(apkfile, extract_signatures=None): +def scan_binary(apkfile): """Scan output of dexdump for known non-free classes.""" logging.info(_('Scanning APK with dexdump for known non-free classes.')) result = get_embedded_classes(apkfile) From e967fc61cfd3733f840c69e365b8f1a8389ae35d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 6 Oct 2022 14:50:30 +0200 Subject: [PATCH 20/21] scanner: add regression tests for signautre load logic --- tests/scanner.TestCase | 171 +++++++++++++++++++++++++++++++++++------ 1 file changed, 146 insertions(+), 25 deletions(-) diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index 85c55cf2..b301b9ea 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -450,7 +450,9 @@ class Test_scan_binary(unittest.TestCase): fdroidserver.scanner._SCANNER_TOOL = mock.Mock() fdroidserver.scanner._SCANNER_TOOL.regexs = {} fdroidserver.scanner._SCANNER_TOOL.regexs['err_code_signatures'] = { - "java/lang/Object": re.compile(r'.*java/lang/Object', re.IGNORECASE | re.UNICODE) + "java/lang/Object": re.compile( + r'.*java/lang/Object', re.IGNORECASE | re.UNICODE + ) } fdroidserver.scanner._SCANNER_TOOL.regexs['warn_code_signatures'] = {} @@ -459,7 +461,12 @@ class Test_scan_binary(unittest.TestCase): self.assertEqual( 1, fdroidserver.scanner.scan_binary(apkfile), - "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner._SCANNER_TOOL.regexs['err_code_signatures'].values(), apkfile), + "Did not find expected code signature '{}' in binary '{}'".format( + fdroidserver.scanner._SCANNER_TOOL.regexs[ + 'err_code_signatures' + ].values(), + apkfile, + ), ) @unittest.skipIf( @@ -472,14 +479,20 @@ class Test_scan_binary(unittest.TestCase): apkfile = os.path.join(self.basedir, 'apk.embedded_1.apk') fdroidserver.scanner._SCANNER_TOOL.regexs['err_code_signatures'] = { "org/bitbucket/tickytacky/mirrormirror/MainActivity": re.compile( - r'.*org/bitbucket/tickytacky/mirrormirror/MainActivity', re.IGNORECASE | re.UNICODE + r'.*org/bitbucket/tickytacky/mirrormirror/MainActivity', + re.IGNORECASE | re.UNICODE, ) } self.assertEqual( 1, fdroidserver.scanner.scan_binary(apkfile), - "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner._SCANNER_TOOL.regexs['err_code_signatures'].values(), apkfile), + "Did not find expected code signature '{}' in binary '{}'".format( + fdroidserver.scanner._SCANNER_TOOL.regexs[ + 'err_code_signatures' + ].values(), + apkfile, + ), ) def test_top_level_signature_embedded_apk_present(self): @@ -492,14 +505,21 @@ class Test_scan_binary(unittest.TestCase): self.assertEqual( 1, fdroidserver.scanner.scan_binary(apkfile), - "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner._SCANNER_TOOL.regexs['err_code_signatures'].values(), apkfile), + "Did not find expected code signature '{}' in binary '{}'".format( + fdroidserver.scanner._SCANNER_TOOL.regexs[ + 'err_code_signatures' + ].values(), + apkfile, + ), ) class Test_SignatureDataController(unittest.TestCase): # __init__ def test_init(self): - sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') + sdc = fdroidserver.scanner.SignatureDataController( + 'nnn', 'fff.yml', 'https://example.com/test.json' + ) self.assertEqual(sdc.name, 'nnn') self.assertEqual(sdc.filename, 'fff.yml') self.assertEqual(sdc.cache_duration, timedelta(999999)) @@ -507,51 +527,144 @@ class Test_SignatureDataController(unittest.TestCase): # check_last_updated def test_check_last_updated_ok(self): - sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') + sdc = fdroidserver.scanner.SignatureDataController( + 'nnn', 'fff.yml', 'https://example.com/test.json' + ) sdc.data['last_updated'] = datetime.utcnow().timestamp() sdc.check_last_updated() def test_check_last_updated_exception_cache_outdated(self): - sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') + sdc = fdroidserver.scanner.SignatureDataController( + 'nnn', 'fff.yml', 'https://example.com/test.json' + ) sdc.cache_duration = timedelta(days=7) sdc.data['last_updated'] = (datetime.utcnow() - timedelta(days=30)).timestamp() with self.assertRaises(fdroidserver.scanner.SignatureDataOutdatedException): sdc.check_last_updated() def test_check_last_updated_exception_not_string(self): - sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') + sdc = fdroidserver.scanner.SignatureDataController( + 'nnn', 'fff.yml', 'https://example.com/test.json' + ) sdc.data['last_updated'] = 'sepp' with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException): sdc.check_last_updated() def test_check_last_updated_exception_not_iso_formatted_string(self): - sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') + sdc = fdroidserver.scanner.SignatureDataController( + 'nnn', 'fff.yml', 'https://example.com/test.json' + ) sdc.data['last_updated'] = '01/09/2002 10:11' with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException): sdc.check_last_updated() def test_check_last_updated_no_exception_missing_when_last_updated_not_set(self): - sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') + sdc = fdroidserver.scanner.SignatureDataController( + 'nnn', 'fff.yml', 'https://example.com/test.json' + ) sdc.check_last_updated() # check_data_version def test_check_data_version_ok(self): - sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') + sdc = fdroidserver.scanner.SignatureDataController( + 'nnn', 'fff.yml', 'https://example.com/test.json' + ) sdc.data['version'] = fdroidserver.scanner.SCANNER_CACHE_VERSION sdc.check_data_version() def test_check_data_version_exception(self): - sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') - with self.assertRaises(fdroidserver.scanner.SignatureDataVersionMismatchException): + sdc = fdroidserver.scanner.SignatureDataController( + 'nnn', 'fff.yml', 'https://example.com/test.json' + ) + with self.assertRaises( + fdroidserver.scanner.SignatureDataVersionMismatchException + ): sdc.check_data_version() + def test_load_ok(self): + sdc = fdroidserver.scanner.SignatureDataController( + 'nnn', 'fff.yml', 'https://example.com/test.json' + ) + func_lfc = mock.Mock() + func_vd = mock.Mock() + func_clu = mock.Mock() + with mock.patch( + 'fdroidserver.scanner.SignatureDataController.load_from_cache', + func_lfc, + ), mock.patch( + 'fdroidserver.scanner.SignatureDataController.verify_data', + func_vd, + ), mock.patch( + 'fdroidserver.scanner.SignatureDataController.check_last_updated', + func_clu, + ): + sdc.load() + func_lfc.assert_called_once_with() + func_vd.assert_called_once_with() + func_clu.assert_called_once_with() + + def test_load_initial_cache_miss(self): + sdc = fdroidserver.scanner.SignatureDataController( + 'nnn', 'fff.yml', 'https://example.com/test.json' + ) + func_lfc = mock.Mock( + side_effect=fdroidserver.scanner.SignatureDataCacheMissException + ) + func_lfd = mock.Mock() + with mock.patch( + 'fdroidserver.scanner.SignatureDataController.load_from_cache', + func_lfc, + ), mock.patch( + 'fdroidserver.scanner.SignatureDataController.load_from_defaults', + func_lfd, + ): + sdc.load() + func_lfc.assert_called_once_with() + func_lfd.assert_called_once_with() + + def test_load_cache_auto_refresh(self): + sdc = fdroidserver.scanner.SignatureDataController( + 'nnn', 'fff.yml', 'https://example.com/test.json' + ) + func_lfc = mock.Mock() + func_vd = mock.Mock() + func_clu = mock.Mock( + side_effect=fdroidserver.scanner.SignatureDataOutdatedException() + ) + func_fsfw = mock.Mock() + func_wtc = mock.Mock() + with mock.patch( + 'fdroidserver.scanner.SignatureDataController.load_from_cache', + func_lfc, + ), mock.patch( + 'fdroidserver.scanner.SignatureDataController.verify_data', + func_vd, + ), mock.patch( + 'fdroidserver.scanner.SignatureDataController.check_last_updated', + func_clu, + ), mock.patch( + 'fdroidserver.scanner.SignatureDataController.fetch_signatures_from_web', + func_fsfw, + ), mock.patch( + 'fdroidserver.scanner.SignatureDataController.write_to_cache', + func_wtc, + ): + sdc.load() + func_lfc.assert_called_once_with() + func_vd.assert_called_once_with() + func_clu.assert_called_once_with() + func_fsfw.assert_called_once_with() + func_wtc.assert_called_once_with() + @unittest.skipIf( sys.version_info < (3, 9, 0), "mock_open doesn't allow easy access to written data in older python versions", ) def test_write_to_cache(self): open_func = mock.mock_open() - sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') + sdc = fdroidserver.scanner.SignatureDataController( + 'nnn', 'fff.yml', 'https://example.com/test.json' + ) sdc.data = {"mocked": "data"} with mock.patch("builtins.open", open_func), mock.patch( @@ -561,10 +674,16 @@ class Test_SignatureDataController(unittest.TestCase): sdc.write_to_cache() open_func.assert_called_with(pathlib.Path('fff.yml'), 'w', encoding="utf-8") - self.assertEqual( - mock_open_to_str(open_func), - """{\n "mocked": "data"\n}""" - ) + self.assertEqual(mock_open_to_str(open_func), """{\n "mocked": "data"\n}""") + + +class Test_ScannerTool(unittest.TestCase): + def test_load(self): + st = mock.Mock() + st.sdcs = [mock.Mock(), mock.Mock()] + fdroidserver.scanner.ScannerTool.load(st) + st.sdcs[0].load.assert_called_once_with() + st.sdcs[1].load.assert_called_once_with() class Test_main(unittest.TestCase): @@ -630,10 +749,12 @@ if __name__ == "__main__": (fdroidserver.common.options, args) = parser.parse_args(['--verbose']) newSuite = unittest.TestSuite() - newSuite.addTests([ - unittest.makeSuite(ScannerTest), - unittest.makeSuite(Test_scan_binary), - unittest.makeSuite(Test_SignatureDataController), - unittest.makeSuite(Test_main), - ]) + newSuite.addTests( + [ + unittest.makeSuite(ScannerTest), + unittest.makeSuite(Test_scan_binary), + unittest.makeSuite(Test_SignatureDataController), + unittest.makeSuite(Test_main), + ] + ) unittest.main(failfast=False) From 9ccf583061e22b04bf3ee48ab4827a4b3d3ebe43 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 6 Oct 2022 13:55:38 +0000 Subject: [PATCH 21/21] Apply 3 suggestion(s) to 2 file(s) --- fdroidserver/common.py | 2 +- fdroidserver/scanner.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 0f001019..421e6a54 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -325,7 +325,7 @@ def fill_config_defaults(thisconfig): def get_config(opts=None): - """Get config instace. This function takes care of initaling config data before returning it.""" + """Get config instace. This function takes care of initializing config data before returning it.""" global config, options if config is not None: diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 39bffd67..e14aadae 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -353,7 +353,7 @@ def _get_tool(): """ Lazy loading function for getting a ScannerTool instance. - ScannerTool initialization need to access `common.config` values. Those are only available after initialization through `common.read_config()` So this factory assumes config was called at an erlier point in time + ScannerTool initialization need to access `common.config` values. Those are only available after initialization through `common.read_config()`. So this factory assumes config was called at an erlier point in time. """ if not scanner._SCANNER_TOOL: scanner._SCANNER_TOOL = ScannerTool() @@ -682,7 +682,7 @@ def main(): parser.add_argument("--json", action="store_true", default=False, help=_("Output JSON to stdout.")) parser.add_argument("--refresh", "-r", action="store_true", default=False, - help=_("fetach the latest version of signatures from the web")) + help=_("fetch the latest version of signatures from the web")) metadata.add_metadata_arguments(parser) options = parser.parse_args() metadata.warnings_action = options.W