diff --git a/.gitignore b/.gitignore index 44799e9a..98c5b90b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ *.box TAGS .idea +.ropeproject/ # files generated by build build/ @@ -70,4 +71,4 @@ makebuildserver.config.py locale/*/LC_MESSAGES/fdroidserver.mo # sphinx -public/ \ No newline at end of file +public/ diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 7f4aa846..05b9c2b8 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -36,6 +36,7 @@ from . import _ from . import common from . import metadata from .exception import BuildException, VCSException +from . import scanner config = None options = None @@ -144,7 +145,7 @@ def get_embedded_classes(apkfile, depth=0): # taken from exodus_core -def _compile_signatures(signatures): +def _exodus_compile_signatures(signatures): """ Compiles the regex associated to each signature, in order to speed up the trackers detection. @@ -161,7 +162,7 @@ def _compile_signatures(signatures): # taken from exodus_core -def load_trackers_signatures(): +def load_exodus_trackers_signatures(): """ Load trackers signatures from the official Exodus database. @@ -178,7 +179,7 @@ def load_trackers_signatures(): ) ) logging.debug('{} trackers signatures loaded'.format(len(signatures))) - return signatures, _compile_signatures(signatures) + return signatures, scanner._exodus_compile_signatures(signatures) def scan_binary(apkfile, extract_signatures=None): @@ -514,7 +515,7 @@ def main(): # Parse command line... parser = ArgumentParser( - usage="%(prog)s [options] [APPID[:VERCODE] path/to.apk ...]" + usage="%(prog)s [options] [(APPID[:VERCODE] | path/to.apk) ...]" ) common.setup_global_opts(parser) parser.add_argument("appid", nargs='*', help=_("application ID with optional versionCode in the form APPID[:VERCODE]")) @@ -544,12 +545,12 @@ def main(): exodus = [] if options.exodus: - exodus = load_trackers_signatures() + exodus = load_exodus_trackers_signatures() appids = [] for apk in options.appid: if os.path.isfile(apk): - count = scan_binary(apk, exodus) + count = scanner.scan_binary(apk, exodus) if count > 0: logging.warning( _('Scanner found {count} problems in {apk}:').format( @@ -564,6 +565,7 @@ def main(): return # Read all app and srclib metadata + allapps = metadata.read_metadata() apps = common.read_app_args(appids, allapps, True) diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index 5e4c0666..2e5f6305 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -13,6 +13,8 @@ import textwrap import unittest import uuid import yaml +import collections +import pathlib from unittest import mock localmodule = os.path.realpath( @@ -26,6 +28,7 @@ import fdroidserver.build import fdroidserver.common import fdroidserver.metadata import fdroidserver.scanner +from testcommon import TmpCwd class ScannerTest(unittest.TestCase): @@ -204,34 +207,6 @@ class ScannerTest(unittest.TestCase): self.assertTrue(f in files['infos'], f + ' should be removed with an info message') - def test_scan_binary(self): - config = dict() - fdroidserver.common.fill_config_defaults(config) - fdroidserver.common.config = config - fdroidserver.common.options = mock.Mock() - fdroidserver.common.options.verbose = False - - apkfile = os.path.join(self.basedir, 'no_targetsdk_minsdk1_unsigned.apk') - self.assertEqual( - 0, - fdroidserver.scanner.scan_binary(apkfile), - 'Found false positives in binary', - ) - fdroidserver.scanner.CODE_SIGNATURES["java/lang/Object"] = re.compile( - r'.*java/lang/Object', re.IGNORECASE | re.UNICODE - ) - self.assertEqual( - 1, - fdroidserver.scanner.scan_binary(apkfile), - 'Did not find bad code signature in binary', - ) - apkfile = os.path.join(self.basedir, 'apk.embedded_1.apk') - self.assertEqual( - 1, - fdroidserver.scanner.scan_binary(apkfile), - 'Did not find bad code signature in binary', - ) - def test_build_local_scanner(self): """`fdroid build` calls scanner functions, test them here""" testdir = tempfile.mkdtemp( @@ -338,6 +313,173 @@ class ScannerTest(unittest.TestCase): self.assertEqual(0, count, 'there should be this many errors') +class Test_scan_binary(unittest.TestCase): + + def setUp(self): + self.basedir = os.path.join(localmodule, 'tests') + config = dict() + fdroidserver.common.fill_config_defaults(config) + fdroidserver.common.config = config + fdroidserver.common.options = mock.Mock() + + def test_code_signature_match(self): + apkfile = os.path.join(self.basedir, 'no_targetsdk_minsdk1_unsigned.apk') + with mock.patch("fdroidserver.scanner.CODE_SIGNATURES", {"java/lang/Object": re.compile( + r'.*java/lang/Object', re.IGNORECASE | re.UNICODE + )}): + self.assertEqual( + 1, + fdroidserver.scanner.scan_binary(apkfile), + "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner.CODE_SIGNATURES.values(), apkfile), + ) + + @unittest.skipIf( + sys.version_info < (3, 9), + "Our implementation for traversing zip files will silently fail to work" + "on older python versions, also see: " + "https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1110#note_932026766" + ) + def test_bottom_level_embedded_apk_code_signature(self): + apkfile = os.path.join(self.basedir, 'apk.embedded_1.apk') + with mock.patch("fdroidserver.scanner.CODE_SIGNATURES", {"org/bitbucket/tickytacky/mirrormirror/MainActivity": re.compile( + r'.*org/bitbucket/tickytacky/mirrormirror/MainActivity', re.IGNORECASE | re.UNICODE + )}): + self.assertEqual( + 1, + fdroidserver.scanner.scan_binary(apkfile), + "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner.CODE_SIGNATURES.values(), apkfile), + ) + + def test_top_level_signature_embedded_apk_present(self): + apkfile = os.path.join(self.basedir, 'apk.embedded_1.apk') + with mock.patch("fdroidserver.scanner.CODE_SIGNATURES", {"org/fdroid/ci/BuildConfig": re.compile( + r'.*org/fdroid/ci/BuildConfig', re.IGNORECASE | re.UNICODE + )}): + self.assertEqual( + 1, + fdroidserver.scanner.scan_binary(apkfile), + "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner.CODE_SIGNATURES.values(), apkfile), + ) + + def test_no_match(self): + apkfile = os.path.join(self.basedir, 'no_targetsdk_minsdk1_unsigned.apk') + result = fdroidserver.scanner.scan_binary(apkfile) + self.assertEqual(0, result, "Found false positives in binary '{}'".format(apkfile)) + + +class Test__exodus_compile_signatures(unittest.TestCase): + + def setUp(self): + self.m1 = mock.Mock() + self.m1.code_signature = r"^random\sregex$" + self.m2 = mock.Mock() + self.m2.code_signature = r"^another.+regex$" + self.mock_sigs = [self.m1, self.m2] + + def test_ok(self): + result = fdroidserver.scanner._exodus_compile_signatures(self.mock_sigs) + self.assertListEqual(result, [ + re.compile(self.m1.code_signature), + re.compile(self.m2.code_signature), + ]) + + def test_not_iterable(self): + result = fdroidserver.scanner._exodus_compile_signatures(123) + self.assertListEqual(result, []) + + +class Test_load_exodus_trackers_signatures(unittest.TestCase): + + def setUp(self): + self.requests_ret = mock.Mock() + self.requests_ret.json = mock.Mock(return_value={ + "trackers": { + "1": { + "id": 1, + "name": "Steyer Puch 1", + "description": "blah blah blah", + "creation_date": "1956-01-01", + "code_signature": "com.puch.|com.steyer.", + "network_signature": "pst\\.com", + "website": "https://pst.com", + "categories": ["tracker"], + "documentation": [], + }, + "2": { + "id": 2, + "name": "Steyer Puch 2", + "description": "blah blah blah", + "creation_date": "1956-01-01", + "code_signature": "com.puch.|com.steyer.", + "network_signature": "pst\\.com", + "website": "https://pst.com", + "categories": ["tracker"], + "documentation": [], + } + }, + }) + self.requests_func = mock.Mock(return_value=self.requests_ret) + self.compilesig_func = mock.Mock(return_value="mocked return value") + + def test_ok(self): + with mock.patch("requests.get", self.requests_func), mock.patch( + "fdroidserver.scanner._exodus_compile_signatures", self.compilesig_func + ): + result_sigs, result_regex = fdroidserver.scanner.load_exodus_trackers_signatures() + self.requests_func.assert_called_once_with("https://reports.exodus-privacy.eu.org/api/trackers") + self.assertEqual(len(result_sigs), 2) + self.assertListEqual([1, 2], sorted([x.id for x in result_sigs])) + + self.compilesig_func.assert_called_once_with(result_sigs) + self.assertEqual(result_regex, "mocked return value") + + +class Test_main(unittest.TestCase): + + def setUp(self): + self.args = ["com.example.app", "local/additional.apk", "another.apk"] + self.exit_func = mock.Mock() + self.read_app_args_func = mock.Mock(return_value={}) + self.scan_binary_func = mock.Mock(return_value=0) + + def test_parsing_appid(self): + """ + This test verifies that app id get parsed correctly + (doesn't test how they get processed) + """ + self.args = ["com.example.app"] + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir), mock.patch( + "sys.exit", self.exit_func + ), mock.patch("sys.argv", ["fdroid scanner", *self.args]), mock.patch( + "fdroidserver.common.read_app_args", self.read_app_args_func + ), mock.patch("fdroidserver.scanner.scan_binary", self.scan_binary_func): + fdroidserver.scanner.main() + + self.exit_func.assert_not_called() + self.read_app_args_func.assert_called_once_with( + ['com.example.app'], collections.OrderedDict(), True + ) + self.scan_binary_func.assert_not_called() + + def test_parsing_apkpath(self): + """ + This test verifies that apk paths get parsed correctly + (doesn't test how they get processed) + """ + self.args = ["local.application.apk"] + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir), mock.patch( + "sys.exit", self.exit_func + ), mock.patch("sys.argv", ["fdroid scanner", *self.args]), mock.patch( + "fdroidserver.common.read_app_args", self.read_app_args_func + ), mock.patch("fdroidserver.scanner.scan_binary", self.scan_binary_func): + pathlib.Path(self.args[0]).touch() + fdroidserver.scanner.main() + + self.exit_func.assert_not_called() + self.read_app_args_func.assert_not_called() + self.scan_binary_func.assert_called_once_with('local.application.apk', []) + + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) @@ -352,5 +494,11 @@ if __name__ == "__main__": (fdroidserver.common.options, args) = parser.parse_args(['--verbose']) newSuite = unittest.TestSuite() - newSuite.addTest(unittest.makeSuite(ScannerTest)) + newSuite.addTests([ + unittest.makeSuite(ScannerTest), + unittest.makeSuite(Test_scan_binary), + unittest.makeSuite(Test__exodus_compile_signatures), + unittest.makeSuite(Test_load_exodus_trackers_signatures), + unittest.makeSuite(Test_main), + ]) unittest.main(failfast=False)