scanner: add a simple scan for blacklisted classes after build step

add com.android.billing to blacklist, see
https://gitlab.com/fdroid/fdroiddata/-/issues/2070#note_360611289

completion/bash-completion

@@ -83,7 +83,7 @@ __complete_options() {
 __complete_build() {
 	opts="-v -q -l -s -t -f -a -w"
 
-	lopts="--verbose --quiet --latest --stop --test --server --reset-server --skip-scan --no-tarball --force --all --wiki --no-refresh"
+	lopts="--verbose --quiet --latest --stop --test --server --reset-server --skip-scan --scan-binary --no-tarball --force --all --wiki --no-refresh"
 	case "${prev}" in
 		:)
 			__vercode

examples/config.py

@@ -50,6 +50,9 @@
 # Defaults to using an internal gradle wrapper (gradlew-fdroid).
 # gradle = "gradle"
 
+# Always scan the APKs produced by `fdroid build` for known non-free classes
+# scan_binary = True
+
 # Set the maximum age (in days) of an index that a client should accept from
 # this repo. Setting it to 0 or not setting it at all disables this
 # functionality. If you do set this to a non-zero value, you need to ensure

fdroidserver/build.py

@@ -806,6 +806,9 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
                                   " Expected: '%s' / '%s'")
                                  % (version, str(vercode), build.versionName,
                                     str(build.versionCode)))
+        if (options.scan_binary or config.get('scan_binary')) and not options.skipscan:
+            if scanner.scan_binary(src):
+                raise BuildException("Found blacklisted packages in final apk!")
 
     # Copy the unsigned apk to our destination directory for further
     # processing (by publish.py)...
@@ -899,6 +902,8 @@ def parse_commandline():
                         help=argparse.SUPPRESS)
     parser.add_argument("--skip-scan", dest="skipscan", action="store_true", default=False,
                         help=_("Skip scanning the source code for binaries and other problems"))
+    parser.add_argument("--scan-binary", action="store_true", default=False,
+                        help=_("Scan the resulting APK(s) for known non-free classes."))
     parser.add_argument("--no-tarball", dest="notarball", action="store_true", default=False,
                         help=_("Don't create a source tarball, useful when testing a build"))
     parser.add_argument("--no-refresh", dest="refresh", action="store_false", default=True,

fdroidserver/common.py

@@ -114,6 +114,7 @@ default_config = {
     'build_tools': MINIMUM_AAPT_VERSION,
     'force_build_tools': False,
     'java_paths': None,
+    'scan_binary': False,
     'ant': "ant",
     'mvn3': "mvn",
     'gradle': os.path.join(FDROID_PATH, 'gradlew-fdroid'),
@@ -432,6 +433,7 @@ def find_sdk_tools_cmd(cmd):
         sdk_tools = os.path.join(config['sdk_path'], 'tools')
         if os.path.exists(sdk_tools):
             tooldirs.append(sdk_tools)
+            tooldirs.append(os.path.join(sdk_tools, 'bin'))
         sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
         if os.path.exists(sdk_platform_tools):
             tooldirs.append(sdk_platform_tools)

fdroidserver/scanner.py

@@ -58,6 +58,36 @@ def get_gradle_compile_commands(build):
     return [re.compile(r'\s*' + c, re.IGNORECASE) for c in commands]
 
 
+def scan_binary(apkfile):
+    usual_suspects = {
+        # The `apkanalyzer dex packages` output looks like this:
+        # M d 1   1       93      <packagename> <other stuff>
+        # The first column has P/C/M/F for package, class, methos or field
+        # The second column has x/k/r/d for removed, kept, referenced and defined.
+        # We already filter for defined only in the apkanalyzer call. 'r' will be
+        # for things referenced but not distributed in the apk.
+        exp: re.compile(r'.[\s]*d[\s]*[0-9]*[\s]*[0-9*][\s]*[0-9]*[\s]*' + exp, re.IGNORECASE) for exp in [
+            r'(com\.google\.firebase[^\s]*)',
+            r'(com\.google\.android\.gms[^\s]*)',
+            r'(com\.google\.tagmanager[^\s]*)',
+            r'(com\.google\.analytics[^\s]*)',
+            r'(com\.android\.billing[^\s]*)',
+        ]
+    }
+    logging.info("Scanning APK for known non-free classes.")
+    result = common.SdkToolsPopen(["apkanalyzer", "dex", "packages", "--defined-only", apkfile], output=False)
+    problems = 0
+    for suspect, regexp in usual_suspects.items():
+        matches = regexp.findall(result.output)
+        if matches:
+            for m in set(matches):
+                logging.debug("Found class '%s'" % m)
+            problems += 1
+    if problems:
+        logging.critical("Found problems in %s" % apkfile)
+    return problems
+
+
 def scan_source(build_dir, build=metadata.Build()):
     """Scan the source code in the given directory (and all subdirectories)
     and return the number of fatal problems encountered
@@ -308,7 +338,6 @@ def scan_source(build_dir, build=metadata.Build()):
 
 
 def main():
-
     global config, options, json_per_build
 
     # Parse command line...

tests/build.TestCase

@@ -131,7 +131,9 @@ class BuildTest(unittest.TestCase):
         config = dict()
         fdroidserver.common.fill_config_defaults(config)
         fdroidserver.common.config = config
+        fdroidserver.build.config = config
         fdroidserver.build.options = mock.Mock()
+        fdroidserver.build.options.scan_binary = False
         fdroidserver.build.options.notarball = True
         fdroidserver.build.options.skipscan = False
 

tests/scanner.TestCase

@@ -194,8 +194,10 @@ class ScannerTest(unittest.TestCase):
         config = dict()
         fdroidserver.common.fill_config_defaults(config)
         fdroidserver.common.config = config
+        fdroidserver.build.config = config
         fdroidserver.build.options = mock.Mock()
         fdroidserver.build.options.json = False
+        fdroidserver.build.options.scan_binary = False
         fdroidserver.build.options.notarball = True
         fdroidserver.build.options.skipscan = False
         fdroidserver.scanner.options = fdroidserver.build.options