From a0e3b01e94e156a79d5eac4c1f9fa5adfeb8f221 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 14 May 2020 15:02:19 +0200 Subject: [PATCH 1/2] metadata: parsed srclibs must always return a dict as the container --- fdroidserver/metadata.py | 3 +++ tests/metadata.TestCase | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 99764ac4..5531d369 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -763,6 +763,9 @@ def parse_yaml_srclib(metadatapath): with open(metadatapath, "r", encoding="utf-8") as f: try: data = yaml.load(f, Loader=SafeLoader) + if type(data) is not dict: + raise yaml.error.YAMLError(_('{file} is blank or corrupt!') + .format(file=metadatapath)) except yaml.error.YAMLError as e: warn_or_exception(_("Invalid srclib metadata: could not " "parse '{file}'") diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index d52cd991..c2c9aa57 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -227,6 +227,19 @@ class MetadataTest(unittest.TestCase): with self.assertRaises(MetaDataException): fdroidserver.metadata.parse_yaml_metadata(mf, {}) + def test_parse_yaml_srclib_corrupt_file(self): + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + srclibfile = os.path.join(testdir, 'srclib', 'mock.yml') + os.mkdir(os.path.dirname(srclibfile)) + with open(srclibfile, 'w') as fp: + fp.write(textwrap.dedent(""" + - RepoType: git + - Repo: https://github.com/realm/realm-js.git + """)) + with mock.patch('fdroidserver.metadata.warnings_action', 'error'): + with self.assertRaises(MetaDataException): + fdroidserver.metadata.parse_yaml_srclib(srclibfile) + def test_write_yaml_with_placeholder_values(self): mf = io.StringIO() From cdaf62e5d9ec2448e2aedf8989c1390d11f66370 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 14 May 2020 16:08:56 +0200 Subject: [PATCH 2/2] scanner: add --json option for outputting machine readable results * makes per-build entries in per-app entries * `fdroid scanner --json --verbose` will output logging messages to stderr * removed " at line N" from one message to make them uniform keys * this will be used in issuebot --- fdroidserver/scanner.py | 50 ++++++++++++++++++++++++++++++++++------- tests/scanner.TestCase | 2 ++ 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 3d5b700f..6c163698 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -16,8 +16,10 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import json import os import re +import sys import traceback from argparse import ArgumentParser import logging @@ -31,6 +33,8 @@ from .exception import BuildException, VCSException config = None options = None +json_per_build = None + def get_gradle_compile_commands(build): compileCommands = ['compile', @@ -141,10 +145,12 @@ def scan_source(build_dir, build=metadata.Build()): def ignoreproblem(what, path_in_build_dir): logging.info('Ignoring %s at %s' % (what, path_in_build_dir)) + json_per_build['infos'].append([what, path_in_build_dir]) return 0 def removeproblem(what, path_in_build_dir, filepath): logging.info('Removing %s at %s' % (what, path_in_build_dir)) + json_per_build['infos'].append([what, path_in_build_dir]) os.remove(filepath) return 0 @@ -152,13 +158,17 @@ def scan_source(build_dir, build=metadata.Build()): if toignore(path_in_build_dir): return logging.warn('Found %s at %s' % (what, path_in_build_dir)) + json_per_build['warnings'].append([what, path_in_build_dir]) def handleproblem(what, path_in_build_dir, filepath): if toignore(path_in_build_dir): return ignoreproblem(what, path_in_build_dir) if todelete(path_in_build_dir): return removeproblem(what, path_in_build_dir, filepath) - logging.error('Found %s at %s' % (what, path_in_build_dir)) + if options.json: + json_per_build['errors'].append([what, path_in_build_dir]) + if not options.json or options.verbose: + logging.error('Found %s at %s' % (what, path_in_build_dir)) return 1 def is_executable(path): @@ -249,7 +259,8 @@ def scan_source(build_dir, build=metadata.Build()): for i, line in enumerate(lines): if is_used_by_gradle(line): for name in suspects_found(line): - count += handleproblem('usual suspect \'%s\' at line %d' % (name, i + 1), path_in_build_dir, filepath) + count += handleproblem("usual suspect \'%s\'" % (name), + path_in_build_dir, filepath) noncomment_lines = [line for line in lines if not common.gradle_comment.match(line)] joined = re.sub(r'[\n\r\s]+', ' ', ' '.join(noncomment_lines)) for m in gradle_mavenrepo.finditer(joined): @@ -280,7 +291,7 @@ def scan_source(build_dir, build=metadata.Build()): def main(): - global config, options + global config, options, json_per_build # Parse command line... parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") @@ -288,10 +299,19 @@ def main(): parser.add_argument("appid", nargs='*', help=_("applicationId with optional versionCode in the form APPID[:VERCODE]")) parser.add_argument("-f", "--force", action="store_true", default=False, help=_("Force scan of disabled apps and builds.")) + parser.add_argument("--json", action="store_true", default=False, + help=_("Output JSON to stdout.")) metadata.add_metadata_arguments(parser) options = parser.parse_args() metadata.warnings_action = options.W + json_output = dict() + if options.json: + if options.verbose: + logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) + else: + logging.getLogger().setLevel(logging.ERROR) + config = common.read_config(options) # Read all app and srclib metadata @@ -309,8 +329,11 @@ def main(): for appid, app in apps.items(): + json_per_appid = dict() + if app.Disabled and not options.force: logging.info(_("Skipping {appid}: disabled").format(appid=appid)) + json_per_appid = json_per_appid['infos'].append('Skipping: disabled') continue try: @@ -321,20 +344,23 @@ def main(): if app.builds: logging.info(_("Processing {appid}").format(appid=appid)) + # Set up vcs interface and make sure we have the latest code... + vcs = common.getvcs(app.RepoType, app.Repo, build_dir) else: logging.info(_("{appid}: no builds specified, running on current source state") .format(appid=appid)) + json_per_build = {'errors': [], 'warnings': [], 'infos': []} + json_per_appid['current-source-state'] = json_per_build count = scan_source(build_dir) if count > 0: logging.warn(_('Scanner found {count} problems in {appid}:') .format(count=count, appid=appid)) probcount += count - continue - - # Set up vcs interface and make sure we have the latest code... - vcs = common.getvcs(app.RepoType, app.Repo, build_dir) + app.builds = [] for build in app.builds: + json_per_build = {'errors': [], 'warnings': [], 'infos': []} + json_per_appid[build.versionCode] = json_per_build if build.disable and not options.force: logging.info("...skipping version %s - %s" % ( @@ -365,8 +391,16 @@ def main(): appid, traceback.format_exc())) probcount += 1 + for k, v in json_per_appid.items(): + if len(v['errors']) or len(v['warnings']) or len(v['infos']): + json_output[appid] = json_per_appid + break + logging.info(_("Finished")) - print(_("%d problems found") % probcount) + if options.json: + print(json.dumps(json_output)) + else: + print(_("%d problems found") % probcount) if __name__ == "__main__": diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index cbaa9de2..72710dd5 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -26,6 +26,8 @@ class ScannerTest(unittest.TestCase): self.basedir = os.path.join(localmodule, 'tests') def test_scan_source_files(self): + fdroidserver.scanner.options = type('', (), {})() + fdroidserver.scanner.options.json = False source_files = os.path.join(self.basedir, 'source-files') projects = { 'cn.wildfirechat.chat': 4,