Merge branch 'scanner-json' into 'master'

`fdroid scanner --json` for machine readable output

See merge request fdroid/fdroidserver!748

fdroidserver/metadata.py

@@ -763,6 +763,9 @@ def parse_yaml_srclib(metadatapath):
     with open(metadatapath, "r", encoding="utf-8") as f:
         try:
             data = yaml.load(f, Loader=SafeLoader)
+            if type(data) is not dict:
+                raise yaml.error.YAMLError(_('{file} is blank or corrupt!')
+                                           .format(file=metadatapath))
         except yaml.error.YAMLError as e:
             warn_or_exception(_("Invalid srclib metadata: could not "
                                 "parse '{file}'")

fdroidserver/scanner.py

@@ -16,8 +16,10 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import json
 import os
 import re
+import sys
 import traceback
 from argparse import ArgumentParser
 import logging
@@ -31,6 +33,8 @@ from .exception import BuildException, VCSException
 config = None
 options = None
 
+json_per_build = None
+
 
 def get_gradle_compile_commands(build):
     compileCommands = ['compile',
@@ -141,10 +145,12 @@ def scan_source(build_dir, build=metadata.Build()):
 
     def ignoreproblem(what, path_in_build_dir):
         logging.info('Ignoring %s at %s' % (what, path_in_build_dir))
+        json_per_build['infos'].append([what, path_in_build_dir])
         return 0
 
     def removeproblem(what, path_in_build_dir, filepath):
         logging.info('Removing %s at %s' % (what, path_in_build_dir))
+        json_per_build['infos'].append([what, path_in_build_dir])
         os.remove(filepath)
         return 0
 
@@ -152,13 +158,17 @@ def scan_source(build_dir, build=metadata.Build()):
         if toignore(path_in_build_dir):
             return
         logging.warn('Found %s at %s' % (what, path_in_build_dir))
+        json_per_build['warnings'].append([what, path_in_build_dir])
 
     def handleproblem(what, path_in_build_dir, filepath):
         if toignore(path_in_build_dir):
             return ignoreproblem(what, path_in_build_dir)
         if todelete(path_in_build_dir):
             return removeproblem(what, path_in_build_dir, filepath)
-        logging.error('Found %s at %s' % (what, path_in_build_dir))
+        if options.json:
+            json_per_build['errors'].append([what, path_in_build_dir])
+        if not options.json or options.verbose:
+            logging.error('Found %s at %s' % (what, path_in_build_dir))
         return 1
 
     def is_executable(path):
@@ -249,7 +259,8 @@ def scan_source(build_dir, build=metadata.Build()):
                 for i, line in enumerate(lines):
                     if is_used_by_gradle(line):
                         for name in suspects_found(line):
-                            count += handleproblem('usual suspect \'%s\' at line %d' % (name, i + 1), path_in_build_dir, filepath)
+                            count += handleproblem("usual suspect \'%s\'" % (name),
+                                                   path_in_build_dir, filepath)
                 noncomment_lines = [line for line in lines if not common.gradle_comment.match(line)]
                 joined = re.sub(r'[\n\r\s]+', ' ', ' '.join(noncomment_lines))
                 for m in gradle_mavenrepo.finditer(joined):
@@ -280,7 +291,7 @@ def scan_source(build_dir, build=metadata.Build()):
 
 def main():
 
-    global config, options
+    global config, options, json_per_build
 
     # Parse command line...
     parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
@@ -288,10 +299,19 @@ def main():
     parser.add_argument("appid", nargs='*', help=_("applicationId with optional versionCode in the form APPID[:VERCODE]"))
     parser.add_argument("-f", "--force", action="store_true", default=False,
                         help=_("Force scan of disabled apps and builds."))
+    parser.add_argument("--json", action="store_true", default=False,
+                        help=_("Output JSON to stdout."))
     metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
     metadata.warnings_action = options.W
 
+    json_output = dict()
+    if options.json:
+        if options.verbose:
+            logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
+        else:
+            logging.getLogger().setLevel(logging.ERROR)
+
     config = common.read_config(options)
 
     # Read all app and srclib metadata
@@ -309,8 +329,11 @@ def main():
 
     for appid, app in apps.items():
 
+        json_per_appid = dict()
+
         if app.Disabled and not options.force:
             logging.info(_("Skipping {appid}: disabled").format(appid=appid))
+            json_per_appid = json_per_appid['infos'].append('Skipping: disabled')
             continue
 
         try:
@@ -321,20 +344,23 @@ def main():
 
             if app.builds:
                 logging.info(_("Processing {appid}").format(appid=appid))
+                # Set up vcs interface and make sure we have the latest code...
+                vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
             else:
                 logging.info(_("{appid}: no builds specified, running on current source state")
                              .format(appid=appid))
+                json_per_build = {'errors': [], 'warnings': [], 'infos': []}
+                json_per_appid['current-source-state'] = json_per_build
                 count = scan_source(build_dir)
                 if count > 0:
                     logging.warn(_('Scanner found {count} problems in {appid}:')
                                  .format(count=count, appid=appid))
                     probcount += count
-                continue
+                app.builds = []
-
-            # Set up vcs interface and make sure we have the latest code...
-            vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
 
             for build in app.builds:
+                json_per_build = {'errors': [], 'warnings': [], 'infos': []}
+                json_per_appid[build.versionCode] = json_per_build
 
                 if build.disable and not options.force:
                     logging.info("...skipping version %s - %s" % (
@@ -365,8 +391,16 @@ def main():
                 appid, traceback.format_exc()))
             probcount += 1
 
+        for k, v in json_per_appid.items():
+            if len(v['errors']) or len(v['warnings']) or len(v['infos']):
+                json_output[appid] = json_per_appid
+                break
+
     logging.info(_("Finished"))
-    print(_("%d problems found") % probcount)
+    if options.json:
+        print(json.dumps(json_output))
+    else:
+        print(_("%d problems found") % probcount)
 
 
 if __name__ == "__main__":

tests/metadata.TestCase

@@ -227,6 +227,19 @@ class MetadataTest(unittest.TestCase):
             with self.assertRaises(MetaDataException):
                 fdroidserver.metadata.parse_yaml_metadata(mf, {})
 
+    def test_parse_yaml_srclib_corrupt_file(self):
+        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+        srclibfile = os.path.join(testdir, 'srclib', 'mock.yml')
+        os.mkdir(os.path.dirname(srclibfile))
+        with open(srclibfile, 'w') as fp:
+            fp.write(textwrap.dedent("""
+            - RepoType: git
+            - Repo: https://github.com/realm/realm-js.git
+            """))
+        with mock.patch('fdroidserver.metadata.warnings_action', 'error'):
+            with self.assertRaises(MetaDataException):
+                fdroidserver.metadata.parse_yaml_srclib(srclibfile)
+
     def test_write_yaml_with_placeholder_values(self):
         mf = io.StringIO()
 

tests/scanner.TestCase

@@ -26,6 +26,8 @@ class ScannerTest(unittest.TestCase):
         self.basedir = os.path.join(localmodule, 'tests')
 
     def test_scan_source_files(self):
+        fdroidserver.scanner.options = type('', (), {})()
+        fdroidserver.scanner.options.json = False
         source_files = os.path.join(self.basedir, 'source-files')
         projects = {
             'cn.wildfirechat.chat': 4,