Merge branch 'verify-handle-corrupt-json' into 'master'

verify: handle corrupt verified.json

See merge request fdroid/fdroidserver!1580

fdroidserver/verify.py

@@ -80,6 +80,30 @@ def _add_diffoscope_info(d):
         pass
 
 
+def get_verified_json(path):
+    """Get the full collection of reports that is written out to verified.json."""
+    if os.path.exists(path):
+        try:
+            with open(path) as fp:
+                return json.load(fp)
+        except Exception as e:
+            logging.info(f'{path}: {e}')
+
+    data = OrderedDict()
+    data['packages'] = OrderedDict()
+
+    for f in glob.glob(os.path.join(os.path.dirname(path), '*.apk.json')):
+        with open(f) as fp:
+            reports = json.load(fp)
+        for report in reports.values():
+            packageName = report['local']['packageName']
+            if packageName not in data['packages']:
+                data['packages'][packageName] = []
+            data['packages'][packageName].append(report)
+
+    return data
+
+
 def write_json_report(url, remote_apk, unsigned_apk, compare_result):
     """Write out the results of the verify run to JSON.
 
@@ -122,12 +146,7 @@ def write_json_report(url, remote_apk, unsigned_apk, compare_result):
 
     if output['verified']:
         jsonfile = 'unsigned/verified.json'
-        if os.path.exists(jsonfile):
+        data = get_verified_json(jsonfile)
-            with open(jsonfile) as fp:
-                data = json.load(fp)
-        else:
-            data = OrderedDict()
-            data['packages'] = OrderedDict()
         packageName = output['local']['packageName']
 
         if packageName not in data['packages']:

tests/org.fdroid.fdroid_1019051.apk.json

@@ -0,0 +1,43 @@
+{
+  "1708238023.6572325": {
+    "diffoscope": {
+      "Available-in-Debian-packages": [
+        "coreboot-utils",
+        "ghc",
+        "libxmlb-dev",
+        "radare2"
+      ],
+      "External-Tools-Required": [
+        "cbfstool",
+        "ghc",
+        "lipo",
+        "otool",
+        "radare2",
+        "xb-tool"
+      ],
+      "Missing-Python-Modules": [
+        "pypdf",
+        "r2pipe"
+      ],
+      "VERSION": "240"
+    },
+    "local": {
+      "file": "unsigned/org.fdroid.fdroid_1019051.apk",
+      "packageName": "org.fdroid.fdroid",
+      "sha256": "0eec78236ec5ebb8f416c611717bd659b75d2b6600ef71a50c922efc99dbdca2",
+      "timestamp": 1708238023.6572325,
+      "versionCode": 1019051,
+      "versionName": "1.19.1"
+    },
+    "remote": {
+      "file": "tmp/org.fdroid.fdroid_1019051.apk",
+      "packageName": "org.fdroid.fdroid",
+      "sha256": "162cb14b93bd9b665fff4256b4fc91cfe75fb72335a02b1d0febe606220b50f4",
+      "timestamp": 1715356428.522411,
+      "versionCode": 1019051,
+      "versionName": "1.19.1"
+    },
+    "url": "https://f-droid.org/repo/org.fdroid.fdroid_1019051.apk",
+    "verified": true
+  }
+}

tests/test_verify.py

@@ -44,10 +44,53 @@ class VerifyTest(unittest.TestCase):
         os.chdir(self.tempdir.name)
         self.repodir = Path('repo')
         self.repodir.mkdir()
+        self.apk_reports_json = basedir / 'org.fdroid.fdroid_1019051.apk.json'
 
     def tearDown(self):
         self.tempdir.cleanup()
 
+    def test_get_verified_json_creation(self):
+        self.assertEqual({'packages': {}}, verify.get_verified_json('does-not-exist'))
+
+    def test_get_verified_json_existing(self):
+        f = 'verified.json'
+        reports = {'packages': {'placeholder': {}}}
+        with open(f, 'w') as fp:
+            json.dump(reports, fp)
+        self.assertEqual(reports, verify.get_verified_json(f))
+
+    def test_get_verified_json_pull_in_one_report(self):
+        shutil.copy(self.apk_reports_json, self.tempdir.name)
+        with open(self.apk_reports_json) as fp:
+            reports = json.load(fp)
+        self.assertEqual(
+            {'packages': {'org.fdroid.fdroid': [reports['1708238023.6572325']]}},
+            verify.get_verified_json('does-not-exist'),
+        )
+
+    def test_get_verified_json_ignore_corrupt(self):
+        f = 'verified.json'
+        with open(f, 'w') as fp:
+            fp.write("""{"packages": {"placeholder": {""")
+        shutil.copy(self.apk_reports_json, self.tempdir.name)
+        with open(self.apk_reports_json) as fp:
+            reports = json.load(fp)
+        self.assertEqual(
+            {'packages': {'org.fdroid.fdroid': [reports['1708238023.6572325']]}},
+            verify.get_verified_json(f),
+        )
+
+    def test_get_verified_json_ignore_apk_reports(self):
+        """When an intact verified.json exists, it should ignore the .apk.json reports."""
+        f = 'verified.json'
+        placeholder = {'packages': {'placeholder': {}}}
+        with open(f, 'w') as fp:
+            json.dump(placeholder, fp)
+        shutil.copy(self.apk_reports_json, self.tempdir.name)
+        with open(self.apk_reports_json) as fp:
+            json.load(fp)
+        self.assertEqual(placeholder, verify.get_verified_json(f))
+
     @patch('fdroidserver.common.sha256sum')
     def test_write_json_report(self, sha256sum):
         sha256sum.return_value = (