From 5b1b1d12a1f0ce394ba68647c70835adc407a0c2 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 16 May 2024 11:53:00 +0200 Subject: [PATCH] verify: handle corrupt verified.json verified.json can get quite large on verification.f-droid.org, and for some unknown reason, it sometimes corrupts it when writing it out. All the data is already available in all the other JSON files, so this just automatically reconstructs it. Its a hack, but it took me much less time than I've already spent trying to troubleshoot why it writes out corrupt verified.json. --- fdroidserver/verify.py | 31 +++++++++++++---- tests/org.fdroid.fdroid_1019051.apk.json | 43 ++++++++++++++++++++++++ tests/test_verify.py | 43 ++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 6 deletions(-) create mode 100644 tests/org.fdroid.fdroid_1019051.apk.json diff --git a/fdroidserver/verify.py b/fdroidserver/verify.py index 9ed46407..63540147 100644 --- a/fdroidserver/verify.py +++ b/fdroidserver/verify.py @@ -80,6 +80,30 @@ def _add_diffoscope_info(d): pass +def get_verified_json(path): + """Get the full collection of reports that is written out to verified.json.""" + if os.path.exists(path): + try: + with open(path) as fp: + return json.load(fp) + except Exception as e: + logging.info(f'{path}: {e}') + + data = OrderedDict() + data['packages'] = OrderedDict() + + for f in glob.glob(os.path.join(os.path.dirname(path), '*.apk.json')): + with open(f) as fp: + reports = json.load(fp) + for report in reports.values(): + packageName = report['local']['packageName'] + if packageName not in data['packages']: + data['packages'][packageName] = [] + data['packages'][packageName].append(report) + + return data + + def write_json_report(url, remote_apk, unsigned_apk, compare_result): """Write out the results of the verify run to JSON. @@ -122,12 +146,7 @@ def write_json_report(url, remote_apk, unsigned_apk, compare_result): if output['verified']: jsonfile = 'unsigned/verified.json' - if os.path.exists(jsonfile): - with open(jsonfile) as fp: - data = json.load(fp) - else: - data = OrderedDict() - data['packages'] = OrderedDict() + data = get_verified_json(jsonfile) packageName = output['local']['packageName'] if packageName not in data['packages']: diff --git a/tests/org.fdroid.fdroid_1019051.apk.json b/tests/org.fdroid.fdroid_1019051.apk.json new file mode 100644 index 00000000..e71c1704 --- /dev/null +++ b/tests/org.fdroid.fdroid_1019051.apk.json @@ -0,0 +1,43 @@ +{ + "1708238023.6572325": { + "diffoscope": { + "Available-in-Debian-packages": [ + "coreboot-utils", + "ghc", + "libxmlb-dev", + "radare2" + ], + "External-Tools-Required": [ + "cbfstool", + "ghc", + "lipo", + "otool", + "radare2", + "xb-tool" + ], + "Missing-Python-Modules": [ + "pypdf", + "r2pipe" + ], + "VERSION": "240" + }, + "local": { + "file": "unsigned/org.fdroid.fdroid_1019051.apk", + "packageName": "org.fdroid.fdroid", + "sha256": "0eec78236ec5ebb8f416c611717bd659b75d2b6600ef71a50c922efc99dbdca2", + "timestamp": 1708238023.6572325, + "versionCode": 1019051, + "versionName": "1.19.1" + }, + "remote": { + "file": "tmp/org.fdroid.fdroid_1019051.apk", + "packageName": "org.fdroid.fdroid", + "sha256": "162cb14b93bd9b665fff4256b4fc91cfe75fb72335a02b1d0febe606220b50f4", + "timestamp": 1715356428.522411, + "versionCode": 1019051, + "versionName": "1.19.1" + }, + "url": "https://f-droid.org/repo/org.fdroid.fdroid_1019051.apk", + "verified": true + } +} diff --git a/tests/test_verify.py b/tests/test_verify.py index 041fc82c..30fde2ef 100755 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -44,10 +44,53 @@ class VerifyTest(unittest.TestCase): os.chdir(self.tempdir.name) self.repodir = Path('repo') self.repodir.mkdir() + self.apk_reports_json = basedir / 'org.fdroid.fdroid_1019051.apk.json' def tearDown(self): self.tempdir.cleanup() + def test_get_verified_json_creation(self): + self.assertEqual({'packages': {}}, verify.get_verified_json('does-not-exist')) + + def test_get_verified_json_existing(self): + f = 'verified.json' + reports = {'packages': {'placeholder': {}}} + with open(f, 'w') as fp: + json.dump(reports, fp) + self.assertEqual(reports, verify.get_verified_json(f)) + + def test_get_verified_json_pull_in_one_report(self): + shutil.copy(self.apk_reports_json, self.tempdir.name) + with open(self.apk_reports_json) as fp: + reports = json.load(fp) + self.assertEqual( + {'packages': {'org.fdroid.fdroid': [reports['1708238023.6572325']]}}, + verify.get_verified_json('does-not-exist'), + ) + + def test_get_verified_json_ignore_corrupt(self): + f = 'verified.json' + with open(f, 'w') as fp: + fp.write("""{"packages": {"placeholder": {""") + shutil.copy(self.apk_reports_json, self.tempdir.name) + with open(self.apk_reports_json) as fp: + reports = json.load(fp) + self.assertEqual( + {'packages': {'org.fdroid.fdroid': [reports['1708238023.6572325']]}}, + verify.get_verified_json(f), + ) + + def test_get_verified_json_ignore_apk_reports(self): + """When an intact verified.json exists, it should ignore the .apk.json reports.""" + f = 'verified.json' + placeholder = {'packages': {'placeholder': {}}} + with open(f, 'w') as fp: + json.dump(placeholder, fp) + shutil.copy(self.apk_reports_json, self.tempdir.name) + with open(self.apk_reports_json) as fp: + json.load(fp) + self.assertEqual(placeholder, verify.get_verified_json(f)) + @patch('fdroidserver.common.sha256sum') def test_write_json_report(self, sha256sum): sha256sum.return_value = (