diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index 0555c498..149c0c0e 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -69,6 +69,7 @@ COMMANDS_INTERNAL = [ "exec", "pull", "push", + "schedule_verify", "up", ] diff --git a/fdroidserver/schedule_verify.py b/fdroidserver/schedule_verify.py new file mode 100644 index 00000000..a6876e64 --- /dev/null +++ b/fdroidserver/schedule_verify.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# schedule_verify.py - part of the FDroid server tools +# Copyright (C) 2024-2025, Hans-Christoph Steiner +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Schedule packages to build and verify. + +This creates a list of Application ID/Version Code that need the +verify process to be run. + +Since this is an internal command, the strings are not localized. + +""" + +import collections +import json +import logging +import os +import sys +import subprocess +from argparse import ArgumentParser +from pathlib import Path + +from . import common, index, metadata + + +def get_versions( + repo='https://f-droid.org/repo?fingerprint=43238D512C1E5EB2D6569F4A3AFBF5523418B82E0A3ED1552770ABB9A9C9CCAB', +): + """Get the list of versions that need to be built, newest first. + + Newest are built first because older versions that are still not + built are most likely to be failing builds. Repeating failed + builds mostly results in another failure. + + If there are versions with the same versionCode but different + signers, there will still only be a single entry returned. If the + rebuild matches one signature, that is enough. + + """ + data, _ignored = index.download_repo_index_v2(repo) + to_schedule = collections.defaultdict(list) + for appid, package in data['packages'].items(): + for version in package['versions'].values(): + versionCode = version['manifest']['versionCode'] + ext = common.get_file_extension(version['file']['name']) + jsonf = f'unsigned/{appid}_{versionCode}.{ext}.json' + if not os.path.exists(jsonf): + to_schedule[version['added']].append( + {'applicationId': appid, 'versionCode': versionCode} + ) + + ret = list() + for added in sorted(to_schedule, reverse=True): + for i in to_schedule[added]: + if i not in ret: + ret.append(i) + return ret + + +def get_scheduled(versions): + """Get versions that need to be built and there is local build metadata for it.""" + apps = metadata.read_metadata() + schedule = [] + for version in versions: + app = apps.get(version['applicationId']) + if app and not app.get("Disabled"): + for build in app.get("Builds", []): + versionCode = build['versionCode'] + if versionCode == version['versionCode'] and not build.get("disable"): + schedule.append( + { + "applicationId": app.id, + "versionCode": versionCode, + } + ) + return schedule + + +def sendchange(scheduled, verbose=False): + """Use `buildbot sendchange` to submit builds to the queue. + + This requires the automatically generated password to authenticate + to the buildbot instance, which is created at a static path by the + buildbot master: + https://gitlab.com/fdroid/buildbot/-/merge_requests/1 + + The passwd file's path is hardcoded in the server setup, which is + defined outside of fdroidserver. Think of the path as a variable + name for accessing a value from the filesystem. + + """ + git_revision = common.get_head_commit_id('.') + passwd = Path('/tmp/fdroid-buildbot-sendchange/passwd').read_text().strip() # nosec + for d in scheduled: + command = [ + 'buildbot', + 'sendchange', + '--master=127.0.0.1:9999', + f'--auth=fdroid:{passwd}', + '--branch=master', + '--repository=https://gitlab.com/fdroid/fdroiddata', + f'--revision={git_revision}', + '--category=verify', + f"--who={d['applicationId']}", + f"--project={d['applicationId']}", + f"--property=versionCode:{d['versionCode']}", + f"--property=packageName:{d['applicationId']}", + f"metadata/{d['applicationId']}.yml", + ] + if verbose: + logging.info(' '.join(command)) + subprocess.run(command, check=True) + + +def main(): + parser = ArgumentParser(description="Schedule packages to build and verify.") + common.setup_global_opts(parser) + parser.add_argument( + "url", + default='https://f-droid.org/repo?fingerprint=43238D512C1E5EB2D6569F4A3AFBF5523418B82E0A3ED1552770ABB9A9C9CCAB', + nargs='?', + help='Base URL to mirror, can include the index signing key using the query string: ?fingerprint=', + ) + parser.add_argument( + '--sendchange', + action="store_true", + help='Call buildbot sendchange with the results instead of printing to stdout.', + ) + options = common.parse_args(parser) + common.get_config() + common.set_console_logging(options.verbose) + + # TODO support priority list, and ignore list (see buildbot-sendchange-build) + if not os.path.exists('metadata'): + logging.error("'metadata/' directory does not exist!") + sys.exit(1) + + versions = get_versions(options.url) + scheduled = get_scheduled(versions) + + if options.sendchange: + sendchange(scheduled, options.verbose) + else: + print(json.dumps(scheduled)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_schedule_verify.py b/tests/test_schedule_verify.py new file mode 100755 index 00000000..83e4df39 --- /dev/null +++ b/tests/test_schedule_verify.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +import json +import os +import unittest + +from pathlib import Path +from unittest import mock, skipUnless + +from fdroidserver import common, schedule_verify +from .shared_test_code import mkdtemp + + +basedir = Path(__file__).parent + +FULL_LIST = [ + {'applicationId': 'org.maxsdkversion', 'versionCode': 4}, + {'applicationId': 'info.zwanenburg.caffeinetile', 'versionCode': 4}, + {'applicationId': 'no.min.target.sdk', 'versionCode': 987}, + {'applicationId': 'souch.smsbypass', 'versionCode': 9}, + {'applicationId': 'duplicate.permisssions', 'versionCode': 9999999}, + {'applicationId': 'com.politedroid', 'versionCode': 6}, + {'applicationId': 'com.politedroid', 'versionCode': 5}, + {'applicationId': 'com.politedroid', 'versionCode': 4}, + {'applicationId': 'com.politedroid', 'versionCode': 3}, + {'applicationId': 'obb.mainpatch.current', 'versionCode': 1619}, + {'applicationId': 'info.guardianproject.urzip', 'versionCode': 100}, + {'applicationId': 'obb.main.twoversions', 'versionCode': 1101617}, + {'applicationId': 'fake.ota.update', 'versionCode': 1234}, + {'applicationId': 'obb.main.twoversions', 'versionCode': 1101615}, + {'applicationId': 'obb.main.twoversions', 'versionCode': 1101613}, + {'applicationId': 'obb.main.oldversion', 'versionCode': 1444412523}, +] + + +def _mock(repo): # pylint: disable=unused-argument + indexf = basedir / 'repo' / 'index-v2.json' + return json.loads(indexf.read_text()), None + + +class Schedule_verifyTest(unittest.TestCase): + def setUp(self): + self._td = mkdtemp() + self.testdir = self._td.name + os.chdir(self.testdir) + os.mkdir('unsigned') + + def tearDown(self): + self._td.cleanup() + common.config = None + + +@skipUnless(False, 'This involves downloading the full index') +class Schedule_verify_main(Schedule_verifyTest): + def test_main_smokecheck(self): + schedule_verify.main() + + +class Schedule_verify_get_versions(Schedule_verifyTest): + def setUp(self): + super().setUp() + common.config = {'sdk_path': os.getenv('ANDROID_HOME')} + common.config['jarsigner'] = common.find_sdk_tools_cmd('jarsigner') + + @mock.patch('fdroidserver.index.download_repo_index_v2', _mock) + def test_get_versions_none_exist(self): + self.assertEqual(FULL_LIST, schedule_verify.get_versions()) + + @mock.patch('fdroidserver.index.download_repo_index_v2', _mock) + def test_get_versions_all_json_exist(self): + for d in FULL_LIST: + appid = d['applicationId'] + if appid == 'fake.ota.update': + ext = 'zip' + else: + ext = 'apk' + Path(f"unsigned/{appid}_{d['versionCode']}.{ext}.json").write_text('{}') + self.assertEqual([], schedule_verify.get_versions()) + + @mock.patch('fdroidserver.index.download_repo_index_v2', _mock) + def test_get_versions_all_apks_exist(self): + for d in FULL_LIST: + appid = d['applicationId'] + if appid != 'fake.ota.update': + Path(f"unsigned/{appid}_{d['versionCode']}.apk.json").write_text('{}') + self.assertEqual( + [{'applicationId': 'fake.ota.update', 'versionCode': 1234}], + schedule_verify.get_versions(), + ) + + +class Schedule_verify_get_scheduled(Schedule_verifyTest): + def setUp(self): + super().setUp() + os.chdir(basedir) + common.config = {'sdk_path': os.getenv('ANDROID_HOME')} + common.config['jarsigner'] = common.find_sdk_tools_cmd('jarsigner') + + @mock.patch('fdroidserver.index.download_repo_index_v2', _mock) + def test_get_scheduled_none_exist(self): + versions = schedule_verify.get_versions(basedir / 'repo') + self.assertEqual( + [ + {'applicationId': 'souch.smsbypass', 'versionCode': 9}, + {'applicationId': 'com.politedroid', 'versionCode': 6}, + {'applicationId': 'com.politedroid', 'versionCode': 5}, + {'applicationId': 'com.politedroid', 'versionCode': 4}, + {'applicationId': 'com.politedroid', 'versionCode': 3}, + ], + schedule_verify.get_scheduled(versions), + ) diff --git a/tests/test_signatures.py b/tests/test_signatures.py index 4f7bd105..a6b3805d 100755 --- a/tests/test_signatures.py +++ b/tests/test_signatures.py @@ -4,56 +4,62 @@ import hashlib import os import sys import unittest -from tempfile import TemporaryDirectory from fdroidserver import common, signatures -from .shared_test_code import TmpCwd +from .shared_test_code import mkdtemp basedir = os.path.dirname(__file__) class SignaturesTest(unittest.TestCase): def setUp(self): + self._td = mkdtemp() + self.testdir = self._td.name + os.chdir(self.testdir) + common.config = None config = common.read_config() config['jarsigner'] = common.find_sdk_tools_cmd('jarsigner') common.config = config + def tearDown(self): + self._td.cleanup() + common.config = None + @unittest.skipIf(sys.byteorder == 'big', "androguard is not ported to big-endian") def test_main(self): class OptionsFixture: APK = [os.path.join(basedir, 'repo', 'com.politedroid_3.apk')] - with TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): - signatures.extract(OptionsFixture) + signatures.extract(OptionsFixture) - # check if extracted signatures are where they are supposed to be - # also verify weather if extracted file contain what they should - filesAndHashes = ( - ( - os.path.join( - 'metadata', 'com.politedroid', 'signatures', '3', 'MANIFEST.MF' - ), - '7dcd83f0c41a75457fd2311bf3c4578f80d684362d74ba8dc52838d353f31cf2', + # check if extracted signatures are where they are supposed to be + # also verify weather if extracted file contain what they should + filesAndHashes = ( + ( + os.path.join( + 'metadata', 'com.politedroid', 'signatures', '3', 'MANIFEST.MF' ), - ( - os.path.join( - 'metadata', 'com.politedroid', 'signatures', '3', 'RELEASE.RSA' - ), - '883ef3d5a6e0bf69d2a58d9e255a7930f08a49abc38e216ed054943c99c8fdb4', + '7dcd83f0c41a75457fd2311bf3c4578f80d684362d74ba8dc52838d353f31cf2', + ), + ( + os.path.join( + 'metadata', 'com.politedroid', 'signatures', '3', 'RELEASE.RSA' ), - ( - os.path.join( - 'metadata', 'com.politedroid', 'signatures', '3', 'RELEASE.SF' - ), - '99fbb3211ef5d7c1253f3a7ad4836eadc9905103ce6a75916c40de2831958284', + '883ef3d5a6e0bf69d2a58d9e255a7930f08a49abc38e216ed054943c99c8fdb4', + ), + ( + os.path.join( + 'metadata', 'com.politedroid', 'signatures', '3', 'RELEASE.SF' ), + '99fbb3211ef5d7c1253f3a7ad4836eadc9905103ce6a75916c40de2831958284', + ), + ) + for path, checksum in filesAndHashes: + self.assertTrue( + os.path.isfile(path), + f'check whether {path!r} was extracted correctly.', ) - for path, checksum in filesAndHashes: - self.assertTrue( - os.path.isfile(path), - f'check whether {path!r} was extracted correctly.', - ) - with open(path, 'rb') as f: - self.assertEqual(hashlib.sha256(f.read()).hexdigest(), checksum) + with open(path, 'rb') as f: + self.assertEqual(hashlib.sha256(f.read()).hexdigest(), checksum)