From b1b30d13a4a26e0b64237701a49a1e6868039e5d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 29 Jan 2025 14:05:52 +0100 Subject: [PATCH] set up config['ndk_paths'] on demand by reading source.properties ndk_paths is set up right after installing the NDK and right before ndk_paths is used in Build.ndk_path(). A proper NDK install always has a _source.properties_ which declares the revision and release strings. That is used as the source of revision and release. This only reads from the local filesystem, and reads the official source of version info (_source.properties_), thereby avoiding any potential network calls that sdkmanager might do. This implementation was inspired by @uniqx's: https://gitlab.com/uniqx/fdroidserver/-/commit/ed931d47d728291ba6b428e0fcf7b53e52476d98#27b997d583cd1c0a05b2213437e0d9fe6536bfe3 --- MANIFEST.in | 3 + fdroidserver/common.py | 114 +++++++++++++++++- fdroidserver/metadata.py | 1 + .../ndk/11.1.2683735/source.properties | 2 + .../ndk/29.0.14033849/source.properties | 4 + .../ndk/29.0.14206865/source.properties | 4 + tests/test_common.py | 54 ++++++++- tests/test_metadata.py | 2 +- 8 files changed, 178 insertions(+), 6 deletions(-) create mode 100644 tests/source-files/ndk/11.1.2683735/source.properties create mode 100644 tests/source-files/ndk/29.0.14033849/source.properties create mode 100644 tests/source-files/ndk/29.0.14206865/source.properties diff --git a/MANIFEST.in b/MANIFEST.in index 93307ace..f4778927 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -818,6 +818,9 @@ include tests/source-files/lockfile.test/rust/subdir/Cargo.lock include tests/source-files/lockfile.test/rust/subdir/Cargo.toml include tests/source-files/lockfile.test/rust/subdir/subdir/subdir/Cargo.toml include tests/source-files/lockfile.test/rust/subdir2/Cargo.toml +include tests/source-files/ndk/11.1.2683735/source.properties +include tests/source-files/ndk/29.0.14033849/source.properties +include tests/source-files/ndk/29.0.14206865/source.properties include tests/source-files/open-keychain/open-keychain/build.gradle include tests/source-files/open-keychain/open-keychain/OpenKeychain/build.gradle include tests/source-files/org.mozilla.rocket/app/build.gradle diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 97fc1f50..6911e2fe 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -4963,17 +4963,123 @@ def _install_ndk(ndk): sdk_path = config['sdk_path'] sdkmanager.install(f'ndk;{ndk}', sdk_path) + # we've installed a new NDK so add it to known paths in config + init_ndk_paths() + + +def init_ndk_paths(): + """Scan sdkmanager's NDK path for installs and add them to global config.""" + sdk_path = config['sdk_path'] for found in glob.glob(f'{sdk_path}/ndk/*'): - version = get_ndk_version(found) + revision, release = _parse_ndk_revision_release(found) if 'ndk_paths' not in config: config['ndk_paths'] = dict() - config['ndk_paths'][ndk] = found - config['ndk_paths'][version] = found + if revision: + config['ndk_paths'][revision] = found + if release: + config['ndk_paths'][release] = found logging.info( - _('Set NDK {release} ({version}) up').format(release=ndk, version=version) + _('Set up NDK {release} ({revision})').format( + release=release, revision=revision + ) ) +# Pkg.ReleaseName was not included in source.properties until r26d +# (26.3.11579264), so this provides a static map of ReleaseName to +# Revision for the earlier NDK releases. +# +# Past NDK releases will never change, we have a log of them here: +# https://gitlab.com/fdroid/android-sdk-transparency-log/-/blob/master/checksums.json +NDK_REVISION_RELEASE = { + "11.0.2655954": "r11", + "11.1.2683735": "r11b", + "11.2.2725575": "r11c", + "12.0.2931149": "r12", + "12.1.2977051": "r12b", + "13.0.3315539": "r13", + "13.1.3345770": "r13b", + "14.0.3770861": "r14", + "14.1.3816874": "r14b", + "15.0.4075724": "r15", + "15.1.4119039": "r15b", + "15.2.4203891": "r15c", + "16.0.4442984": "r16", + "16.1.4479499": "r16b", + "17.0.4754217": "r17", + "17.1.4828580": "r17b", + "17.2.4988734": "r17c", + "18.1.5063045": "r18b", + "19.0.5232133": "r19", + "19.1.5304403": "r19b", + "19.2.5345600": "r19c", + "20.0.5392854-beta2": "r20-beta2", + "20.0.5471264-beta3": "r20-beta3", + "20.0.5594570": "r20", + "20.1.5948944": "r20b", + "21.0.6011959-beta2": "r21-beta2", + "21.0.6113669": "r21", + "21.1.6210238-beta1": "r21b-beta1", + "21.1.6273396-beta2": "r21b-beta2", + "21.1.6363665-beta3": "r21b-beta3", + "21.1.6352462": "r21b", + "21.2.6472646": "r21c", + "21.3.6528147": "r21d", + "21.4.7075529": "r21e", + "22.0.6917172-beta1": "r22-beta1", + "22.0.7026061": "r22", + "22.1.7171670": "r22b", + "23.0.7123448-beta1": "r23-beta1", + "23.0.7196353-beta2": "r23-beta2", + "23.0.7272597-beta3": "r23-beta3", + "23.0.7344513-beta4": "r23-beta4", + "23.0.7421159-beta5": "r23-beta5", + "23.0.7530507-beta6": "r23-beta6", + "23.0.7599858": "r23", + "23.1.7779620": "r23b", + "23.2.8568313": "r23c", + "24.0.7856742-beta1": "r24-beta1", + "24.0.7956693-beta2": "r24-beta2", + "24.0.8215888": "r24", + "24.0.8079956-beta3": "r24-rc1", + "25.0.8141415-beta1": "r25-beta1", + "25.0.8151533-beta1": "r25-beta1", + "25.0.8221429-beta2": "r25-beta2", + "25.0.8355429-beta3": "r25-beta3", + "25.0.8528842-beta4": "r25-beta4", + "25.0.8775105": "r25", + "25.1.8937393": "r25b", + "25.2.9519653": "r25c", + "26.0.10404224-beta1": "r26-beta1", + "26.0.10792818": "r26", + "26.0.10636728-beta2": "r26-rc1", + "26.1.10909125": "r26b", + "26.2.11394342": "r26c", +} + + +def _parse_ndk_revision_release(path): + """Parse NDK at path for revision and release versions. + + When Google started labeling NDK releases as "beta", they added + "-betaX" to the pkg.revision string, but kept using only the semver + revision value for the path, so they added pkg.baserevision for the + path. + + """ + import sdkmanager + + with open(os.path.join(path, 'source.properties')) as fp: + source_properties = sdkmanager.get_properties_dict(fp.read()) + revision = source_properties.get( + 'pkg.baserevision', source_properties.get('pkg.revision') + ) + release = source_properties.get( + 'pkg.releasename', NDK_REVISION_RELEASE.get(revision) + ) + return revision, release + + def calculate_archive_policy(app, default): """Calculate the archive policy from the metadata and default config.""" if app.get('ArchivePolicy') is not None: diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index c9c592ca..a0403fe4 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -326,6 +326,7 @@ class Build(dict): ndk = self.ndk if isinstance(ndk, list): ndk = self.ndk[0] + common.init_ndk_paths() path = common.config['ndk_paths'].get(ndk) if path and not isinstance(path, str): raise TypeError('NDK path is not string') diff --git a/tests/source-files/ndk/11.1.2683735/source.properties b/tests/source-files/ndk/11.1.2683735/source.properties new file mode 100644 index 00000000..97322096 --- /dev/null +++ b/tests/source-files/ndk/11.1.2683735/source.properties @@ -0,0 +1,2 @@ +Pkg.Desc = Android NDK +Pkg.Revision = 11.1.2683735 diff --git a/tests/source-files/ndk/29.0.14033849/source.properties b/tests/source-files/ndk/29.0.14033849/source.properties new file mode 100644 index 00000000..92a56adb --- /dev/null +++ b/tests/source-files/ndk/29.0.14033849/source.properties @@ -0,0 +1,4 @@ +Pkg.Desc = Android NDK +Pkg.Revision = 29.0.14033849-beta4 +Pkg.BaseRevision = 29.0.14033849 +Pkg.ReleaseName = r29-beta4 diff --git a/tests/source-files/ndk/29.0.14206865/source.properties b/tests/source-files/ndk/29.0.14206865/source.properties new file mode 100644 index 00000000..5eb143b8 --- /dev/null +++ b/tests/source-files/ndk/29.0.14206865/source.properties @@ -0,0 +1,4 @@ +Pkg.Desc = Android NDK +Pkg.Revision = 29.0.14206865 +Pkg.BaseRevision = 29.0.14206865 +Pkg.ReleaseName = r29 diff --git a/tests/test_common.py b/tests/test_common.py index c32a5b35..eb7c0396 100755 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -2522,6 +2522,7 @@ class CommonTest(SetUpTearDownMixin, unittest.TestCase): def test_ndk_paths_in_config_must_be_strings(self): """All paths in config must be strings, and never pathlib.Path instances""" fdroidserver.common.config = { + 'sdk_path': 'ignored', 'ndk_paths': {'r21d': Path('/opt/android-sdk/ndk/r21d')} } build = fdroidserver.metadata.Build() @@ -2610,7 +2611,7 @@ class CommonTest(SetUpTearDownMixin, unittest.TestCase): ) def test_no_zero_length_ndk_path_prefixes(self): - fdroidserver.common.config = {'ndk_paths': {}} + fdroidserver.common.config = {'sdk_path': 'ignored', 'ndk_paths': dict()} build = fdroidserver.metadata.Build() with mock.patch.dict(os.environ, clear=True): @@ -3705,3 +3706,54 @@ class VirtContainerTypeTest(unittest.TestCase): with self.assertRaises(SystemExit): fdroidserver.common.get_virt_container_type(self.options) self.assertIn(testvalue, logs.output[0]) + + +class ParseNdkTest(unittest.TestCase): + """Test parsing installed NDKs.""" + + def setUp(self): + os.chdir(os.path.join(basedir, 'source-files/ndk')) + + def tearDown(self): + fdroidserver.common.config = None + + def test_parse_ndk_revision_release_release(self): + testvalue = '29.0.14206865' + self.assertEqual( + (testvalue, 'r29'), + fdroidserver.common._parse_ndk_revision_release(testvalue), + ) + + def test_parse_ndk_revision_release_beta(self): + testvalue = '29.0.14033849' + self.assertEqual( + (testvalue, 'r29-beta4'), + fdroidserver.common._parse_ndk_revision_release(testvalue), + ) + + def test_init_ndk_paths(self): + sdk_path = os.path.dirname(os.getcwd()) + fdroidserver.common.config = {'sdk_path': sdk_path} + fdroidserver.common.init_ndk_paths() + self.assertEqual( + fdroidserver.common.config, + { + 'ndk_paths': { + '11.1.2683735': f'{sdk_path}/ndk/11.1.2683735', + '29.0.14033849': f'{sdk_path}/ndk/29.0.14033849', + '29.0.14206865': f'{sdk_path}/ndk/29.0.14206865', + 'r11b': f'{sdk_path}/ndk/11.1.2683735', + 'r29': f'{sdk_path}/ndk/29.0.14206865', + 'r29-beta4': f'{sdk_path}/ndk/29.0.14033849', + }, + 'sdk_path': sdk_path, + }, + ) + + def test_build_ndk_path(self): + testvalue = '29.0.14033849' + sdk_path = os.path.dirname(os.getcwd()) + fdroidserver.common.config = {'sdk_path': sdk_path} + build = fdroidserver.metadata.Build() + build.ndk = testvalue + self.assertEqual(build.ndk_path(), f'{sdk_path}/ndk/{testvalue}') diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 85ebb0de..90793c16 100755 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -1850,7 +1850,7 @@ class MetadataTest(unittest.TestCase): def test_build_ndk_path_only_accepts_str(self): """Paths in the config must be strings, never pathlib.Path instances""" - config = {'ndk_paths': {'r24': Path('r24')}} + config = {'sdk_path': 'ignored', 'ndk_paths': {'r24': Path('r24')}} fdroidserver.common.config = config build = fdroidserver.metadata.Build() build.ndk = 'r24'