From 4686c06f628b5d7ebc742a187c503e99f11f5977 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 26 May 2021 10:58:46 +0200 Subject: [PATCH 1/4] metadata: allow `ndk:` to be str or list of release or revision There are two version numbers used for NDKs: the "release" and the "revision". The "release" is used in the download URL and zipball and the "revision" is used in the source.properties and the gradle ndkVersion field. Also, there are some builds which need multiple NDKs installed, so this makes it possible to have a list of release/revision entries in build.ndk. This does not yet add full support since _fdroidserver/build.py_ will also need changes. --- fdroidserver/common.py | 26 ++++++++++++++++++++++++++ fdroidserver/metadata.py | 6 +++++- tests/common.TestCase | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 01244f03..60b21f1b 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -4021,6 +4021,32 @@ def auto_install_ndk(build): ndk = build.get('ndk') if not ndk: return + if isinstance(ndk, str): + _install_ndk(ndk) + elif isinstance(ndk, list): + for n in ndk: + _install_ndk(n) + else: + BuildException(_('Invalid ndk: entry in build: "{ndk}"') + .format(ndk=str(ndk))) + + +def _install_ndk(ndk): + """Install specified NDK if it is not already installed + + Parameters + ---------- + + ndk + The NDK version to install, either in "release" form (r21e) or + "revision" form (21.4.7075529). + """ + if re.match(r'[1-9][0-9.]+[0-9]', ndk): + for ndkdict in NDKS: + if ndk == ndkdict['revision']: + ndk = ndkdict['release'] + break + ndk_path = config.get(ndk) if ndk_path and os.path.isdir(ndk_path): return diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index b72d6c9d..21147e30 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -326,7 +326,11 @@ class Build(dict): return 'ant' def ndk_path(self): - return fdroidserver.common.config['ndk_paths'].get(self.ndk, '') + """Returns the path to the first configured NDK or an empty string""" + ndk = self.ndk + if isinstance(ndk, list): + ndk = self.ndk[0] + return fdroidserver.common.config['ndk_paths'].get(ndk, '') flagtypes = { diff --git a/tests/common.TestCase b/tests/common.TestCase index a58ca0a3..9e12ff9a 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -1796,6 +1796,42 @@ class CommonTest(unittest.TestCase): fdroidserver.common.metadata_find_developer_signing_files(appid, vc) ) + def test_auto_install_ndk(self): + """Test all possible field data types for build.ndk""" + build = fdroidserver.metadata.Build() + + none_entry = mock.Mock() + with mock.patch('fdroidserver.common._install_ndk', none_entry): + fdroidserver.common.auto_install_ndk(build) + none_entry.assert_not_called() + + empty_list = mock.Mock() + build.ndk = [] + with mock.patch('fdroidserver.common._install_ndk', empty_list): + fdroidserver.common.auto_install_ndk(build) + empty_list.assert_not_called() + + release_entry = mock.Mock() + build.ndk = 'r21e' + with mock.patch('fdroidserver.common._install_ndk', release_entry): + fdroidserver.common.auto_install_ndk(build) + release_entry.assert_called_once_with('r21e') + + revision_entry = mock.Mock() + build.ndk = '21.4.7075529' + with mock.patch('fdroidserver.common._install_ndk', revision_entry): + fdroidserver.common.auto_install_ndk(build) + revision_entry.assert_called_once_with('21.4.7075529') + + list_entry = mock.Mock() + calls = [] + build.ndk = ['11.0.2655954', 'r12b', 'r21e'] + for n in build.ndk: + calls.append(mock.call(n)) + with mock.patch('fdroidserver.common._install_ndk', list_entry): + fdroidserver.common.auto_install_ndk(build) + list_entry.assert_has_calls(calls) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From 9f77044d0d3f36d9b1e65a2068b51ea35254f991 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 26 May 2021 12:52:54 +0200 Subject: [PATCH 2/4] auto-detect NDKs installed in standard paths 'ndk_paths' will be automatically filled out from well known sources like $ANDROID_HOME/ndk-bundle and $ANDROID_HOME/ndk/*. If a required version is missing in the buildserver VM, it will be automatically downloaded and installed into the standard $ANDROID_HOME/ndk/ directory. Manually setting it here will override the auto-detected values. The keys can either be the "release" (e.g. r21e) or the "revision" (e.g. 21.4.7075529). https://developer.android.com/studio/projects/configure-agp-ndk#agp_version_41 * sdkmanager installs "ndk;12.3.4567890" into $ANDROID_SDK_ROOT/ndk/ * sdkmanager installs "ndk-bundle" into $ANDROID_SDK_ROOT/ndk-bundle/ --- buildserver/config.buildserver.yml | 4 ---- examples/config.yml | 28 ++++++++++++---------------- fdroidserver/common.py | 30 ++++++++++++++++++++++++++++++ tests/common.TestCase | 22 ++++++++++++++++++++++ 4 files changed, 64 insertions(+), 20 deletions(-) diff --git a/buildserver/config.buildserver.yml b/buildserver/config.buildserver.yml index 3073faf7..f5fff843 100644 --- a/buildserver/config.buildserver.yml +++ b/buildserver/config.buildserver.yml @@ -1,8 +1,4 @@ sdk_path: /opt/android-sdk -ndk_paths: - r10e: /opt/android-sdk/ndk/r10e - r21e: /opt/android-sdk/ndk/21.4.7075529 - r22b: /opt/android-sdk/ndk/22.0.7026061 java_paths: 8: /usr/lib/jvm/java-8-openjdk-amd64 diff --git a/examples/config.yml b/examples/config.yml index ee62c9ac..ca818414 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -5,24 +5,20 @@ # Custom path to the Android SDK, defaults to $ANDROID_HOME # sdk_path: $ANDROID_HOME -# Paths to various installed versions of the Android NDK. If a -# required version is missing in the buildserver VM, it will be -# automatically downloaded and installed into a temporary dir. +# Paths to installed versions of the Android NDK. This will be +# automatically filled out from well known sources like +# $ANDROID_HOME/ndk-bundle and $ANDROID_HOME/ndk/*. If a required +# version is missing in the buildserver VM, it will be automatically +# downloaded and installed into the standard $ANDROID_HOME/ndk/ +# directory. Manually setting it here will override the auto-detected +# values. The keys can either be the "release" (e.g. r21e) or the +# "revision" (e.g. 21.4.7075529). # # ndk_paths: -# r10e: None -# r11c: None -# r12b: None -# r13b: None -# r14b: None -# r15c: None -# r16b: None -# r17c: None -# r18b: None -# r19c: None -# r20b: None -# r21e: None -# r22b: None +# r10e: $ANDROID_HOME/android-ndk-r10e +# r17: "" +# 21.4.7075529: ~/Android/Ndk +# r22b: null # Directory to store downloaded tools in (i.e. gradle versions) # By default, these are stored in ~/.cache/fdroidserver diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 60b21f1b..8e6f5ac1 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -214,6 +214,14 @@ def _add_java_paths_to_config(pathlist, thisconfig): def fill_config_defaults(thisconfig): + """Fill in the global config dict with relevant defaults + + For config values that have a path that can be expanded, e.g. an + env var or a ~/, this will store the original value using "_orig" + appended to the key name so that if the config gets written out, + it will preserve the original, unexpanded string. + + """ for k, v in default_config.items(): if k not in thisconfig: thisconfig[k] = v @@ -281,6 +289,28 @@ def fill_config_defaults(thisconfig): thisconfig[k][k2] = exp thisconfig[k][k2 + '_orig'] = v + ndk_paths = thisconfig.get('ndk_paths', {}) + + ndk_bundle = os.path.join(thisconfig['sdk_path'], 'ndk-bundle') + if os.path.exists(ndk_bundle): + version = get_ndk_version(ndk_bundle) + if version not in ndk_paths: + ndk_paths[version] = ndk_bundle + + ndk_dir = os.path.join(thisconfig['sdk_path'], 'ndk') + if os.path.exists(ndk_dir): + for ndk in glob.glob(os.path.join(ndk_dir, '*')): + version = get_ndk_version(ndk) + if version not in ndk_paths: + ndk_paths[version] = ndk + + for k in list(ndk_paths.keys()): + if not re.match(r'r[1-9][0-9]*[a-z]?', k): + for ndkdict in NDKS: + if k == ndkdict['revision']: + ndk_paths[ndkdict['release']] = ndk_paths.pop(k) + break + def regsub_file(pattern, repl, path): with open(path, 'rb') as f: diff --git a/tests/common.TestCase b/tests/common.TestCase index 9e12ff9a..6eca64bc 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -1832,6 +1832,28 @@ class CommonTest(unittest.TestCase): fdroidserver.common.auto_install_ndk(build) list_entry.assert_has_calls(calls) + def test_fill_config_defaults(self): + """Test the auto-detection of NDKs installed in standard paths""" + sdk_path = tempfile.mkdtemp( + prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir + ) + + ndk_bundle = os.path.join(sdk_path, 'ndk-bundle') + os.makedirs(ndk_bundle) + with open(os.path.join(ndk_bundle, 'source.properties'), 'w') as fp: + fp.write('Pkg.Desc = Android NDK\nPkg.Revision = 17.2.4988734\n') + config = {'sdk_path': sdk_path} + fdroidserver.common.fill_config_defaults(config) + self.assertEqual({'r17c': ndk_bundle}, config['ndk_paths']) + + r21e = os.path.join(sdk_path, 'ndk', '21.4.7075529') + os.makedirs(r21e) + with open(os.path.join(r21e, 'source.properties'), 'w') as fp: + fp.write('Pkg.Desc = Android NDK\nPkg.Revision = 21.4.7075529\n') + config = {'sdk_path': sdk_path} + fdroidserver.common.fill_config_defaults(config) + self.assertEqual({'r17c': ndk_bundle, 'r21e': r21e}, config['ndk_paths']) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From 7a1d236c8dabe0f12b759c74f3ab931be67f1fd3 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 26 May 2021 17:12:28 +0200 Subject: [PATCH 3/4] only support zipballs in NDK provisioning Since I discovered there is an r10e zipball, this can now get all NDKs in zipball form. fdroid/android-sdk-transparency-log@447fea86e719295af6cd8a3bbee4529d114ece2d closes #902 --- buildserver/provision-android-ndk | 5 ----- buildserver/provision-apt-get-install | 1 - fdroidserver/common.py | 18 +++++++++++++++++- makebuildserver | 2 -- tests/common.TestCase | 26 +++++++++++++++++++++++++- 5 files changed, 42 insertions(+), 10 deletions(-) diff --git a/buildserver/provision-android-ndk b/buildserver/provision-android-ndk index 655bf06f..99e7c32e 100644 --- a/buildserver/provision-android-ndk +++ b/buildserver/provision-android-ndk @@ -10,11 +10,6 @@ NDK_BASE=$1 test -e $NDK_BASE || mkdir -p $NDK_BASE cd $NDK_BASE -if [ ! -e $NDK_BASE/r10e ]; then - 7zr x /vagrant/cache/android-ndk-r10e-linux-x86_64.bin > /dev/null - mv android-ndk-r10e r10e -fi - for version in r21e r22b; do if [ ! -e ${NDK_BASE}/${version} ]; then unzip /vagrant/cache/android-ndk-${version}-linux-x86_64.zip > /dev/null diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index 1767bb13..070fb6b3 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -96,7 +96,6 @@ packages=" openjdk-8-jre-headless openjdk-8-jdk-headless optipng - p7zip pkg-config python-gnupg python-lxml diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 8e6f5ac1..9faa8490 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -307,7 +307,7 @@ def fill_config_defaults(thisconfig): for k in list(ndk_paths.keys()): if not re.match(r'r[1-9][0-9]*[a-z]?', k): for ndkdict in NDKS: - if k == ndkdict['revision']: + if k == ndkdict.get('revision'): ndk_paths[ndkdict['release']] = ndk_paths.pop(k) break @@ -4015,12 +4015,23 @@ def sha256base64(filename): def get_ndk_version(ndk_path): + """Get the version info from the metadata in the NDK package + + Since r11, the info is nice and easy to find in + sources.properties. Before, there was a kludgey format in + RELEASE.txt. This is only needed for r10e. + + """ source_properties = os.path.join(ndk_path, 'source.properties') + release_txt = os.path.join(ndk_path, 'RELEASE.TXT') if os.path.exists(source_properties): with open(source_properties) as fp: m = re.search(r'^Pkg.Revision *= *(.+)', fp.read(), flags=re.MULTILINE) if m: return m.group(1) + elif os.path.exists(release_txt): + with open(release_txt) as fp: + return fp.read().split('-')[0] def auto_install_ndk(build): @@ -4118,6 +4129,11 @@ def _install_ndk(ndk): """Derived from https://gitlab.com/fdroid/android-sdk-transparency-log/-/blob/master/checksums.json""" NDKS = [ + { + "release": "r10e", + "sha256": "ee5f405f3b57c4f5c3b3b8b5d495ae12b660e03d2112e4ed5c728d349f1e520c", + "url": "https://dl.google.com/android/repository/android-ndk-r10e-linux-x86_64.zip" + }, { "release": "r11", "revision": "11.0.2655954", diff --git a/makebuildserver b/makebuildserver index 4f1864a8..d1298720 100755 --- a/makebuildserver +++ b/makebuildserver @@ -291,8 +291,6 @@ CACHE_FILES = [ 'dccda8aa069563c8ba2f6cdfd0777df0e34a5b4d15138ca8b9757e94f4e8a8cb'), ('https://services.gradle.org/distributions/gradle-7.0.2-bin.zip', '0e46229820205440b48a5501122002842b82886e76af35f0f3a069243dca4b3c'), - ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin', - '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'), ('https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip', 'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e'), ('https://dl.google.com/android/repository/android-ndk-r22b-linux-x86_64.zip', diff --git a/tests/common.TestCase b/tests/common.TestCase index 6eca64bc..d73bc497 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -1825,13 +1825,27 @@ class CommonTest(unittest.TestCase): list_entry = mock.Mock() calls = [] - build.ndk = ['11.0.2655954', 'r12b', 'r21e'] + build.ndk = ['r10e', '11.0.2655954', 'r12b', 'r21e'] for n in build.ndk: calls.append(mock.call(n)) with mock.patch('fdroidserver.common._install_ndk', list_entry): fdroidserver.common.auto_install_ndk(build) list_entry.assert_has_calls(calls) + @unittest.skip("This test downloads and unzips a 1GB file.") + def test_install_ndk(self): + """NDK r10e is a special case since its missing source.properties""" + sdk_path = tempfile.mkdtemp( + prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir + ) + config = {'sdk_path': sdk_path} + fdroidserver.common.config = config + fdroidserver.common._install_ndk('r10e') + r10e = os.path.join(sdk_path, 'ndk', 'r10e') + self.assertEqual('r10e', fdroidserver.common.get_ndk_version(r10e)) + fdroidserver.common.fill_config_defaults(config) + self.assertEqual({'r10e': r10e}, config['ndk_paths']) + def test_fill_config_defaults(self): """Test the auto-detection of NDKs installed in standard paths""" sdk_path = tempfile.mkdtemp( @@ -1854,6 +1868,16 @@ class CommonTest(unittest.TestCase): fdroidserver.common.fill_config_defaults(config) self.assertEqual({'r17c': ndk_bundle, 'r21e': r21e}, config['ndk_paths']) + r10e = os.path.join(sdk_path, 'ndk', 'r10e') + os.makedirs(r10e) + with open(os.path.join(r10e, 'RELEASE.TXT'), 'w') as fp: + fp.write('r10e-rc4 (64-bit)\n') + config = {'sdk_path': sdk_path} + fdroidserver.common.fill_config_defaults(config) + self.assertEqual( + {'r10e': r10e, 'r17c': ndk_bundle, 'r21e': r21e}, config['ndk_paths'] + ) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From 09fa49a7a3af313e60013ed69107137df928f15e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 26 May 2021 17:35:39 +0200 Subject: [PATCH 4/4] make get_android_tools_versions() search ndk_paths from config --- fdroidserver/build.py | 6 +++--- fdroidserver/common.py | 14 ++++++-------- tests/build.TestCase | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index efe7ab3f..6c3fa790 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -237,7 +237,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): try: cmd_stdout = chan.makefile('rb', 1024) output = bytes() - output += common.get_android_tools_version_log(build.ndk_path()).encode() + output += common.get_android_tools_version_log().encode() while not chan.exit_status_ready(): line = cmd_stdout.readline() if line: @@ -402,7 +402,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext log_path = os.path.join(log_dir, common.get_toolsversion_logname(app, build)) with open(log_path, 'w') as f: - f.write(common.get_android_tools_version_log(build.ndk_path())) + f.write(common.get_android_tools_version_log()) else: if build.sudo: logging.warning('%s:%s runs this on the buildserver with sudo:\n\t%s\nThese commands were skipped because fdroid build is not running on a dedicated build server.' @@ -1088,7 +1088,7 @@ def main(): build_starttime = common.get_wiki_timestamp() tools_version_log = '' if not options.onserver: - tools_version_log = common.get_android_tools_version_log(build.ndk_path()) + tools_version_log = common.get_android_tools_version_log() common.write_running_status_json(status_output) try: diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 9faa8490..851fdea5 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -3872,7 +3872,7 @@ def get_wiki_timestamp(timestamp=None): return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp) -def get_android_tools_versions(ndk_path=None): +def get_android_tools_versions(): '''get a list of the versions of all installed Android SDK/NDK components''' global config @@ -3880,11 +3880,9 @@ def get_android_tools_versions(ndk_path=None): if sdk_path[-1] != '/': sdk_path += '/' components = [] - if ndk_path: - ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT') - if os.path.isfile(ndk_release_txt): - with open(ndk_release_txt, 'r') as fp: - components.append((os.path.basename(ndk_path), fp.read()[:-1])) + for ndk_path in config.get('ndk_paths', []): + version = get_ndk_version(ndk_path) + components.append((os.path.basename(ndk_path), str(version))) pattern = re.compile(r'^Pkg.Revision *= *(.+)', re.MULTILINE) for root, dirs, files in os.walk(sdk_path): @@ -3898,10 +3896,10 @@ def get_android_tools_versions(ndk_path=None): return components -def get_android_tools_version_log(ndk_path=None): +def get_android_tools_version_log(): '''get a list of the versions of all installed Android SDK/NDK components''' log = '== Installed Android Tools ==\n\n' - components = get_android_tools_versions(ndk_path) + components = get_android_tools_versions() for name, version in sorted(components): log += '* ' + name + ' (' + version + ')\n' diff --git a/tests/build.TestCase b/tests/build.TestCase index f700b74c..809ed588 100755 --- a/tests/build.TestCase +++ b/tests/build.TestCase @@ -362,7 +362,7 @@ class BuildTest(unittest.TestCase): with mock.patch( 'fdroidserver.common.force_exit', lambda *args: None ) as a, mock.patch( - 'fdroidserver.common.get_android_tools_version_log', lambda s: 'fake' + 'fdroidserver.common.get_android_tools_version_log', lambda: 'fake' ) as b, mock.patch( 'fdroidserver.common.FDroidPopen', FakeProcess ) as c, mock.patch(