diff --git a/buildserver/config.buildserver.yml b/buildserver/config.buildserver.yml index 3073faf7..f5fff843 100644 --- a/buildserver/config.buildserver.yml +++ b/buildserver/config.buildserver.yml @@ -1,8 +1,4 @@ sdk_path: /opt/android-sdk -ndk_paths: - r10e: /opt/android-sdk/ndk/r10e - r21e: /opt/android-sdk/ndk/21.4.7075529 - r22b: /opt/android-sdk/ndk/22.0.7026061 java_paths: 8: /usr/lib/jvm/java-8-openjdk-amd64 diff --git a/buildserver/provision-android-ndk b/buildserver/provision-android-ndk index 655bf06f..99e7c32e 100644 --- a/buildserver/provision-android-ndk +++ b/buildserver/provision-android-ndk @@ -10,11 +10,6 @@ NDK_BASE=$1 test -e $NDK_BASE || mkdir -p $NDK_BASE cd $NDK_BASE -if [ ! -e $NDK_BASE/r10e ]; then - 7zr x /vagrant/cache/android-ndk-r10e-linux-x86_64.bin > /dev/null - mv android-ndk-r10e r10e -fi - for version in r21e r22b; do if [ ! -e ${NDK_BASE}/${version} ]; then unzip /vagrant/cache/android-ndk-${version}-linux-x86_64.zip > /dev/null diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index 1767bb13..070fb6b3 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -96,7 +96,6 @@ packages=" openjdk-8-jre-headless openjdk-8-jdk-headless optipng - p7zip pkg-config python-gnupg python-lxml diff --git a/examples/config.yml b/examples/config.yml index ee62c9ac..ca818414 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -5,24 +5,20 @@ # Custom path to the Android SDK, defaults to $ANDROID_HOME # sdk_path: $ANDROID_HOME -# Paths to various installed versions of the Android NDK. If a -# required version is missing in the buildserver VM, it will be -# automatically downloaded and installed into a temporary dir. +# Paths to installed versions of the Android NDK. This will be +# automatically filled out from well known sources like +# $ANDROID_HOME/ndk-bundle and $ANDROID_HOME/ndk/*. If a required +# version is missing in the buildserver VM, it will be automatically +# downloaded and installed into the standard $ANDROID_HOME/ndk/ +# directory. Manually setting it here will override the auto-detected +# values. The keys can either be the "release" (e.g. r21e) or the +# "revision" (e.g. 21.4.7075529). # # ndk_paths: -# r10e: None -# r11c: None -# r12b: None -# r13b: None -# r14b: None -# r15c: None -# r16b: None -# r17c: None -# r18b: None -# r19c: None -# r20b: None -# r21e: None -# r22b: None +# r10e: $ANDROID_HOME/android-ndk-r10e +# r17: "" +# 21.4.7075529: ~/Android/Ndk +# r22b: null # Directory to store downloaded tools in (i.e. gradle versions) # By default, these are stored in ~/.cache/fdroidserver diff --git a/fdroidserver/build.py b/fdroidserver/build.py index efe7ab3f..6c3fa790 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -237,7 +237,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): try: cmd_stdout = chan.makefile('rb', 1024) output = bytes() - output += common.get_android_tools_version_log(build.ndk_path()).encode() + output += common.get_android_tools_version_log().encode() while not chan.exit_status_ready(): line = cmd_stdout.readline() if line: @@ -402,7 +402,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext log_path = os.path.join(log_dir, common.get_toolsversion_logname(app, build)) with open(log_path, 'w') as f: - f.write(common.get_android_tools_version_log(build.ndk_path())) + f.write(common.get_android_tools_version_log()) else: if build.sudo: logging.warning('%s:%s runs this on the buildserver with sudo:\n\t%s\nThese commands were skipped because fdroid build is not running on a dedicated build server.' @@ -1088,7 +1088,7 @@ def main(): build_starttime = common.get_wiki_timestamp() tools_version_log = '' if not options.onserver: - tools_version_log = common.get_android_tools_version_log(build.ndk_path()) + tools_version_log = common.get_android_tools_version_log() common.write_running_status_json(status_output) try: diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 01244f03..851fdea5 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -214,6 +214,14 @@ def _add_java_paths_to_config(pathlist, thisconfig): def fill_config_defaults(thisconfig): + """Fill in the global config dict with relevant defaults + + For config values that have a path that can be expanded, e.g. an + env var or a ~/, this will store the original value using "_orig" + appended to the key name so that if the config gets written out, + it will preserve the original, unexpanded string. + + """ for k, v in default_config.items(): if k not in thisconfig: thisconfig[k] = v @@ -281,6 +289,28 @@ def fill_config_defaults(thisconfig): thisconfig[k][k2] = exp thisconfig[k][k2 + '_orig'] = v + ndk_paths = thisconfig.get('ndk_paths', {}) + + ndk_bundle = os.path.join(thisconfig['sdk_path'], 'ndk-bundle') + if os.path.exists(ndk_bundle): + version = get_ndk_version(ndk_bundle) + if version not in ndk_paths: + ndk_paths[version] = ndk_bundle + + ndk_dir = os.path.join(thisconfig['sdk_path'], 'ndk') + if os.path.exists(ndk_dir): + for ndk in glob.glob(os.path.join(ndk_dir, '*')): + version = get_ndk_version(ndk) + if version not in ndk_paths: + ndk_paths[version] = ndk + + for k in list(ndk_paths.keys()): + if not re.match(r'r[1-9][0-9]*[a-z]?', k): + for ndkdict in NDKS: + if k == ndkdict.get('revision'): + ndk_paths[ndkdict['release']] = ndk_paths.pop(k) + break + def regsub_file(pattern, repl, path): with open(path, 'rb') as f: @@ -3842,7 +3872,7 @@ def get_wiki_timestamp(timestamp=None): return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp) -def get_android_tools_versions(ndk_path=None): +def get_android_tools_versions(): '''get a list of the versions of all installed Android SDK/NDK components''' global config @@ -3850,11 +3880,9 @@ def get_android_tools_versions(ndk_path=None): if sdk_path[-1] != '/': sdk_path += '/' components = [] - if ndk_path: - ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT') - if os.path.isfile(ndk_release_txt): - with open(ndk_release_txt, 'r') as fp: - components.append((os.path.basename(ndk_path), fp.read()[:-1])) + for ndk_path in config.get('ndk_paths', []): + version = get_ndk_version(ndk_path) + components.append((os.path.basename(ndk_path), str(version))) pattern = re.compile(r'^Pkg.Revision *= *(.+)', re.MULTILINE) for root, dirs, files in os.walk(sdk_path): @@ -3868,10 +3896,10 @@ def get_android_tools_versions(ndk_path=None): return components -def get_android_tools_version_log(ndk_path=None): +def get_android_tools_version_log(): '''get a list of the versions of all installed Android SDK/NDK components''' log = '== Installed Android Tools ==\n\n' - components = get_android_tools_versions(ndk_path) + components = get_android_tools_versions() for name, version in sorted(components): log += '* ' + name + ' (' + version + ')\n' @@ -3985,12 +4013,23 @@ def sha256base64(filename): def get_ndk_version(ndk_path): + """Get the version info from the metadata in the NDK package + + Since r11, the info is nice and easy to find in + sources.properties. Before, there was a kludgey format in + RELEASE.txt. This is only needed for r10e. + + """ source_properties = os.path.join(ndk_path, 'source.properties') + release_txt = os.path.join(ndk_path, 'RELEASE.TXT') if os.path.exists(source_properties): with open(source_properties) as fp: m = re.search(r'^Pkg.Revision *= *(.+)', fp.read(), flags=re.MULTILINE) if m: return m.group(1) + elif os.path.exists(release_txt): + with open(release_txt) as fp: + return fp.read().split('-')[0] def auto_install_ndk(build): @@ -4021,6 +4060,32 @@ def auto_install_ndk(build): ndk = build.get('ndk') if not ndk: return + if isinstance(ndk, str): + _install_ndk(ndk) + elif isinstance(ndk, list): + for n in ndk: + _install_ndk(n) + else: + BuildException(_('Invalid ndk: entry in build: "{ndk}"') + .format(ndk=str(ndk))) + + +def _install_ndk(ndk): + """Install specified NDK if it is not already installed + + Parameters + ---------- + + ndk + The NDK version to install, either in "release" form (r21e) or + "revision" form (21.4.7075529). + """ + if re.match(r'[1-9][0-9.]+[0-9]', ndk): + for ndkdict in NDKS: + if ndk == ndkdict['revision']: + ndk = ndkdict['release'] + break + ndk_path = config.get(ndk) if ndk_path and os.path.isdir(ndk_path): return @@ -4062,6 +4127,11 @@ def auto_install_ndk(build): """Derived from https://gitlab.com/fdroid/android-sdk-transparency-log/-/blob/master/checksums.json""" NDKS = [ + { + "release": "r10e", + "sha256": "ee5f405f3b57c4f5c3b3b8b5d495ae12b660e03d2112e4ed5c728d349f1e520c", + "url": "https://dl.google.com/android/repository/android-ndk-r10e-linux-x86_64.zip" + }, { "release": "r11", "revision": "11.0.2655954", diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index b72d6c9d..21147e30 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -326,7 +326,11 @@ class Build(dict): return 'ant' def ndk_path(self): - return fdroidserver.common.config['ndk_paths'].get(self.ndk, '') + """Returns the path to the first configured NDK or an empty string""" + ndk = self.ndk + if isinstance(ndk, list): + ndk = self.ndk[0] + return fdroidserver.common.config['ndk_paths'].get(ndk, '') flagtypes = { diff --git a/makebuildserver b/makebuildserver index 4f1864a8..d1298720 100755 --- a/makebuildserver +++ b/makebuildserver @@ -291,8 +291,6 @@ CACHE_FILES = [ 'dccda8aa069563c8ba2f6cdfd0777df0e34a5b4d15138ca8b9757e94f4e8a8cb'), ('https://services.gradle.org/distributions/gradle-7.0.2-bin.zip', '0e46229820205440b48a5501122002842b82886e76af35f0f3a069243dca4b3c'), - ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin', - '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'), ('https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip', 'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e'), ('https://dl.google.com/android/repository/android-ndk-r22b-linux-x86_64.zip', diff --git a/tests/build.TestCase b/tests/build.TestCase index f700b74c..809ed588 100755 --- a/tests/build.TestCase +++ b/tests/build.TestCase @@ -362,7 +362,7 @@ class BuildTest(unittest.TestCase): with mock.patch( 'fdroidserver.common.force_exit', lambda *args: None ) as a, mock.patch( - 'fdroidserver.common.get_android_tools_version_log', lambda s: 'fake' + 'fdroidserver.common.get_android_tools_version_log', lambda: 'fake' ) as b, mock.patch( 'fdroidserver.common.FDroidPopen', FakeProcess ) as c, mock.patch( diff --git a/tests/common.TestCase b/tests/common.TestCase index a58ca0a3..d73bc497 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -1796,6 +1796,88 @@ class CommonTest(unittest.TestCase): fdroidserver.common.metadata_find_developer_signing_files(appid, vc) ) + def test_auto_install_ndk(self): + """Test all possible field data types for build.ndk""" + build = fdroidserver.metadata.Build() + + none_entry = mock.Mock() + with mock.patch('fdroidserver.common._install_ndk', none_entry): + fdroidserver.common.auto_install_ndk(build) + none_entry.assert_not_called() + + empty_list = mock.Mock() + build.ndk = [] + with mock.patch('fdroidserver.common._install_ndk', empty_list): + fdroidserver.common.auto_install_ndk(build) + empty_list.assert_not_called() + + release_entry = mock.Mock() + build.ndk = 'r21e' + with mock.patch('fdroidserver.common._install_ndk', release_entry): + fdroidserver.common.auto_install_ndk(build) + release_entry.assert_called_once_with('r21e') + + revision_entry = mock.Mock() + build.ndk = '21.4.7075529' + with mock.patch('fdroidserver.common._install_ndk', revision_entry): + fdroidserver.common.auto_install_ndk(build) + revision_entry.assert_called_once_with('21.4.7075529') + + list_entry = mock.Mock() + calls = [] + build.ndk = ['r10e', '11.0.2655954', 'r12b', 'r21e'] + for n in build.ndk: + calls.append(mock.call(n)) + with mock.patch('fdroidserver.common._install_ndk', list_entry): + fdroidserver.common.auto_install_ndk(build) + list_entry.assert_has_calls(calls) + + @unittest.skip("This test downloads and unzips a 1GB file.") + def test_install_ndk(self): + """NDK r10e is a special case since its missing source.properties""" + sdk_path = tempfile.mkdtemp( + prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir + ) + config = {'sdk_path': sdk_path} + fdroidserver.common.config = config + fdroidserver.common._install_ndk('r10e') + r10e = os.path.join(sdk_path, 'ndk', 'r10e') + self.assertEqual('r10e', fdroidserver.common.get_ndk_version(r10e)) + fdroidserver.common.fill_config_defaults(config) + self.assertEqual({'r10e': r10e}, config['ndk_paths']) + + def test_fill_config_defaults(self): + """Test the auto-detection of NDKs installed in standard paths""" + sdk_path = tempfile.mkdtemp( + prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir + ) + + ndk_bundle = os.path.join(sdk_path, 'ndk-bundle') + os.makedirs(ndk_bundle) + with open(os.path.join(ndk_bundle, 'source.properties'), 'w') as fp: + fp.write('Pkg.Desc = Android NDK\nPkg.Revision = 17.2.4988734\n') + config = {'sdk_path': sdk_path} + fdroidserver.common.fill_config_defaults(config) + self.assertEqual({'r17c': ndk_bundle}, config['ndk_paths']) + + r21e = os.path.join(sdk_path, 'ndk', '21.4.7075529') + os.makedirs(r21e) + with open(os.path.join(r21e, 'source.properties'), 'w') as fp: + fp.write('Pkg.Desc = Android NDK\nPkg.Revision = 21.4.7075529\n') + config = {'sdk_path': sdk_path} + fdroidserver.common.fill_config_defaults(config) + self.assertEqual({'r17c': ndk_bundle, 'r21e': r21e}, config['ndk_paths']) + + r10e = os.path.join(sdk_path, 'ndk', 'r10e') + os.makedirs(r10e) + with open(os.path.join(r10e, 'RELEASE.TXT'), 'w') as fp: + fp.write('r10e-rc4 (64-bit)\n') + config = {'sdk_path': sdk_path} + fdroidserver.common.fill_config_defaults(config) + self.assertEqual( + {'r10e': r10e, 'r17c': ndk_bundle, 'r21e': r21e}, config['ndk_paths'] + ) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__))