From 0fbd04f1c25170d2bd566225af17cc97c2a9d5e8 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 11 May 2021 20:25:51 +0200 Subject: [PATCH 1/6] remove redundant imports and pointless code --- fdroidserver/index.py | 1 - fdroidserver/init.py | 3 --- fdroidserver/metadata.py | 4 ---- fdroidserver/update.py | 1 - locale/pick-complete-translations.py | 1 - 5 files changed, 10 deletions(-) diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 10dea357..1f30bd48 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -957,7 +957,6 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing % repo_icon) os.makedirs(os.path.dirname(iconfilename), exist_ok=True) try: - import qrcode qrcode.make(common.config['repo_url']).save(iconfilename) except Exception: exampleicon = os.path.join(common.get_examples_dir(), diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 8291612d..e66d4daa 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -66,7 +66,6 @@ def main(): help=_("Do not prompt for Android SDK path, just fail")) options = parser.parse_args() - aapt = None fdroiddir = os.getcwd() test_config = dict() examplesdir = common.get_examples_dir() @@ -227,8 +226,6 @@ def main(): msg = '\n' msg += _('Built repo based in "%s" with this config:') % fdroiddir msg += '\n\n Android SDK:\t\t\t' + config['sdk_path'] - if aapt: - msg += '\n Android SDK Build Tools:\t' + os.path.dirname(aapt) msg += '\n Android NDK r12b (optional):\t$ANDROID_NDK' msg += '\n ' + _('Keystore for signing key:\t') + keystore if repo_keyalias is not None: diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index d5911a8d..597c4cd2 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -890,10 +890,6 @@ def write_yaml(mf, app): _yaml_bools_plus_lists.extend(_yaml_bools_false) _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_false]) - def _class_as_dict_representer(dumper, data): - '''Creates a YAML representation of a App/Build instance''' - return dumper.represent_dict(data) - def _field_to_yaml(typ, value): if typ is TYPE_STRING: if value in _yaml_bools_plus_lists: diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 0866d9f0..9c563381 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -2067,7 +2067,6 @@ def create_metadata_from_template(apk): template: field sort order, empty field value, formatting, etc. ''' - import yaml if os.path.exists('template.yml'): with open('template.yml') as f: metatxt = f.read() diff --git a/locale/pick-complete-translations.py b/locale/pick-complete-translations.py index 99c1c125..3f343685 100755 --- a/locale/pick-complete-translations.py +++ b/locale/pick-complete-translations.py @@ -7,7 +7,6 @@ import os import re import requests import subprocess -import sys projectbasedir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) From 7a7ecbf9dc3027042459b8c63539790835c77d68 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 12 May 2021 09:43:48 +0200 Subject: [PATCH 2/6] move sha256sum() and sha256base64() to common --- fdroidserver/common.py | 25 +++++++++++++++++++++++++ fdroidserver/update.py | 35 +++++------------------------------ 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index df884b6d..ea9c7c26 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -58,6 +58,7 @@ try: except ImportError: import xml.etree.ElementTree as XMLElementTree # nosec this is a fallback only +from base64 import urlsafe_b64encode from binascii import hexlify from datetime import datetime, timedelta, timezone from distutils.version import LooseVersion @@ -3964,3 +3965,27 @@ def run_yamllint(path, indent=0): for problem in problems: result.append(' ' * indent + path + ':' + str(problem.line) + ': ' + problem.message) return '\n'.join(result) + + +def sha256sum(filename): + '''Calculate the sha256 of the given file''' + sha = hashlib.sha256() + with open(filename, 'rb') as f: + while True: + t = f.read(16384) + if len(t) == 0: + break + sha.update(t) + return sha.hexdigest() + + +def sha256base64(filename): + '''Calculate the sha256 of the given file as URL-safe base64''' + hasher = hashlib.sha256() + with open(filename, 'rb') as f: + while True: + t = f.read(16384) + if len(t) == 0: + break + hasher.update(t) + return urlsafe_b64encode(hasher.digest()).decode() diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 9c563381..1c3a1ed7 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -36,7 +36,6 @@ import yaml import copy from datetime import datetime from argparse import ArgumentParser -from base64 import urlsafe_b64encode try: from yaml import CSafeLoader as SafeLoader except ImportError: @@ -590,30 +589,6 @@ def get_icon_bytes(apkzip, iconsrc): return apkzip.read(iconsrc.encode('utf-8').decode('cp437')) -def sha256sum(filename): - '''Calculate the sha256 of the given file''' - sha = hashlib.sha256() - with open(filename, 'rb') as f: - while True: - t = f.read(16384) - if len(t) == 0: - break - sha.update(t) - return sha.hexdigest() - - -def sha256base64(filename): - '''Calculate the sha256 of the given file as URL-safe base64''' - hasher = hashlib.sha256() - with open(filename, 'rb') as f: - while True: - t = f.read(16384) - if len(t) == 0: - break - hasher.update(t) - return urlsafe_b64encode(hasher.digest()).decode() - - def has_known_vulnerability(filename): """checks for known vulnerabilities in the APK @@ -723,7 +698,7 @@ def insert_obbs(repodir, apps, apks): obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:') .format(integer=str(versionCode))) continue - obbsha256 = sha256sum(f) + obbsha256 = common.sha256sum(f) obbs.append((packagename, versionCode, obbfile, obbsha256)) for apk in apks: @@ -1267,7 +1242,7 @@ def insert_localized_app_metadata(apps): if not os.path.samefile(f, basepath): os.unlink(f) else: - sha256 = sha256base64(f) + sha256 = common.sha256base64(f) filename = base + '_' + sha256 + '.' + extension index_file = os.path.join(os.path.dirname(f), filename) if not os.path.exists(index_file): @@ -1313,7 +1288,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): raise FDroidException(_('{path} is zero size!') .format(path=filename)) - shasum = sha256sum(filename) + shasum = common.sha256sum(filename) usecache = False if name_utf8 in apkcache: repo_file = apkcache[name_utf8] @@ -1378,7 +1353,7 @@ def scan_apk(apk_file): :return A dict containing APK metadata """ apk = { - 'hash': sha256sum(apk_file), + 'hash': common.sha256sum(apk_file), 'hashType': 'sha256', 'uses-permission': [], 'uses-permission-sdk-23': [], @@ -1613,7 +1588,7 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal usecache = False if apkfilename in apkcache: apk = apkcache[apkfilename] - if apk.get('hash') == sha256sum(apkfile): + if apk.get('hash') == common.sha256sum(apkfile): logging.debug(_("Reading {apkfilename} from cache") .format(apkfilename=apkfilename)) usecache = True From 69fcd6a0249aac630bc1d59b77f780951b3d9716 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 12 May 2021 11:28:25 +0200 Subject: [PATCH 3/6] build: auto-download missing NDKS if they're known and can be verified refs #517 #717 --- fdroidserver/build.py | 5 +- fdroidserver/common.py | 262 +++++++++++++++++++++++++++++++++++++++++ tests/build.TestCase | 122 +++++++++++++++++-- 3 files changed, 378 insertions(+), 11 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 76f4d930..3e0b2243 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -367,7 +367,10 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext if k.endswith("_orig"): continue logging.critical(" %s: %s" % (k, v)) - raise FDroidException() + if onserver: + common.auto_install_ndk(build) + else: + raise FDroidException() elif not os.path.isdir(ndk_path): logging.critical("Android NDK '%s' is not a directory!" % ndk_path) raise FDroidException() diff --git a/fdroidserver/common.py b/fdroidserver/common.py index ea9c7c26..1194402e 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -69,6 +69,7 @@ from pyasn1.codec.der import decoder, encoder from pyasn1_modules import rfc2315 from pyasn1.error import PyAsn1Error +from . import net import fdroidserver.metadata import fdroidserver.lint from fdroidserver import _ @@ -3989,3 +3990,264 @@ def sha256base64(filename): break hasher.update(t) return urlsafe_b64encode(hasher.digest()).decode() + + +def get_ndk_version(ndk_path): + source_properties = os.path.join(ndk_path, 'source.properties') + if os.path.exists(source_properties): + with open(source_properties) as fp: + m = re.search(r'^Pkg.Revision *= *(.+)', fp.read(), flags=re.MULTILINE) + if m: + return m.group(1) + + +def auto_install_ndk(build): + """auto-install the NDK in the build, this assumes its in a buildserver guest VM + + Download, verify, and install the NDK version as specified via the + "ndk:" field in the build entry. As it uncompresses the zipball, + this forces the permissions to work for all users, since this + might uncompress as root and then be used from a different user. + + This needs to be able to install multiple versions of the NDK, + since this is also used in CI builds, where multiple `fdroid build + --onserver` calls can run in a single session. The production + buildserver is reset between every build. + + The default ANDROID_HOME base dir of /home/vagrant/android-sdk is + hard-coded in buildserver/Vagrantfile. The "ndk" subdir is where + Android Studio will install the NDK into versioned subdirs. + https://developer.android.com/studio/projects/configure-agp-ndk#agp_version_41 + + Also, r10e and older cannot be handled via this mechanism because + they are packaged differently. + + """ + global config + if build.get('disable'): + return + ndk = build.get('ndk') + if not ndk: + return + ndk_path = config.get(ndk) + if ndk_path and os.path.isdir(ndk_path): + return + for ndkdict in NDKS: + if ndk == ndkdict['release']: + url = ndkdict['url'] + sha256 = ndkdict['sha256'] + break + ndk_base = os.path.join(config['sdk_path'], 'ndk') + logging.info(_('Downloading %s') % url) + zipball = os.path.join( + tempfile.mkdtemp(prefix='android-ndk-'), + os.path.basename(url) + ) + net.download_file(url, zipball) + if sha256 != sha256sum(zipball): + raise FDroidException('SHA-256 %s does not match expected for %s' % (sha256, url)) + logging.info(_('Unzipping to %s') % ndk_base) + with zipfile.ZipFile(zipball) as zipfp: + for info in zipfp.infolist(): + permbits = info.external_attr >> 16 + if stat.S_ISDIR(permbits) or stat.S_IXUSR & permbits: + zipfp.extract(info.filename, path=ndk_base) + os.chmod(os.path.join(ndk_base, info.filename), 0o755) # nosec bandit B103 + else: + zipfp.extract(info.filename, path=ndk_base) + os.chmod(os.path.join(ndk_base, info.filename), 0o644) # nosec bandit B103 + os.remove(zipball) + extracted = glob.glob(os.path.join(ndk_base, '*'))[0] + version = get_ndk_version(extracted) + ndk_dir = os.path.join(ndk_base, version) + os.rename(extracted, ndk_dir) + if 'ndk_paths' not in config: + config['ndk_paths'] = dict() + config['ndk_paths'][ndk] = ndk_dir + logging.info(_('Set NDK {release} ({version}) up') + .format(release=ndk, version=version)) + + +"""Derived from https://gitlab.com/fdroid/android-sdk-transparency-log/-/blob/master/checksums.json""" +NDKS = [ + { + "release": "r11", + "revision": "11.0.2655954", + "sha256": "59ab44f7ee6201df4381844736fdc456134c7f7660151003944a3017a0dcce97", + "url": "https://dl.google.com/android/repository/android-ndk-r11-linux-x86_64.zip" + }, + { + "release": "r11b", + "revision": "11.1.2683735", + "sha256": "51d429bfda8bbe038683ed7ae7acc03b39604b84711901b555fe18c698867e53", + "url": "https://dl.google.com/android/repository/android-ndk-r11b-linux-x86_64.zip" + }, + { + "release": "r11c", + "revision": "11.2.2725575", + "sha256": "ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94", + "url": "https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip" + }, + { + "release": "r12", + "revision": "12.0.2931149", + "sha256": "7876e3b99f3596a3215ecf4e9f152d24b82dfdf2bbe7d3a38c423ae6a3edee79", + "url": "https://dl.google.com/android/repository/android-ndk-r12-linux-x86_64.zip" + }, + { + "release": "r12b", + "revision": "12.1.2977051", + "sha256": "eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e", + "url": "https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip" + }, + { + "release": "r13", + "revision": "13.0.3315539", + "sha256": "0a1dbd216386399e2979c17a48f65b962bf7ddc0c2311ef35d902b90c298c400", + "url": "https://dl.google.com/android/repository/android-ndk-r13-linux-x86_64.zip" + }, + { + "release": "r13b", + "revision": "13.1.3345770", + "sha256": "3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c", + "url": "https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip" + }, + { + "release": "r14", + "revision": "14.0.3770861", + "sha256": "3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44", + "url": "https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip" + }, + { + "release": "r14b", + "revision": "14.1.3816874", + "sha256": "0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024", + "url": "https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip" + }, + { + "release": "r15", + "revision": "15.0.4075724", + "sha256": "078eb7d28c3fcf45841f5baf6e6582e7fd5b73d8e8c4e0101df490f51abd37b6", + "url": "https://dl.google.com/android/repository/android-ndk-r15-linux-x86_64.zip" + }, + { + "release": "r15b", + "revision": "15.1.4119039", + "sha256": "d1ce63f68cd806b5a992d4e5aa60defde131c243bf523cdfc5b67990ef0ee0d3", + "url": "https://dl.google.com/android/repository/android-ndk-r15b-linux-x86_64.zip" + }, + { + "release": "r15c", + "revision": "15.2.4203891", + "sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c", + "url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip" + }, + { + "release": "r16", + "revision": "16.0.4442984", + "sha256": "a8550b81771c67cc6ab7b479a6918d29aa78de3482901762b4f9e0132cd9672e", + "url": "https://dl.google.com/android/repository/android-ndk-r16-linux-x86_64.zip" + }, + { + "release": "r16b", + "revision": "16.1.4479499", + "sha256": "bcdea4f5353773b2ffa85b5a9a2ae35544ce88ec5b507301d8cf6a76b765d901", + "url": "https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip" + }, + { + "release": "r17", + "revision": "17.0.4754217", + "sha256": "ba3d813b47de75bc32a2f3de087f72599c6cb36fdc9686b96f517f5492ff43ca", + "url": "https://dl.google.com/android/repository/android-ndk-r17-linux-x86_64.zip" + }, + { + "release": "r17b", + "revision": "17.1.4828580", + "sha256": "5dfbbdc2d3ba859fed90d0e978af87c71a91a5be1f6e1c40ba697503d48ccecd", + "url": "https://dl.google.com/android/repository/android-ndk-r17b-linux-x86_64.zip" + }, + { + "release": "r17c", + "revision": "17.2.4988734", + "sha256": "3f541adbd0330a9205ba12697f6d04ec90752c53d6b622101a2a8a856e816589", + "url": "https://dl.google.com/android/repository/android-ndk-r17c-linux-x86_64.zip" + }, + { + "release": "r18b", + "revision": "18.1.5063045", + "sha256": "4f61cbe4bbf6406aa5ef2ae871def78010eed6271af72de83f8bd0b07a9fd3fd", + "url": "https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip" + }, + { + "release": "r19", + "revision": "19.0.5232133", + "sha256": "c0a2425206191252197b97ea5fcc7eab9f693a576e69ef4773a9ed1690feed53", + "url": "https://dl.google.com/android/repository/android-ndk-r19-linux-x86_64.zip" + }, + { + "release": "r19b", + "revision": "19.1.5304403", + "sha256": "0fbb1645d0f1de4dde90a4ff79ca5ec4899c835e729d692f433fda501623257a", + "url": "https://dl.google.com/android/repository/android-ndk-r19b-linux-x86_64.zip" + }, + { + "release": "r19c", + "revision": "19.2.5345600", + "sha256": "4c62514ec9c2309315fd84da6d52465651cdb68605058f231f1e480fcf2692e1", + "url": "https://dl.google.com/android/repository/android-ndk-r19c-linux-x86_64.zip" + }, + { + "release": "r20", + "revision": "20.0.5594570", + "sha256": "57435158f109162f41f2f43d5563d2164e4d5d0364783a9a6fab3ef12cb06ce0", + "url": "https://dl.google.com/android/repository/android-ndk-r20-linux-x86_64.zip" + }, + { + "release": "r20b", + "revision": "20.1.5948944", + "sha256": "8381c440fe61fcbb01e209211ac01b519cd6adf51ab1c2281d5daad6ca4c8c8c", + "url": "https://dl.google.com/android/repository/android-ndk-r20b-linux-x86_64.zip" + }, + { + "release": "r21", + "revision": "21.0.6113669", + "sha256": "b65ea2d5c5b68fb603626adcbcea6e4d12c68eb8a73e373bbb9d23c252fc647b", + "url": "https://dl.google.com/android/repository/android-ndk-r21-linux-x86_64.zip" + }, + { + "release": "r21b", + "revision": "21.1.6352462", + "sha256": "0c7af5dd23c5d2564915194e71b1053578438ac992958904703161c7672cbed7", + "url": "https://dl.google.com/android/repository/android-ndk-r21b-linux-x86_64.zip" + }, + { + "release": "r21c", + "revision": "21.2.6472646", + "sha256": "214ebfcfa5108ba78f5b2cc8db4d575068f9c973ac7f27d2fa1987dfdb76c9e7", + "url": "https://dl.google.com/android/repository/android-ndk-r21c-linux-x86_64.zip" + }, + { + "release": "r21d", + "revision": "21.3.6528147", + "sha256": "dd6dc090b6e2580206c64bcee499bc16509a5d017c6952dcd2bed9072af67cbd", + "url": "https://dl.google.com/android/repository/android-ndk-r21d-linux-x86_64.zip" + }, + { + "release": "r21e", + "revision": "21.4.7075529", + "sha256": "ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e", + "url": "https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip" + }, + { + "release": "r22", + "revision": "22.0.7026061", + "sha256": "d37fc69cd81e5660234a686e20adef39bc0244086e4d66525a40af771c020718", + "url": "https://dl.google.com/android/repository/android-ndk-r22-linux-x86_64.zip" + }, + { + "release": "r22b", + "revision": "22.1.7171670", + "sha256": "ac3a0421e76f71dd330d0cd55f9d99b9ac864c4c034fc67e0d671d022d4e806b", + "url": "https://dl.google.com/android/repository/android-ndk-r22b-linux-x86_64.zip" + } +] diff --git a/tests/build.TestCase b/tests/build.TestCase index 9645f583..f700b74c 100755 --- a/tests/build.TestCase +++ b/tests/build.TestCase @@ -12,10 +12,12 @@ import tempfile import textwrap import unittest import yaml +import zipfile from unittest import mock localmodule = os.path.realpath( - os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) + os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..') +) print('localmodule: ' + localmodule) if localmodule not in sys.path: sys.path.insert(0, localmodule) @@ -26,6 +28,14 @@ import fdroidserver.metadata import fdroidserver.scanner +class FakeProcess: + output = 'fake output' + returncode = 0 + + def __init__(self, args, **kwargs): + print('FakeFDroidPopen', args, kwargs) + + class BuildTest(unittest.TestCase): '''fdroidserver/build.py''' @@ -94,9 +104,106 @@ class BuildTest(unittest.TestCase): self.assertEqual(versionCode, vc) self.assertEqual(versionName, vn) + def test_build_local_ndk(self): + """Test if `fdroid build` detects installed NDKs and auto-installs when missing""" + testdir = tempfile.mkdtemp( + prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir + ) + os.chdir(testdir) + + config = {'ndk_paths': {}, 'sdk_path': tempfile.mkdtemp(prefix='android-sdk-')} + fdroidserver.common.config = config + fdroidserver.build.config = config + fdroidserver.build.options = mock.Mock() + fdroidserver.build.options.scan_binary = False + fdroidserver.build.options.notarball = True + fdroidserver.build.options.skipscan = True + + app = fdroidserver.metadata.App() + app.id = 'mocked.app.id' + build = fdroidserver.metadata.Build() + build.commit = '1.0' + build.output = app.id + '.apk' + build.versionCode = '1' + build.versionName = '1.0' + build.ndk = 'r21e' # aka 21.4.7075529 + vcs = mock.Mock() + + def make_fake_apk(output, build): + with open(build.output, 'w') as fp: + fp.write('APK PLACEHOLDER') + return output + + def fake_download_file(_ignored, local_filename): + _ignored # silence the linters + with zipfile.ZipFile(local_filename, 'x') as zipfp: + zipfp.writestr( + 'android-ndk-r21e/source.properties', + 'Pkg.Revision = 21.4.7075529\n', + ) + + # use "as _ignored" just to make a pretty layout + with mock.patch( + 'fdroidserver.common.replace_build_vars', wraps=make_fake_apk + ) as _ignored, mock.patch( + 'fdroidserver.common.get_native_code', return_value='x86' + ) as _ignored, mock.patch( + 'fdroidserver.common.get_apk_id', + return_value=(app.id, build.versionCode, build.versionName), + ) as _ignored, mock.patch( + 'fdroidserver.common.is_apk_and_debuggable', return_value=False + ) as _ignored, mock.patch( + 'fdroidserver.common.sha256sum', + return_value='ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e', + ) as _ignored, mock.patch( + 'fdroidserver.common.is_apk_and_debuggable', return_value=False + ) as _ignored, mock.patch( + 'fdroidserver.build.FDroidPopen', FakeProcess + ) as _ignored, mock.patch( + 'fdroidserver.net.download_file', wraps=fake_download_file + ) as _ignored: + _ignored # silence the linters + with self.assertRaises( + fdroidserver.exception.FDroidException, + msg="No NDK setup, `fdroid build` should fail with error", + ): + fdroidserver.build.build_local( + app, + build, + vcs, + build_dir=testdir, + output_dir=testdir, + log_dir=None, + srclib_dir=None, + extlib_dir=None, + tmp_dir=None, + force=False, + onserver=False, + refresh=False, + ) + # now run `fdroid buid --onserver` + self.assertTrue('r21e' not in config['ndk_paths']) + fdroidserver.build.build_local( + app, + build, + vcs, + build_dir=testdir, + output_dir=testdir, + log_dir=os.getcwd(), + srclib_dir=None, + extlib_dir=None, + tmp_dir=None, + force=False, + onserver=True, + refresh=False, + ) + self.assertTrue(os.path.exists(config['ndk_paths']['r21e'])) + def test_build_local_clean(self): """Test if `fdroid build` cleans ant and gradle build products""" - testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + testdir = tempfile.mkdtemp( + prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir + ) os.chdir(testdir) config = dict() @@ -214,14 +321,9 @@ class BuildTest(unittest.TestCase): def test_failed_verifies_are_not_in_unsigned(self): - class FakeProcess: - output = 'fake output' - returncode = 0 - - def __init__(self, args, **kwargs): - print('FakeFDroidPopen', args, kwargs) - - testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + testdir = tempfile.mkdtemp( + prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir + ) os.chdir(testdir) sdk_path = os.path.join(testdir, 'android-sdk') self.create_fake_android_home(sdk_path) From 9fc2a2371353f5a3962e8a4f833b4d3ebf958cc8 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 12 May 2021 14:24:05 +0200 Subject: [PATCH 4/6] build: remove default NDK, closes #717 --- examples/config.yml | 10 +++++----- fdroidserver/build.py | 2 +- fdroidserver/common.py | 10 +--------- fdroidserver/init.py | 1 - fdroidserver/metadata.py | 8 +------- 5 files changed, 8 insertions(+), 23 deletions(-) diff --git a/examples/config.yml b/examples/config.yml index 36a66c22..ee62c9ac 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -5,14 +5,14 @@ # Custom path to the Android SDK, defaults to $ANDROID_HOME # sdk_path: $ANDROID_HOME -# Custom paths to various versions of the Android NDK, defaults to 'r12b' set -# to $ANDROID_NDK. Most users will have the latest at $ANDROID_NDK, which is -# used by default. If a version is missing or assigned to None, it is assumed -# not installed. +# Paths to various installed versions of the Android NDK. If a +# required version is missing in the buildserver VM, it will be +# automatically downloaded and installed into a temporary dir. +# # ndk_paths: # r10e: None # r11c: None -# r12b: $ANDROID_NDK +# r12b: None # r13b: None # r14b: None # r15c: None diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 3e0b2243..efe7ab3f 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -361,7 +361,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext ndk_path = build.ndk_path() if build.ndk or (build.buildjni and build.buildjni != ['no']): if not ndk_path: - logging.critical("Android NDK version '%s' could not be found!" % build.ndk or 'r12b') + logging.critical("Android NDK version '%s' could not be found!" % build.ndk) logging.critical("Configured versions:") for k, v in config['ndk_paths'].items(): if k.endswith("_orig"): diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 1194402e..fba87e99 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -118,15 +118,7 @@ orig_path = None default_config = { 'sdk_path': "$ANDROID_HOME", - 'ndk_paths': { - 'r10e': None, - 'r11c': None, - 'r12b': "$ANDROID_NDK", - 'r13b': None, - 'r14b': None, - 'r15c': None, - 'r16b': None, - }, + 'ndk_paths': {}, 'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), 'java_paths': None, 'scan_binary': False, diff --git a/fdroidserver/init.py b/fdroidserver/init.py index e66d4daa..c1230d9c 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -226,7 +226,6 @@ def main(): msg = '\n' msg += _('Built repo based in "%s" with this config:') % fdroiddir msg += '\n\n Android SDK:\t\t\t' + config['sdk_path'] - msg += '\n Android NDK r12b (optional):\t$ANDROID_NDK' msg += '\n ' + _('Keystore for signing key:\t') + keystore if repo_keyalias is not None: msg += '\n Alias for key in store:\t' + repo_keyalias diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 597c4cd2..b72d6c9d 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -326,13 +326,7 @@ class Build(dict): return 'ant' def ndk_path(self): - version = self.ndk - if not version: - version = 'r12b' # falls back to latest - paths = fdroidserver.common.config['ndk_paths'] - if version not in paths: - return '' - return paths[version] + return fdroidserver.common.config['ndk_paths'].get(self.ndk, '') flagtypes = { From ec2cace22207329eaa1a288c6966cc086faf2353 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 12 May 2021 15:04:24 +0200 Subject: [PATCH 5/6] buildserver: trim pre-installed NDK list down to the bare minimum This keeps the Long Term Support release and the latest release installed. r10e was kept in because it needs a special extraction method, since it is a .bin file, not a .zip. r12b is kept in because it is the old default. Here is a survey of the NDK versions used in the most recent Builds entry in each app that uses the NDK: {'r10e': 6, 'r12b': 93, 'r13b': 4, 'r14b': 5, 'r15c': 7, 'r16b': 14, 'r17b': 4, 'r17c': 7, 'r18b': 9, 'r19c': 17, 'r20': 1, 'r20b': 22, 'r21': 3, 'r21d': 56, 'r21e': 65, 'r22': 9, 'r22b': 15, 'r9b': 1} #517 import glob import os import yaml try: from yaml import CSafeLoader as SafeLoader except ImportError: from yaml import SafeLoader ndks = dict() for f in glob.glob('metadata/*.yml'): with open(f) as fp: app = yaml.load(fp, Loader=SafeLoader) if app.get('Disable'): continue build = app.get('Builds', [])[-1] if build.get('disabled'): continue ndk = build.get('ndk') if ndk and ndk[1] == '9': print(f, build) elif ndk and int(ndk[2:3]) < 18: print(f, build) if ndk: print(f, ndk) if ndk not in ndks: ndks[ndk] = 0 ndks[ndk] += 1 import pprint pprint.pprint(ndks) --- buildserver/config.buildserver.yml | 10 ---------- buildserver/provision-android-ndk | 2 +- makebuildserver | 20 -------------------- 3 files changed, 1 insertion(+), 31 deletions(-) diff --git a/buildserver/config.buildserver.yml b/buildserver/config.buildserver.yml index cc139f96..2a3cb3fa 100644 --- a/buildserver/config.buildserver.yml +++ b/buildserver/config.buildserver.yml @@ -1,16 +1,6 @@ sdk_path: /home/vagrant/android-sdk ndk_paths: r10e: /home/vagrant/android-ndk/r10e - r11c: /home/vagrant/android-ndk/r11c - r12b: /home/vagrant/android-ndk/r12b - r13b: /home/vagrant/android-ndk/r13b - r14b: /home/vagrant/android-ndk/r14b - r15c: /home/vagrant/android-ndk/r15c - r16b: /home/vagrant/android-ndk/r16b - r17c: /home/vagrant/android-ndk/r17c - r18b: /home/vagrant/android-ndk/r18b - r19c: /home/vagrant/android-ndk/r19c - r20b: /home/vagrant/android-ndk/r20b r21e: /home/vagrant/android-ndk/r21e r22b: /home/vagrant/android-ndk/r22b diff --git a/buildserver/provision-android-ndk b/buildserver/provision-android-ndk index 754dd797..1fcabf93 100644 --- a/buildserver/provision-android-ndk +++ b/buildserver/provision-android-ndk @@ -15,7 +15,7 @@ if [ ! -e $NDK_BASE/r10e ]; then mv android-ndk-r10e r10e fi -for version in r11c r12b r13b r14b r15c r16b r17c r18b r19c r20b r21e r22b; do +for version in r21e r22b; do if [ ! -e ${NDK_BASE}/${version} ]; then unzip /vagrant/cache/android-ndk-${version}-linux-x86_64.zip > /dev/null mv android-ndk-${version} ${version} diff --git a/makebuildserver b/makebuildserver index 01b45a30..4f1864a8 100755 --- a/makebuildserver +++ b/makebuildserver @@ -293,26 +293,6 @@ CACHE_FILES = [ '0e46229820205440b48a5501122002842b82886e76af35f0f3a069243dca4b3c'), ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin', '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'), - ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip', - 'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'), - ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip', - 'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'), - ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip', - '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'), - ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip', - '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'), - ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip', - 'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'), - ('https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip', - 'bcdea4f5353773b2ffa85b5a9a2ae35544ce88ec5b507301d8cf6a76b765d901'), - ('https://dl.google.com/android/repository/android-ndk-r17c-linux-x86_64.zip', - '3f541adbd0330a9205ba12697f6d04ec90752c53d6b622101a2a8a856e816589'), - ('https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip', - '4f61cbe4bbf6406aa5ef2ae871def78010eed6271af72de83f8bd0b07a9fd3fd'), - ('https://dl.google.com/android/repository/android-ndk-r19c-linux-x86_64.zip', - '4c62514ec9c2309315fd84da6d52465651cdb68605058f231f1e480fcf2692e1'), - ('https://dl.google.com/android/repository/android-ndk-r20b-linux-x86_64.zip', - '8381c440fe61fcbb01e209211ac01b519cd6adf51ab1c2281d5daad6ca4c8c8c'), ('https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip', 'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e'), ('https://dl.google.com/android/repository/android-ndk-r22b-linux-x86_64.zip', From 9d44fa79190811d70166bc6813e9d12d7e8e9555 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 14 May 2021 00:03:16 +0200 Subject: [PATCH 6/6] gitlab-ci: auto-generate merge request when NDK release found Following the pattern of the gradle bot, this will check the transparency log for any new NDK release. If there are any, it will make a merge request from @fdroid-bot. --- .gitlab-ci.yml | 6 +- tests/ndk-release-checksums.py | 140 +++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 2 deletions(-) create mode 100755 tests/ndk-release-checksums.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3fae5c80..149b34f9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -261,7 +261,7 @@ fedora_latest: "cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests" -gradle: +gradle/ndk: image: debian:bullseye <<: *apt-template variables: @@ -276,16 +276,18 @@ gradle: python3-git python3-gitlab python3-requests - # if this is a merge request fork, then only check if makebuildserver or gradlew-fdroid changed + # if this is a merge request fork, then only check if relevant files changed - if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then git fetch https://gitlab.com/fdroid/fdroidserver.git; for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; do test "$f" == "makebuildserver" && export CHANGED="yes"; test "$f" == "gradlew-fdroid" && export CHANGED="yes"; + test "$f" == "fdroidserver/common.py" && export CHANGED="yes"; done; test -z "$CHANGED" && exit; fi - ./tests/gradle-release-checksums.py + - ./tests/ndk-release-checksums.py # Run an actual build in a simple, faked version of the buildserver guest VM. diff --git a/tests/ndk-release-checksums.py b/tests/ndk-release-checksums.py new file mode 100755 index 00000000..6e99d6a0 --- /dev/null +++ b/tests/ndk-release-checksums.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 + +import git +import gitlab +import json +import os +import re +import requests +import subprocess +from colorama import Fore, Style + + +checksums = None +versions = dict() + +while not checksums: + r = requests.get( + 'https://gitlab.com/fdroid/android-sdk-transparency-log/-/raw/master/checksums.json' + ) + if r.status_code == 200: + checksums = r.json() + +with open('fdroidserver/common.py') as fp: + common_py = fp.read() + +to_compile = re.search(r'\nNDKS = [^\]]+\]', common_py).group() +if not to_compile: + exit(1) +code = compile(to_compile, '', 'exec') +config = {} +exec(code, None, config) # nosec this is just a CI script + +ndks = [] +errors = 0 +release = None +revision = None +for k, entries in checksums.items(): + if k.startswith('https://dl.google.com/android/repository/android-ndk'): + m = re.search(r'-(r[1-9][0-9]?[a-z]?)-linux', k) + if m: + for entry in entries: + if 'source.properties' in entry: + n = re.search( + r'[1-9][0-9]\.[0-9]\.[0-9]{7}', entry['source.properties'] + ) + if n: + release = m.group(1) + revision = n.group() + ndks.append( + { + 'url': k, + 'release': release, + 'revision': revision, + 'sha256': checksums[k][0]['sha256'], + } + ) + for d in config['NDKS']: + if k == d['url']: + sha256 = d['sha256'] + found = False + for entry in entries: + if sha256 == entry['sha256']: + found = True + if not found: + print( + Fore.RED + + ( + 'ERROR: checksum mismatch: %s != %s' + % (sha256, entry['sha256']) + ) + + Style.RESET_ALL + ) + errors += 1 + +with open('fdroidserver/common.py', 'w') as fp: + fp.write( + common_py.replace( + to_compile, '\nNDKS = ' + json.dumps(ndks, indent=4, sort_keys=True) + ) + ) + +if os.getenv('CI_PROJECT_NAMESPACE') != 'fdroid': + p = subprocess.run(['git', '--no-pager', 'diff']) + print(p.stdout) + exit(errors) + + +# This only runs after commits are pushed to fdroid/fdroidserver +git_repo = git.repo.Repo('.') +modified = git_repo.git().ls_files(modified=True).split() +if git_repo.is_dirty() and 'fdroidserver/common.py' in modified: + branch = git_repo.create_head(os.path.basename(__file__), force=True) + branch.checkout() + git_repo.index.add(['fdroidserver/common.py']) + author = git.Actor('fdroid-bot', 'fdroid-bot@f-droid.org') + git_repo.index.commit('Android NDK %s (%s)' % (release, revision), author=author) + project_path = 'fdroid-bot/' + os.getenv('CI_PROJECT_NAME') + url = 'https://gitlab-ci-token:%s@%s/%s.git' % ( + os.getenv('PERSONAL_ACCESS_TOKEN'), + os.getenv('CI_SERVER_HOST'), + project_path, + ) + remote_name = 'fdroid-bot' + try: + remote = git_repo.create_remote(remote_name, url) + except git.exc.GitCommandError: + remote = git.remote.Remote(git_repo, remote_name) + remote.set_url(url) + remote.push(force=True) + git.remote.Remote.rm(git_repo, remote_name) + + private_token = os.getenv('PERSONAL_ACCESS_TOKEN') + if not private_token: + print( + Fore.RED + + 'ERROR: GitLab Token not found in PERSONAL_ACCESS_TOKEN!' + + Style.RESET_ALL + ) + exit(1) + gl = gitlab.Gitlab( + os.getenv('CI_SERVER_URL'), api_version=4, private_token=private_token + ) + project = gl.projects.get(project_path, lazy=True) + description = ( + 'see ' + '\n\n

generated by GitLab CI Job #%s

' + % (os.getenv('CI_PROJECT_URL'), os.getenv('CI_JOB_ID'), os.getenv('CI_JOB_ID')) + ) + mr = project.mergerequests.create( + { + 'source_branch': branch.name, + 'target_project_id': 36527, # fdroid/fdroidserver + 'target_branch': 'master', + 'title': 'update to gradle v' + version, + 'description': description, + 'labels': ['fdroid-bot', 'buildserver'], + 'remove_source_branch': True, + } + ) + mr.save()