diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3fae5c80..149b34f9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -261,7 +261,7 @@ fedora_latest: "cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests" -gradle: +gradle/ndk: image: debian:bullseye <<: *apt-template variables: @@ -276,16 +276,18 @@ gradle: python3-git python3-gitlab python3-requests - # if this is a merge request fork, then only check if makebuildserver or gradlew-fdroid changed + # if this is a merge request fork, then only check if relevant files changed - if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then git fetch https://gitlab.com/fdroid/fdroidserver.git; for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; do test "$f" == "makebuildserver" && export CHANGED="yes"; test "$f" == "gradlew-fdroid" && export CHANGED="yes"; + test "$f" == "fdroidserver/common.py" && export CHANGED="yes"; done; test -z "$CHANGED" && exit; fi - ./tests/gradle-release-checksums.py + - ./tests/ndk-release-checksums.py # Run an actual build in a simple, faked version of the buildserver guest VM. diff --git a/buildserver/config.buildserver.yml b/buildserver/config.buildserver.yml index cc139f96..2a3cb3fa 100644 --- a/buildserver/config.buildserver.yml +++ b/buildserver/config.buildserver.yml @@ -1,16 +1,6 @@ sdk_path: /home/vagrant/android-sdk ndk_paths: r10e: /home/vagrant/android-ndk/r10e - r11c: /home/vagrant/android-ndk/r11c - r12b: /home/vagrant/android-ndk/r12b - r13b: /home/vagrant/android-ndk/r13b - r14b: /home/vagrant/android-ndk/r14b - r15c: /home/vagrant/android-ndk/r15c - r16b: /home/vagrant/android-ndk/r16b - r17c: /home/vagrant/android-ndk/r17c - r18b: /home/vagrant/android-ndk/r18b - r19c: /home/vagrant/android-ndk/r19c - r20b: /home/vagrant/android-ndk/r20b r21e: /home/vagrant/android-ndk/r21e r22b: /home/vagrant/android-ndk/r22b diff --git a/buildserver/provision-android-ndk b/buildserver/provision-android-ndk index 754dd797..1fcabf93 100644 --- a/buildserver/provision-android-ndk +++ b/buildserver/provision-android-ndk @@ -15,7 +15,7 @@ if [ ! -e $NDK_BASE/r10e ]; then mv android-ndk-r10e r10e fi -for version in r11c r12b r13b r14b r15c r16b r17c r18b r19c r20b r21e r22b; do +for version in r21e r22b; do if [ ! -e ${NDK_BASE}/${version} ]; then unzip /vagrant/cache/android-ndk-${version}-linux-x86_64.zip > /dev/null mv android-ndk-${version} ${version} diff --git a/examples/config.yml b/examples/config.yml index 36a66c22..ee62c9ac 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -5,14 +5,14 @@ # Custom path to the Android SDK, defaults to $ANDROID_HOME # sdk_path: $ANDROID_HOME -# Custom paths to various versions of the Android NDK, defaults to 'r12b' set -# to $ANDROID_NDK. Most users will have the latest at $ANDROID_NDK, which is -# used by default. If a version is missing or assigned to None, it is assumed -# not installed. +# Paths to various installed versions of the Android NDK. If a +# required version is missing in the buildserver VM, it will be +# automatically downloaded and installed into a temporary dir. +# # ndk_paths: # r10e: None # r11c: None -# r12b: $ANDROID_NDK +# r12b: None # r13b: None # r14b: None # r15c: None diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 76f4d930..efe7ab3f 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -361,13 +361,16 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext ndk_path = build.ndk_path() if build.ndk or (build.buildjni and build.buildjni != ['no']): if not ndk_path: - logging.critical("Android NDK version '%s' could not be found!" % build.ndk or 'r12b') + logging.critical("Android NDK version '%s' could not be found!" % build.ndk) logging.critical("Configured versions:") for k, v in config['ndk_paths'].items(): if k.endswith("_orig"): continue logging.critical(" %s: %s" % (k, v)) - raise FDroidException() + if onserver: + common.auto_install_ndk(build) + else: + raise FDroidException() elif not os.path.isdir(ndk_path): logging.critical("Android NDK '%s' is not a directory!" % ndk_path) raise FDroidException() diff --git a/fdroidserver/common.py b/fdroidserver/common.py index df884b6d..fba87e99 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -58,6 +58,7 @@ try: except ImportError: import xml.etree.ElementTree as XMLElementTree # nosec this is a fallback only +from base64 import urlsafe_b64encode from binascii import hexlify from datetime import datetime, timedelta, timezone from distutils.version import LooseVersion @@ -68,6 +69,7 @@ from pyasn1.codec.der import decoder, encoder from pyasn1_modules import rfc2315 from pyasn1.error import PyAsn1Error +from . import net import fdroidserver.metadata import fdroidserver.lint from fdroidserver import _ @@ -116,15 +118,7 @@ orig_path = None default_config = { 'sdk_path': "$ANDROID_HOME", - 'ndk_paths': { - 'r10e': None, - 'r11c': None, - 'r12b': "$ANDROID_NDK", - 'r13b': None, - 'r14b': None, - 'r15c': None, - 'r16b': None, - }, + 'ndk_paths': {}, 'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), 'java_paths': None, 'scan_binary': False, @@ -3964,3 +3958,288 @@ def run_yamllint(path, indent=0): for problem in problems: result.append(' ' * indent + path + ':' + str(problem.line) + ': ' + problem.message) return '\n'.join(result) + + +def sha256sum(filename): + '''Calculate the sha256 of the given file''' + sha = hashlib.sha256() + with open(filename, 'rb') as f: + while True: + t = f.read(16384) + if len(t) == 0: + break + sha.update(t) + return sha.hexdigest() + + +def sha256base64(filename): + '''Calculate the sha256 of the given file as URL-safe base64''' + hasher = hashlib.sha256() + with open(filename, 'rb') as f: + while True: + t = f.read(16384) + if len(t) == 0: + break + hasher.update(t) + return urlsafe_b64encode(hasher.digest()).decode() + + +def get_ndk_version(ndk_path): + source_properties = os.path.join(ndk_path, 'source.properties') + if os.path.exists(source_properties): + with open(source_properties) as fp: + m = re.search(r'^Pkg.Revision *= *(.+)', fp.read(), flags=re.MULTILINE) + if m: + return m.group(1) + + +def auto_install_ndk(build): + """auto-install the NDK in the build, this assumes its in a buildserver guest VM + + Download, verify, and install the NDK version as specified via the + "ndk:" field in the build entry. As it uncompresses the zipball, + this forces the permissions to work for all users, since this + might uncompress as root and then be used from a different user. + + This needs to be able to install multiple versions of the NDK, + since this is also used in CI builds, where multiple `fdroid build + --onserver` calls can run in a single session. The production + buildserver is reset between every build. + + The default ANDROID_HOME base dir of /home/vagrant/android-sdk is + hard-coded in buildserver/Vagrantfile. The "ndk" subdir is where + Android Studio will install the NDK into versioned subdirs. + https://developer.android.com/studio/projects/configure-agp-ndk#agp_version_41 + + Also, r10e and older cannot be handled via this mechanism because + they are packaged differently. + + """ + global config + if build.get('disable'): + return + ndk = build.get('ndk') + if not ndk: + return + ndk_path = config.get(ndk) + if ndk_path and os.path.isdir(ndk_path): + return + for ndkdict in NDKS: + if ndk == ndkdict['release']: + url = ndkdict['url'] + sha256 = ndkdict['sha256'] + break + ndk_base = os.path.join(config['sdk_path'], 'ndk') + logging.info(_('Downloading %s') % url) + zipball = os.path.join( + tempfile.mkdtemp(prefix='android-ndk-'), + os.path.basename(url) + ) + net.download_file(url, zipball) + if sha256 != sha256sum(zipball): + raise FDroidException('SHA-256 %s does not match expected for %s' % (sha256, url)) + logging.info(_('Unzipping to %s') % ndk_base) + with zipfile.ZipFile(zipball) as zipfp: + for info in zipfp.infolist(): + permbits = info.external_attr >> 16 + if stat.S_ISDIR(permbits) or stat.S_IXUSR & permbits: + zipfp.extract(info.filename, path=ndk_base) + os.chmod(os.path.join(ndk_base, info.filename), 0o755) # nosec bandit B103 + else: + zipfp.extract(info.filename, path=ndk_base) + os.chmod(os.path.join(ndk_base, info.filename), 0o644) # nosec bandit B103 + os.remove(zipball) + extracted = glob.glob(os.path.join(ndk_base, '*'))[0] + version = get_ndk_version(extracted) + ndk_dir = os.path.join(ndk_base, version) + os.rename(extracted, ndk_dir) + if 'ndk_paths' not in config: + config['ndk_paths'] = dict() + config['ndk_paths'][ndk] = ndk_dir + logging.info(_('Set NDK {release} ({version}) up') + .format(release=ndk, version=version)) + + +"""Derived from https://gitlab.com/fdroid/android-sdk-transparency-log/-/blob/master/checksums.json""" +NDKS = [ + { + "release": "r11", + "revision": "11.0.2655954", + "sha256": "59ab44f7ee6201df4381844736fdc456134c7f7660151003944a3017a0dcce97", + "url": "https://dl.google.com/android/repository/android-ndk-r11-linux-x86_64.zip" + }, + { + "release": "r11b", + "revision": "11.1.2683735", + "sha256": "51d429bfda8bbe038683ed7ae7acc03b39604b84711901b555fe18c698867e53", + "url": "https://dl.google.com/android/repository/android-ndk-r11b-linux-x86_64.zip" + }, + { + "release": "r11c", + "revision": "11.2.2725575", + "sha256": "ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94", + "url": "https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip" + }, + { + "release": "r12", + "revision": "12.0.2931149", + "sha256": "7876e3b99f3596a3215ecf4e9f152d24b82dfdf2bbe7d3a38c423ae6a3edee79", + "url": "https://dl.google.com/android/repository/android-ndk-r12-linux-x86_64.zip" + }, + { + "release": "r12b", + "revision": "12.1.2977051", + "sha256": "eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e", + "url": "https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip" + }, + { + "release": "r13", + "revision": "13.0.3315539", + "sha256": "0a1dbd216386399e2979c17a48f65b962bf7ddc0c2311ef35d902b90c298c400", + "url": "https://dl.google.com/android/repository/android-ndk-r13-linux-x86_64.zip" + }, + { + "release": "r13b", + "revision": "13.1.3345770", + "sha256": "3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c", + "url": "https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip" + }, + { + "release": "r14", + "revision": "14.0.3770861", + "sha256": "3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44", + "url": "https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip" + }, + { + "release": "r14b", + "revision": "14.1.3816874", + "sha256": "0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024", + "url": "https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip" + }, + { + "release": "r15", + "revision": "15.0.4075724", + "sha256": "078eb7d28c3fcf45841f5baf6e6582e7fd5b73d8e8c4e0101df490f51abd37b6", + "url": "https://dl.google.com/android/repository/android-ndk-r15-linux-x86_64.zip" + }, + { + "release": "r15b", + "revision": "15.1.4119039", + "sha256": "d1ce63f68cd806b5a992d4e5aa60defde131c243bf523cdfc5b67990ef0ee0d3", + "url": "https://dl.google.com/android/repository/android-ndk-r15b-linux-x86_64.zip" + }, + { + "release": "r15c", + "revision": "15.2.4203891", + "sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c", + "url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip" + }, + { + "release": "r16", + "revision": "16.0.4442984", + "sha256": "a8550b81771c67cc6ab7b479a6918d29aa78de3482901762b4f9e0132cd9672e", + "url": "https://dl.google.com/android/repository/android-ndk-r16-linux-x86_64.zip" + }, + { + "release": "r16b", + "revision": "16.1.4479499", + "sha256": "bcdea4f5353773b2ffa85b5a9a2ae35544ce88ec5b507301d8cf6a76b765d901", + "url": "https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip" + }, + { + "release": "r17", + "revision": "17.0.4754217", + "sha256": "ba3d813b47de75bc32a2f3de087f72599c6cb36fdc9686b96f517f5492ff43ca", + "url": "https://dl.google.com/android/repository/android-ndk-r17-linux-x86_64.zip" + }, + { + "release": "r17b", + "revision": "17.1.4828580", + "sha256": "5dfbbdc2d3ba859fed90d0e978af87c71a91a5be1f6e1c40ba697503d48ccecd", + "url": "https://dl.google.com/android/repository/android-ndk-r17b-linux-x86_64.zip" + }, + { + "release": "r17c", + "revision": "17.2.4988734", + "sha256": "3f541adbd0330a9205ba12697f6d04ec90752c53d6b622101a2a8a856e816589", + "url": "https://dl.google.com/android/repository/android-ndk-r17c-linux-x86_64.zip" + }, + { + "release": "r18b", + "revision": "18.1.5063045", + "sha256": "4f61cbe4bbf6406aa5ef2ae871def78010eed6271af72de83f8bd0b07a9fd3fd", + "url": "https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip" + }, + { + "release": "r19", + "revision": "19.0.5232133", + "sha256": "c0a2425206191252197b97ea5fcc7eab9f693a576e69ef4773a9ed1690feed53", + "url": "https://dl.google.com/android/repository/android-ndk-r19-linux-x86_64.zip" + }, + { + "release": "r19b", + "revision": "19.1.5304403", + "sha256": "0fbb1645d0f1de4dde90a4ff79ca5ec4899c835e729d692f433fda501623257a", + "url": "https://dl.google.com/android/repository/android-ndk-r19b-linux-x86_64.zip" + }, + { + "release": "r19c", + "revision": "19.2.5345600", + "sha256": "4c62514ec9c2309315fd84da6d52465651cdb68605058f231f1e480fcf2692e1", + "url": "https://dl.google.com/android/repository/android-ndk-r19c-linux-x86_64.zip" + }, + { + "release": "r20", + "revision": "20.0.5594570", + "sha256": "57435158f109162f41f2f43d5563d2164e4d5d0364783a9a6fab3ef12cb06ce0", + "url": "https://dl.google.com/android/repository/android-ndk-r20-linux-x86_64.zip" + }, + { + "release": "r20b", + "revision": "20.1.5948944", + "sha256": "8381c440fe61fcbb01e209211ac01b519cd6adf51ab1c2281d5daad6ca4c8c8c", + "url": "https://dl.google.com/android/repository/android-ndk-r20b-linux-x86_64.zip" + }, + { + "release": "r21", + "revision": "21.0.6113669", + "sha256": "b65ea2d5c5b68fb603626adcbcea6e4d12c68eb8a73e373bbb9d23c252fc647b", + "url": "https://dl.google.com/android/repository/android-ndk-r21-linux-x86_64.zip" + }, + { + "release": "r21b", + "revision": "21.1.6352462", + "sha256": "0c7af5dd23c5d2564915194e71b1053578438ac992958904703161c7672cbed7", + "url": "https://dl.google.com/android/repository/android-ndk-r21b-linux-x86_64.zip" + }, + { + "release": "r21c", + "revision": "21.2.6472646", + "sha256": "214ebfcfa5108ba78f5b2cc8db4d575068f9c973ac7f27d2fa1987dfdb76c9e7", + "url": "https://dl.google.com/android/repository/android-ndk-r21c-linux-x86_64.zip" + }, + { + "release": "r21d", + "revision": "21.3.6528147", + "sha256": "dd6dc090b6e2580206c64bcee499bc16509a5d017c6952dcd2bed9072af67cbd", + "url": "https://dl.google.com/android/repository/android-ndk-r21d-linux-x86_64.zip" + }, + { + "release": "r21e", + "revision": "21.4.7075529", + "sha256": "ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e", + "url": "https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip" + }, + { + "release": "r22", + "revision": "22.0.7026061", + "sha256": "d37fc69cd81e5660234a686e20adef39bc0244086e4d66525a40af771c020718", + "url": "https://dl.google.com/android/repository/android-ndk-r22-linux-x86_64.zip" + }, + { + "release": "r22b", + "revision": "22.1.7171670", + "sha256": "ac3a0421e76f71dd330d0cd55f9d99b9ac864c4c034fc67e0d671d022d4e806b", + "url": "https://dl.google.com/android/repository/android-ndk-r22b-linux-x86_64.zip" + } +] diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 10dea357..1f30bd48 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -957,7 +957,6 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing % repo_icon) os.makedirs(os.path.dirname(iconfilename), exist_ok=True) try: - import qrcode qrcode.make(common.config['repo_url']).save(iconfilename) except Exception: exampleicon = os.path.join(common.get_examples_dir(), diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 8291612d..c1230d9c 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -66,7 +66,6 @@ def main(): help=_("Do not prompt for Android SDK path, just fail")) options = parser.parse_args() - aapt = None fdroiddir = os.getcwd() test_config = dict() examplesdir = common.get_examples_dir() @@ -227,9 +226,6 @@ def main(): msg = '\n' msg += _('Built repo based in "%s" with this config:') % fdroiddir msg += '\n\n Android SDK:\t\t\t' + config['sdk_path'] - if aapt: - msg += '\n Android SDK Build Tools:\t' + os.path.dirname(aapt) - msg += '\n Android NDK r12b (optional):\t$ANDROID_NDK' msg += '\n ' + _('Keystore for signing key:\t') + keystore if repo_keyalias is not None: msg += '\n Alias for key in store:\t' + repo_keyalias diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index d5911a8d..b72d6c9d 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -326,13 +326,7 @@ class Build(dict): return 'ant' def ndk_path(self): - version = self.ndk - if not version: - version = 'r12b' # falls back to latest - paths = fdroidserver.common.config['ndk_paths'] - if version not in paths: - return '' - return paths[version] + return fdroidserver.common.config['ndk_paths'].get(self.ndk, '') flagtypes = { @@ -890,10 +884,6 @@ def write_yaml(mf, app): _yaml_bools_plus_lists.extend(_yaml_bools_false) _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_false]) - def _class_as_dict_representer(dumper, data): - '''Creates a YAML representation of a App/Build instance''' - return dumper.represent_dict(data) - def _field_to_yaml(typ, value): if typ is TYPE_STRING: if value in _yaml_bools_plus_lists: diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 0866d9f0..1c3a1ed7 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -36,7 +36,6 @@ import yaml import copy from datetime import datetime from argparse import ArgumentParser -from base64 import urlsafe_b64encode try: from yaml import CSafeLoader as SafeLoader except ImportError: @@ -590,30 +589,6 @@ def get_icon_bytes(apkzip, iconsrc): return apkzip.read(iconsrc.encode('utf-8').decode('cp437')) -def sha256sum(filename): - '''Calculate the sha256 of the given file''' - sha = hashlib.sha256() - with open(filename, 'rb') as f: - while True: - t = f.read(16384) - if len(t) == 0: - break - sha.update(t) - return sha.hexdigest() - - -def sha256base64(filename): - '''Calculate the sha256 of the given file as URL-safe base64''' - hasher = hashlib.sha256() - with open(filename, 'rb') as f: - while True: - t = f.read(16384) - if len(t) == 0: - break - hasher.update(t) - return urlsafe_b64encode(hasher.digest()).decode() - - def has_known_vulnerability(filename): """checks for known vulnerabilities in the APK @@ -723,7 +698,7 @@ def insert_obbs(repodir, apps, apks): obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:') .format(integer=str(versionCode))) continue - obbsha256 = sha256sum(f) + obbsha256 = common.sha256sum(f) obbs.append((packagename, versionCode, obbfile, obbsha256)) for apk in apks: @@ -1267,7 +1242,7 @@ def insert_localized_app_metadata(apps): if not os.path.samefile(f, basepath): os.unlink(f) else: - sha256 = sha256base64(f) + sha256 = common.sha256base64(f) filename = base + '_' + sha256 + '.' + extension index_file = os.path.join(os.path.dirname(f), filename) if not os.path.exists(index_file): @@ -1313,7 +1288,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): raise FDroidException(_('{path} is zero size!') .format(path=filename)) - shasum = sha256sum(filename) + shasum = common.sha256sum(filename) usecache = False if name_utf8 in apkcache: repo_file = apkcache[name_utf8] @@ -1378,7 +1353,7 @@ def scan_apk(apk_file): :return A dict containing APK metadata """ apk = { - 'hash': sha256sum(apk_file), + 'hash': common.sha256sum(apk_file), 'hashType': 'sha256', 'uses-permission': [], 'uses-permission-sdk-23': [], @@ -1613,7 +1588,7 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal usecache = False if apkfilename in apkcache: apk = apkcache[apkfilename] - if apk.get('hash') == sha256sum(apkfile): + if apk.get('hash') == common.sha256sum(apkfile): logging.debug(_("Reading {apkfilename} from cache") .format(apkfilename=apkfilename)) usecache = True @@ -2067,7 +2042,6 @@ def create_metadata_from_template(apk): template: field sort order, empty field value, formatting, etc. ''' - import yaml if os.path.exists('template.yml'): with open('template.yml') as f: metatxt = f.read() diff --git a/locale/pick-complete-translations.py b/locale/pick-complete-translations.py index 99c1c125..3f343685 100755 --- a/locale/pick-complete-translations.py +++ b/locale/pick-complete-translations.py @@ -7,7 +7,6 @@ import os import re import requests import subprocess -import sys projectbasedir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) diff --git a/makebuildserver b/makebuildserver index 01b45a30..4f1864a8 100755 --- a/makebuildserver +++ b/makebuildserver @@ -293,26 +293,6 @@ CACHE_FILES = [ '0e46229820205440b48a5501122002842b82886e76af35f0f3a069243dca4b3c'), ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin', '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'), - ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip', - 'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'), - ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip', - 'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'), - ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip', - '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'), - ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip', - '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'), - ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip', - 'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'), - ('https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip', - 'bcdea4f5353773b2ffa85b5a9a2ae35544ce88ec5b507301d8cf6a76b765d901'), - ('https://dl.google.com/android/repository/android-ndk-r17c-linux-x86_64.zip', - '3f541adbd0330a9205ba12697f6d04ec90752c53d6b622101a2a8a856e816589'), - ('https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip', - '4f61cbe4bbf6406aa5ef2ae871def78010eed6271af72de83f8bd0b07a9fd3fd'), - ('https://dl.google.com/android/repository/android-ndk-r19c-linux-x86_64.zip', - '4c62514ec9c2309315fd84da6d52465651cdb68605058f231f1e480fcf2692e1'), - ('https://dl.google.com/android/repository/android-ndk-r20b-linux-x86_64.zip', - '8381c440fe61fcbb01e209211ac01b519cd6adf51ab1c2281d5daad6ca4c8c8c'), ('https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip', 'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e'), ('https://dl.google.com/android/repository/android-ndk-r22b-linux-x86_64.zip', diff --git a/tests/build.TestCase b/tests/build.TestCase index 9645f583..f700b74c 100755 --- a/tests/build.TestCase +++ b/tests/build.TestCase @@ -12,10 +12,12 @@ import tempfile import textwrap import unittest import yaml +import zipfile from unittest import mock localmodule = os.path.realpath( - os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) + os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..') +) print('localmodule: ' + localmodule) if localmodule not in sys.path: sys.path.insert(0, localmodule) @@ -26,6 +28,14 @@ import fdroidserver.metadata import fdroidserver.scanner +class FakeProcess: + output = 'fake output' + returncode = 0 + + def __init__(self, args, **kwargs): + print('FakeFDroidPopen', args, kwargs) + + class BuildTest(unittest.TestCase): '''fdroidserver/build.py''' @@ -94,9 +104,106 @@ class BuildTest(unittest.TestCase): self.assertEqual(versionCode, vc) self.assertEqual(versionName, vn) + def test_build_local_ndk(self): + """Test if `fdroid build` detects installed NDKs and auto-installs when missing""" + testdir = tempfile.mkdtemp( + prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir + ) + os.chdir(testdir) + + config = {'ndk_paths': {}, 'sdk_path': tempfile.mkdtemp(prefix='android-sdk-')} + fdroidserver.common.config = config + fdroidserver.build.config = config + fdroidserver.build.options = mock.Mock() + fdroidserver.build.options.scan_binary = False + fdroidserver.build.options.notarball = True + fdroidserver.build.options.skipscan = True + + app = fdroidserver.metadata.App() + app.id = 'mocked.app.id' + build = fdroidserver.metadata.Build() + build.commit = '1.0' + build.output = app.id + '.apk' + build.versionCode = '1' + build.versionName = '1.0' + build.ndk = 'r21e' # aka 21.4.7075529 + vcs = mock.Mock() + + def make_fake_apk(output, build): + with open(build.output, 'w') as fp: + fp.write('APK PLACEHOLDER') + return output + + def fake_download_file(_ignored, local_filename): + _ignored # silence the linters + with zipfile.ZipFile(local_filename, 'x') as zipfp: + zipfp.writestr( + 'android-ndk-r21e/source.properties', + 'Pkg.Revision = 21.4.7075529\n', + ) + + # use "as _ignored" just to make a pretty layout + with mock.patch( + 'fdroidserver.common.replace_build_vars', wraps=make_fake_apk + ) as _ignored, mock.patch( + 'fdroidserver.common.get_native_code', return_value='x86' + ) as _ignored, mock.patch( + 'fdroidserver.common.get_apk_id', + return_value=(app.id, build.versionCode, build.versionName), + ) as _ignored, mock.patch( + 'fdroidserver.common.is_apk_and_debuggable', return_value=False + ) as _ignored, mock.patch( + 'fdroidserver.common.sha256sum', + return_value='ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e', + ) as _ignored, mock.patch( + 'fdroidserver.common.is_apk_and_debuggable', return_value=False + ) as _ignored, mock.patch( + 'fdroidserver.build.FDroidPopen', FakeProcess + ) as _ignored, mock.patch( + 'fdroidserver.net.download_file', wraps=fake_download_file + ) as _ignored: + _ignored # silence the linters + with self.assertRaises( + fdroidserver.exception.FDroidException, + msg="No NDK setup, `fdroid build` should fail with error", + ): + fdroidserver.build.build_local( + app, + build, + vcs, + build_dir=testdir, + output_dir=testdir, + log_dir=None, + srclib_dir=None, + extlib_dir=None, + tmp_dir=None, + force=False, + onserver=False, + refresh=False, + ) + # now run `fdroid buid --onserver` + self.assertTrue('r21e' not in config['ndk_paths']) + fdroidserver.build.build_local( + app, + build, + vcs, + build_dir=testdir, + output_dir=testdir, + log_dir=os.getcwd(), + srclib_dir=None, + extlib_dir=None, + tmp_dir=None, + force=False, + onserver=True, + refresh=False, + ) + self.assertTrue(os.path.exists(config['ndk_paths']['r21e'])) + def test_build_local_clean(self): """Test if `fdroid build` cleans ant and gradle build products""" - testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + testdir = tempfile.mkdtemp( + prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir + ) os.chdir(testdir) config = dict() @@ -214,14 +321,9 @@ class BuildTest(unittest.TestCase): def test_failed_verifies_are_not_in_unsigned(self): - class FakeProcess: - output = 'fake output' - returncode = 0 - - def __init__(self, args, **kwargs): - print('FakeFDroidPopen', args, kwargs) - - testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + testdir = tempfile.mkdtemp( + prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir + ) os.chdir(testdir) sdk_path = os.path.join(testdir, 'android-sdk') self.create_fake_android_home(sdk_path) diff --git a/tests/ndk-release-checksums.py b/tests/ndk-release-checksums.py new file mode 100755 index 00000000..6e99d6a0 --- /dev/null +++ b/tests/ndk-release-checksums.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 + +import git +import gitlab +import json +import os +import re +import requests +import subprocess +from colorama import Fore, Style + + +checksums = None +versions = dict() + +while not checksums: + r = requests.get( + 'https://gitlab.com/fdroid/android-sdk-transparency-log/-/raw/master/checksums.json' + ) + if r.status_code == 200: + checksums = r.json() + +with open('fdroidserver/common.py') as fp: + common_py = fp.read() + +to_compile = re.search(r'\nNDKS = [^\]]+\]', common_py).group() +if not to_compile: + exit(1) +code = compile(to_compile, '', 'exec') +config = {} +exec(code, None, config) # nosec this is just a CI script + +ndks = [] +errors = 0 +release = None +revision = None +for k, entries in checksums.items(): + if k.startswith('https://dl.google.com/android/repository/android-ndk'): + m = re.search(r'-(r[1-9][0-9]?[a-z]?)-linux', k) + if m: + for entry in entries: + if 'source.properties' in entry: + n = re.search( + r'[1-9][0-9]\.[0-9]\.[0-9]{7}', entry['source.properties'] + ) + if n: + release = m.group(1) + revision = n.group() + ndks.append( + { + 'url': k, + 'release': release, + 'revision': revision, + 'sha256': checksums[k][0]['sha256'], + } + ) + for d in config['NDKS']: + if k == d['url']: + sha256 = d['sha256'] + found = False + for entry in entries: + if sha256 == entry['sha256']: + found = True + if not found: + print( + Fore.RED + + ( + 'ERROR: checksum mismatch: %s != %s' + % (sha256, entry['sha256']) + ) + + Style.RESET_ALL + ) + errors += 1 + +with open('fdroidserver/common.py', 'w') as fp: + fp.write( + common_py.replace( + to_compile, '\nNDKS = ' + json.dumps(ndks, indent=4, sort_keys=True) + ) + ) + +if os.getenv('CI_PROJECT_NAMESPACE') != 'fdroid': + p = subprocess.run(['git', '--no-pager', 'diff']) + print(p.stdout) + exit(errors) + + +# This only runs after commits are pushed to fdroid/fdroidserver +git_repo = git.repo.Repo('.') +modified = git_repo.git().ls_files(modified=True).split() +if git_repo.is_dirty() and 'fdroidserver/common.py' in modified: + branch = git_repo.create_head(os.path.basename(__file__), force=True) + branch.checkout() + git_repo.index.add(['fdroidserver/common.py']) + author = git.Actor('fdroid-bot', 'fdroid-bot@f-droid.org') + git_repo.index.commit('Android NDK %s (%s)' % (release, revision), author=author) + project_path = 'fdroid-bot/' + os.getenv('CI_PROJECT_NAME') + url = 'https://gitlab-ci-token:%s@%s/%s.git' % ( + os.getenv('PERSONAL_ACCESS_TOKEN'), + os.getenv('CI_SERVER_HOST'), + project_path, + ) + remote_name = 'fdroid-bot' + try: + remote = git_repo.create_remote(remote_name, url) + except git.exc.GitCommandError: + remote = git.remote.Remote(git_repo, remote_name) + remote.set_url(url) + remote.push(force=True) + git.remote.Remote.rm(git_repo, remote_name) + + private_token = os.getenv('PERSONAL_ACCESS_TOKEN') + if not private_token: + print( + Fore.RED + + 'ERROR: GitLab Token not found in PERSONAL_ACCESS_TOKEN!' + + Style.RESET_ALL + ) + exit(1) + gl = gitlab.Gitlab( + os.getenv('CI_SERVER_URL'), api_version=4, private_token=private_token + ) + project = gl.projects.get(project_path, lazy=True) + description = ( + 'see ' + '\n\n

generated by GitLab CI Job #%s

' + % (os.getenv('CI_PROJECT_URL'), os.getenv('CI_JOB_ID'), os.getenv('CI_JOB_ID')) + ) + mr = project.mergerequests.create( + { + 'source_branch': branch.name, + 'target_project_id': 36527, # fdroid/fdroidserver + 'target_branch': 'master', + 'title': 'update to gradle v' + version, + 'description': description, + 'labels': ['fdroid-bot', 'buildserver'], + 'remove_source_branch': True, + } + ) + mr.save()