From 0dd5a7db648133e3f5ea2dfe1acc381b0541738b Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 14 Jan 2022 12:50:44 +0100 Subject: [PATCH] remove NDK download handling in favor of fdroid/sdkmanager --- .gitlab-ci.yml | 5 +- fdroidserver/common.py | 315 ++------------------------------- setup.py | 1 + tests/build.TestCase | 111 ++++++++++-- tests/common.TestCase | 166 ++++++----------- tests/ndk-release-checksums.py | 139 --------------- 6 files changed, 172 insertions(+), 565 deletions(-) delete mode 100755 tests/ndk-release-checksums.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 094e0d42..6a57b907 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -276,7 +276,6 @@ black: tests/key-tricks.py tests/lint.TestCase tests/metadata.TestCase - tests/ndk-release-checksums.py tests/nightly.TestCase tests/rewritemeta.TestCase tests/scanner.TestCase @@ -329,7 +328,7 @@ fedora_latest: "cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests" -gradle/ndk: +gradle: image: debian:bullseye <<: *apt-template variables: @@ -350,12 +349,10 @@ gradle/ndk: for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; do test "$f" == "makebuildserver" && export CHANGED="yes"; test "$f" == "gradlew-fdroid" && export CHANGED="yes"; - test "$f" == "fdroidserver/common.py" && export CHANGED="yes"; done; test -z "$CHANGED" && exit; fi - ./tests/gradle-release-checksums.py - - ./tests/ndk-release-checksums.py # Run an actual build in a simple, faked version of the buildserver guest VM. diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 942e8fe6..e980f7fd 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -29,6 +29,7 @@ # libraries here as they will become a requirement for all commands. import git +import glob import io import os import sys @@ -36,7 +37,6 @@ import re import ast import gzip import shutil -import glob import stat import subprocess import time @@ -64,7 +64,6 @@ from pyasn1.codec.der import decoder, encoder from pyasn1_modules import rfc2315 from pyasn1.error import PyAsn1Error -from . import net import fdroidserver.metadata import fdroidserver.lint from fdroidserver import _ @@ -310,13 +309,6 @@ def fill_config_defaults(thisconfig): if version not in ndk_paths: ndk_paths[version] = ndk - for k in list(ndk_paths.keys()): - if not re.match(r'r[1-9][0-9]*[a-z]?', k): - for ndkdict in NDKS: - if k == ndkdict.get('revision'): - ndk_paths[ndkdict['release']] = ndk_paths.pop(k) - break - if 'cachedir_scanner' not in thisconfig: thisconfig['cachedir_scanner'] = str(Path(thisconfig['cachedir']) / 'scanner') if 'gradle_version_dir' not in thisconfig: @@ -4226,6 +4218,8 @@ def auto_install_ndk(build): they are packaged differently. """ + import sdkmanager + global config if build.get('disable'): return @@ -4233,8 +4227,10 @@ def auto_install_ndk(build): if not ndk: return if isinstance(ndk, str): + sdkmanager.build_package_list(use_net=True) _install_ndk(ndk) elif isinstance(ndk, list): + sdkmanager.build_package_list(use_net=True) for n in ndk: _install_ndk(n) else: @@ -4251,295 +4247,16 @@ def _install_ndk(ndk): The NDK version to install, either in "release" form (r21e) or "revision" form (21.4.7075529). """ - if re.match(r'[1-9][0-9.]+[0-9]', ndk): - for ndkdict in NDKS: - if ndk == ndkdict.get('revision'): - ndk = ndkdict['release'] - break + import sdkmanager - ndk_path = config.get(ndk) - if ndk_path and os.path.isdir(ndk_path): - return - for ndkdict in NDKS: - if ndk == ndkdict['release']: - url = ndkdict['url'] - sha256 = ndkdict['sha256'] - break - else: - raise FDroidException("NDK %s not found" % ndk) - ndk_base = os.path.join(config['sdk_path'], 'ndk') - logging.info(_('Downloading %s') % url) - with tempfile.TemporaryDirectory(prefix='android-ndk-') as ndk_dir: - zipball = os.path.join( - ndk_dir, - os.path.basename(url) + sdk_path = config['sdk_path'] + sdkmanager.install(f'ndk;{ndk}', sdk_path) + for found in glob.glob(f'{sdk_path}/ndk/*'): + version = get_ndk_version(found) + if 'ndk_paths' not in config: + config['ndk_paths'] = dict() + config['ndk_paths'][ndk] = found + config['ndk_paths'][version] = found + logging.info( + _('Set NDK {release} ({version}) up').format(release=ndk, version=version) ) - net.download_file(url, zipball) - calced = sha256sum(zipball) - if sha256 != calced: - raise FDroidException('SHA-256 %s does not match expected for %s (%s)' % (calced, url, sha256)) - logging.info(_('Unzipping to %s') % ndk_base) - with zipfile.ZipFile(zipball) as zipfp: - for info in zipfp.infolist(): - permbits = info.external_attr >> 16 - if stat.S_ISLNK(permbits): - link = os.path.join(ndk_base, info.filename) - link_target = zipfp.read(info).decode() - link_dir = os.path.dirname(link) - os.makedirs(link_dir, 0o755, True) # ensure intermediate directories are created - os.symlink(link_target, link) - - real_target = os.path.realpath(link) - if not real_target.startswith(ndk_base): - os.remove(link) - logging.error(_('Unexpected symlink target: {link} -> {target}') - .format(link=link, target=real_target)) - elif stat.S_ISDIR(permbits) or stat.S_IXUSR & permbits: - zipfp.extract(info.filename, path=ndk_base) - os.chmod(os.path.join(ndk_base, info.filename), 0o755) # nosec bandit B103 - else: - zipfp.extract(info.filename, path=ndk_base) - os.chmod(os.path.join(ndk_base, info.filename), 0o644) # nosec bandit B103 - os.remove(zipball) - for extracted in glob.glob(os.path.join(ndk_base, '*')): - version = get_ndk_version(extracted) - if os.path.basename(extracted) != version: - ndk_dir = os.path.join(ndk_base, version) - os.rename(extracted, ndk_dir) - if 'ndk_paths' not in config: - config['ndk_paths'] = dict() - config['ndk_paths'][ndk] = ndk_dir - logging.info(_('Set NDK {release} ({version}) up') - .format(release=ndk, version=version)) - - -"""Derived from https://gitlab.com/fdroid/android-sdk-transparency-log/-/blob/master/checksums.json""" -NDKS = [ - { - "release": "r10e", - "sha256": "ee5f405f3b57c4f5c3b3b8b5d495ae12b660e03d2112e4ed5c728d349f1e520c", - "url": "https://dl.google.com/android/repository/android-ndk-r10e-linux-x86_64.zip" - }, - { - "release": "r11", - "revision": "11.0.2655954", - "sha256": "59ab44f7ee6201df4381844736fdc456134c7f7660151003944a3017a0dcce97", - "url": "https://dl.google.com/android/repository/android-ndk-r11-linux-x86_64.zip" - }, - { - "release": "r11b", - "revision": "11.1.2683735", - "sha256": "51d429bfda8bbe038683ed7ae7acc03b39604b84711901b555fe18c698867e53", - "url": "https://dl.google.com/android/repository/android-ndk-r11b-linux-x86_64.zip" - }, - { - "release": "r11c", - "revision": "11.2.2725575", - "sha256": "ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94", - "url": "https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip" - }, - { - "release": "r12", - "revision": "12.0.2931149", - "sha256": "7876e3b99f3596a3215ecf4e9f152d24b82dfdf2bbe7d3a38c423ae6a3edee79", - "url": "https://dl.google.com/android/repository/android-ndk-r12-linux-x86_64.zip" - }, - { - "release": "r12b", - "revision": "12.1.2977051", - "sha256": "eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e", - "url": "https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip" - }, - { - "release": "r13", - "revision": "13.0.3315539", - "sha256": "0a1dbd216386399e2979c17a48f65b962bf7ddc0c2311ef35d902b90c298c400", - "url": "https://dl.google.com/android/repository/android-ndk-r13-linux-x86_64.zip" - }, - { - "release": "r13b", - "revision": "13.1.3345770", - "sha256": "3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c", - "url": "https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip" - }, - { - "release": "r14", - "revision": "14.0.3770861", - "sha256": "3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44", - "url": "https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip" - }, - { - "release": "r14b", - "revision": "14.1.3816874", - "sha256": "0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024", - "url": "https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip" - }, - { - "release": "r15", - "revision": "15.0.4075724", - "sha256": "078eb7d28c3fcf45841f5baf6e6582e7fd5b73d8e8c4e0101df490f51abd37b6", - "url": "https://dl.google.com/android/repository/android-ndk-r15-linux-x86_64.zip" - }, - { - "release": "r15b", - "revision": "15.1.4119039", - "sha256": "d1ce63f68cd806b5a992d4e5aa60defde131c243bf523cdfc5b67990ef0ee0d3", - "url": "https://dl.google.com/android/repository/android-ndk-r15b-linux-x86_64.zip" - }, - { - "release": "r15c", - "revision": "15.2.4203891", - "sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c", - "url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip" - }, - { - "release": "r16", - "revision": "16.0.4442984", - "sha256": "a8550b81771c67cc6ab7b479a6918d29aa78de3482901762b4f9e0132cd9672e", - "url": "https://dl.google.com/android/repository/android-ndk-r16-linux-x86_64.zip" - }, - { - "release": "r16b", - "revision": "16.1.4479499", - "sha256": "bcdea4f5353773b2ffa85b5a9a2ae35544ce88ec5b507301d8cf6a76b765d901", - "url": "https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip" - }, - { - "release": "r17", - "revision": "17.0.4754217", - "sha256": "ba3d813b47de75bc32a2f3de087f72599c6cb36fdc9686b96f517f5492ff43ca", - "url": "https://dl.google.com/android/repository/android-ndk-r17-linux-x86_64.zip" - }, - { - "release": "r17b", - "revision": "17.1.4828580", - "sha256": "5dfbbdc2d3ba859fed90d0e978af87c71a91a5be1f6e1c40ba697503d48ccecd", - "url": "https://dl.google.com/android/repository/android-ndk-r17b-linux-x86_64.zip" - }, - { - "release": "r17c", - "revision": "17.2.4988734", - "sha256": "3f541adbd0330a9205ba12697f6d04ec90752c53d6b622101a2a8a856e816589", - "url": "https://dl.google.com/android/repository/android-ndk-r17c-linux-x86_64.zip" - }, - { - "release": "r18b", - "revision": "18.1.5063045", - "sha256": "4f61cbe4bbf6406aa5ef2ae871def78010eed6271af72de83f8bd0b07a9fd3fd", - "url": "https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip" - }, - { - "release": "r19", - "revision": "19.0.5232133", - "sha256": "c0a2425206191252197b97ea5fcc7eab9f693a576e69ef4773a9ed1690feed53", - "url": "https://dl.google.com/android/repository/android-ndk-r19-linux-x86_64.zip" - }, - { - "release": "r19b", - "revision": "19.1.5304403", - "sha256": "0fbb1645d0f1de4dde90a4ff79ca5ec4899c835e729d692f433fda501623257a", - "url": "https://dl.google.com/android/repository/android-ndk-r19b-linux-x86_64.zip" - }, - { - "release": "r19c", - "revision": "19.2.5345600", - "sha256": "4c62514ec9c2309315fd84da6d52465651cdb68605058f231f1e480fcf2692e1", - "url": "https://dl.google.com/android/repository/android-ndk-r19c-linux-x86_64.zip" - }, - { - "release": "r20", - "revision": "20.0.5594570", - "sha256": "57435158f109162f41f2f43d5563d2164e4d5d0364783a9a6fab3ef12cb06ce0", - "url": "https://dl.google.com/android/repository/android-ndk-r20-linux-x86_64.zip" - }, - { - "release": "r20b", - "revision": "20.1.5948944", - "sha256": "8381c440fe61fcbb01e209211ac01b519cd6adf51ab1c2281d5daad6ca4c8c8c", - "url": "https://dl.google.com/android/repository/android-ndk-r20b-linux-x86_64.zip" - }, - { - "release": "r21", - "revision": "21.0.6113669", - "sha256": "b65ea2d5c5b68fb603626adcbcea6e4d12c68eb8a73e373bbb9d23c252fc647b", - "url": "https://dl.google.com/android/repository/android-ndk-r21-linux-x86_64.zip" - }, - { - "release": "r21b", - "revision": "21.1.6352462", - "sha256": "0c7af5dd23c5d2564915194e71b1053578438ac992958904703161c7672cbed7", - "url": "https://dl.google.com/android/repository/android-ndk-r21b-linux-x86_64.zip" - }, - { - "release": "r21c", - "revision": "21.2.6472646", - "sha256": "214ebfcfa5108ba78f5b2cc8db4d575068f9c973ac7f27d2fa1987dfdb76c9e7", - "url": "https://dl.google.com/android/repository/android-ndk-r21c-linux-x86_64.zip" - }, - { - "release": "r21d", - "revision": "21.3.6528147", - "sha256": "dd6dc090b6e2580206c64bcee499bc16509a5d017c6952dcd2bed9072af67cbd", - "url": "https://dl.google.com/android/repository/android-ndk-r21d-linux-x86_64.zip" - }, - { - "release": "r21e", - "revision": "21.4.7075529", - "sha256": "ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e", - "url": "https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip" - }, - { - "release": "r22", - "revision": "22.0.7026061", - "sha256": "d37fc69cd81e5660234a686e20adef39bc0244086e4d66525a40af771c020718", - "url": "https://dl.google.com/android/repository/android-ndk-r22-linux-x86_64.zip" - }, - { - "release": "r22b", - "revision": "22.1.7171670", - "sha256": "ac3a0421e76f71dd330d0cd55f9d99b9ac864c4c034fc67e0d671d022d4e806b", - "url": "https://dl.google.com/android/repository/android-ndk-r22b-linux-x86_64.zip" - }, - { - "release": "r23", - "revision": "23.0.7599858", - "sha256": "e3eacf80016b91d4cd2c8ca9f34eebd32df912bb799c859cc5450b6b19277b4f", - "url": "https://dl.google.com/android/repository/android-ndk-r23-linux.zip" - }, - { - "release": "r23b", - "revision": "23.1.7779620", - "sha256": "c6e97f9c8cfe5b7be0a9e6c15af8e7a179475b7ded23e2d1c1fa0945d6fb4382", - "url": "https://dl.google.com/android/repository/android-ndk-r23b-linux.zip" - }, - { - "release": "r23c", - "revision": "23.2.8568313", - "sha256": "6ce94604b77d28113ecd588d425363624a5228d9662450c48d2e4053f8039242", - "url": "https://dl.google.com/android/repository/android-ndk-r23c-linux.zip" - }, - { - "release": "r24", - "revision": "24.0.8215888", - "sha256": "caac638f060347c9aae994e718ba00bb18413498d8e0ad4e12e1482964032997", - "url": "https://dl.google.com/android/repository/android-ndk-r24-linux.zip" - }, - { - "release": "r25", - "revision": "25.0.8775105", - "sha256": "cd661aeda5d9b7cfb6e64bd80737c274d7c1c0d026df2f85be3bf3327b25e545", - "url": "https://dl.google.com/android/repository/android-ndk-r25-linux.zip" - }, - { - "release": "r25b", - "revision": "25.1.8937393", - "sha256": "403ac3e3020dd0db63a848dcaba6ceb2603bf64de90949d5c4361f848e44b005", - "url": "https://dl.google.com/android/repository/android-ndk-r25b-linux.zip" - }, - { - "release": "r25c", - "revision": "25.2.9519653", - "sha256": "769ee342ea75f80619d985c2da990c48b3d8eaf45f48783a2d48870d04b46108", - "url": "https://dl.google.com/android/repository/android-ndk-r25c-linux.zip" - } -] diff --git a/setup.py b/setup.py index a64f7df2..065c1c28 100755 --- a/setup.py +++ b/setup.py @@ -105,6 +105,7 @@ setup( 'qrcode', 'ruamel.yaml >= 0.15', 'requests >= 2.5.2, != 2.11.0, != 2.12.2, != 2.18.0', + 'sdkmanager >= 0.6.4', 'yamllint', ], extras_require={ diff --git a/tests/build.TestCase b/tests/build.TestCase index 197cf48b..7129d1bf 100755 --- a/tests/build.TestCase +++ b/tests/build.TestCase @@ -12,7 +12,7 @@ import tempfile import textwrap import unittest import yaml -import zipfile +from pathlib import Path from unittest import mock localmodule = os.path.realpath( @@ -214,6 +214,7 @@ class BuildTest(unittest.TestCase): self.assertEqual(versionCode, vc) self.assertEqual(versionName, vn) + @mock.patch('sdkmanager.build_package_list', lambda use_net: None) def test_build_local_ndk(self): """Test if `fdroid build` detects installed NDKs and auto-installs when missing""" with tempfile.TemporaryDirectory() as testdir, TmpCwd( @@ -235,6 +236,8 @@ class BuildTest(unittest.TestCase): build.versionCode = 1 build.versionName = '1.0' build.ndk = 'r21e' # aka 21.4.7075529 + ndk_version = '21.4.7075529' + ndk_dir = Path(config['sdk_path']) / 'ndk' / ndk_version vcs = mock.Mock() def make_fake_apk(output, build): @@ -242,13 +245,12 @@ class BuildTest(unittest.TestCase): fp.write('APK PLACEHOLDER') return output - def fake_download_file(_ignored, local_filename): - _ignored # silence the linters - with zipfile.ZipFile(local_filename, 'x') as zipfp: - zipfp.writestr( - 'android-ndk-r21e/source.properties', - 'Pkg.Revision = 21.4.7075529\n', - ) + # pylint: disable=unused-argument + def fake_sdkmanager_install(to_install, android_home=None): + ndk_dir.mkdir(parents=True) + self.assertNotEqual(ndk_version, to_install) # converts r21e to version + with (ndk_dir / 'source.properties').open('w') as fp: + fp.write('Pkg.Revision = %s\n' % ndk_version) # use "as _ignored" just to make a pretty layout with mock.patch( @@ -258,8 +260,6 @@ class BuildTest(unittest.TestCase): ) as _ignored, mock.patch( 'fdroidserver.common.get_apk_id', return_value=(app.id, build.versionCode, build.versionName), - ) as _ignored, mock.patch( - 'fdroidserver.common.is_apk_and_debuggable', return_value=False ) as _ignored, mock.patch( 'fdroidserver.common.sha256sum', return_value='ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e', @@ -268,7 +268,7 @@ class BuildTest(unittest.TestCase): ) as _ignored, mock.patch( 'fdroidserver.build.FDroidPopen', FakeProcess ) as _ignored, mock.patch( - 'fdroidserver.net.download_file', wraps=fake_download_file + 'sdkmanager.install', wraps=fake_sdkmanager_install ) as _ignored: _ignored # silence the linters with self.assertRaises( @@ -290,7 +290,10 @@ class BuildTest(unittest.TestCase): refresh=False, ) # now run `fdroid build --onserver` - self.assertTrue('r21e' not in config['ndk_paths']) + print('now run `fdroid build --onserver`') + self.assertFalse(ndk_dir.exists()) + self.assertFalse('r21e' in config['ndk_paths']) + self.assertFalse(ndk_version in config['ndk_paths']) fdroidserver.build.build_local( app, build, @@ -305,7 +308,89 @@ class BuildTest(unittest.TestCase): onserver=True, refresh=False, ) - self.assertTrue(os.path.exists(config['ndk_paths']['r21e'])) + self.assertTrue(ndk_dir.exists()) + self.assertTrue(os.path.exists(config['ndk_paths'][ndk_version])) + # All paths in the config must be strings, never pathlib.Path instances + self.assertIsInstance(config['ndk_paths'][ndk_version], str) + + @mock.patch('sdkmanager.build_package_list', lambda use_net: None) + @mock.patch('fdroidserver.build.FDroidPopen', FakeProcess) + @mock.patch('fdroidserver.common.get_native_code', lambda _ignored: 'x86') + @mock.patch('fdroidserver.common.is_apk_and_debuggable', lambda _ignored: False) + @mock.patch( + 'fdroidserver.common.sha256sum', + lambda f: 'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e', + ) + def test_build_local_ndk_some_installed(self): + """Test if `fdroid build` detects installed NDKs and auto-installs when missing""" + with tempfile.TemporaryDirectory() as testdir, TmpCwd( + testdir + ), tempfile.TemporaryDirectory() as sdk_path: + ndk_r24 = os.path.join(sdk_path, 'ndk', '24.0.8215888') + os.makedirs(ndk_r24) + with open(os.path.join(ndk_r24, 'source.properties'), 'w') as fp: + fp.write('Pkg.Revision = 24.0.8215888\n') + config = {'ndk_paths': {'r24': ndk_r24}, 'sdk_path': sdk_path} + fdroidserver.common.config = config + fdroidserver.build.config = config + fdroidserver.build.options = mock.Mock() + fdroidserver.build.options.scan_binary = False + fdroidserver.build.options.notarball = True + fdroidserver.build.options.skipscan = True + + app = fdroidserver.metadata.App() + app.id = 'mocked.app.id' + build = fdroidserver.metadata.Build() + build.commit = '1.0' + build.output = app.id + '.apk' + build.versionCode = 1 + build.versionName = '1.0' + build.ndk = 'r21e' # aka 21.4.7075529 + ndk_version = '21.4.7075529' + ndk_dir = Path(config['sdk_path']) / 'ndk' / ndk_version + vcs = mock.Mock() + + def make_fake_apk(output, build): + with open(build.output, 'w') as fp: + fp.write('APK PLACEHOLDER') + return output + + # pylint: disable=unused-argument + def fake_sdkmanager_install(to_install, android_home=None): + ndk_dir.mkdir(parents=True) + self.assertNotEqual(ndk_version, to_install) # converts r21e to version + with (ndk_dir / 'source.properties').open('w') as fp: + fp.write('Pkg.Revision = %s\n' % ndk_version) + + # use "as _ignored" just to make a pretty layout + with mock.patch( + 'fdroidserver.common.replace_build_vars', wraps=make_fake_apk + ) as _ignored, mock.patch( + 'fdroidserver.common.get_apk_id', + return_value=(app.id, build.versionCode, build.versionName), + ) as _ignored, mock.patch( + 'sdkmanager.install', wraps=fake_sdkmanager_install + ) as _ignored: + _ignored # silence the linters + self.assertFalse(ndk_dir.exists()) + self.assertFalse('r21e' in config['ndk_paths']) + self.assertFalse(ndk_version in config['ndk_paths']) + fdroidserver.build.build_local( + app, + build, + vcs, + build_dir=testdir, + output_dir=testdir, + log_dir=os.getcwd(), + srclib_dir=None, + extlib_dir=None, + tmp_dir=None, + force=False, + onserver=True, + refresh=False, + ) + self.assertTrue(ndk_dir.exists()) + self.assertTrue(os.path.exists(config['ndk_paths'][ndk_version])) def test_build_local_clean(self): """Test if `fdroid build` cleans ant and gradle build products""" diff --git a/tests/common.TestCase b/tests/common.TestCase index 11b1da66..0d17a879 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -5,6 +5,7 @@ import difflib import git import glob +import importlib import inspect import json import logging @@ -20,9 +21,8 @@ import unittest import textwrap import yaml import gzip -import stat from distutils.version import LooseVersion -from zipfile import BadZipFile, ZipFile, ZipInfo +from zipfile import BadZipFile, ZipFile from unittest import mock from pathlib import Path @@ -2164,7 +2164,7 @@ class CommonTest(unittest.TestCase): ) self.assertEqual( {'com.example': [12345, 67890], 'org.fdroid.fdroid': [1]}, - fdroidserver.common.read_pkg_args(appid_versionCode_pairs, allow_vercodes) + fdroidserver.common.read_pkg_args(appid_versionCode_pairs, allow_vercodes), ) appid_versionCode_pairs = ( 'com.example:67890', @@ -2210,42 +2210,79 @@ class CommonTest(unittest.TestCase): fdroidserver.common.metadata_find_developer_signing_files(appid, vc), ) + @mock.patch('sdkmanager.build_package_list', lambda use_net: None) def test_auto_install_ndk(self): """Test all possible field data types for build.ndk""" + fdroidserver.common.config = {'sdk_path': self.testdir} + sdk_path = self.testdir build = fdroidserver.metadata.Build() none_entry = mock.Mock() - with mock.patch('fdroidserver.common._install_ndk', none_entry): + with mock.patch('sdkmanager.install', none_entry): fdroidserver.common.auto_install_ndk(build) none_entry.assert_not_called() empty_list = mock.Mock() build.ndk = [] - with mock.patch('fdroidserver.common._install_ndk', empty_list): + with mock.patch('sdkmanager.install', empty_list): fdroidserver.common.auto_install_ndk(build) empty_list.assert_not_called() release_entry = mock.Mock() build.ndk = 'r21e' - with mock.patch('fdroidserver.common._install_ndk', release_entry): + with mock.patch('sdkmanager.install', release_entry): fdroidserver.common.auto_install_ndk(build) - release_entry.assert_called_once_with('r21e') + release_entry.assert_called_once_with('ndk;r21e', sdk_path) revision_entry = mock.Mock() build.ndk = '21.4.7075529' - with mock.patch('fdroidserver.common._install_ndk', revision_entry): + with mock.patch('sdkmanager.install', revision_entry): fdroidserver.common.auto_install_ndk(build) - revision_entry.assert_called_once_with('21.4.7075529') + revision_entry.assert_called_once_with('ndk;21.4.7075529', sdk_path) list_entry = mock.Mock() calls = [] build.ndk = ['r10e', '11.0.2655954', 'r12b', 'r21e'] for n in build.ndk: - calls.append(mock.call(n)) - with mock.patch('fdroidserver.common._install_ndk', list_entry): + calls.append(mock.call(f'ndk;{n}', sdk_path)) + with mock.patch('sdkmanager.install', list_entry): fdroidserver.common.auto_install_ndk(build) list_entry.assert_has_calls(calls) + @unittest.skipIf(importlib.util.find_spec('sdkmanager') is None, 'needs sdkmanager') + @mock.patch('sdkmanager.build_package_list', lambda use_net: None) + @mock.patch('sdkmanager._install_zipball_from_cache', lambda a, b: None) + @mock.patch('sdkmanager._generate_package_xml', lambda a, b, c: None) + def test_auto_install_ndk_mock_dl(self): + """Test NDK installs by actually calling sdkmanager""" + import sdkmanager + import pkg_resources + + sdkmanager_version = LooseVersion( + pkg_resources.get_distribution('sdkmanager').version + ) + if sdkmanager_version < LooseVersion('0.6.4'): + raise unittest.SkipTest('needs fdroid sdkmanager >= 0.6.4') + + fdroidserver.common.config = {'sdk_path': 'placeholder'} + build = fdroidserver.metadata.Build() + url = 'https://dl.google.com/android/repository/android-ndk-r24-linux.zip' + path = sdkmanager.get_cachedir() / os.path.basename(url) + sdkmanager.packages = { + ('ndk', '24.0.8215888'): url, + ('ndk', 'r24'): url, + } + build.ndk = 'r24' + firstrun = mock.Mock() + with mock.patch('sdkmanager.download_file', firstrun): + fdroidserver.common.auto_install_ndk(build) + firstrun.assert_called_once_with(url, path) + build.ndk = '24.0.8215888' + secondrun = mock.Mock() + with mock.patch('sdkmanager.download_file', secondrun): + fdroidserver.common.auto_install_ndk(build) + secondrun.assert_called_once_with(url, path) + @unittest.skip("This test downloads and unzips a 1GB file.") def test_install_ndk(self): """NDK r10e is a special case since its missing source.properties""" @@ -2257,101 +2294,6 @@ class CommonTest(unittest.TestCase): fdroidserver.common.fill_config_defaults(config) self.assertEqual({'r10e': r10e}, config['ndk_paths']) - def test_install_ndk_versions(self): - """Test whether NDK version parsing is working properly""" - - def fake_download(url, zipball): - print(url, zipball) - with ZipFile(zipball, 'w') as zipfp: - zipfp.writestr(os.path.basename(url), url) - - with tempfile.TemporaryDirectory( - prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir - ) as sdk_path: - config = {'sdk_path': sdk_path} - fdroidserver.common.config = config - for r, sha256 in ( - ( - 'r10e', - 'ee5f405f3b57c4f5c3b3b8b5d495ae12b660e03d2112e4ed5c728d349f1e520c', - ), - ('r20', '57435158f109162f41f2f43d5563d2164e4d5d0364783a9a6fab3ef12cb06ce0'), - ( - '23.0.7599858', - 'e3eacf80016b91d4cd2c8ca9f34eebd32df912bb799c859cc5450b6b19277b4f', - ), - ): - with mock.patch( - 'fdroidserver.net.download_file', side_effect=fake_download - ) as _ignored, mock.patch( - 'fdroidserver.common.get_ndk_version', return_value=r - ) as _ignored, mock.patch( - 'fdroidserver.common.sha256sum', return_value=sha256 - ): - _ignored # silence the linters - fdroidserver.common._install_ndk(r) - - def test_install_ndk_with_symlinks(self): - """Some NDK zipballs might have symlinks in them.""" - - def fake_download(url, zipball): - print(url, zipball) - unix_st_mode = ( - stat.S_IFLNK - | stat.S_IRUSR - | stat.S_IWUSR - | stat.S_IXUSR - | stat.S_IRGRP - | stat.S_IWGRP - | stat.S_IXGRP - | stat.S_IROTH - | stat.S_IWOTH - | stat.S_IXOTH - ) - with ZipFile(zipball, 'w') as zipfp: - zipfp.writestr('ndk/' + os.path.basename(url), url) - - zipInfo = ZipInfo('ndk/basename') - zipInfo.create_system = 3 - zipInfo.external_attr = unix_st_mode << 16 - zipfp.writestr(zipInfo, os.path.basename(url)) - - zipInfo = ZipInfo('ndk/bad_abs_link') - zipInfo.create_system = 3 - zipInfo.external_attr = unix_st_mode << 16 - zipfp.writestr(zipInfo, '/etc/passwd') - - zipInfo = ZipInfo('ndk/bad_rel_link') - zipInfo.create_system = 3 - zipInfo.external_attr = unix_st_mode << 16 - zipfp.writestr(zipInfo, '../../../../../../../etc/passwd') - - zipInfo = ZipInfo('ndk/bad_rel_link2') - zipInfo.create_system = 3 - zipInfo.external_attr = unix_st_mode << 16 - zipfp.writestr(zipInfo, 'foo/../../../../../../../../../etc/passwd') - - config = {'sdk_path': self.tmpdir} - fdroidserver.common.config = config - r = 'r20' - sha256 = '57435158f109162f41f2f43d5563d2164e4d5d0364783a9a6fab3ef12cb06ce0' - with mock.patch( - 'fdroidserver.net.download_file', side_effect=fake_download - ) as _ignored, mock.patch( - 'fdroidserver.common.get_ndk_version', return_value=r - ) as _ignored, mock.patch( - 'fdroidserver.common.sha256sum', return_value=sha256 - ): - _ignored # silence the linters - fdroidserver.common._install_ndk(r) - - self.assertTrue(os.path.exists(os.path.join(self.tmpdir, 'ndk', 'r20'))) - self.assertTrue(os.path.exists(os.path.join(self.tmpdir, 'ndk', 'r20', 'basename'))) - self.assertFalse(os.path.exists(os.path.join(self.tmpdir, 'ndk', 'r20', 'bad_abs_link'))) - self.assertFalse(os.path.exists(os.path.join(self.tmpdir, 'ndk', 'r20', 'bad_rel_link'))) - self.assertFalse(os.path.exists(os.path.join(self.tmpdir, 'ndk', 'r20', 'bad_rel_link2'))) - os.system('ls -l ' + os.path.join(self.tmpdir, 'ndk', 'r20')) - def test_fill_config_defaults(self): """Test the auto-detection of NDKs installed in standard paths""" ndk_bundle = os.path.join(self.tmpdir, 'ndk-bundle') @@ -2360,7 +2302,7 @@ class CommonTest(unittest.TestCase): fp.write('Pkg.Desc = Android NDK\nPkg.Revision = 17.2.4988734\n') config = {'sdk_path': self.tmpdir} fdroidserver.common.fill_config_defaults(config) - self.assertEqual({'r17c': ndk_bundle}, config['ndk_paths']) + self.assertEqual({'17.2.4988734': ndk_bundle}, config['ndk_paths']) r21e = os.path.join(self.tmpdir, 'ndk', '21.4.7075529') os.makedirs(r21e) @@ -2368,7 +2310,10 @@ class CommonTest(unittest.TestCase): fp.write('Pkg.Desc = Android NDK\nPkg.Revision = 21.4.7075529\n') config = {'sdk_path': self.tmpdir} fdroidserver.common.fill_config_defaults(config) - self.assertEqual({'r17c': ndk_bundle, 'r21e': r21e}, config['ndk_paths']) + self.assertEqual( + {'17.2.4988734': ndk_bundle, '21.4.7075529': r21e}, + config['ndk_paths'], + ) r10e = os.path.join(self.tmpdir, 'ndk', 'r10e') os.makedirs(r10e) @@ -2377,7 +2322,8 @@ class CommonTest(unittest.TestCase): config = {'sdk_path': self.tmpdir} fdroidserver.common.fill_config_defaults(config) self.assertEqual( - {'r10e': r10e, 'r17c': ndk_bundle, 'r21e': r21e}, config['ndk_paths'] + {'r10e': r10e, '17.2.4988734': ndk_bundle, '21.4.7075529': r21e}, + config['ndk_paths'], ) @unittest.skipIf(not os.path.isdir('/usr/lib/jvm/default-java'), 'uses Debian path') @@ -2411,7 +2357,7 @@ class CommonTest(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): repo = git.Repo.init(Path.cwd()) f = Path("test") - date = 10**9 + date = 10 ** 9 for tag in tags: date += 1 f.write_text(tag) diff --git a/tests/ndk-release-checksums.py b/tests/ndk-release-checksums.py deleted file mode 100755 index 97b7038b..00000000 --- a/tests/ndk-release-checksums.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python3 - -import git -import gitlab -import json -import os -import re -import requests -import subprocess -import sys -from colorama import Fore, Style - - -checksums = None -versions = dict() - -while not checksums: - r = requests.get( - 'https://gitlab.com/fdroid/android-sdk-transparency-log/-/raw/master/checksums.json', - timeout=300, - ) - if r.status_code == 200: - checksums = r.json() - -with open('fdroidserver/common.py') as fp: - common_py = fp.read() - -to_compile = re.search(r'\nNDKS = [^\]]+\]', common_py).group() -if not to_compile: - sys.exit(1) -code = compile(to_compile, '', 'exec') -config = {} -exec(code, None, config) # nosec this is just a CI script - -ndks = [] -errors = 0 -release = None -revision = None -for k, entries in checksums.items(): - if k.endswith('.zip') and k.startswith( - 'https://dl.google.com/android/repository/android-ndk' - ): - m = re.search(r'-(r[1-9][0-9]?[a-z]?)-linux', k) - if m: - d = {'url': k, 'release': m.group(1), 'sha256': checksums[k][0]['sha256']} - for entry in entries: - if 'source.properties' in entry: - n = re.search( - r'[1-9][0-9]\.[0-9]\.[0-9]{7}', entry['source.properties'] - ) - if n: - d['revision'] = n.group() - ndks.append(d) - for d in config['NDKS']: - if k == d['url']: - sha256 = d['sha256'] - found = False - for entry in entries: - if sha256 == entry['sha256']: - found = True - if not found: - print( - Fore.RED - + ( - 'ERROR: checksum mismatch: %s != %s' - % (sha256, entry['sha256']) - ) - + Style.RESET_ALL - ) - errors += 1 - -with open('fdroidserver/common.py', 'w') as fp: - fp.write( - common_py.replace( - to_compile, '\nNDKS = ' + json.dumps(ndks, indent=4, sort_keys=True) - ) - ) - -if os.getenv('CI_PROJECT_NAMESPACE') != 'fdroid': - p = subprocess.run(['git', '--no-pager', 'diff']) - print(p.stdout) - sys.exit(errors) - - -# This only runs after commits are pushed to fdroid/fdroidserver -git_repo = git.repo.Repo('.') -modified = git_repo.git().ls_files(modified=True).split() -if git_repo.is_dirty() and 'fdroidserver/common.py' in modified: - branch = git_repo.create_head(os.path.basename(__file__), force=True) - branch.checkout() - git_repo.index.add(['fdroidserver/common.py']) - author = git.Actor('fdroid-bot', 'fdroid-bot@f-droid.org') - git_repo.index.commit('Android NDK %s (%s)' % (release, revision), author=author) - project_path = 'fdroid-bot/' + os.getenv('CI_PROJECT_NAME') - url = 'https://gitlab-ci-token:%s@%s/%s.git' % ( - os.getenv('PERSONAL_ACCESS_TOKEN'), - os.getenv('CI_SERVER_HOST'), - project_path, - ) - remote_name = 'fdroid-bot' - try: - remote = git_repo.create_remote(remote_name, url) - # See https://github.com/PyCQA/pylint/issues/2856 . - # pylint: disable-next=no-member - except git.exc.GitCommandError: - remote = git.remote.Remote(git_repo, remote_name) - remote.set_url(url) - remote.push(force=True) - git.remote.Remote.rm(git_repo, remote_name) - - private_token = os.getenv('PERSONAL_ACCESS_TOKEN') - if not private_token: - print( - Fore.RED - + 'ERROR: GitLab Token not found in PERSONAL_ACCESS_TOKEN!' - + Style.RESET_ALL - ) - sys.exit(1) - gl = gitlab.Gitlab( - os.getenv('CI_SERVER_URL'), api_version=4, private_token=private_token - ) - project = gl.projects.get(project_path, lazy=True) - description = ( - 'see ' - '\n\n

generated by GitLab CI Job #%s

' - % (os.getenv('CI_PROJECT_URL'), os.getenv('CI_JOB_ID'), os.getenv('CI_JOB_ID')) - ) - mr = project.mergerequests.create( - { - 'source_branch': branch.name, - 'target_project_id': 36527, # fdroid/fdroidserver - 'target_branch': 'master', - 'title': 'update NDK', - 'description': description, - 'labels': ['fdroid-bot', 'buildserver'], - 'remove_source_branch': True, - } - ) - mr.save()