mirrors/fdroidserver
1
0
Fork 0
mirror of https://github.com/f-droid/fdroidserver.git synced 2025-09-13 14:32:28 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

remove NDK download handling in favor of fdroid/sdkmanager

Browse source
This commit is contained in:
Hans-Christoph Steiner 2022-01-14 12:50:44 +01:00
parent 66d750e5fa
commit 0dd5a7db64
No known key found for this signature in database
GPG key ID: 3E177817BA1B9BFA
6 changed files with 172 additions and 565 deletions
Download patch file Download diff file Expand all files Collapse all files

5
.gitlab-ci.yml
View file

@ -276,7 +276,6 @@ black:
tests/key-tricks.py tests/key-tricks.py
tests/lint.TestCase tests/lint.TestCase
tests/metadata.TestCase tests/metadata.TestCase
tests/ndk-release-checksums.py
tests/nightly.TestCase tests/nightly.TestCase
tests/rewritemeta.TestCase tests/rewritemeta.TestCase
tests/scanner.TestCase tests/scanner.TestCase
@ -329,7 +328,7 @@ fedora_latest:
"cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests" "cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests"
gradle/ndk: gradle:
image: debian:bullseye image: debian:bullseye
<<: *apt-template <<: *apt-template
variables: variables:
@ -350,12 +349,10 @@ gradle/ndk:
for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; do for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; do
test "$f" == "makebuildserver" && export CHANGED="yes"; test "$f" == "makebuildserver" && export CHANGED="yes";
test "$f" == "gradlew-fdroid" && export CHANGED="yes"; test "$f" == "gradlew-fdroid" && export CHANGED="yes";
test "$f" == "fdroidserver/common.py" && export CHANGED="yes";
done; done;
test -z "$CHANGED" && exit; test -z "$CHANGED" && exit;
fi fi
- ./tests/gradle-release-checksums.py - ./tests/gradle-release-checksums.py
- ./tests/ndk-release-checksums.py
# Run an actual build in a simple, faked version of the buildserver guest VM. # Run an actual build in a simple, faked version of the buildserver guest VM.

315
fdroidserver/common.py
View file

@ -29,6 +29,7 @@
# libraries here as they will become a requirement for all commands. # libraries here as they will become a requirement for all commands.
import git import git
import glob
import io import io
import os import os
import sys import sys
@ -36,7 +37,6 @@ import re
import ast import ast
import gzip import gzip
import shutil import shutil
import glob
import stat import stat
import subprocess import subprocess
import time import time
@ -64,7 +64,6 @@ from pyasn1.codec.der import decoder, encoder
from pyasn1_modules import rfc2315 from pyasn1_modules import rfc2315
from pyasn1.error import PyAsn1Error from pyasn1.error import PyAsn1Error
from . import net
import fdroidserver.metadata import fdroidserver.metadata
import fdroidserver.lint import fdroidserver.lint
from fdroidserver import _ from fdroidserver import _
@ -310,13 +309,6 @@ def fill_config_defaults(thisconfig):
if version not in ndk_paths: if version not in ndk_paths:
ndk_paths[version] = ndk ndk_paths[version] = ndk
for k in list(ndk_paths.keys()):
if not re.match(r'r[1-9][0-9]*[a-z]?', k):
for ndkdict in NDKS:
if k == ndkdict.get('revision'):
ndk_paths[ndkdict['release']] = ndk_paths.pop(k)
break
if 'cachedir_scanner' not in thisconfig: if 'cachedir_scanner' not in thisconfig:
thisconfig['cachedir_scanner'] = str(Path(thisconfig['cachedir']) / 'scanner') thisconfig['cachedir_scanner'] = str(Path(thisconfig['cachedir']) / 'scanner')
if 'gradle_version_dir' not in thisconfig: if 'gradle_version_dir' not in thisconfig:
@ -4226,6 +4218,8 @@ def auto_install_ndk(build):
they are packaged differently. they are packaged differently.
""" """
import sdkmanager
global config global config
if build.get('disable'): if build.get('disable'):
return return
@ -4233,8 +4227,10 @@ def auto_install_ndk(build):
if not ndk: if not ndk:
return return
if isinstance(ndk, str): if isinstance(ndk, str):
sdkmanager.build_package_list(use_net=True)
_install_ndk(ndk) _install_ndk(ndk)
elif isinstance(ndk, list): elif isinstance(ndk, list):
sdkmanager.build_package_list(use_net=True)
for n in ndk: for n in ndk:
_install_ndk(n) _install_ndk(n)
else: else:
@ -4251,295 +4247,16 @@ def _install_ndk(ndk):
The NDK version to install, either in "release" form (r21e) or The NDK version to install, either in "release" form (r21e) or
"revision" form (21.4.7075529). "revision" form (21.4.7075529).
""" """
if re.match(r'[1-9][0-9.]+[0-9]', ndk): import sdkmanager
for ndkdict in NDKS:
if ndk == ndkdict.get('revision'):
ndk = ndkdict['release']
break
ndk_path = config.get(ndk) sdk_path = config['sdk_path']
if ndk_path and os.path.isdir(ndk_path): sdkmanager.install(f'ndk;{ndk}', sdk_path)
return for found in glob.glob(f'{sdk_path}/ndk/*'):
for ndkdict in NDKS: version = get_ndk_version(found)
if ndk == ndkdict['release']: if 'ndk_paths' not in config:
url = ndkdict['url'] config['ndk_paths'] = dict()
sha256 = ndkdict['sha256'] config['ndk_paths'][ndk] = found
break config['ndk_paths'][version] = found
else: logging.info(
raise FDroidException("NDK %s not found" % ndk) _('Set NDK {release} ({version}) up').format(release=ndk, version=version)
ndk_base = os.path.join(config['sdk_path'], 'ndk')
logging.info(_('Downloading %s') % url)
with tempfile.TemporaryDirectory(prefix='android-ndk-') as ndk_dir:
zipball = os.path.join(
ndk_dir,
os.path.basename(url)
) )
net.download_file(url, zipball)
calced = sha256sum(zipball)
if sha256 != calced:
raise FDroidException('SHA-256 %s does not match expected for %s (%s)' % (calced, url, sha256))
logging.info(_('Unzipping to %s') % ndk_base)
with zipfile.ZipFile(zipball) as zipfp:
for info in zipfp.infolist():
permbits = info.external_attr >> 16
if stat.S_ISLNK(permbits):
link = os.path.join(ndk_base, info.filename)
link_target = zipfp.read(info).decode()
link_dir = os.path.dirname(link)
os.makedirs(link_dir, 0o755, True) # ensure intermediate directories are created
os.symlink(link_target, link)
real_target = os.path.realpath(link)
if not real_target.startswith(ndk_base):
os.remove(link)
logging.error(_('Unexpected symlink target: {link} -> {target}')
.format(link=link, target=real_target))
elif stat.S_ISDIR(permbits) or stat.S_IXUSR & permbits:
zipfp.extract(info.filename, path=ndk_base)
os.chmod(os.path.join(ndk_base, info.filename), 0o755) # nosec bandit B103
else:
zipfp.extract(info.filename, path=ndk_base)
os.chmod(os.path.join(ndk_base, info.filename), 0o644) # nosec bandit B103
os.remove(zipball)
for extracted in glob.glob(os.path.join(ndk_base, '*')):
version = get_ndk_version(extracted)
if os.path.basename(extracted) != version:
ndk_dir = os.path.join(ndk_base, version)
os.rename(extracted, ndk_dir)
if 'ndk_paths' not in config:
config['ndk_paths'] = dict()
config['ndk_paths'][ndk] = ndk_dir
logging.info(_('Set NDK {release} ({version}) up')
.format(release=ndk, version=version))
"""Derived from https://gitlab.com/fdroid/android-sdk-transparency-log/-/blob/master/checksums.json"""
NDKS = [
{
"release": "r10e",
"sha256": "ee5f405f3b57c4f5c3b3b8b5d495ae12b660e03d2112e4ed5c728d349f1e520c",
"url": "https://dl.google.com/android/repository/android-ndk-r10e-linux-x86_64.zip"
},
{
"release": "r11",
"revision": "11.0.2655954",
"sha256": "59ab44f7ee6201df4381844736fdc456134c7f7660151003944a3017a0dcce97",
"url": "https://dl.google.com/android/repository/android-ndk-r11-linux-x86_64.zip"
},
{
"release": "r11b",
"revision": "11.1.2683735",
"sha256": "51d429bfda8bbe038683ed7ae7acc03b39604b84711901b555fe18c698867e53",
"url": "https://dl.google.com/android/repository/android-ndk-r11b-linux-x86_64.zip"
},
{
"release": "r11c",
"revision": "11.2.2725575",
"sha256": "ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94",
"url": "https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip"
},
{
"release": "r12",
"revision": "12.0.2931149",
"sha256": "7876e3b99f3596a3215ecf4e9f152d24b82dfdf2bbe7d3a38c423ae6a3edee79",
"url": "https://dl.google.com/android/repository/android-ndk-r12-linux-x86_64.zip"
},
{
"release": "r12b",
"revision": "12.1.2977051",
"sha256": "eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e",
"url": "https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip"
},
{
"release": "r13",
"revision": "13.0.3315539",
"sha256": "0a1dbd216386399e2979c17a48f65b962bf7ddc0c2311ef35d902b90c298c400",
"url": "https://dl.google.com/android/repository/android-ndk-r13-linux-x86_64.zip"
},
{
"release": "r13b",
"revision": "13.1.3345770",
"sha256": "3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c",
"url": "https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip"
},
{
"release": "r14",
"revision": "14.0.3770861",
"sha256": "3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44",
"url": "https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip"
},
{
"release": "r14b",
"revision": "14.1.3816874",
"sha256": "0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024",
"url": "https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip"
},
{
"release": "r15",
"revision": "15.0.4075724",
"sha256": "078eb7d28c3fcf45841f5baf6e6582e7fd5b73d8e8c4e0101df490f51abd37b6",
"url": "https://dl.google.com/android/repository/android-ndk-r15-linux-x86_64.zip"
},
{
"release": "r15b",
"revision": "15.1.4119039",
"sha256": "d1ce63f68cd806b5a992d4e5aa60defde131c243bf523cdfc5b67990ef0ee0d3",
"url": "https://dl.google.com/android/repository/android-ndk-r15b-linux-x86_64.zip"
},
{
"release": "r15c",
"revision": "15.2.4203891",
"sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c",
"url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip"
},
{
"release": "r16",
"revision": "16.0.4442984",
"sha256": "a8550b81771c67cc6ab7b479a6918d29aa78de3482901762b4f9e0132cd9672e",
"url": "https://dl.google.com/android/repository/android-ndk-r16-linux-x86_64.zip"
},
{
"release": "r16b",
"revision": "16.1.4479499",
"sha256": "bcdea4f5353773b2ffa85b5a9a2ae35544ce88ec5b507301d8cf6a76b765d901",
"url": "https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip"
},
{
"release": "r17",
"revision": "17.0.4754217",
"sha256": "ba3d813b47de75bc32a2f3de087f72599c6cb36fdc9686b96f517f5492ff43ca",
"url": "https://dl.google.com/android/repository/android-ndk-r17-linux-x86_64.zip"
},
{
"release": "r17b",
"revision": "17.1.4828580",
"sha256": "5dfbbdc2d3ba859fed90d0e978af87c71a91a5be1f6e1c40ba697503d48ccecd",
"url": "https://dl.google.com/android/repository/android-ndk-r17b-linux-x86_64.zip"
},
{
"release": "r17c",
"revision": "17.2.4988734",
"sha256": "3f541adbd0330a9205ba12697f6d04ec90752c53d6b622101a2a8a856e816589",
"url": "https://dl.google.com/android/repository/android-ndk-r17c-linux-x86_64.zip"
},
{
"release": "r18b",
"revision": "18.1.5063045",
"sha256": "4f61cbe4bbf6406aa5ef2ae871def78010eed6271af72de83f8bd0b07a9fd3fd",
"url": "https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip"
},
{
"release": "r19",
"revision": "19.0.5232133",
"sha256": "c0a2425206191252197b97ea5fcc7eab9f693a576e69ef4773a9ed1690feed53",
"url": "https://dl.google.com/android/repository/android-ndk-r19-linux-x86_64.zip"
},
{
"release": "r19b",
"revision": "19.1.5304403",
"sha256": "0fbb1645d0f1de4dde90a4ff79ca5ec4899c835e729d692f433fda501623257a",
"url": "https://dl.google.com/android/repository/android-ndk-r19b-linux-x86_64.zip"
},
{
"release": "r19c",
"revision": "19.2.5345600",
"sha256": "4c62514ec9c2309315fd84da6d52465651cdb68605058f231f1e480fcf2692e1",
"url": "https://dl.google.com/android/repository/android-ndk-r19c-linux-x86_64.zip"
},
{
"release": "r20",
"revision": "20.0.5594570",
"sha256": "57435158f109162f41f2f43d5563d2164e4d5d0364783a9a6fab3ef12cb06ce0",
"url": "https://dl.google.com/android/repository/android-ndk-r20-linux-x86_64.zip"
},
{
"release": "r20b",
"revision": "20.1.5948944",
"sha256": "8381c440fe61fcbb01e209211ac01b519cd6adf51ab1c2281d5daad6ca4c8c8c",
"url": "https://dl.google.com/android/repository/android-ndk-r20b-linux-x86_64.zip"
},
{
"release": "r21",
"revision": "21.0.6113669",
"sha256": "b65ea2d5c5b68fb603626adcbcea6e4d12c68eb8a73e373bbb9d23c252fc647b",
"url": "https://dl.google.com/android/repository/android-ndk-r21-linux-x86_64.zip"
},
{
"release": "r21b",
"revision": "21.1.6352462",
"sha256": "0c7af5dd23c5d2564915194e71b1053578438ac992958904703161c7672cbed7",
"url": "https://dl.google.com/android/repository/android-ndk-r21b-linux-x86_64.zip"
},
{
"release": "r21c",
"revision": "21.2.6472646",
"sha256": "214ebfcfa5108ba78f5b2cc8db4d575068f9c973ac7f27d2fa1987dfdb76c9e7",
"url": "https://dl.google.com/android/repository/android-ndk-r21c-linux-x86_64.zip"
},
{
"release": "r21d",
"revision": "21.3.6528147",
"sha256": "dd6dc090b6e2580206c64bcee499bc16509a5d017c6952dcd2bed9072af67cbd",
"url": "https://dl.google.com/android/repository/android-ndk-r21d-linux-x86_64.zip"
},
{
"release": "r21e",
"revision": "21.4.7075529",
"sha256": "ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e",
"url": "https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip"
},
{
"release": "r22",
"revision": "22.0.7026061",
"sha256": "d37fc69cd81e5660234a686e20adef39bc0244086e4d66525a40af771c020718",
"url": "https://dl.google.com/android/repository/android-ndk-r22-linux-x86_64.zip"
},
{
"release": "r22b",
"revision": "22.1.7171670",
"sha256": "ac3a0421e76f71dd330d0cd55f9d99b9ac864c4c034fc67e0d671d022d4e806b",
"url": "https://dl.google.com/android/repository/android-ndk-r22b-linux-x86_64.zip"
},
{
"release": "r23",
"revision": "23.0.7599858",
"sha256": "e3eacf80016b91d4cd2c8ca9f34eebd32df912bb799c859cc5450b6b19277b4f",
"url": "https://dl.google.com/android/repository/android-ndk-r23-linux.zip"
},
{
"release": "r23b",
"revision": "23.1.7779620",
"sha256": "c6e97f9c8cfe5b7be0a9e6c15af8e7a179475b7ded23e2d1c1fa0945d6fb4382",
"url": "https://dl.google.com/android/repository/android-ndk-r23b-linux.zip"
},
{
"release": "r23c",
"revision": "23.2.8568313",
"sha256": "6ce94604b77d28113ecd588d425363624a5228d9662450c48d2e4053f8039242",
"url": "https://dl.google.com/android/repository/android-ndk-r23c-linux.zip"
},
{
"release": "r24",
"revision": "24.0.8215888",
"sha256": "caac638f060347c9aae994e718ba00bb18413498d8e0ad4e12e1482964032997",
"url": "https://dl.google.com/android/repository/android-ndk-r24-linux.zip"
},
{
"release": "r25",
"revision": "25.0.8775105",
"sha256": "cd661aeda5d9b7cfb6e64bd80737c274d7c1c0d026df2f85be3bf3327b25e545",
"url": "https://dl.google.com/android/repository/android-ndk-r25-linux.zip"
},
{
"release": "r25b",
"revision": "25.1.8937393",
"sha256": "403ac3e3020dd0db63a848dcaba6ceb2603bf64de90949d5c4361f848e44b005",
"url": "https://dl.google.com/android/repository/android-ndk-r25b-linux.zip"
},
{
"release": "r25c",
"revision": "25.2.9519653",
"sha256": "769ee342ea75f80619d985c2da990c48b3d8eaf45f48783a2d48870d04b46108",
"url": "https://dl.google.com/android/repository/android-ndk-r25c-linux.zip"
}
]

1
setup.py
View file

@ -105,6 +105,7 @@ setup(
'qrcode', 'qrcode',
'ruamel.yaml >= 0.15', 'ruamel.yaml >= 0.15',
'requests >= 2.5.2, != 2.11.0, != 2.12.2, != 2.18.0', 'requests >= 2.5.2, != 2.11.0, != 2.12.2, != 2.18.0',
'sdkmanager >= 0.6.4',
'yamllint', 'yamllint',
], ],
extras_require={ extras_require={

111
tests/build.TestCase
View file

@ -12,7 +12,7 @@ import tempfile
import textwrap import textwrap
import unittest import unittest
import yaml import yaml
import zipfile from pathlib import Path
from unittest import mock from unittest import mock
localmodule = os.path.realpath( localmodule = os.path.realpath(
@ -214,6 +214,7 @@ class BuildTest(unittest.TestCase):
self.assertEqual(versionCode, vc) self.assertEqual(versionCode, vc)
self.assertEqual(versionName, vn) self.assertEqual(versionName, vn)
@mock.patch('sdkmanager.build_package_list', lambda use_net: None)
def test_build_local_ndk(self): def test_build_local_ndk(self):
"""Test if `fdroid build` detects installed NDKs and auto-installs when missing""" """Test if `fdroid build` detects installed NDKs and auto-installs when missing"""
with tempfile.TemporaryDirectory() as testdir, TmpCwd( with tempfile.TemporaryDirectory() as testdir, TmpCwd(
@ -235,6 +236,8 @@ class BuildTest(unittest.TestCase):
build.versionCode = 1 build.versionCode = 1
build.versionName = '1.0' build.versionName = '1.0'
build.ndk = 'r21e' # aka 21.4.7075529 build.ndk = 'r21e' # aka 21.4.7075529
ndk_version = '21.4.7075529'
ndk_dir = Path(config['sdk_path']) / 'ndk' / ndk_version
vcs = mock.Mock() vcs = mock.Mock()
def make_fake_apk(output, build): def make_fake_apk(output, build):
@ -242,13 +245,12 @@ class BuildTest(unittest.TestCase):
fp.write('APK PLACEHOLDER') fp.write('APK PLACEHOLDER')
return output return output
def fake_download_file(_ignored, local_filename): # pylint: disable=unused-argument
_ignored # silence the linters def fake_sdkmanager_install(to_install, android_home=None):
with zipfile.ZipFile(local_filename, 'x') as zipfp: ndk_dir.mkdir(parents=True)
zipfp.writestr( self.assertNotEqual(ndk_version, to_install) # converts r21e to version
'android-ndk-r21e/source.properties', with (ndk_dir / 'source.properties').open('w') as fp:
'Pkg.Revision = 21.4.7075529\n', fp.write('Pkg.Revision = %s\n' % ndk_version)
)
# use "as _ignored" just to make a pretty layout # use "as _ignored" just to make a pretty layout
with mock.patch( with mock.patch(
@ -258,8 +260,6 @@ class BuildTest(unittest.TestCase):
) as _ignored, mock.patch( ) as _ignored, mock.patch(
'fdroidserver.common.get_apk_id', 'fdroidserver.common.get_apk_id',
return_value=(app.id, build.versionCode, build.versionName), return_value=(app.id, build.versionCode, build.versionName),
) as _ignored, mock.patch(
'fdroidserver.common.is_apk_and_debuggable', return_value=False
) as _ignored, mock.patch( ) as _ignored, mock.patch(
'fdroidserver.common.sha256sum', 'fdroidserver.common.sha256sum',
return_value='ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e', return_value='ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e',
@ -268,7 +268,7 @@ class BuildTest(unittest.TestCase):
) as _ignored, mock.patch( ) as _ignored, mock.patch(
'fdroidserver.build.FDroidPopen', FakeProcess 'fdroidserver.build.FDroidPopen', FakeProcess
) as _ignored, mock.patch( ) as _ignored, mock.patch(
'fdroidserver.net.download_file', wraps=fake_download_file 'sdkmanager.install', wraps=fake_sdkmanager_install
) as _ignored: ) as _ignored:
_ignored # silence the linters _ignored # silence the linters
with self.assertRaises( with self.assertRaises(
@ -290,7 +290,10 @@ class BuildTest(unittest.TestCase):
refresh=False, refresh=False,
) )
# now run `fdroid build --onserver` # now run `fdroid build --onserver`
self.assertTrue('r21e' not in config['ndk_paths']) print('now run `fdroid build --onserver`')
self.assertFalse(ndk_dir.exists())
self.assertFalse('r21e' in config['ndk_paths'])
self.assertFalse(ndk_version in config['ndk_paths'])
fdroidserver.build.build_local( fdroidserver.build.build_local(
app, app,
build, build,
@ -305,7 +308,89 @@ class BuildTest(unittest.TestCase):
onserver=True, onserver=True,
refresh=False, refresh=False,
) )
self.assertTrue(os.path.exists(config['ndk_paths']['r21e'])) self.assertTrue(ndk_dir.exists())
self.assertTrue(os.path.exists(config['ndk_paths'][ndk_version]))
# All paths in the config must be strings, never pathlib.Path instances
self.assertIsInstance(config['ndk_paths'][ndk_version], str)
@mock.patch('sdkmanager.build_package_list', lambda use_net: None)
@mock.patch('fdroidserver.build.FDroidPopen', FakeProcess)
@mock.patch('fdroidserver.common.get_native_code', lambda _ignored: 'x86')
@mock.patch('fdroidserver.common.is_apk_and_debuggable', lambda _ignored: False)
@mock.patch(
'fdroidserver.common.sha256sum',
lambda f: 'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e',
)
def test_build_local_ndk_some_installed(self):
"""Test if `fdroid build` detects installed NDKs and auto-installs when missing"""
with tempfile.TemporaryDirectory() as testdir, TmpCwd(
testdir
), tempfile.TemporaryDirectory() as sdk_path:
ndk_r24 = os.path.join(sdk_path, 'ndk', '24.0.8215888')
os.makedirs(ndk_r24)
with open(os.path.join(ndk_r24, 'source.properties'), 'w') as fp:
fp.write('Pkg.Revision = 24.0.8215888\n')
config = {'ndk_paths': {'r24': ndk_r24}, 'sdk_path': sdk_path}
fdroidserver.common.config = config
fdroidserver.build.config = config
fdroidserver.build.options = mock.Mock()
fdroidserver.build.options.scan_binary = False
fdroidserver.build.options.notarball = True
fdroidserver.build.options.skipscan = True
app = fdroidserver.metadata.App()
app.id = 'mocked.app.id'
build = fdroidserver.metadata.Build()
build.commit = '1.0'
build.output = app.id + '.apk'
build.versionCode = 1
build.versionName = '1.0'
build.ndk = 'r21e' # aka 21.4.7075529
ndk_version = '21.4.7075529'
ndk_dir = Path(config['sdk_path']) / 'ndk' / ndk_version
vcs = mock.Mock()
def make_fake_apk(output, build):
with open(build.output, 'w') as fp:
fp.write('APK PLACEHOLDER')
return output
# pylint: disable=unused-argument
def fake_sdkmanager_install(to_install, android_home=None):
ndk_dir.mkdir(parents=True)
self.assertNotEqual(ndk_version, to_install) # converts r21e to version
with (ndk_dir / 'source.properties').open('w') as fp:
fp.write('Pkg.Revision = %s\n' % ndk_version)
# use "as _ignored" just to make a pretty layout
with mock.patch(
'fdroidserver.common.replace_build_vars', wraps=make_fake_apk
) as _ignored, mock.patch(
'fdroidserver.common.get_apk_id',
return_value=(app.id, build.versionCode, build.versionName),
) as _ignored, mock.patch(
'sdkmanager.install', wraps=fake_sdkmanager_install
) as _ignored:
_ignored # silence the linters
self.assertFalse(ndk_dir.exists())
self.assertFalse('r21e' in config['ndk_paths'])
self.assertFalse(ndk_version in config['ndk_paths'])
fdroidserver.build.build_local(
app,
build,
vcs,
build_dir=testdir,
output_dir=testdir,
log_dir=os.getcwd(),
srclib_dir=None,
extlib_dir=None,
tmp_dir=None,
force=False,
onserver=True,
refresh=False,
)
self.assertTrue(ndk_dir.exists())
self.assertTrue(os.path.exists(config['ndk_paths'][ndk_version]))
def test_build_local_clean(self): def test_build_local_clean(self):
"""Test if `fdroid build` cleans ant and gradle build products""" """Test if `fdroid build` cleans ant and gradle build products"""

166
tests/common.TestCase
View file

@ -5,6 +5,7 @@
import difflib import difflib
import git import git
import glob import glob
import importlib
import inspect import inspect
import json import json
import logging import logging
@ -20,9 +21,8 @@ import unittest
import textwrap import textwrap
import yaml import yaml
import gzip import gzip
import stat
from distutils.version import LooseVersion from distutils.version import LooseVersion
from zipfile import BadZipFile, ZipFile, ZipInfo from zipfile import BadZipFile, ZipFile
from unittest import mock from unittest import mock
from pathlib import Path from pathlib import Path
@ -2164,7 +2164,7 @@ class CommonTest(unittest.TestCase):
) )
self.assertEqual( self.assertEqual(
{'com.example': [12345, 67890], 'org.fdroid.fdroid': [1]}, {'com.example': [12345, 67890], 'org.fdroid.fdroid': [1]},
fdroidserver.common.read_pkg_args(appid_versionCode_pairs, allow_vercodes) fdroidserver.common.read_pkg_args(appid_versionCode_pairs, allow_vercodes),
) )
appid_versionCode_pairs = ( appid_versionCode_pairs = (
'com.example:67890', 'com.example:67890',
@ -2210,42 +2210,79 @@ class CommonTest(unittest.TestCase):
fdroidserver.common.metadata_find_developer_signing_files(appid, vc), fdroidserver.common.metadata_find_developer_signing_files(appid, vc),
) )
@mock.patch('sdkmanager.build_package_list', lambda use_net: None)
def test_auto_install_ndk(self): def test_auto_install_ndk(self):
"""Test all possible field data types for build.ndk""" """Test all possible field data types for build.ndk"""
fdroidserver.common.config = {'sdk_path': self.testdir}
sdk_path = self.testdir
build = fdroidserver.metadata.Build() build = fdroidserver.metadata.Build()
none_entry = mock.Mock() none_entry = mock.Mock()
with mock.patch('fdroidserver.common._install_ndk', none_entry): with mock.patch('sdkmanager.install', none_entry):
fdroidserver.common.auto_install_ndk(build) fdroidserver.common.auto_install_ndk(build)
none_entry.assert_not_called() none_entry.assert_not_called()
empty_list = mock.Mock() empty_list = mock.Mock()
build.ndk = [] build.ndk = []
with mock.patch('fdroidserver.common._install_ndk', empty_list): with mock.patch('sdkmanager.install', empty_list):
fdroidserver.common.auto_install_ndk(build) fdroidserver.common.auto_install_ndk(build)
empty_list.assert_not_called() empty_list.assert_not_called()
release_entry = mock.Mock() release_entry = mock.Mock()
build.ndk = 'r21e' build.ndk = 'r21e'
with mock.patch('fdroidserver.common._install_ndk', release_entry): with mock.patch('sdkmanager.install', release_entry):
fdroidserver.common.auto_install_ndk(build) fdroidserver.common.auto_install_ndk(build)
release_entry.assert_called_once_with('r21e') release_entry.assert_called_once_with('ndk;r21e', sdk_path)
revision_entry = mock.Mock() revision_entry = mock.Mock()
build.ndk = '21.4.7075529' build.ndk = '21.4.7075529'
with mock.patch('fdroidserver.common._install_ndk', revision_entry): with mock.patch('sdkmanager.install', revision_entry):
fdroidserver.common.auto_install_ndk(build) fdroidserver.common.auto_install_ndk(build)
revision_entry.assert_called_once_with('21.4.7075529') revision_entry.assert_called_once_with('ndk;21.4.7075529', sdk_path)
list_entry = mock.Mock() list_entry = mock.Mock()
calls = [] calls = []
build.ndk = ['r10e', '11.0.2655954', 'r12b', 'r21e'] build.ndk = ['r10e', '11.0.2655954', 'r12b', 'r21e']
for n in build.ndk: for n in build.ndk:
calls.append(mock.call(n)) calls.append(mock.call(f'ndk;{n}', sdk_path))
with mock.patch('fdroidserver.common._install_ndk', list_entry): with mock.patch('sdkmanager.install', list_entry):
fdroidserver.common.auto_install_ndk(build) fdroidserver.common.auto_install_ndk(build)
list_entry.assert_has_calls(calls) list_entry.assert_has_calls(calls)
@unittest.skipIf(importlib.util.find_spec('sdkmanager') is None, 'needs sdkmanager')
@mock.patch('sdkmanager.build_package_list', lambda use_net: None)
@mock.patch('sdkmanager._install_zipball_from_cache', lambda a, b: None)
@mock.patch('sdkmanager._generate_package_xml', lambda a, b, c: None)
def test_auto_install_ndk_mock_dl(self):
"""Test NDK installs by actually calling sdkmanager"""
import sdkmanager
import pkg_resources
sdkmanager_version = LooseVersion(
pkg_resources.get_distribution('sdkmanager').version
)
if sdkmanager_version < LooseVersion('0.6.4'):
raise unittest.SkipTest('needs fdroid sdkmanager >= 0.6.4')
fdroidserver.common.config = {'sdk_path': 'placeholder'}
build = fdroidserver.metadata.Build()
url = 'https://dl.google.com/android/repository/android-ndk-r24-linux.zip'
path = sdkmanager.get_cachedir() / os.path.basename(url)
sdkmanager.packages = {
('ndk', '24.0.8215888'): url,
('ndk', 'r24'): url,
}
build.ndk = 'r24'
firstrun = mock.Mock()
with mock.patch('sdkmanager.download_file', firstrun):
fdroidserver.common.auto_install_ndk(build)
firstrun.assert_called_once_with(url, path)
build.ndk = '24.0.8215888'
secondrun = mock.Mock()
with mock.patch('sdkmanager.download_file', secondrun):
fdroidserver.common.auto_install_ndk(build)
secondrun.assert_called_once_with(url, path)
@unittest.skip("This test downloads and unzips a 1GB file.") @unittest.skip("This test downloads and unzips a 1GB file.")
def test_install_ndk(self): def test_install_ndk(self):
"""NDK r10e is a special case since its missing source.properties""" """NDK r10e is a special case since its missing source.properties"""
@ -2257,101 +2294,6 @@ class CommonTest(unittest.TestCase):
fdroidserver.common.fill_config_defaults(config) fdroidserver.common.fill_config_defaults(config)
self.assertEqual({'r10e': r10e}, config['ndk_paths']) self.assertEqual({'r10e': r10e}, config['ndk_paths'])
def test_install_ndk_versions(self):
"""Test whether NDK version parsing is working properly"""
def fake_download(url, zipball):
print(url, zipball)
with ZipFile(zipball, 'w') as zipfp:
zipfp.writestr(os.path.basename(url), url)
with tempfile.TemporaryDirectory(
prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
) as sdk_path:
config = {'sdk_path': sdk_path}
fdroidserver.common.config = config
for r, sha256 in (
(
'r10e',
'ee5f405f3b57c4f5c3b3b8b5d495ae12b660e03d2112e4ed5c728d349f1e520c',
),
('r20', '57435158f109162f41f2f43d5563d2164e4d5d0364783a9a6fab3ef12cb06ce0'),
(
'23.0.7599858',
'e3eacf80016b91d4cd2c8ca9f34eebd32df912bb799c859cc5450b6b19277b4f',
),
):
with mock.patch(
'fdroidserver.net.download_file', side_effect=fake_download
) as _ignored, mock.patch(
'fdroidserver.common.get_ndk_version', return_value=r
) as _ignored, mock.patch(
'fdroidserver.common.sha256sum', return_value=sha256
):
_ignored # silence the linters
fdroidserver.common._install_ndk(r)
def test_install_ndk_with_symlinks(self):
"""Some NDK zipballs might have symlinks in them."""
def fake_download(url, zipball):
print(url, zipball)
unix_st_mode = (
stat.S_IFLNK
| stat.S_IRUSR
| stat.S_IWUSR
| stat.S_IXUSR
| stat.S_IRGRP
| stat.S_IWGRP
| stat.S_IXGRP
| stat.S_IROTH
| stat.S_IWOTH
| stat.S_IXOTH
)
with ZipFile(zipball, 'w') as zipfp:
zipfp.writestr('ndk/' + os.path.basename(url), url)
zipInfo = ZipInfo('ndk/basename')
zipInfo.create_system = 3
zipInfo.external_attr = unix_st_mode << 16
zipfp.writestr(zipInfo, os.path.basename(url))
zipInfo = ZipInfo('ndk/bad_abs_link')
zipInfo.create_system = 3
zipInfo.external_attr = unix_st_mode << 16
zipfp.writestr(zipInfo, '/etc/passwd')
zipInfo = ZipInfo('ndk/bad_rel_link')
zipInfo.create_system = 3
zipInfo.external_attr = unix_st_mode << 16
zipfp.writestr(zipInfo, '../../../../../../../etc/passwd')
zipInfo = ZipInfo('ndk/bad_rel_link2')
zipInfo.create_system = 3
zipInfo.external_attr = unix_st_mode << 16
zipfp.writestr(zipInfo, 'foo/../../../../../../../../../etc/passwd')
config = {'sdk_path': self.tmpdir}
fdroidserver.common.config = config
r = 'r20'
sha256 = '57435158f109162f41f2f43d5563d2164e4d5d0364783a9a6fab3ef12cb06ce0'
with mock.patch(
'fdroidserver.net.download_file', side_effect=fake_download
) as _ignored, mock.patch(
'fdroidserver.common.get_ndk_version', return_value=r
) as _ignored, mock.patch(
'fdroidserver.common.sha256sum', return_value=sha256
):
_ignored # silence the linters
fdroidserver.common._install_ndk(r)
self.assertTrue(os.path.exists(os.path.join(self.tmpdir, 'ndk', 'r20')))
self.assertTrue(os.path.exists(os.path.join(self.tmpdir, 'ndk', 'r20', 'basename')))
self.assertFalse(os.path.exists(os.path.join(self.tmpdir, 'ndk', 'r20', 'bad_abs_link')))
self.assertFalse(os.path.exists(os.path.join(self.tmpdir, 'ndk', 'r20', 'bad_rel_link')))
self.assertFalse(os.path.exists(os.path.join(self.tmpdir, 'ndk', 'r20', 'bad_rel_link2')))
os.system('ls -l ' + os.path.join(self.tmpdir, 'ndk', 'r20'))
def test_fill_config_defaults(self): def test_fill_config_defaults(self):
"""Test the auto-detection of NDKs installed in standard paths""" """Test the auto-detection of NDKs installed in standard paths"""
ndk_bundle = os.path.join(self.tmpdir, 'ndk-bundle') ndk_bundle = os.path.join(self.tmpdir, 'ndk-bundle')
@ -2360,7 +2302,7 @@ class CommonTest(unittest.TestCase):
fp.write('Pkg.Desc = Android NDK\nPkg.Revision = 17.2.4988734\n') fp.write('Pkg.Desc = Android NDK\nPkg.Revision = 17.2.4988734\n')
config = {'sdk_path': self.tmpdir} config = {'sdk_path': self.tmpdir}
fdroidserver.common.fill_config_defaults(config) fdroidserver.common.fill_config_defaults(config)
self.assertEqual({'r17c': ndk_bundle}, config['ndk_paths']) self.assertEqual({'17.2.4988734': ndk_bundle}, config['ndk_paths'])
r21e = os.path.join(self.tmpdir, 'ndk', '21.4.7075529') r21e = os.path.join(self.tmpdir, 'ndk', '21.4.7075529')
os.makedirs(r21e) os.makedirs(r21e)
@ -2368,7 +2310,10 @@ class CommonTest(unittest.TestCase):
fp.write('Pkg.Desc = Android NDK\nPkg.Revision = 21.4.7075529\n') fp.write('Pkg.Desc = Android NDK\nPkg.Revision = 21.4.7075529\n')
config = {'sdk_path': self.tmpdir} config = {'sdk_path': self.tmpdir}
fdroidserver.common.fill_config_defaults(config) fdroidserver.common.fill_config_defaults(config)
self.assertEqual({'r17c': ndk_bundle, 'r21e': r21e}, config['ndk_paths']) self.assertEqual(
{'17.2.4988734': ndk_bundle, '21.4.7075529': r21e},
config['ndk_paths'],
)
r10e = os.path.join(self.tmpdir, 'ndk', 'r10e') r10e = os.path.join(self.tmpdir, 'ndk', 'r10e')
os.makedirs(r10e) os.makedirs(r10e)
@ -2377,7 +2322,8 @@ class CommonTest(unittest.TestCase):
config = {'sdk_path': self.tmpdir} config = {'sdk_path': self.tmpdir}
fdroidserver.common.fill_config_defaults(config) fdroidserver.common.fill_config_defaults(config)
self.assertEqual( self.assertEqual(
{'r10e': r10e, 'r17c': ndk_bundle, 'r21e': r21e}, config['ndk_paths'] {'r10e': r10e, '17.2.4988734': ndk_bundle, '21.4.7075529': r21e},
config['ndk_paths'],
) )
@unittest.skipIf(not os.path.isdir('/usr/lib/jvm/default-java'), 'uses Debian path') @unittest.skipIf(not os.path.isdir('/usr/lib/jvm/default-java'), 'uses Debian path')
@ -2411,7 +2357,7 @@ class CommonTest(unittest.TestCase):
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
repo = git.Repo.init(Path.cwd()) repo = git.Repo.init(Path.cwd())
f = Path("test") f = Path("test")
date = 10**9 date = 10 ** 9
for tag in tags: for tag in tags:
date += 1 date += 1
f.write_text(tag) f.write_text(tag)

139
tests/ndk-release-checksums.py
View file

@ -1,139 +0,0 @@
#!/usr/bin/env python3
import git
import gitlab
import json
import os
import re
import requests
import subprocess
import sys
from colorama import Fore, Style
checksums = None
versions = dict()
while not checksums:
r = requests.get(
'https://gitlab.com/fdroid/android-sdk-transparency-log/-/raw/master/checksums.json',
timeout=300,
)
if r.status_code == 200:
checksums = r.json()
with open('fdroidserver/common.py') as fp:
common_py = fp.read()
to_compile = re.search(r'\nNDKS = [^\]]+\]', common_py).group()
if not to_compile:
sys.exit(1)
code = compile(to_compile, '<string>', 'exec')
config = {}
exec(code, None, config) # nosec this is just a CI script
ndks = []
errors = 0
release = None
revision = None
for k, entries in checksums.items():
if k.endswith('.zip') and k.startswith(
'https://dl.google.com/android/repository/android-ndk'
):
m = re.search(r'-(r[1-9][0-9]?[a-z]?)-linux', k)
if m:
d = {'url': k, 'release': m.group(1), 'sha256': checksums[k][0]['sha256']}
for entry in entries:
if 'source.properties' in entry:
n = re.search(
r'[1-9][0-9]\.[0-9]\.[0-9]{7}', entry['source.properties']
)
if n:
d['revision'] = n.group()
ndks.append(d)
for d in config['NDKS']:
if k == d['url']:
sha256 = d['sha256']
found = False
for entry in entries:
if sha256 == entry['sha256']:
found = True
if not found:
print(
Fore.RED
+ (
'ERROR: checksum mismatch: %s != %s'
% (sha256, entry['sha256'])
)
+ Style.RESET_ALL
)
errors += 1
with open('fdroidserver/common.py', 'w') as fp:
fp.write(
common_py.replace(
to_compile, '\nNDKS = ' + json.dumps(ndks, indent=4, sort_keys=True)
)
)
if os.getenv('CI_PROJECT_NAMESPACE') != 'fdroid':
p = subprocess.run(['git', '--no-pager', 'diff'])
print(p.stdout)
sys.exit(errors)
# This only runs after commits are pushed to fdroid/fdroidserver
git_repo = git.repo.Repo('.')
modified = git_repo.git().ls_files(modified=True).split()
if git_repo.is_dirty() and 'fdroidserver/common.py' in modified:
branch = git_repo.create_head(os.path.basename(__file__), force=True)
branch.checkout()
git_repo.index.add(['fdroidserver/common.py'])
author = git.Actor('fdroid-bot', 'fdroid-bot@f-droid.org')
git_repo.index.commit('Android NDK %s (%s)' % (release, revision), author=author)
project_path = 'fdroid-bot/' + os.getenv('CI_PROJECT_NAME')
url = 'https://gitlab-ci-token:%s@%s/%s.git' % (
os.getenv('PERSONAL_ACCESS_TOKEN'),
os.getenv('CI_SERVER_HOST'),
project_path,
)
remote_name = 'fdroid-bot'
try:
remote = git_repo.create_remote(remote_name, url)
# See https://github.com/PyCQA/pylint/issues/2856 .
# pylint: disable-next=no-member
except git.exc.GitCommandError:
remote = git.remote.Remote(git_repo, remote_name)
remote.set_url(url)
remote.push(force=True)
git.remote.Remote.rm(git_repo, remote_name)
private_token = os.getenv('PERSONAL_ACCESS_TOKEN')
if not private_token:
print(
Fore.RED
+ 'ERROR: GitLab Token not found in PERSONAL_ACCESS_TOKEN!'
+ Style.RESET_ALL
)
sys.exit(1)
gl = gitlab.Gitlab(
os.getenv('CI_SERVER_URL'), api_version=4, private_token=private_token
)
project = gl.projects.get(project_path, lazy=True)
description = (
'see <https://gitlab.com/fdroid/android-sdk-transparency-log/-/blob/master/checksums.json>'
'\n\n<p><small>generated by <a href="%s/-/jobs/%s">GitLab CI Job #%s</a></small></p>'
% (os.getenv('CI_PROJECT_URL'), os.getenv('CI_JOB_ID'), os.getenv('CI_JOB_ID'))
)
mr = project.mergerequests.create(
{
'source_branch': branch.name,
'target_project_id': 36527, # fdroid/fdroidserver
'target_branch': 'master',
'title': 'update NDK',
'description': description,
'labels': ['fdroid-bot', 'buildserver'],
'remove_source_branch': True,
}
)
mr.save()