diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index 149c0c0e..ced3d6b2 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -65,8 +65,12 @@ COMMANDS = OrderedDict([ # interactively because they rely on the presense of a VM/container, # modify the local environment, or even run things as root. COMMANDS_INTERNAL = [ + "build_local_run", "destroy", "exec", + "install_ndk", + "make_source_tarball", + "prepare_source", "pull", "push", "schedule_verify", diff --git a/fdroidserver/build_local_run.py b/fdroidserver/build_local_run.py new file mode 100644 index 00000000..accd79d3 --- /dev/null +++ b/fdroidserver/build_local_run.py @@ -0,0 +1,757 @@ +#!/usr/bin/env python3 +# +# build_local_run.py - part of the F-Droid server tools +# Copyright (C) 2017-2025 Michael Pöhn +# Copyright (C) 2021-2025 linsui <2873532-linsui@users.noreply.gitlab.com> +# Copyright (C) 2021-2025 Jochen Sprickerhof +# Copyright (C) 2014-2025 Hans-Christoph Steiner +# Copyright (C) 2024 g0t mi1k +# Copyright (C) 2024 Gregor Düster +# Copyright (C) 2023 cvzi +# Copyright (C) 2023 Jason A. Donenfeld +# Copyright (C) 2023 FestplattenSchnitzel +# Copyright (C) 2022 proletarius101 +# Copyright (C) 2021 Benedikt Brückmann <64bit+git@posteo.de> +# Copyright (C) 2021 Felix C. Stegerman +# Copyright (C) 2017-2021 relan +# Copyright (C) 2017-2020 Marcus Hoffmann +# Copyright (C) 2016, 2017, 2020 mimi89999 +# Copyright (C) 2019 Michael von Glasow +# Copyright (C) 2018 csagan5 <32685696+csagan5@users.noreply.github.com> +# Copyright (C) 2018 Areeb Jamal +# Copyright (C) 2017-2018 Torsten Grote +# Copyright (C) 2017 thez3ro +# Copyright (C) 2017 lb@lb520 +# Copyright (C) 2017 Jan Berkel +# Copyright (C) 2013-2016 Daniel Martí +# Copyright (C) 2010-2016 Ciaran Gultnieks, ciaran@ciarang.com +# Copyright (C) 2016 Kevin C. Krinke +# Copyright (C) 2015 أحمد المحمودي (Ahmed El-Mahmoudy) +# Copyright (C) 2015 nero-tux +# Copyright (C) 2015 Rancor +# Copyright (C) 2015 Lode Hoste +# Copyright (C) 2013 Simon Josefsson +# Copyright (C) 2013 Frans Gifford +# Copyright (C) 2013 Christopher +# Copyright (C) 2012-2013 Paul Sokolovsky +# Copyright (C) 2012 Tias Guns +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Build one specific in app, assuming all required build tools are installed and configured.""" + +import os +import re +import sys +import glob +import shutil +import logging +import pathlib +import argparse +import traceback + +from fdroidserver import common, exception, metadata + + +def rlimit_check(apps_count=1): + """Make sure Linux is configured to allow for enough simultaneously open files. + + TODO: check if this is obsolete + + Parameters + ---------- + apps_count + In the past this used to be `len(apps)` In this context we're + always buidling just one app so this is always 1 + """ + try: + import resource # not available on Windows + + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if apps_count > soft: + try: + soft = apps_count * 2 + if soft > hard: + soft = hard + resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) + logging.debug(f'Set open file limit to {soft}') + except (OSError, ValueError) as e: + logging.warning('Setting open file limit failed: ' + str(e)) + except ImportError: + pass + + +def get_build_root_dir(app, build): + if build.subdir: + return os.path.join(common.get_build_dir(app), build.subdir) + return common.get_build_dir(app) + + +def transform_first_char(string, method): + """Use method() on the first character of string.""" + if len(string) == 0: + return string + if len(string) == 1: + return method(string) + return method(string[0]) + string[1:] + + +def get_flavours_cmd(build): + """Get flavor string, preformatted for gradle cli. + + Reads build flavors form metadata if any and reformats and concatenates + them to be ready for use as CLI arguments to gradle. This will treat the + vlue 'yes' as if there were not particular build flavor selected. + + Parameters + ---------- + build + The metadata build entry you'd like to read flavors from + + Returns + ------- + A string containing the build flavor for this build. If it's the default + flavor ("yes" in metadata) this returns an empty string. Returns None if + it's not a gradle build. + """ + flavours = build.gradle + + if flavours == ['yes']: + flavours = [] + + flavours_cmd = ''.join([transform_first_char(flav, str.upper) for flav in flavours]) + + return flavours_cmd + + +def init_build(app, build, config): + root_dir = get_build_root_dir(app, build) + + p = None + gradletasks = [] + + # We need to clean via the build tool in case the binary dirs are + # different from the default ones + + bmethod = build.build_method() + if bmethod == 'maven': + logging.info("Cleaning Maven project...") + cmd = [config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']] + + if '@' in build.maven: + maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1]) + maven_dir = os.path.normpath(maven_dir) + else: + maven_dir = root_dir + + p = common.FDroidPopen(cmd, cwd=maven_dir) + + elif bmethod == 'gradle': + logging.info("Cleaning Gradle project...") + + if build.preassemble: + gradletasks += build.preassemble + + flavours_cmd = get_flavours_cmd(build) + + gradletasks += ['assemble' + flavours_cmd + 'Release'] + + cmd = [config['gradle']] + if build.gradleprops: + cmd += ['-P' + kv for kv in build.gradleprops] + + cmd += ['clean'] + p = common.FDroidPopen( + cmd, + cwd=root_dir, + envs={ + "GRADLE_VERSION_DIR": config['gradle_version_dir'], + "CACHEDIR": config['cachedir'], + }, + ) + + elif bmethod == 'ant': + logging.info("Cleaning Ant project...") + p = common.FDroidPopen(['ant', 'clean'], cwd=root_dir) + + if p is not None and p.returncode != 0: + raise exception.BuildException( + "Error cleaning %s:%s" % (app.id, build.versionCode), p.output + ) + + return gradletasks + + +def sanitize_build_dir(app): + """Delete build output directories. + + This function deletes the default build/binary/target/... output + directories for follwoing build tools: gradle, maven, ant, jni. It also + deletes gradle-wrapper if present. It just uses parths, hardcoded here, + it doesn't call and build system clean routines. + + Parameters + ---------- + app + The metadata of the app to sanitize + """ + build_dir = common.get_build_dir(app) + for root, dirs, files in os.walk(build_dir): + + def del_dirs(dl): + for d in dl: + shutil.rmtree(os.path.join(root, d), ignore_errors=True) + + def del_files(fl): + for f in fl: + if f in files: + os.remove(os.path.join(root, f)) + + if any( + f in files + for f in [ + 'build.gradle', + 'build.gradle.kts', + 'settings.gradle', + 'settings.gradle.kts', + ] + ): + # Even when running clean, gradle stores task/artifact caches in + # .gradle/ as binary files. To avoid overcomplicating the scanner, + # manually delete them, just like `gradle clean` should have removed + # the build/* dirs. + del_dirs( + [ + os.path.join('build', 'android-profile'), + os.path.join('build', 'generated'), + os.path.join('build', 'intermediates'), + os.path.join('build', 'outputs'), + os.path.join('build', 'reports'), + os.path.join('build', 'tmp'), + os.path.join('buildSrc', 'build'), + '.gradle', + ] + ) + del_files(['gradlew', 'gradlew.bat']) + + if 'pom.xml' in files: + del_dirs(['target']) + + if any( + f in files for f in ['ant.properties', 'project.properties', 'build.xml'] + ): + del_dirs(['bin', 'gen']) + + if 'jni' in dirs: + del_dirs(['obj']) + + +def execute_build_commands(app, build): + """Execute `bulid` commands if present in metadata. + + see: https://f-droid.org/docs/Build_Metadata_Reference/#build_build + + Parameters + ---------- + app + metadata app object + build + metadata build object + """ + root_dir = get_build_root_dir(app, build) + srclibpaths = get_srclibpaths(app, build) + + if build.build: + logging.info("Running 'build' commands in %s" % root_dir) + cmd = common.replace_config_vars("; ".join(build.build), build) + + # Substitute source library paths into commands... + for name, number, libpath in srclibpaths: + cmd = cmd.replace('$$' + name + '$$', os.path.join(os.getcwd(), libpath)) + + p = common.FDroidPopen( + ['bash', '-e', '-u', '-o', 'pipefail', '-x', '-c', cmd], cwd=root_dir + ) + + if p.returncode != 0: + raise exception.BuildException( + f"Error running build command for {app.id}:{build.versionCode}", + p.output, + ) + + +def get_srclibpaths(app, build): + """Get srclibpaths list of tuples. + + This will just assemble the srclibpaths list of tuples, it won't fetch + or checkout any source code, identical to return value of + common.prepare_souce(). + + Parameters + ---------- + app + metadata app object + build + metadata build object + + Returns + ------- + List of srclibpath tuples + """ + vcs, _ = common.setup_vcs(app) + + srclibpaths = [] + if build.srclibs: + logging.info("Collecting source libraries") + for lib in build.srclibs: + srclibpaths.append( + common.getsrclib( + lib, + "./build/srclib", + build=build, + prepare=False, + refresh=False, + preponly=True, + ) + ) + + basesrclib = vcs.getsrclib() + if basesrclib: + srclibpaths.append(basesrclib) + + return srclibpaths + + +def execute_buildjni_commands(app, build): + root_dir = get_build_root_dir(app, build) + ndk_path = build.ndk_path() + if build.buildjni and build.buildjni != ['no']: + logging.info("Building the native code") + jni_components = build.buildjni + + if jni_components == ['yes']: + jni_components = [''] + cmd = [os.path.join(ndk_path, "ndk-build"), "-j1"] + for d in jni_components: + if d: + logging.info("Building native code in '%s'" % d) + else: + logging.info("Building native code in the main project") + manifest = os.path.join(root_dir, d, 'AndroidManifest.xml') + if os.path.exists(manifest): + # Read and write the whole AM.xml to fix newlines and avoid + # the ndk r8c or later 'wordlist' errors. The outcome of this + # under gnu/linux is the same as when using tools like + # dos2unix, but the native python way is faster and will + # work in non-unix systems. + manifest_text = open(manifest, 'U').read() + open(manifest, 'w').write(manifest_text) + # In case the AM.xml read was big, free the memory + del manifest_text + p = common.FDroidPopen(cmd, cwd=os.path.join(root_dir, d)) + if p.returncode != 0: + raise exception.BuildException( + "NDK build failed for %s:%s" % (app.id, build.versionName), p.output + ) + + +def execute_build(app, build, config, gradletasks): + root_dir = get_build_root_dir(app, build) + + p = None + bindir = None + bmethod = build.build_method() + if bmethod == 'maven': + logging.info("Building Maven project...") + + if '@' in build.maven: + maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1]) + else: + maven_dir = root_dir + + mvncmd = [ + config['mvn3'], + '-Dandroid.sdk.path=' + config['sdk_path'], + '-Dmaven.jar.sign.skip=true', + '-Dmaven.test.skip=true', + '-Dandroid.sign.debug=false', + '-Dandroid.release=true', + 'package', + ] + if build.target: + target = build.target.split('-')[1] + common.regsub_file( + r'[0-9]*', + r'%s' % target, + os.path.join(root_dir, 'pom.xml'), + ) + if '@' in build.maven: + common.regsub_file( + r'[0-9]*', + r'%s' % target, + os.path.join(maven_dir, 'pom.xml'), + ) + + p = common.FDroidPopen(mvncmd, cwd=maven_dir) + + bindir = os.path.join(root_dir, 'target') + + elif bmethod == 'gradle': + logging.info("Building Gradle project...") + + cmd = [config['gradle']] + if build.gradleprops: + cmd += ['-P' + kv for kv in build.gradleprops] + + cmd += gradletasks + + p = common.FDroidPopen( + cmd, + cwd=root_dir, + envs={ + "GRADLE_VERSION_DIR": config['gradle_version_dir'], + "CACHEDIR": config['cachedir'], + }, + ) + + elif bmethod == 'ant': + logging.info("Building Ant project...") + cmd = ['ant'] + if build.antcommands: + cmd += build.antcommands + else: + cmd += ['release'] + p = common.FDroidPopen(cmd, cwd=root_dir) + + bindir = os.path.join(root_dir, 'bin') + + return p, bindir + + +def collect_build_output(app, build, p, bindir): + root_dir = get_build_root_dir(app, build) + + omethod = build.output_method() + src = None + if omethod == 'maven': + stdout_apk = '\n'.join( + [ + line + for line in p.output.splitlines() + if any(a in line for a in ('.apk', '.ap_', '.jar')) + ] + ) + m = re.match( + r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", stdout_apk, re.S | re.M + ) + if not m: + m = re.match( + r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]", + stdout_apk, + re.S | re.M, + ) + if not m: + m = re.match( + r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' + + bindir + + r'/([^/]+)\.ap[_k][,\]]', + stdout_apk, + re.S | re.M, + ) + + if not m: + m = re.match( + r".*^\[INFO\] Building jar: .*/" + bindir + r"/(.+)\.jar", + stdout_apk, + re.S | re.M, + ) + if not m: + raise exception.BuildException('Failed to find output') + src = m.group(1) + src = os.path.join(bindir, src) + '.apk' + + elif omethod == 'gradle': + src = None + apk_dirs = [ + # gradle plugin >= 3.0 + os.path.join(root_dir, 'build', 'outputs', 'apk', 'release'), + # gradle plugin < 3.0 and >= 0.11 + os.path.join(root_dir, 'build', 'outputs', 'apk'), + # really old path + os.path.join(root_dir, 'build', 'apk'), + ] + # If we build with gradle flavours with gradle plugin >= 3.0 the APK will be in + # a subdirectory corresponding to the flavour command used, but with different + # capitalization. + flavours_cmd = get_flavours_cmd(build) + if flavours_cmd: + apk_dirs.append( + os.path.join( + root_dir, + 'build', + 'outputs', + 'apk', + transform_first_char(flavours_cmd, str.lower), + 'release', + ) + ) + for apks_dir in apk_dirs: + for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']: + apks = glob.glob(os.path.join(apks_dir, apkglob)) + + if len(apks) > 1: + raise exception.BuildException( + 'More than one resulting apks found in %s' % apks_dir, + '\n'.join(apks), + ) + if len(apks) == 1: + src = apks[0] + break + if src is not None: + break + + if src is None: + raise exception.BuildException('Failed to find any output apks') + + elif omethod == 'ant': + stdout_apk = '\n'.join( + [line for line in p.output.splitlines() if '.apk' in line] + ) + src = re.match( + r".*^.*Creating (.+) for release.*$.*", stdout_apk, re.S | re.M + ).group(1) + src = os.path.join(bindir, src) + elif omethod == 'raw': + output_path = common.replace_build_vars(build.output, build) + globpath = os.path.join(root_dir, output_path) + apks = glob.glob(globpath) + if len(apks) > 1: + raise exception.BuildException( + 'Multiple apks match %s' % globpath, '\n'.join(apks) + ) + if len(apks) < 1: + raise exception.BuildException('No apks match %s' % globpath) + src = os.path.normpath(apks[0]) + return src + + +def check_build_success(app, build, p): + build_dir = common.get_build_dir(app) + + if os.path.isdir(os.path.join(build_dir, '.git')): + import git + + commit_id = common.get_head_commit_id(git.repo.Repo(build_dir)) + else: + commit_id = build.commit + + if p is not None and p.returncode != 0: + raise exception.BuildException( + "Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id), + p.output, + ) + logging.info( + "Successfully built version {versionName} of {appid} from {commit_id}".format( + versionName=build.versionName, appid=app.id, commit_id=commit_id + ) + ) + + +def execute_postbuild(app, build, src): + root_dir = get_build_root_dir(app, build) + srclibpaths = get_srclibpaths(app, build) + + if build.postbuild: + logging.info(f"Running 'postbuild' commands in {root_dir}") + cmd = common.replace_config_vars("; ".join(build.postbuild), build) + + # Substitute source library paths into commands... + for name, number, libpath in srclibpaths: + cmd = cmd.replace(f"$${name}$$", str(pathlib.Path.cwd() / libpath)) + + cmd = cmd.replace('$$OUT$$', str(pathlib.Path(src).resolve())) + + p = common.FDroidPopen( + ['bash', '-e', '-u', '-o', 'pipefail', '-x', '-c', cmd], cwd=root_dir + ) + + if p.returncode != 0: + raise exception.BuildException( + "Error running postbuild command for " f"{app.id}:{build.versionName}", + p.output, + ) + + +def get_metadata_from_apk(app, build, apkfile): + """Get the required metadata from the built APK. + + VersionName is allowed to be a blank string, i.e. '' + + Parameters + ---------- + app + The app metadata used to build the APK. + build + The build that resulted in the APK. + apkfile + The path of the APK file. + + Returns + ------- + versionCode + The versionCode from the APK or from the metadata is build.novcheck is + set. + versionName + The versionName from the APK or from the metadata is build.novcheck is + set. + + Raises + ------ + :exc:`~exception.BuildException` + If native code should have been built but was not packaged, no version + information or no package ID could be found or there is a mismatch + between the package ID in the metadata and the one found in the APK. + """ + appid, versionCode, versionName = common.get_apk_id(apkfile) + native_code = common.get_native_code(apkfile) + + if build.buildjni and build.buildjni != ['no'] and not native_code: + raise exception.BuildException( + "Native code should have been built but none was packaged" + ) + if build.novcheck: + versionCode = build.versionCode + versionName = build.versionName + if not versionCode or versionName is None: + raise exception.BuildException( + "Could not find version information in build in output" + ) + if not appid: + raise exception.BuildException("Could not find package ID in output") + if appid != app.id: + raise exception.BuildException( + "Wrong package ID - build " + appid + " but expected " + app.id + ) + + return versionCode, versionName + + +def validate_build_artifacts(app, build, src): + # Make sure it's not debuggable... + if common.is_debuggable_or_testOnly(src): + raise exception.BuildException( + "%s: debuggable or testOnly set in AndroidManifest.xml" % src + ) + + # By way of a sanity check, make sure the version and version + # code in our new APK match what we expect... + logging.debug("Checking " + src) + if not os.path.exists(src): + raise exception.BuildException( + "Unsigned APK is not at expected location of " + src + ) + + if common.get_file_extension(src) == 'apk': + vercode, version = get_metadata_from_apk(app, build, src) + if version != build.versionName or vercode != build.versionCode: + raise exception.BuildException( + ( + "Unexpected version/version code in output;" + " APK: '%s' / '%d', " + " Expected: '%s' / '%d'" + ) + % (version, vercode, build.versionName, build.versionCode) + ) + + +def move_build_output(app, build, src, output_dir="unsigned"): + # Copy the unsigned APK to our destination directory for further + # processing (by publish.py)... + dest = os.path.join( + output_dir, + common.get_release_filename(app, build, common.get_file_extension(src)), + ) + shutil.copyfile(src, dest) + + +def build_local_run_wrapper(appid, versionCode, config): + """Run build for one specific version of an app localy. + + :raises: various exceptions in case and of the pre-required conditions for the requested build are not met + """ + app, build = metadata.get_single_build(appid, versionCode) + + # not sure if this makes any sense to change open file limits since we know + # that this script will only ever build one app + rlimit_check() + + logging.info( + "Building version %s (%s) of %s" + % (build.versionName, build.versionCode, app.id) + ) + + # init fdroid Popen wrapper + common.set_FDroidPopen_env(app=app, build=build) + gradletasks = init_build(app, build, config) + + sanitize_build_dir(app) + + # this is where we'd call scanner.scan_source() in old build.py + + # Run a build command if one is required... + execute_build_commands(app, build) + + # Build native stuff if required... + execute_buildjni_commands(app, build) + + # Build the release... + p, bindir = execute_build(app, build, config, gradletasks) + check_build_success(app, build, p) + src = collect_build_output(app, build, p, bindir) + + # Run a postbuild command if one is required... + execute_postbuild(app, build, src) + + validate_build_artifacts(app, build, src) + + # this is where we'd call scanner.scan_binary() in old build.py + + tmp_dir = pathlib.Path("tmp") + tmp_dir.mkdir(exist_ok=True) + move_build_output(app, build, src, tmp_dir) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + common.setup_global_opts(parser) + parser.add_argument( + "APPID:VERCODE", + help="Application ID with Version Code in the form APPID:VERCODE", + ) + options = common.parse_args(parser) + config = common.get_config() + + try: + appid, versionCode = common.split_pkg_arg(options.__dict__['APPID:VERCODE']) + build_local_run_wrapper(appid, versionCode, config) + except Exception as e: + if options.verbose: + traceback.print_exc() + else: + print(e) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/fdroidserver/install_ndk.py b/fdroidserver/install_ndk.py new file mode 100644 index 00000000..5ce0c12d --- /dev/null +++ b/fdroidserver/install_ndk.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# +# install_ndk.py - part of the F-Droid server tools +# Copyright (C) 2024-2025 Michael Pöhn +# Copyright (C) 2025 Hans-Christoph Steiner +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Read the "ndk:" field from the build metadata and install the right NDK packages. + +For more info, see: +https://f-droid.org/docs/Build_Metadata_Reference/#build_ndk + +""" + +import argparse +import logging +import os + +from fdroidserver import common, exception, metadata + + +def install_ndk_wrapper(build, ndk_paths=dict()): + """Make sure the requested NDK version is or gets installed. + + Parameters + ---------- + build + metadata.Build instance entry that may contain the + requested NDK version + ndk_paths + dictionary holding the currently installed NDKs + """ + ndk_path = build.ndk_path() + if build.ndk or (build.buildjni and build.buildjni != ['no']): + if not ndk_path: + for k, v in ndk_paths.items(): + if k.endswith("_orig"): + continue + common.auto_install_ndk(build) + ndk_path = build.ndk_path() + if not os.path.isdir(ndk_path): + logging.critical("Android NDK '%s' is not installed!" % ndk_path) + raise exception.FDroidException() + return ndk_path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + common.setup_global_opts(parser) + parser.add_argument( + "APPID:VERCODE", + help="Application ID with Version Code in the form APPID:VERCODE", + ) + + options = common.parse_args(parser) + config = common.get_config() + appid, versionCode = common.split_pkg_arg(options.__dict__['APPID:VERCODE']) + app, build = metadata.get_single_build(appid, versionCode) + install_ndk_wrapper(build, config.get('ndk_paths', dict())) + + +if __name__ == "__main__": + main() diff --git a/fdroidserver/make_source_tarball.py b/fdroidserver/make_source_tarball.py new file mode 100644 index 00000000..bc03a8c1 --- /dev/null +++ b/fdroidserver/make_source_tarball.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# +# make_source_tarball.py - part of the F-Droid server tools +# Copyright (C) 2024-2025, Michael Pöhn +# Copyright (C) 2024-2025, Hans-Christoph Steiner +# Copyright (C) 2018, Areeb Jamal +# Copyright (C) 2013-2014, Daniel Martí +# Copyright (C) 2010-2015, Ciaran Gultnieks +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Make a source tarball from a single app's checked out source code. + +This assumes that the app's source code is already checked out into the +repo, and that it is in a clean state. It makes a src.tar.gz in the +repo's tmp/ directory. + +""" + +import argparse +import logging +import os +import pathlib +import sys +import tarfile +import traceback + +from fdroidserver import common, metadata + + +def make_source_tarball(app, build, output_dir=pathlib.Path('unsigned')): + if not output_dir.exists(): + output_dir.mkdir() + build_dir = common.get_build_dir(app) + # Build the source tarball right before we build the release... + logging.info("Creating source tarball...") + tarname = common.get_src_tarball_name(app.id, build.versionCode) + tarball = tarfile.open(os.path.join(output_dir, tarname), "w:gz") + + def tarexc(t): + return ( + None + if any(t.name.endswith(s) for s in ['.svn', '.git', '.hg', '.bzr']) + else t + ) + + tarball.add(build_dir, tarname, filter=tarexc) + tarball.close() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + common.setup_global_opts(parser) + parser.add_argument( + "APPID:VERCODE", + help="Application ID with Version Code in the form APPID:VERCODE", + ) + options = common.parse_args(parser) + + try: + appid, versionCode = common.split_pkg_arg(options.APPID_VERCODE) + make_source_tarball(metadata.get_single_build(appid, versionCode)) + except Exception as e: + if options.verbose: + traceback.print_exc() + else: + print(e) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/fdroidserver/prepare_source.py b/fdroidserver/prepare_source.py new file mode 100644 index 00000000..4d3e65b1 --- /dev/null +++ b/fdroidserver/prepare_source.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# +# prepare_source.py - part of the F-Droid server tools +# Copyright (C) 2024-2025 Michael Pöhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Prepare the source code directory for a particular build.""" + +import argparse +import pathlib + +from fdroidserver import common, metadata + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + common.setup_global_opts(parser) + parser.add_argument( + "APPID:VERCODE", + help="Application ID with Version Code in the form APPID:VERCODE", + ) + options = common.parse_args(parser) + common.get_config() + appid, versionCode = common.split_pkg_arg(options.__dict__['APPID:VERCODE']) + app, build = metadata.get_single_build(appid, versionCode) + + # prepare folders for git/vcs checkout + vcs, build_dir = common.setup_vcs(app) + srclib_dir = pathlib.Path('./build/srclib') + extlib_dir = pathlib.Path('./build/extlib') + log_dir = pathlib.Path('./logs') + output_dir = pathlib.Path('./unsigned') + for d in (srclib_dir, extlib_dir, log_dir, output_dir): + d.mkdir(exist_ok=True, parents=True) + + # do git/vcs checkout + common.prepare_source( + vcs, + app, + build, + build_dir, + str(srclib_dir), + str(extlib_dir), + refresh=False, + onserver=True, + ) + + +if __name__ == "__main__": + main() diff --git a/tests/test_install_ndk.py b/tests/test_install_ndk.py new file mode 100755 index 00000000..500a0341 --- /dev/null +++ b/tests/test_install_ndk.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +import os +import unittest + +from pathlib import Path +from unittest import mock + +from fdroidserver import common, metadata, install_ndk +from .shared_test_code import mkdtemp, APPID, VERCODE, APPID_VERCODE + +NDK_RELEASE = 'r24' +NDK_REVISION = '24.0.8215888' + + +class InstallNdkTest(unittest.TestCase): + basedir = Path(__file__).resolve().parent + + def setUp(self): + self._td = mkdtemp() + self.testdir = self._td.name + os.chdir(self.testdir) + common.config = {'ndk_paths': {}, 'sdk_path': self.testdir} + + def tearDown(self): + self._td.cleanup() + common.config = None + + +def mock_sdkmanager_install(to_install, android_home=None): + path = f'{android_home}/{to_install.replace(";", "/")}' + ndk_dir = Path(path) + ndk_dir.mkdir(parents=True) + (ndk_dir / 'source.properties').write_text(f'Pkg.Revision = {NDK_REVISION}\n') + + +@mock.patch('sdkmanager.build_package_list', lambda use_net: None) +class InstallNdk_wrapper(InstallNdkTest): + @mock.patch('sdkmanager.install') + def test_with_ndk(self, sdkmanager_install): + sdkmanager_install.side_effect = mock_sdkmanager_install + build = metadata.Build({'versionCode': VERCODE, 'ndk': NDK_RELEASE}) + install_ndk.install_ndk_wrapper(build) + sdkmanager_install.assert_called_once() + + @mock.patch('fdroidserver.common.auto_install_ndk') + def test_without_ndk(self, auto_install_ndk): + build = metadata.Build({'versionCode': VERCODE}) + install_ndk.install_ndk_wrapper(build) + auto_install_ndk.assert_not_called() + + +@mock.patch('sys.argv', ['fdroid ndk', APPID_VERCODE]) +@mock.patch('sdkmanager.build_package_list', lambda use_net: None) +@mock.patch('sdkmanager.install') +class InstallNdk_main(InstallNdkTest): + def setUp(self): + super().setUp() + metadatapath = Path(common.get_metadatapath(APPID)) + metadatapath.parent.mkdir() + metadatapath.write_text( + f'Builds:\n - versionCode: {VERCODE}\n ndk: {NDK_RELEASE}\n' + ) + + def test_ndk_main(self, sdkmanager_install): + sdkmanager_install.side_effect = mock_sdkmanager_install + install_ndk.main() + sdkmanager_install.assert_called_once()