From 9fa719362059d74bc6acd968456f1f16c4baa183 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 8 Oct 2025 11:12:34 +0200 Subject: [PATCH] new subcommand "push": podman/vagrant implementation --- fdroidserver/__main__.py | 1 + fdroidserver/common.py | 4 +- fdroidserver/push.py | 208 +++++++++++++++++++++++++++++++++++++++ tests/test_push.py | 87 ++++++++++++++++ 4 files changed, 298 insertions(+), 2 deletions(-) create mode 100644 fdroidserver/push.py create mode 100755 tests/test_push.py diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index bbfa1ebb..73d32b97 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -67,6 +67,7 @@ COMMANDS = OrderedDict([ COMMANDS_INTERNAL = [ "destroy", "exec", + "push", "up", ] diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 745eec1c..25053d72 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -4,10 +4,10 @@ # # Copyright (C) 2010-2016, Ciaran Gultnieks, ciaran@ciarang.com # Copyright (C) 2013-2017, Daniel Martí -# Copyright (C) 2013-2021, Hans-Christoph Steiner +# Copyright (C) 2013-2025, Hans-Christoph Steiner # Copyright (C) 2017-2018, Torsten Grote # Copyright (C) 2017, tobiasKaminsky -# Copyright (C) 2017-2021, Michael Pöhn +# Copyright (C) 2017-2025, Michael Pöhn # Copyright (C) 2017,2021, mimi89999 # Copyright (C) 2019-2021, Jochen Sprickerhof # Copyright (C) 2021, Felix C. Stegerman diff --git a/fdroidserver/push.py b/fdroidserver/push.py new file mode 100644 index 00000000..1719936f --- /dev/null +++ b/fdroidserver/push.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +# +# push.py - part of the FDroid server tools +# Copyright (C) 2024-2025, Hans-Christoph Steiner +# Copyright (C) 2024-2025, Michael Pöhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Push files into the build setup for a given ApplicationID:versionCode. + +This is a carefully constructed method to copy files from the host +filesystem to inside the buildserver VM/container to run the build. The +source paths are generated based on the build metadata, and the +destination is forced to build home. This is not meant as a generic +method for copying files, that is already provided by each VM/container +system (e.g. `podman cp`). + +Since this is an internal command, the strings are not localized. + +""" + +import logging +import os +import sys +import tarfile +import tempfile +import traceback +from argparse import ArgumentParser +from pathlib import Path + +from . import common, metadata +from .exception import BuildException + + +def podman_push(paths, appid, vercode, as_root=False): + """Push relative paths into the podman container using the tar method. + + This builds up a tar file from the supplied paths to send into + container via put_archive(). This assumes it is running in the + base of fdroiddata and it will push into common.BUILD_HOME in the + container. + + This is a version of podman.api.create_tar() that builds up via + adding paths rather than giving a base dir and an exclude list. + + """ + + def _tar_perms_filter(tarinfo): + """Force perms to something safe.""" + if '__pycache__' in tarinfo.name: + return + + if as_root: + tarinfo.uid = tarinfo.gid = 0 + tarinfo.uname = tarinfo.gname = 'root' + else: + tarinfo.uid = tarinfo.gid = 1000 + tarinfo.uname = tarinfo.gname = common.BUILD_USER + + if tarinfo.isdir(): + tarinfo.mode = 0o0755 + elif tarinfo.isfile(): + if tarinfo.mode & 0o111: + tarinfo.mode = 0o0755 + else: + tarinfo.mode = 0o0644 + elif not tarinfo.issym(): # symlinks shouldn't need perms set + raise BuildException(f'{tarinfo.name} is not a file or directory!') + return tarinfo + + if isinstance(paths, Path): + paths = [str(paths)] + if isinstance(paths, str): + paths = [paths] + + container = common.get_podman_container(appid, vercode) + with tempfile.TemporaryFile() as tf: + with tarfile.TarFile(fileobj=tf, mode='w') as tar: + for f in paths: + if Path(f).is_absolute(): + raise BuildException(f'{f} must be relative to {Path.cwd()}') + # throw ValueError on bad path + f = (Path.cwd() / f).resolve().relative_to(Path.cwd()) + tar.add(f, filter=_tar_perms_filter) + tf.seek(0) + container.put_archive(common.BUILD_HOME, tf.read()) + + +def vagrant_push(paths, appid, vercode): + """Push files into a build specific vagrant VM.""" + # TODO implement with `vagrant ssh` and rsync? + + +# TODO split out shared code into _podman.py, _vagrant.py, etc + + +def push_wrapper(paths, appid, vercode, virt_container_type): + """Push standard set of files into VM/container.""" + if virt_container_type == 'vagrant': + vagrant_push(paths, appid, vercode) + elif virt_container_type == 'podman': + podman_push(paths, appid, vercode) + + +def make_file_list(appid, vercode): + """Assemble list of files/folders that go into this specific build.""" + files = [f'build/{appid}', f'metadata/{appid}.yml'] + app_dir = f'metadata/{appid}' + if Path(app_dir).exists(): + files.append(app_dir) + app, build = metadata.get_single_build(appid, vercode) + + for lib in build.srclibs: + srclib = common.getsrclib(lib, 'build/srclib', basepath=True, prepare=False) + if srclib: + # srclib metadata file + files.append(f"srclibs/{srclib[0]}.yml") + # srclib sourcecode + files.append(srclib[2]) + + extlib_paths = [f"build/extlib/{lib}" for lib in build.extlibs] + missing_extlibs = [p for p in extlib_paths if not os.path.exists(p)] + if any(missing_extlibs): + raise Exception( + "error: requested missing extlibs: {}".format(" ".join(missing_extlibs)) + ) + files.extend(extlib_paths) + + return files + + +def create_build_dirs(appid, vercode, virt_container_type): + """Create directories required for running builds.""" + dirs = ('build', 'build/extlib', 'build/srclib', 'metadata', 'fdroidserver') + for dir_name in dirs: + cmd = ['mkdir', '--parents', f'{common.BUILD_HOME}/{dir_name}'] + common.inside_exec( + appid, + vercode, + cmd, + virt_container_type, + ) + + +def full_push_sequence(appid, vercode, virt_container_type): + """Push all files into vm required for specified build.""" + create_build_dirs(appid, vercode, virt_container_type) + push_wrapper(make_file_list(appid, vercode), appid, vercode, virt_container_type) + + # fdroidserver is pushed in every build + cwd = Path('.').absolute() + try: + os.chdir(Path(__file__).parent.parent.parent) + push_wrapper( + [ + 'fdroidserver/fdroidserver', + 'fdroidserver/fdroid', + ], + appid, + vercode, + virt_container_type, + ) + finally: + os.chdir(cwd) + + +def main(): + """CLI main method for this subcommand.""" + parser = ArgumentParser( + description="Push full build setup into the buildserver container/box." + ) + common.setup_global_opts(parser) + common.setup_virt_container_type_opts(parser) + parser.add_argument( + "APPID:VERCODE", + help="Application ID with Version Code in the form APPID:VERCODE", + ) + options = common.parse_args(parser) + common.set_console_logging(options.verbose) + + try: + appid, vercode = common.split_pkg_arg(options.__dict__['APPID:VERCODE']) + full_push_sequence( + appid, + vercode, + common.get_virt_container_type(options), + ) + except Exception as e: + if options.verbose: + logging.error(traceback.format_exc()) + else: + logging.error(e) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/tests/test_push.py b/tests/test_push.py new file mode 100755 index 00000000..6c0bcb7f --- /dev/null +++ b/tests/test_push.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +import importlib +import os +import unittest + +from pathlib import Path +from unittest import mock, skipIf + +from fdroidserver import common, exception, push +from .shared_test_code import mkdtemp, APPID, VERCODE, APPID_VERCODE + + +class PushTest(unittest.TestCase): + basedir = Path(__file__).resolve().parent + + def setUp(self): + self._td = mkdtemp() + self.testdir = self._td.name + os.chdir(self.testdir) + common.config = dict() + + def tearDown(self): + self._td.cleanup() + common.config = None + + +class Push_main(PushTest): + def setUp(self): + super().setUp() + metadatapath = Path(common.get_metadatapath(APPID)) + metadatapath.parent.mkdir() + metadatapath.write_text(f'Name: Test\nBuilds:\n - versionCode: {VERCODE}\n') + + @mock.patch('sys.argv', ['fdroid push', APPID_VERCODE]) + @mock.patch('fdroidserver.push.create_build_dirs') + @mock.patch('fdroidserver.push.podman_push') + def test_podman(self, podman_push, create_build_dirs): + common.config['virt_container_type'] = 'podman' + push.main() + create_build_dirs.assert_called_once() + podman_push.assert_called() + + @mock.patch('sys.argv', ['fdroid push', APPID_VERCODE]) + @mock.patch('fdroidserver.push.create_build_dirs') + @mock.patch('fdroidserver.push.vagrant_push') + def test_vagrant(self, vagrant_push, create_build_dirs): + common.config['virt_container_type'] = 'vagrant' + push.main() + create_build_dirs.assert_called_once() + vagrant_push.assert_called() + + +@skipIf(importlib.util.find_spec("podman") is None, 'Requires podman-py to run.') +class Push_podman_push(PushTest): + def _only_run_if_container_exists(self): + try: + common.get_podman_container(APPID, VERCODE) + except exception.BuildException as e: + # To run these tests, first do: `./fdroid up com.example:123` + self.skipTest(f'Requires Podman container {APPID_VERCODE} to run: {e}') + + def test_no_existing_container(self): + appid = 'should.never.exist' + f = Path(f'metadata/{appid}.yml') + f.parent.mkdir() + f.write_text(f.name) + with self.assertRaises(exception.BuildException): + push.podman_push(f, appid, 9999) + + def test_bad_absolute_path(self): + self._only_run_if_container_exists() + with self.assertRaises(exception.BuildException): + push.podman_push('/etc/passwd', APPID, VERCODE) + + def test_bad_relative_path(self): + self._only_run_if_container_exists() + with self.assertRaises(ValueError): + push.podman_push('../../etc/passwd', APPID, VERCODE) + + def test_existing(self): + self._only_run_if_container_exists() + f = Path(f'metadata/{APPID}.yml') + f.parent.mkdir() + f.write_text(f.name) + push.podman_push(f, APPID, VERCODE) + common.inside_exec(APPID, VERCODE, ['test', '-e', str(f)], 'podman')