From 6cbef943ac1b9a2aede1cb8f1626e0ca0222feab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 1 Oct 2025 15:03:41 +0200 Subject: [PATCH 01/11] set up new mechanism for unlisted subcommands For the build automation, there will be lots of subcommands that are meant to run in scripted environments. They are not intended for interactive use. Although they sometimes might be useful interactively, they might also sometimes be dangerous to run interactively because they rely on the presense of a VM/container, modify the local environment, or even run things as root. --- fdroidserver/__main__.py | 21 +++++++++++++++++++-- fdroidserver/common.py | 21 +++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index 83a0c81c..5a2241f3 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -55,6 +55,18 @@ COMMANDS = OrderedDict([ ("mirror", _("Download complete mirrors of small repos")), ]) +# The list of subcommands that are not advertised as public api, +# intended for the use-case of breaking down builds into atomic steps. +# +# For the build automation, there will be lots of subcommands that are +# meant to run in scripted environments. They are not intended for +# interactive use. Although they sometimes might be useful +# interactively, they might also sometimes be dangerous to run +# interactively because they rely on the presense of a VM/container, +# modify the local environment, or even run things as root. +COMMANDS_INTERNAL = [ +] + def print_help(available_plugins=None): print(_("usage: ") + _("fdroid [] [-h|--help|--version|]")) @@ -136,7 +148,12 @@ def main(): sys.exit(0) command = sys.argv[1] - if command not in COMMANDS and command not in available_plugins: + command_not_found = ( + command not in COMMANDS + and command not in COMMANDS_INTERNAL + and command not in available_plugins + ) + if command_not_found: if command in ('-h', '--help'): print_help(available_plugins=available_plugins) sys.exit(0) @@ -186,7 +203,7 @@ def main(): sys.argv[0] += ' ' + command del sys.argv[1] - if command in COMMANDS.keys(): + if command in COMMANDS.keys() or command in COMMANDS_INTERNAL: # import is named import_subcommand internally b/c import is reserved by Python command = 'import_subcommand' if command == 'import' else command mod = __import__('fdroidserver.' + command, None, None, [command]) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index e3062129..247ee733 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1046,6 +1046,27 @@ def get_local_metadata_files(): return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]') +def split_pkg_arg(appid_versionCode_pair): + """Split 'appid:versionCode' pair into 2 separate values safely. + + :raises ValueError: if argument is not parseable + :return: (appid, versionCode) tuple with the 2 parsed values + """ + tokens = appid_versionCode_pair.split(":") + if len(tokens) != 2: + raise ValueError( + _("'{}' is not a valid pair of the form appId:versionCode pair").format( + appid_versionCode_pair + ) + ) + if not is_valid_package_name(tokens[0]): + raise ValueError( + _("'{}' does not start with a valid appId").format(appid_versionCode_pair) + ) + versionCode = version_code_string_to_int(tokens[1]) + return tokens[0], versionCode + + def read_pkg_args(appid_versionCode_pairs, allow_version_codes=False): """No summary. From 87d0e5a10b9131ea63c1b9dc147d5782da73553d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 14 Oct 2025 10:16:41 +0200 Subject: [PATCH 02/11] specify virt_container_type via config and CLI options --- fdroidserver/common.py | 23 ++++++++++++++++++ tests/test_common.py | 54 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 247ee733..96deb5ea 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -108,6 +108,8 @@ from .looseversion import LooseVersion # The path to this fdroidserver distribution FDROID_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..')) +SUPPORTED_VIRT_CONTAINER_TYPES = ('podman', 'vagrant') + # There needs to be a default, and this is the most common for software. DEFAULT_LOCALE = 'en-US' @@ -280,6 +282,27 @@ def setup_global_opts(parser): ) +def setup_virt_container_type_opts(parser): + parser.add_argument( + "--virt-container-type", + choices=SUPPORTED_VIRT_CONTAINER_TYPES, + help="Set the VM/container type used by the build process.", + ) + + +def get_virt_container_type(options): + if options.virt_container_type: + return options.virt_container_type + vct = get_config().get('virt_container_type') + if vct not in SUPPORTED_VIRT_CONTAINER_TYPES: + supported = ', '.join(sorted(SUPPORTED_VIRT_CONTAINER_TYPES)) + logging.error( + f"'virt_container_type: {vct}' not supported, try: {supported}" + ) + sys.exit(1) + return vct + + class ColorFormatter(logging.Formatter): def __init__(self, msg): diff --git a/tests/test_common.py b/tests/test_common.py index 1dbe79ba..c32a5b35 100755 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -3651,3 +3651,57 @@ class GetHeadCommitIdTest(unittest.TestCase): git_repo.git.add(all=True) git_repo.index.commit("add code") self.assertIsNotNone(fdroidserver.common.get_head_commit_id(git_repo)) + + +class VirtContainerTypeTest(unittest.TestCase): + """Test the logic for choosing which VM/container system to use.""" + + def setUp(self): + self._td = mkdtemp() + self.testdir = self._td.name + os.chdir(self.testdir) + fdroidserver.common.config = None + fdroidserver.common.options = None + # self.options represents the output of argparse.parser() + self.options = mock.Mock() + self.options.virt_container_type = None + + def tearDown(self): + os.chdir(basedir) + self._td.cleanup() + + def test_get_virt_container_type_unset(self): + with self.assertLogs(level=logging.ERROR) as logs: + with self.assertRaises(SystemExit): + fdroidserver.common.get_virt_container_type(self.options) + self.assertIn('virt_container_type', logs.output[0]) + + def test_get_virt_container_type_config(self): + testvalue = 'podman' + Path('config.yml').write_text(f'virt_container_type: {testvalue}\n') + self.assertEqual( + testvalue, fdroidserver.common.get_virt_container_type(self.options) + ) + + def test_get_virt_container_type_options(self): + testvalue = 'podman' + self.options.virt_container_type = testvalue + self.assertEqual( + testvalue, fdroidserver.common.get_virt_container_type(self.options) + ) + + def test_get_virt_container_type_options_override_config(self): + testvalue = 'podman' + self.options.virt_container_type = testvalue + Path('config.yml').write_text('virt_container_type: vagrant\n') + self.assertEqual( + testvalue, fdroidserver.common.get_virt_container_type(self.options) + ) + + def test_get_virt_container_type_config_bad_value(self): + testvalue = 'doesnotexist' + Path('config.yml').write_text(f'virt_container_type: {testvalue}\n') + with self.assertLogs(level=logging.ERROR) as logs: + with self.assertRaises(SystemExit): + fdroidserver.common.get_virt_container_type(self.options) + self.assertIn(testvalue, logs.output[0]) From 76673627fc84e92c6e512bed0450db4fef978017 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 2 Oct 2025 17:38:38 +0200 Subject: [PATCH 03/11] new subcommand "up" for `vagrant up` and `podman run --- .gitlab-ci.yml | 21 +++++ fdroidserver/__main__.py | 1 + fdroidserver/common.py | 83 +++++++++++++++++- fdroidserver/up.py | 180 ++++++++++++++++++++++++++++++++++++++ setup.py | 1 + tests/shared_test_code.py | 5 ++ tests/test_up.py | 127 +++++++++++++++++++++++++++ 7 files changed, 417 insertions(+), 1 deletion(-) create mode 100644 fdroidserver/up.py create mode 100755 tests/test_up.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 65510c45..91598c8a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -852,3 +852,24 @@ PUBLISH: - fdroid gpgsign --verbose - fdroid signindex --verbose - rsync --stats repo/* $serverwebroot/ + + +# This tests the `podman system service` auto-starting and other general podman things. +podman system service: + rules: + - changes: + - .gitlab-ci.yml + - fdroidserver/up.py + - tests/test_up.py + image: debian:trixie-slim + <<: *apt-template + script: + - apt-get install + python3-asn1crypto + python3-defusedxml + python3-git + python3-pillow + python3-podman + python3-ruamel.yaml + python3-yaml + - python3 -m unittest tests/test_up.py --verbose diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index 5a2241f3..c3631213 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -65,6 +65,7 @@ COMMANDS = OrderedDict([ # interactively because they rely on the presense of a VM/container, # modify the local environment, or even run things as root. COMMANDS_INTERNAL = [ + "up", ] diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 96deb5ea..33944b99 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -82,7 +82,7 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from queue import Queue from typing import List -from urllib.parse import urlparse, urlsplit, urlunparse +from urllib.parse import urlparse, urlsplit, urlunparse, unquote from zipfile import ZipFile import defusedxml.ElementTree as XMLElementTree @@ -1303,6 +1303,16 @@ def get_source_date_epoch(build_dir): return repo.git.log('-n1', '--pretty=%ct', '--', metadatapath) +def get_container_name(appid, vercode): + """Return unique name for associating a build with a container or VM.""" + return f'{appid}_{vercode}' + + +def get_pod_name(appid, vercode): + """Return unique name for associating a build with a Podman "pod".""" + return f'{get_container_name(appid, vercode)}_pod' + + def get_build_dir(app): """Get the dir that this app will be built in.""" if app.RepoType == 'srclib': @@ -5056,3 +5066,74 @@ FDROIDORG_MIRRORS = [ FDROIDORG_FINGERPRINT = ( '43238D512C1E5EB2D6569F4A3AFBF5523418B82E0A3ED1552770ABB9A9C9CCAB' ) + + +def get_podman_client(): + """Return an instance of podman-py to work with Podman. + + This assumes that it will use the default local UNIX Domain Socket + to connect to the Podman service. The preferred setup is to first + do `systemctl --user start podman.socket` to keep the Podman UNIX + socket running all the time. If the socket is not present, this + will automatically start the service, which has a default five + second timeout. After the timeout, it shuts itself down. This is + done outside of systemd so it will work where systemd is not + installed. The systemd socket calls the same command anyway. + + https://docs.podman.io/en/latest/markdown/podman-system-service.1.html + + """ + import podman + + client = podman.PodmanClient() + url = client.api.base_url + socket_path = unquote(url.netloc) + if not os.path.exists(socket_path): + logging.info(f'Starting podman system service at {socket_path}') + subprocess.Popen( + ['podman', 'system', 'service'], + close_fds=True, + ) + retried = 0 + while not os.path.exists(socket_path): + time.sleep(0.1) + retried += 1 + if retried > 100: # wait for ten seconds + break + if not client.ping(): + path = f'{url.scheme}://{unquote(url.netloc)}' + logging.error(f'No Podman service found at {path}!') + sys.exit(1) + return client + + +PODMAN_BUILDSERVER_IMAGE = 'registry.gitlab.com/fdroid/fdroidserver:buildserver' + + +def get_podman_container(appid, vercode): + """Singleton getter, since podman-py is just an interface to the podman daemon singleton.""" + container_name = get_container_name(appid, vercode) + client = get_podman_client() + ret = None + for c in client.containers.list(all=True): + if c.name == container_name: + ret = c + break + client.close() + if ret is None: + raise BuildException(f'Container for {appid}:{vercode} not found!') + if PODMAN_BUILDSERVER_IMAGE not in ret.image.tags: + raise BuildException( + f'Container for {appid}:{vercode} has wrong image: {ret.image}' + ) + return ret + + +def get_vagrantfile_path(appid, vercode): + """Return the path for the unique VM for a given build. + + Vagrant doesn't support ':' in VM names, so this uses '_' instead. + Plus filesystems are often grumpy about using `:` in paths. + + """ + return Path('tmp/buildserver', get_container_name(appid, vercode), 'Vagrantfile') diff --git a/fdroidserver/up.py b/fdroidserver/up.py new file mode 100644 index 00000000..b6605fed --- /dev/null +++ b/fdroidserver/up.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +# +# up.py - part of the FDroid server tools +# Copyright (C) 2024-2025, Hans-Christoph Steiner +# Copyright (C) 2024-2025, Michael Pöhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +"""Create dedicated VM/container to run single build, destroying any existing ones. + +The ApplicationID:versionCode argument from the command line should be +used as the unique identifier. This is necessary so that the other +related processes (push, pull, destroy) can find the dedicated +container/VM without there being any other database or file lookup. + +Since this is an internal command, the strings are not localized. + +""" + +import logging +import sys +import textwrap +import traceback +from argparse import ArgumentParser + +from . import common +from .exception import BuildException + + +def run_podman(appid, vercode): + """Create a Podman container env isolated for a single app build. + + This creates a Podman "pod", which is like an isolated box to + create containers in. Then it creates a container in that pod to + run the actual processes. Using the "pod" seems to be a + requirement of Podman. It also further isolates each app build, + so seems fine to use. It is confusing because these containers + won't show up by default when listing containers using defaults. + + The container is set up with an interactive bash process to keep + the container running. + + """ + container_name = common.get_container_name(appid, vercode) + pod_name = common.get_pod_name(appid, vercode) + client = common.get_podman_client() + + logging.debug(f'Pulling {common.PODMAN_BUILDSERVER_IMAGE}...') + image = client.images.pull(common.PODMAN_BUILDSERVER_IMAGE) + + for c in client.containers.list(): + if c.name == container_name: + logging.warning(f'Container {container_name} exists, removing!') + c.remove(force=True) + + for p in client.pods.list(): + if p.name == pod_name: + logging.warning(f'Pod {pod_name} exists, removing!') + p.remove(force=True) + + pod = client.pods.create(pod_name) + container = client.containers.create( + image, + command=['/bin/bash', '-e', '-i', '-l'], + pod=pod, + name=container_name, + detach=True, + remove=True, + stdin_open=True, + ) + pod.start() + pod.reload() + if container.status != 'created': + raise BuildException( + f'Container {container_name} failed to start ({container.status})!' + ) + + +def run_vagrant(appid, vercode, cpus, memory): + import vagrant + + if cpus is None or not isinstance(cpus, int) or not cpus > 0: + raise BuildException( + f"vagrant cpu setting required, '{cpus}' not a valid value!" + ) + if memory is None or not isinstance(memory, int) or not memory > 0: + raise BuildException( + f"vagrant memory setting required, '{memory}' not a valid value!" + ) + + vagrantfile = common.get_vagrantfile_path(appid, vercode) + + # start new dedicated buildserver vagrant vm from scratch + vagrantfile.parent.mkdir(exist_ok=True, parents=True) + vagrantfile.write_text( + textwrap.dedent( + f"""# generated file, do not change. + + Vagrant.configure("2") do |config| + config.vm.box = "buildserver" + config.vm.synced_folder ".", "/vagrant", disabled: true + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = {cpus} + libvirt.memory = {memory} + end + end + """ + ) + ) + + v = vagrant.Vagrant(vagrantfile.parent) + + if not any((b for b in v.box_list() if b.name == 'buildserver')): + raise BuildException("'buildserver' box not added to vagrant") + + v.up() + + +def up_wrapper(appid, vercode, virt_container_type, cpus=None, memory=None): + if virt_container_type == 'vagrant': + run_vagrant(appid, vercode, cpus, memory) + elif virt_container_type == 'podman': + run_podman(appid, vercode) + + +def main(): + parser = ArgumentParser( + description="Create dedicated VM/container to run single build." + ) + common.setup_global_opts(parser) + common.setup_virt_container_type_opts(parser) + parser.add_argument( + "APPID:VERCODE", + help="Application ID with Version Code in the form APPID:VERCODE", + ) + parser.add_argument( + "--cpus", + default=None, + type=int, + help="How many CPUs the Vagrant VM should be allocated.", + ) + parser.add_argument( + "--memory", + default=None, + type=int, + help="How many MB of RAM the Vagrant VM should be allocated.", + ) + options = common.parse_args(parser) + common.set_console_logging(options.verbose) + + try: + appid, vercode = common.split_pkg_arg(options.__dict__['APPID:VERCODE']) + up_wrapper( + appid, + vercode, + common.get_virt_container_type(options), + ) + except Exception as e: + if options.verbose: + logging.error(traceback.format_exc()) + else: + logging.error(e) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 5fa1c9b4..51ac523c 100755 --- a/setup.py +++ b/setup.py @@ -122,6 +122,7 @@ setup( 'pycountry', 'python-magic', ], + 'podman': ['podman'], 'test': ['pyjks', 'html5print', 'testcontainers[minio]'], 'docs': [ 'sphinx', diff --git a/tests/shared_test_code.py b/tests/shared_test_code.py index 3e34900b..adb7763c 100644 --- a/tests/shared_test_code.py +++ b/tests/shared_test_code.py @@ -29,6 +29,11 @@ class VerboseFalseOptions: verbose = False +APPID = 'com.example' +VERCODE = 123 +APPID_VERCODE = f'{APPID}:{VERCODE}' + + class TmpCwd: """Context-manager for temporarily changing the current working directory.""" diff --git a/tests/test_up.py b/tests/test_up.py new file mode 100755 index 00000000..67783296 --- /dev/null +++ b/tests/test_up.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +import importlib +import os +import shutil +import unittest + +from pathlib import Path +from unittest import mock, skipIf, skipUnless + +from fdroidserver import common, exception, up +from .shared_test_code import mkdtemp, APPID, VERCODE, APPID_VERCODE + + +class UpTest(unittest.TestCase): + basedir = Path(__file__).resolve().parent + + def setUp(self): + self._td = mkdtemp() + self.testdir = self._td.name + os.chdir(self.testdir) + + def tearDown(self): + self._td.cleanup() + common.config = None + + +class Up_main(UpTest): + def setUp(self): + super().setUp() + common.config = dict() + + @skipIf( + importlib.util.find_spec("podman") is None or not shutil.which('podman'), + 'Requires podman and podman-py to run.', + ) + @mock.patch('sys.argv', ['fdroid up', APPID_VERCODE]) + @mock.patch('fdroidserver.common.get_default_cachedir') + @mock.patch('fdroidserver.up.run_podman') + def test_podman(self, run_podman, get_default_cachedir): + get_default_cachedir.return_value = self.testdir + common.config['virt_container_type'] = 'podman' + up.main() + run_podman.assert_called_once() + + @mock.patch('sys.argv', ['fdroid up', APPID_VERCODE]) + @mock.patch('fdroidserver.common.get_default_cachedir') + @mock.patch('fdroidserver.up.run_vagrant') + def test_vagrant(self, run_vagrant, get_default_cachedir): + get_default_cachedir.return_value = self.testdir + common.config['virt_container_type'] = 'vagrant' + up.main() + run_vagrant.assert_called_once() + + +@skipIf( + importlib.util.find_spec("podman") is None or not shutil.which('podman'), + 'Requires podman and podman-py to run.', +) +class Up_run_podman(UpTest): + @skipUnless( + os.path.exists(f'/run/user/{os.getuid()}/podman/podman.sock'), + 'Requires systemd podman.socket to run.', + ) + def test_up_with_systemd_socket(self): + common.get_podman_client() + + @skipIf( + os.path.exists(f'/run/user/{os.getuid()}/podman/podman.sock'), + 'Requires the systemd podman.socket is not present.', + ) + def test_up_with_podman_system_service_start(self): + common.get_podman_client() + + def test_recreate_existing(self): + try: + common.get_podman_container(APPID, VERCODE) + except exception.BuildException as e: + # To run these tests, first do: `./fdroid up com.example:123` + self.skipTest(f'Requires Podman container {APPID_VERCODE} to run: {e}') + + short_id = common.get_podman_container(APPID, VERCODE).short_id + up.run_podman(APPID, VERCODE) + self.assertNotEqual( + short_id, + common.get_podman_container(APPID, VERCODE).short_id, + "This should never reuse an existing container.", + ) + + +@skipIf(importlib.util.find_spec("podman") is None, 'Requires podman-py to run.') +class Up_run_fake_podman(UpTest): + @skipIf( + os.path.exists(f'/run/user/{os.getuid()}/podman/podman.sock'), + 'Requires the systemd podman.socket is not present.', + ) + @mock.patch.dict(os.environ, clear=True) + @mock.patch("podman.PodmanClient.ping") + def test_up_with_podman_system_service_start(self, mock_client_ping): + """Test that the system service gets started if no socket is present.""" + os.environ['PATH'] = os.path.join(self.testdir, 'bin') + os.mkdir('bin') + podman = Path('bin/podman') + podman.write_text('#!/bin/sh\nprintf "$1 $2" > args\n') + os.chmod(podman, 0o700) + common.get_podman_client() + self.assertEqual('system service', Path('args').read_text()) + mock_client_ping.assert_called_once() + + +@skipIf(importlib.util.find_spec("vagrant") is None, 'Requires python-vagrant to run.') +class Up_run_vagrant(UpTest): + def setUp(self): + super().setUp() + b = mock.Mock() + b.name = 'buildserver' + self.box_list_return = [b] + name = common.get_container_name(APPID, VERCODE) + self.vagrantdir = Path('tmp/buildserver') / name + + @mock.patch('vagrant.Vagrant.up') + @mock.patch('vagrant.Vagrant.box_list') + def test_no_existing(self, box_list, vagrant_up): + box_list.return_value = self.box_list_return + up.run_vagrant(APPID, VERCODE, 1, 1) + vagrant_up.assert_called_once() + self.assertTrue((Path(self.testdir) / self.vagrantdir).exists()) From fd60436aa3bd0612246563b3270cb8e6c246fa33 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 8 Oct 2025 15:28:17 +0200 Subject: [PATCH 04/11] new subcommand "destroy": podman implementation --- fdroidserver/__main__.py | 1 + fdroidserver/common.py | 12 ++++++ fdroidserver/destroy.py | 85 ++++++++++++++++++++++++++++++++++++++++ fdroidserver/up.py | 3 ++ tests/test_up.py | 17 ++++++++ 5 files changed, 118 insertions(+) create mode 100644 fdroidserver/destroy.py diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index c3631213..0e5a2386 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -65,6 +65,7 @@ COMMANDS = OrderedDict([ # interactively because they rely on the presense of a VM/container, # modify the local environment, or even run things as root. COMMANDS_INTERNAL = [ + "destroy", "up", ] diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 33944b99..86440109 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -5137,3 +5137,15 @@ def get_vagrantfile_path(appid, vercode): """ return Path('tmp/buildserver', get_container_name(appid, vercode), 'Vagrantfile') + + +def vagrant_destroy(appid, vercode): + import vagrant + + vagrantfile = get_vagrantfile_path(appid, vercode) + if vagrantfile.is_file(): + logging.info(f"Destroying Vagrant buildserver VM ({appid}:{vercode})") + v = vagrant.Vagrant(vagrantfile.parent) + v.destroy() + if vagrantfile.parent.exists(): + shutil.rmtree(vagrantfile.parent, ignore_errors=True) diff --git a/fdroidserver/destroy.py b/fdroidserver/destroy.py new file mode 100644 index 00000000..69eb9e07 --- /dev/null +++ b/fdroidserver/destroy.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# +# destroy.py - part of the FDroid server tools +# Copyright (C) 2024, Hans-Christoph Steiner +# Copyright (C) 2024, Michael Pöhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Destroy any existing per-build container/VM structures. + +After this runs, there should be no trace of the given +ApplicationID:versionCode left in the container/VM system. + +Since this is an internal command, the strings are not localized. + +""" + +import logging +import sys +import traceback +from argparse import ArgumentParser + +from . import common + +# TODO should this track whether it actually removed something? +# What do `podman rm` and `vagrant destroy` do? + + +def podman_rm(appid, vercode): + """Remove a Podman pod and all its containers.""" + pod_name = common.get_pod_name(appid, vercode) + for p in common.get_podman_client().pods.list(): + if p.name == pod_name: + logging.debug(f'Removing {pod_name}.') + p.remove(force=True) + + +def destroy_wrapper(appid, vercode, virt_container_type): + if virt_container_type == 'vagrant': + common.vagrant_destroy(appid, vercode) + elif virt_container_type == 'podman': + podman_rm(appid, vercode) + + +def main(): + parser = ArgumentParser( + description="Push files into the buildserver container/box." + ) + common.setup_global_opts(parser) + common.setup_virt_container_type_opts(parser) + parser.add_argument( + "APPID:VERCODE", + help="Application ID with Version Code in the form APPID:VERCODE", + ) + options = common.parse_args(parser) + common.set_console_logging(options.verbose) + + try: + appid, vercode = common.split_pkg_arg(options.__dict__['APPID:VERCODE']) + destroy_wrapper( + appid, + vercode, + common.get_virt_container_type(options), + ) + except Exception as e: + if options.verbose: + logging.error(traceback.format_exc()) + else: + logging.error(e) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/fdroidserver/up.py b/fdroidserver/up.py index b6605fed..927e123a 100644 --- a/fdroidserver/up.py +++ b/fdroidserver/up.py @@ -102,6 +102,9 @@ def run_vagrant(appid, vercode, cpus, memory): vagrantfile = common.get_vagrantfile_path(appid, vercode) + # cleanup potentially still existsing vagrant VMs/dirs + common.vagrant_destroy(appid, vercode) + # start new dedicated buildserver vagrant vm from scratch vagrantfile.parent.mkdir(exist_ok=True, parents=True) vagrantfile.write_text( diff --git a/tests/test_up.py b/tests/test_up.py index 67783296..e97c9954 100755 --- a/tests/test_up.py +++ b/tests/test_up.py @@ -3,6 +3,7 @@ import importlib import os import shutil +import time import unittest from pathlib import Path @@ -125,3 +126,19 @@ class Up_run_vagrant(UpTest): up.run_vagrant(APPID, VERCODE, 1, 1) vagrant_up.assert_called_once() self.assertTrue((Path(self.testdir) / self.vagrantdir).exists()) + + @mock.patch('vagrant.Vagrant.up') + @mock.patch('vagrant.Vagrant.destroy') + @mock.patch('vagrant.Vagrant.box_list') + def test_existing(self, box_list, vagrant_destroy, vagrant_up): + "This should never reuse an existing VM." + box_list.return_value = self.box_list_return + up.run_vagrant(APPID, VERCODE, 1, 1) + vagrantfile = self.vagrantdir / 'Vagrantfile' + ctime = os.path.getctime(vagrantfile) + vagrant_up.assert_called_once() + + time.sleep(0.01) # ensure reliable failure when testing ctime + up.run_vagrant(APPID, VERCODE, 1, 1) + vagrant_destroy.assert_called_once() + self.assertNotEqual(ctime, os.path.getctime(vagrantfile)) From 63660e1aede3a0917d610ec207d82d504e23230c Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 8 Oct 2025 20:25:25 +0200 Subject: [PATCH 05/11] new subcommand "exec": podman implementation --- fdroidserver/__main__.py | 1 + fdroidserver/common.py | 46 ++++++++++++++++++++++++++++ fdroidserver/exec.py | 66 ++++++++++++++++++++++++++++++++++++++++ tests/test_exec.py | 63 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 176 insertions(+) create mode 100644 fdroidserver/exec.py create mode 100755 tests/test_exec.py diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index 0e5a2386..bbfa1ebb 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -66,6 +66,7 @@ COMMANDS = OrderedDict([ # modify the local environment, or even run things as root. COMMANDS_INTERNAL = [ "destroy", + "exec", "up", ] diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 86440109..627d4cf5 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -67,6 +67,7 @@ import logging import operator import os import re +import shlex import shutil import socket import stat @@ -110,6 +111,11 @@ FDROID_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..')) SUPPORTED_VIRT_CONTAINER_TYPES = ('podman', 'vagrant') +# The path to the homedir where apps are built in the container/VM. +BUILD_HOME = '/home/vagrant' +# Username inside of build containers/VMs used for runing builds. +BUILD_USER = 'vagrant' + # There needs to be a default, and this is the most common for software. DEFAULT_LOCALE = 'en-US' @@ -5129,6 +5135,42 @@ def get_podman_container(appid, vercode): return ret +def inside_exec(appid, vercode, command, virt_container_type): + """Execute the command inside of the VM for the build.""" + if virt_container_type == 'vagrant': + return vagrant_exec(appid, vercode, command) + elif virt_container_type == 'podman': + return podman_exec(appid, vercode, command) + else: + raise Exception( + f"'{virt_container_type}' not supported, currently supported: vagrant, podman" + ) + + +def podman_exec(appid, vercode, command): + """Execute the command inside of a podman container for the build.""" + container_name = get_container_name(appid, vercode) + to_stdin = shlex.join(command) + p = subprocess.run( + [ + 'podman', + 'exec', + '--interactive', + '--user=vagrant', + f'--workdir={BUILD_HOME}', + container_name, + ] + + ['/bin/bash', '-e', '-l', '-x'], # the shell that runs the command + input=to_stdin, + text=True, + stderr=subprocess.STDOUT, + ) + if p.returncode != 0: + raise subprocess.CalledProcessError( + p.returncode, f"{to_stdin} | {' '.join(p.args)}" + ) + + def get_vagrantfile_path(appid, vercode): """Return the path for the unique VM for a given build. @@ -5139,6 +5181,10 @@ def get_vagrantfile_path(appid, vercode): return Path('tmp/buildserver', get_container_name(appid, vercode), 'Vagrantfile') +def vagrant_exec(appid, vercode, command): + """Execute a command in the Vagrant VM via ssh.""" + + def vagrant_destroy(appid, vercode): import vagrant diff --git a/fdroidserver/exec.py b/fdroidserver/exec.py new file mode 100644 index 00000000..2a11e5b7 --- /dev/null +++ b/fdroidserver/exec.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# +# exec.py - part of the FDroid server tools +# Copyright (C) 2024-2025, Hans-Christoph Steiner +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Run an fdroidserver subcommand inside of the VM for the build. + +Since this is an internal command, the strings are not localized. + +""" + +import sys +import logging +import traceback +from argparse import ArgumentParser + +from . import common + + +def main(): + parser = ArgumentParser( + description="Run a subcommand in the buildserver container/box." + ) + common.setup_global_opts(parser) + common.setup_virt_container_type_opts(parser) + parser.add_argument( + "APPID:VERCODE", + help="Application ID with Version Code in the form APPID:VERCODE", + ) + parser.add_argument( + "COMMAND", nargs="*", help="Command to run inside the container/box." + ) + options = common.parse_args(parser) + common.set_console_logging(options.verbose) + + try: + appid, vercode = common.split_pkg_arg(options.__dict__['APPID:VERCODE']) + common.inside_exec( + appid, + vercode, + options.COMMAND, + common.get_virt_container_type(options), + ) + except Exception as e: + if options.verbose: + logging.error(traceback.format_exc()) + else: + logging.error(e) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/tests/test_exec.py b/tests/test_exec.py new file mode 100755 index 00000000..04111942 --- /dev/null +++ b/tests/test_exec.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +import os +import subprocess +import unittest + +from pathlib import Path +from unittest import mock, skipUnless + +from fdroidserver import common, exception, exec +from .shared_test_code import mkdtemp, APPID, VERCODE, APPID_VERCODE + + +class ExecTest(unittest.TestCase): + def setUp(self): + self._td = mkdtemp() + self.testdir = self._td.name + os.chdir(self.testdir) + common.config = dict() + + def tearDown(self): + self._td.cleanup() + common.config = None + + +class Exec_main(ExecTest): + @mock.patch('sys.argv', ['fdroid exec', APPID_VERCODE]) + @mock.patch('fdroidserver.common.get_default_cachedir') + @mock.patch('fdroidserver.common.podman_exec') + def test_podman(self, podman_exec, get_default_cachedir): + get_default_cachedir.return_value = self.testdir + common.config['virt_container_type'] = 'podman' + exec.main() + podman_exec.assert_called_once() + + +@skipUnless(os.path.isdir('/run/podman'), 'Requires Podman to run.') +class Exec_podman_exec(ExecTest): + def _only_run_if_container_exists(self): + try: + common.get_podman_container(APPID, VERCODE) + except exception.BuildException as e: + # To run these tests, first do: `./fdroid up com.example:123` + self.skipTest(f'Requires Podman container {APPID_VERCODE} to run: {e}') + + def test_no_existing_container(self): + appid = 'should.never.exist' + f = Path(f'metadata/{appid}.yml') + f.parent.mkdir() + f.write_text(f.name) + with self.assertRaises(subprocess.CalledProcessError) as e: + common.podman_exec(appid, 9999999999, ['ls']) + self.assertEqual(e.exception.returncode, 1) + + def test_clean_run(self): + self._only_run_if_container_exists() + common.podman_exec(APPID, VERCODE, ['ls']) + + def test_error_run(self): + self._only_run_if_container_exists() + with self.assertRaises(subprocess.CalledProcessError) as e: + common.podman_exec(APPID, VERCODE, ['/bin/false']) + self.assertEqual(e.exception.returncode, 1) From 60f5f8fa1c821ca9565c65ffb622715dd23bc030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 8 Oct 2025 17:21:53 +0200 Subject: [PATCH 06/11] =?UTF-8?q?=F0=9F=A5=8A=20add=20vagrant=20support=20?= =?UTF-8?q?to=20exec=20subcommand?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fdroidserver/common.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 627d4cf5..97bbaedc 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -5183,6 +5183,23 @@ def get_vagrantfile_path(appid, vercode): def vagrant_exec(appid, vercode, command): """Execute a command in the Vagrant VM via ssh.""" + vagrantfile = get_vagrantfile_path(appid, vercode) + to_stdin = shlex.join(command) + p = subprocess.run( + [ + 'vagrant', + 'ssh', + '-c', + 'bash', + ], + input=to_stdin, + text=True, + cwd=vagrantfile.parent, + ) + if p.returncode != 0: + raise subprocess.CalledProcessError( + p.returncode, f"{to_stdin} | {' '.join(p.args)}" + ) def vagrant_destroy(appid, vercode): From 69c67badfbaa4f3e1e41ce434db324effd8af972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Tue, 14 Oct 2025 13:24:42 +0200 Subject: [PATCH 07/11] =?UTF-8?q?=F0=9F=AA=86=20--as-root=20to=20run=20com?= =?UTF-8?q?mand=20inside=20of=20container/VM=20as=20root?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fdroidserver/common.py | 15 ++++++++------- fdroidserver/exec.py | 8 ++++++++ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 97bbaedc..745eec1c 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -5135,28 +5135,29 @@ def get_podman_container(appid, vercode): return ret -def inside_exec(appid, vercode, command, virt_container_type): +def inside_exec(appid, vercode, command, virt_container_type, as_root=False): """Execute the command inside of the VM for the build.""" if virt_container_type == 'vagrant': - return vagrant_exec(appid, vercode, command) + return vagrant_exec(appid, vercode, command, as_root) elif virt_container_type == 'podman': - return podman_exec(appid, vercode, command) + return podman_exec(appid, vercode, command, as_root) else: raise Exception( f"'{virt_container_type}' not supported, currently supported: vagrant, podman" ) -def podman_exec(appid, vercode, command): +def podman_exec(appid, vercode, command, as_root=False): """Execute the command inside of a podman container for the build.""" container_name = get_container_name(appid, vercode) to_stdin = shlex.join(command) + user = 'root' if as_root else BUILD_USER p = subprocess.run( [ 'podman', 'exec', '--interactive', - '--user=vagrant', + f'--user={user}', f'--workdir={BUILD_HOME}', container_name, ] @@ -5181,7 +5182,7 @@ def get_vagrantfile_path(appid, vercode): return Path('tmp/buildserver', get_container_name(appid, vercode), 'Vagrantfile') -def vagrant_exec(appid, vercode, command): +def vagrant_exec(appid, vercode, command, as_root=False): """Execute a command in the Vagrant VM via ssh.""" vagrantfile = get_vagrantfile_path(appid, vercode) to_stdin = shlex.join(command) @@ -5190,7 +5191,7 @@ def vagrant_exec(appid, vercode, command): 'vagrant', 'ssh', '-c', - 'bash', + 'sudo bash' if as_root else 'bash', ], input=to_stdin, text=True, diff --git a/fdroidserver/exec.py b/fdroidserver/exec.py index 2a11e5b7..bbe1bf97 100644 --- a/fdroidserver/exec.py +++ b/fdroidserver/exec.py @@ -2,6 +2,7 @@ # # exec.py - part of the FDroid server tools # Copyright (C) 2024-2025, Hans-Christoph Steiner +# Copyright (C) 2024-2025, Michael Pöhn # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -34,6 +35,12 @@ def main(): parser = ArgumentParser( description="Run a subcommand in the buildserver container/box." ) + parser.add_argument( + '--as-root', + default=False, + action='store_true', + help="run command inside of container/VM as root user", + ) common.setup_global_opts(parser) common.setup_virt_container_type_opts(parser) parser.add_argument( @@ -53,6 +60,7 @@ def main(): vercode, options.COMMAND, common.get_virt_container_type(options), + options.as_root, ) except Exception as e: if options.verbose: From 9fa719362059d74bc6acd968456f1f16c4baa183 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 8 Oct 2025 11:12:34 +0200 Subject: [PATCH 08/11] new subcommand "push": podman/vagrant implementation --- fdroidserver/__main__.py | 1 + fdroidserver/common.py | 4 +- fdroidserver/push.py | 208 +++++++++++++++++++++++++++++++++++++++ tests/test_push.py | 87 ++++++++++++++++ 4 files changed, 298 insertions(+), 2 deletions(-) create mode 100644 fdroidserver/push.py create mode 100755 tests/test_push.py diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index bbfa1ebb..73d32b97 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -67,6 +67,7 @@ COMMANDS = OrderedDict([ COMMANDS_INTERNAL = [ "destroy", "exec", + "push", "up", ] diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 745eec1c..25053d72 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -4,10 +4,10 @@ # # Copyright (C) 2010-2016, Ciaran Gultnieks, ciaran@ciarang.com # Copyright (C) 2013-2017, Daniel Martí -# Copyright (C) 2013-2021, Hans-Christoph Steiner +# Copyright (C) 2013-2025, Hans-Christoph Steiner # Copyright (C) 2017-2018, Torsten Grote # Copyright (C) 2017, tobiasKaminsky -# Copyright (C) 2017-2021, Michael Pöhn +# Copyright (C) 2017-2025, Michael Pöhn # Copyright (C) 2017,2021, mimi89999 # Copyright (C) 2019-2021, Jochen Sprickerhof # Copyright (C) 2021, Felix C. Stegerman diff --git a/fdroidserver/push.py b/fdroidserver/push.py new file mode 100644 index 00000000..1719936f --- /dev/null +++ b/fdroidserver/push.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +# +# push.py - part of the FDroid server tools +# Copyright (C) 2024-2025, Hans-Christoph Steiner +# Copyright (C) 2024-2025, Michael Pöhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Push files into the build setup for a given ApplicationID:versionCode. + +This is a carefully constructed method to copy files from the host +filesystem to inside the buildserver VM/container to run the build. The +source paths are generated based on the build metadata, and the +destination is forced to build home. This is not meant as a generic +method for copying files, that is already provided by each VM/container +system (e.g. `podman cp`). + +Since this is an internal command, the strings are not localized. + +""" + +import logging +import os +import sys +import tarfile +import tempfile +import traceback +from argparse import ArgumentParser +from pathlib import Path + +from . import common, metadata +from .exception import BuildException + + +def podman_push(paths, appid, vercode, as_root=False): + """Push relative paths into the podman container using the tar method. + + This builds up a tar file from the supplied paths to send into + container via put_archive(). This assumes it is running in the + base of fdroiddata and it will push into common.BUILD_HOME in the + container. + + This is a version of podman.api.create_tar() that builds up via + adding paths rather than giving a base dir and an exclude list. + + """ + + def _tar_perms_filter(tarinfo): + """Force perms to something safe.""" + if '__pycache__' in tarinfo.name: + return + + if as_root: + tarinfo.uid = tarinfo.gid = 0 + tarinfo.uname = tarinfo.gname = 'root' + else: + tarinfo.uid = tarinfo.gid = 1000 + tarinfo.uname = tarinfo.gname = common.BUILD_USER + + if tarinfo.isdir(): + tarinfo.mode = 0o0755 + elif tarinfo.isfile(): + if tarinfo.mode & 0o111: + tarinfo.mode = 0o0755 + else: + tarinfo.mode = 0o0644 + elif not tarinfo.issym(): # symlinks shouldn't need perms set + raise BuildException(f'{tarinfo.name} is not a file or directory!') + return tarinfo + + if isinstance(paths, Path): + paths = [str(paths)] + if isinstance(paths, str): + paths = [paths] + + container = common.get_podman_container(appid, vercode) + with tempfile.TemporaryFile() as tf: + with tarfile.TarFile(fileobj=tf, mode='w') as tar: + for f in paths: + if Path(f).is_absolute(): + raise BuildException(f'{f} must be relative to {Path.cwd()}') + # throw ValueError on bad path + f = (Path.cwd() / f).resolve().relative_to(Path.cwd()) + tar.add(f, filter=_tar_perms_filter) + tf.seek(0) + container.put_archive(common.BUILD_HOME, tf.read()) + + +def vagrant_push(paths, appid, vercode): + """Push files into a build specific vagrant VM.""" + # TODO implement with `vagrant ssh` and rsync? + + +# TODO split out shared code into _podman.py, _vagrant.py, etc + + +def push_wrapper(paths, appid, vercode, virt_container_type): + """Push standard set of files into VM/container.""" + if virt_container_type == 'vagrant': + vagrant_push(paths, appid, vercode) + elif virt_container_type == 'podman': + podman_push(paths, appid, vercode) + + +def make_file_list(appid, vercode): + """Assemble list of files/folders that go into this specific build.""" + files = [f'build/{appid}', f'metadata/{appid}.yml'] + app_dir = f'metadata/{appid}' + if Path(app_dir).exists(): + files.append(app_dir) + app, build = metadata.get_single_build(appid, vercode) + + for lib in build.srclibs: + srclib = common.getsrclib(lib, 'build/srclib', basepath=True, prepare=False) + if srclib: + # srclib metadata file + files.append(f"srclibs/{srclib[0]}.yml") + # srclib sourcecode + files.append(srclib[2]) + + extlib_paths = [f"build/extlib/{lib}" for lib in build.extlibs] + missing_extlibs = [p for p in extlib_paths if not os.path.exists(p)] + if any(missing_extlibs): + raise Exception( + "error: requested missing extlibs: {}".format(" ".join(missing_extlibs)) + ) + files.extend(extlib_paths) + + return files + + +def create_build_dirs(appid, vercode, virt_container_type): + """Create directories required for running builds.""" + dirs = ('build', 'build/extlib', 'build/srclib', 'metadata', 'fdroidserver') + for dir_name in dirs: + cmd = ['mkdir', '--parents', f'{common.BUILD_HOME}/{dir_name}'] + common.inside_exec( + appid, + vercode, + cmd, + virt_container_type, + ) + + +def full_push_sequence(appid, vercode, virt_container_type): + """Push all files into vm required for specified build.""" + create_build_dirs(appid, vercode, virt_container_type) + push_wrapper(make_file_list(appid, vercode), appid, vercode, virt_container_type) + + # fdroidserver is pushed in every build + cwd = Path('.').absolute() + try: + os.chdir(Path(__file__).parent.parent.parent) + push_wrapper( + [ + 'fdroidserver/fdroidserver', + 'fdroidserver/fdroid', + ], + appid, + vercode, + virt_container_type, + ) + finally: + os.chdir(cwd) + + +def main(): + """CLI main method for this subcommand.""" + parser = ArgumentParser( + description="Push full build setup into the buildserver container/box." + ) + common.setup_global_opts(parser) + common.setup_virt_container_type_opts(parser) + parser.add_argument( + "APPID:VERCODE", + help="Application ID with Version Code in the form APPID:VERCODE", + ) + options = common.parse_args(parser) + common.set_console_logging(options.verbose) + + try: + appid, vercode = common.split_pkg_arg(options.__dict__['APPID:VERCODE']) + full_push_sequence( + appid, + vercode, + common.get_virt_container_type(options), + ) + except Exception as e: + if options.verbose: + logging.error(traceback.format_exc()) + else: + logging.error(e) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/tests/test_push.py b/tests/test_push.py new file mode 100755 index 00000000..6c0bcb7f --- /dev/null +++ b/tests/test_push.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +import importlib +import os +import unittest + +from pathlib import Path +from unittest import mock, skipIf + +from fdroidserver import common, exception, push +from .shared_test_code import mkdtemp, APPID, VERCODE, APPID_VERCODE + + +class PushTest(unittest.TestCase): + basedir = Path(__file__).resolve().parent + + def setUp(self): + self._td = mkdtemp() + self.testdir = self._td.name + os.chdir(self.testdir) + common.config = dict() + + def tearDown(self): + self._td.cleanup() + common.config = None + + +class Push_main(PushTest): + def setUp(self): + super().setUp() + metadatapath = Path(common.get_metadatapath(APPID)) + metadatapath.parent.mkdir() + metadatapath.write_text(f'Name: Test\nBuilds:\n - versionCode: {VERCODE}\n') + + @mock.patch('sys.argv', ['fdroid push', APPID_VERCODE]) + @mock.patch('fdroidserver.push.create_build_dirs') + @mock.patch('fdroidserver.push.podman_push') + def test_podman(self, podman_push, create_build_dirs): + common.config['virt_container_type'] = 'podman' + push.main() + create_build_dirs.assert_called_once() + podman_push.assert_called() + + @mock.patch('sys.argv', ['fdroid push', APPID_VERCODE]) + @mock.patch('fdroidserver.push.create_build_dirs') + @mock.patch('fdroidserver.push.vagrant_push') + def test_vagrant(self, vagrant_push, create_build_dirs): + common.config['virt_container_type'] = 'vagrant' + push.main() + create_build_dirs.assert_called_once() + vagrant_push.assert_called() + + +@skipIf(importlib.util.find_spec("podman") is None, 'Requires podman-py to run.') +class Push_podman_push(PushTest): + def _only_run_if_container_exists(self): + try: + common.get_podman_container(APPID, VERCODE) + except exception.BuildException as e: + # To run these tests, first do: `./fdroid up com.example:123` + self.skipTest(f'Requires Podman container {APPID_VERCODE} to run: {e}') + + def test_no_existing_container(self): + appid = 'should.never.exist' + f = Path(f'metadata/{appid}.yml') + f.parent.mkdir() + f.write_text(f.name) + with self.assertRaises(exception.BuildException): + push.podman_push(f, appid, 9999) + + def test_bad_absolute_path(self): + self._only_run_if_container_exists() + with self.assertRaises(exception.BuildException): + push.podman_push('/etc/passwd', APPID, VERCODE) + + def test_bad_relative_path(self): + self._only_run_if_container_exists() + with self.assertRaises(ValueError): + push.podman_push('../../etc/passwd', APPID, VERCODE) + + def test_existing(self): + self._only_run_if_container_exists() + f = Path(f'metadata/{APPID}.yml') + f.parent.mkdir() + f.write_text(f.name) + push.podman_push(f, APPID, VERCODE) + common.inside_exec(APPID, VERCODE, ['test', '-e', str(f)], 'podman') From e6e788e533f5d31e8b750769012154a469d5c617 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 8 Oct 2025 19:07:32 +0200 Subject: [PATCH 09/11] =?UTF-8?q?=F0=9F=92=82=20implement=20vagrant=20push?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fdroidserver/common.py | 40 ++++++++++++++++++++++++++++++++++++++++ fdroidserver/push.py | 20 +++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 25053d72..97fc1f50 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -5213,3 +5213,43 @@ def vagrant_destroy(appid, vercode): v.destroy() if vagrantfile.parent.exists(): shutil.rmtree(vagrantfile.parent, ignore_errors=True) + + +def get_vagrant_bin_path(): + p = shutil.which("vagrant") + if p is None: + raise Exception( + "'vagrant' not found, make sure it's installed and added to your path" + ) + return p + + +def get_rsync_bin_path(): + p = shutil.which("rsync") + if p is None: + raise Exception( + "'rsync' not found, make sure it's installed and added to your path" + ) + return p + + +class TmpVagrantSshConf: + """Context manager for getting access to a ssh config of a vagrant VM in form of a temp file.""" + + def __init__(self, vagrant_bin_path, vagrant_vm_dir): + self.vagrant_bin_path = vagrant_bin_path + self.vagrant_vm_dir = vagrant_vm_dir + + def __enter__(self): + self.ssh_config_tf = tempfile.NamedTemporaryFile('wt', encoding="utf8") + self.ssh_config_tf.write( + subprocess.check_output( + [self.vagrant_bin_path, 'ssh-config'], + cwd=self.vagrant_vm_dir, + ).decode('utf8') + ) + self.ssh_config_tf.flush() + return self.ssh_config_tf.name + + def __exit__(self, exc_type, exc_value, traceback): + self.ssh_config_tf.close() diff --git a/fdroidserver/push.py b/fdroidserver/push.py index 1719936f..45e78a35 100644 --- a/fdroidserver/push.py +++ b/fdroidserver/push.py @@ -32,6 +32,7 @@ Since this is an internal command, the strings are not localized. import logging import os +import subprocess import sys import tarfile import tempfile @@ -99,7 +100,24 @@ def podman_push(paths, appid, vercode, as_root=False): def vagrant_push(paths, appid, vercode): """Push files into a build specific vagrant VM.""" - # TODO implement with `vagrant ssh` and rsync? + vagrantbin = common.get_vagrant_bin_path() + rsyncbin = common.get_rsync_bin_path() + + vagrantfile = common.get_vagrantfile_path(appid, vercode) + with common.TmpVagrantSshConf(vagrantbin, vagrantfile.parent) as ssh_config_file: + for path in paths: + cmd = [ + rsyncbin, + '-av', + '--relative', + '-e', + f'ssh -F {ssh_config_file}', + path, + 'default:{}'.format(common.BUILD_HOME), + ] + subprocess.check_call( + cmd, + ) # TODO split out shared code into _podman.py, _vagrant.py, etc From da032d517a5aa908ded509481bba49917b65e131 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 8 Oct 2025 11:13:16 +0200 Subject: [PATCH 10/11] new subcommand "pull": podman/vagrant implementation --- fdroidserver/__main__.py | 1 + fdroidserver/pull.py | 115 ++++++++++++++++++++++++++++++++++++ tests/test_pull.py | 122 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 238 insertions(+) create mode 100644 fdroidserver/pull.py create mode 100755 tests/test_pull.py diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index 73d32b97..0555c498 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -67,6 +67,7 @@ COMMANDS = OrderedDict([ COMMANDS_INTERNAL = [ "destroy", "exec", + "pull", "push", "up", ] diff --git a/fdroidserver/pull.py b/fdroidserver/pull.py new file mode 100644 index 00000000..8144470f --- /dev/null +++ b/fdroidserver/pull.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +# +# pull.py - part of the FDroid server tools +# Copyright (C) 2024-2025, Hans-Christoph Steiner +# Copyright (C) 2024-2025, Michael Pöhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Pull a single file from the build setup for a given ApplicationID:versionCode. + +This is a carefully constructed method to copy files from inside the +buildserver VM/container to the host filesystem for further processing +and publishing. The source path is forced to the within the app's +build dir, and the destination is forced to the repo's unsigned/ dir. +This is not meant as a generic method for getting files, that is already +provided by each VM/container system (e.g. `podman cp`). + +Since this is an internal command, the strings are not localized. + +""" + +import os +import sys +import logging +import tarfile +import tempfile +import traceback +from argparse import ArgumentParser + +from . import common, metadata + + +def podman_pull(appid, vercode, path): + """Implement `fdroid pull` for Podman (e.g. not `podman pull`).""" + path_in_container = os.path.join(common.BUILD_HOME, path) + container = common.get_podman_container(appid, vercode) + stream, stat = container.get_archive(path_in_container) + if not stat['linkTarget'].endswith(path) or stat['name'] != os.path.basename(path): + logging.warning(f'{path} not found!') + return + with tempfile.NamedTemporaryFile(prefix=".fdroidserver_pull_", suffix=".tar") as tf: + for i in stream: + tf.write(i) + tf.seek(0) + with tarfile.TarFile(fileobj=tf, mode='r') as tar: + tar.extract(stat['name'], 'unsigned', set_attrs=False) + + +def vagrant_pull(appid, vercode, path): + """Pull the path from the Vagrant VM.""" + + +def make_file_list(appid, vercode): + app, build = metadata.get_single_build(appid, vercode) + ext = common.get_output_extension(build) + return [ + os.path.join('unsigned', common.get_release_filename(app, build, ext)), + os.path.join( + 'unsigned', common.get_src_tarball_name(app.id, build.versionCode) + ), + ] + + +def pull_wrapper(appid, vercode, virt_container_type): + os.makedirs('unsigned', exist_ok=True) + files = make_file_list(appid, vercode) + for f in files: + logging.info(f"""Pulling {f} from {appid}:{vercode}""") + if virt_container_type == 'vagrant': + vagrant_pull(appid, vercode, f) + elif virt_container_type == 'podman': + podman_pull(appid, vercode, f) + + +def main(): + parser = ArgumentParser( + description="Pull build products from the buildserver container/box." + ) + common.setup_global_opts(parser) + common.setup_virt_container_type_opts(parser) + parser.add_argument( + "APPID:VERCODE", + help="Application ID with Version Code in the form APPID:VERCODE", + ) + options = common.parse_args(parser) + common.set_console_logging(options.verbose) + + try: + appid, vercode = common.split_pkg_arg(options.__dict__['APPID:VERCODE']) + pull_wrapper( + appid, + vercode, + common.get_virt_container_type(options), + ) + except Exception as e: + if options.verbose: + logging.error(traceback.format_exc()) + else: + logging.error(e) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/tests/test_pull.py b/tests/test_pull.py new file mode 100755 index 00000000..6d520818 --- /dev/null +++ b/tests/test_pull.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 + +import importlib +import os +import unittest + +from pathlib import Path +from unittest import mock, skipIf + +from fdroidserver import common, exception, pull +from .shared_test_code import mkdtemp, APPID, VERCODE, APPID_VERCODE + + +class PullTest(unittest.TestCase): + basedir = Path(__file__).resolve().parent + + def setUp(self): + self._td = mkdtemp() + self.testdir = self._td.name + os.chdir(self.testdir) + common.config = dict() + + def tearDown(self): + self._td.cleanup() + common.config = None + + +class Pull_main(PullTest): + def setUp(self): + super().setUp() + metadatapath = Path(common.get_metadatapath(APPID)) + metadatapath.parent.mkdir() + metadatapath.write_text(f'Name: Test\nBuilds:\n - versionCode: {VERCODE}\n') + + @mock.patch('sys.argv', ['fdroid pull', APPID_VERCODE]) + @mock.patch('fdroidserver.pull.podman_pull') + def test_podman(self, podman_pull): + common.config['virt_container_type'] = 'podman' + common.options = mock.Mock() + pull.main() + podman_pull.assert_called() + + @mock.patch('sys.argv', ['fdroid pull', APPID_VERCODE]) + @mock.patch('fdroidserver.pull.vagrant_pull') + def test_vagrant(self, vagrant_pull): + common.config['virt_container_type'] = 'vagrant' + pull.main() + vagrant_pull.assert_called() + + +@skipIf(importlib.util.find_spec("podman") is None, 'Requires podman-py to run.') +class Pull_podman_pull(PullTest): + def setUp(self): + try: + common.get_podman_container(APPID, VERCODE) + except exception.BuildException as e: + self.skipTest(f'Requires Podman container {APPID_VERCODE} to run: {e}') + super().setUp() + + def test_no_existing(self): + appid = 'should.never.exist' + with self.assertRaises(exception.BuildException) as e: + pull.podman_pull(appid, 9999, 'unsigned/foo.apk') + self.assertIn(appid, e.exception.value) + + def test_existing(self): + """Check files get deposited in unsigned/.""" + filename = 'buildserverid' + f = Path('unsigned') / filename + self.assertFalse(f.exists()) + pull.podman_pull(APPID, VERCODE, filename) + self.assertTrue(f.exists()) + + +class Pull_make_file_list(PullTest): + def setUp(self): + super().setUp() + self.metadatapath = Path(common.get_metadatapath(APPID)) + self.metadatapath.parent.mkdir() + + def test_implied(self): + self.metadatapath.write_text(f"""Builds: [versionCode: {VERCODE}]""") + self.assertEqual( + [ + f'unsigned/{APPID}_{VERCODE}.apk', + f'unsigned/{APPID}_{VERCODE}_src.tar.gz', + ], + pull.make_file_list(APPID, VERCODE), + ) + + def test_gradle(self): + self.metadatapath.write_text( + f"""Builds: + - versionCode: {VERCODE} + gradle: fdroid + """ + ) + self.assertEqual( + [ + f'unsigned/{APPID}_{VERCODE}.apk', + f'unsigned/{APPID}_{VERCODE}_src.tar.gz', + ], + pull.make_file_list(APPID, VERCODE), + ) + + def test_raw(self): + ext = 'foo' + Path(common.get_metadatapath(APPID)).write_text( + f"""Builds: + - versionCode: {VERCODE} + versionName: 1.0 + commit: cafebabe123 + output: path/to/output.{ext} + """ + ) + self.assertEqual( + [ + f'unsigned/{APPID}_{VERCODE}.{ext}', + f'unsigned/{APPID}_{VERCODE}_src.tar.gz', + ], + pull.make_file_list(APPID, VERCODE), + ) From 42189c8abcfa7b73fc302f032f6314a156a26b9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 8 Oct 2025 21:30:52 +0200 Subject: [PATCH 11/11] =?UTF-8?q?=F0=9F=90=B4=20implement=20pull=20for=20v?= =?UTF-8?q?agrant?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fdroidserver/pull.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/fdroidserver/pull.py b/fdroidserver/pull.py index 8144470f..4ec21195 100644 --- a/fdroidserver/pull.py +++ b/fdroidserver/pull.py @@ -33,6 +33,7 @@ Since this is an internal command, the strings are not localized. import os import sys import logging +import subprocess import tarfile import tempfile import traceback @@ -59,6 +60,22 @@ def podman_pull(appid, vercode, path): def vagrant_pull(appid, vercode, path): """Pull the path from the Vagrant VM.""" + vagrantfile = common.get_vagrantfile_path(appid, vercode) + path_in_vm = os.path.join(common.BUILD_HOME, path) + + vagrantbin = common.get_vagrant_bin_path() + rsyncbin = common.get_rsync_bin_path() + + with common.TmpVagrantSshConf(vagrantbin, vagrantfile.parent) as ssh_config_file: + cmd = [ + rsyncbin, + '-av', + '-e', + f'ssh -F {ssh_config_file}', + f'default:{path_in_vm}', + './unsigned', + ] + subprocess.check_call(cmd) def make_file_list(appid, vercode):