diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 65510c45..91598c8a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -852,3 +852,24 @@ PUBLISH: - fdroid gpgsign --verbose - fdroid signindex --verbose - rsync --stats repo/* $serverwebroot/ + + +# This tests the `podman system service` auto-starting and other general podman things. +podman system service: + rules: + - changes: + - .gitlab-ci.yml + - fdroidserver/up.py + - tests/test_up.py + image: debian:trixie-slim + <<: *apt-template + script: + - apt-get install + python3-asn1crypto + python3-defusedxml + python3-git + python3-pillow + python3-podman + python3-ruamel.yaml + python3-yaml + - python3 -m unittest tests/test_up.py --verbose diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index 5a2241f3..c3631213 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -65,6 +65,7 @@ COMMANDS = OrderedDict([ # interactively because they rely on the presense of a VM/container, # modify the local environment, or even run things as root. COMMANDS_INTERNAL = [ + "up", ] diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 96deb5ea..33944b99 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -82,7 +82,7 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from queue import Queue from typing import List -from urllib.parse import urlparse, urlsplit, urlunparse +from urllib.parse import urlparse, urlsplit, urlunparse, unquote from zipfile import ZipFile import defusedxml.ElementTree as XMLElementTree @@ -1303,6 +1303,16 @@ def get_source_date_epoch(build_dir): return repo.git.log('-n1', '--pretty=%ct', '--', metadatapath) +def get_container_name(appid, vercode): + """Return unique name for associating a build with a container or VM.""" + return f'{appid}_{vercode}' + + +def get_pod_name(appid, vercode): + """Return unique name for associating a build with a Podman "pod".""" + return f'{get_container_name(appid, vercode)}_pod' + + def get_build_dir(app): """Get the dir that this app will be built in.""" if app.RepoType == 'srclib': @@ -5056,3 +5066,74 @@ FDROIDORG_MIRRORS = [ FDROIDORG_FINGERPRINT = ( '43238D512C1E5EB2D6569F4A3AFBF5523418B82E0A3ED1552770ABB9A9C9CCAB' ) + + +def get_podman_client(): + """Return an instance of podman-py to work with Podman. + + This assumes that it will use the default local UNIX Domain Socket + to connect to the Podman service. The preferred setup is to first + do `systemctl --user start podman.socket` to keep the Podman UNIX + socket running all the time. If the socket is not present, this + will automatically start the service, which has a default five + second timeout. After the timeout, it shuts itself down. This is + done outside of systemd so it will work where systemd is not + installed. The systemd socket calls the same command anyway. + + https://docs.podman.io/en/latest/markdown/podman-system-service.1.html + + """ + import podman + + client = podman.PodmanClient() + url = client.api.base_url + socket_path = unquote(url.netloc) + if not os.path.exists(socket_path): + logging.info(f'Starting podman system service at {socket_path}') + subprocess.Popen( + ['podman', 'system', 'service'], + close_fds=True, + ) + retried = 0 + while not os.path.exists(socket_path): + time.sleep(0.1) + retried += 1 + if retried > 100: # wait for ten seconds + break + if not client.ping(): + path = f'{url.scheme}://{unquote(url.netloc)}' + logging.error(f'No Podman service found at {path}!') + sys.exit(1) + return client + + +PODMAN_BUILDSERVER_IMAGE = 'registry.gitlab.com/fdroid/fdroidserver:buildserver' + + +def get_podman_container(appid, vercode): + """Singleton getter, since podman-py is just an interface to the podman daemon singleton.""" + container_name = get_container_name(appid, vercode) + client = get_podman_client() + ret = None + for c in client.containers.list(all=True): + if c.name == container_name: + ret = c + break + client.close() + if ret is None: + raise BuildException(f'Container for {appid}:{vercode} not found!') + if PODMAN_BUILDSERVER_IMAGE not in ret.image.tags: + raise BuildException( + f'Container for {appid}:{vercode} has wrong image: {ret.image}' + ) + return ret + + +def get_vagrantfile_path(appid, vercode): + """Return the path for the unique VM for a given build. + + Vagrant doesn't support ':' in VM names, so this uses '_' instead. + Plus filesystems are often grumpy about using `:` in paths. + + """ + return Path('tmp/buildserver', get_container_name(appid, vercode), 'Vagrantfile') diff --git a/fdroidserver/up.py b/fdroidserver/up.py new file mode 100644 index 00000000..b6605fed --- /dev/null +++ b/fdroidserver/up.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +# +# up.py - part of the FDroid server tools +# Copyright (C) 2024-2025, Hans-Christoph Steiner +# Copyright (C) 2024-2025, Michael Pöhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +"""Create dedicated VM/container to run single build, destroying any existing ones. + +The ApplicationID:versionCode argument from the command line should be +used as the unique identifier. This is necessary so that the other +related processes (push, pull, destroy) can find the dedicated +container/VM without there being any other database or file lookup. + +Since this is an internal command, the strings are not localized. + +""" + +import logging +import sys +import textwrap +import traceback +from argparse import ArgumentParser + +from . import common +from .exception import BuildException + + +def run_podman(appid, vercode): + """Create a Podman container env isolated for a single app build. + + This creates a Podman "pod", which is like an isolated box to + create containers in. Then it creates a container in that pod to + run the actual processes. Using the "pod" seems to be a + requirement of Podman. It also further isolates each app build, + so seems fine to use. It is confusing because these containers + won't show up by default when listing containers using defaults. + + The container is set up with an interactive bash process to keep + the container running. + + """ + container_name = common.get_container_name(appid, vercode) + pod_name = common.get_pod_name(appid, vercode) + client = common.get_podman_client() + + logging.debug(f'Pulling {common.PODMAN_BUILDSERVER_IMAGE}...') + image = client.images.pull(common.PODMAN_BUILDSERVER_IMAGE) + + for c in client.containers.list(): + if c.name == container_name: + logging.warning(f'Container {container_name} exists, removing!') + c.remove(force=True) + + for p in client.pods.list(): + if p.name == pod_name: + logging.warning(f'Pod {pod_name} exists, removing!') + p.remove(force=True) + + pod = client.pods.create(pod_name) + container = client.containers.create( + image, + command=['/bin/bash', '-e', '-i', '-l'], + pod=pod, + name=container_name, + detach=True, + remove=True, + stdin_open=True, + ) + pod.start() + pod.reload() + if container.status != 'created': + raise BuildException( + f'Container {container_name} failed to start ({container.status})!' + ) + + +def run_vagrant(appid, vercode, cpus, memory): + import vagrant + + if cpus is None or not isinstance(cpus, int) or not cpus > 0: + raise BuildException( + f"vagrant cpu setting required, '{cpus}' not a valid value!" + ) + if memory is None or not isinstance(memory, int) or not memory > 0: + raise BuildException( + f"vagrant memory setting required, '{memory}' not a valid value!" + ) + + vagrantfile = common.get_vagrantfile_path(appid, vercode) + + # start new dedicated buildserver vagrant vm from scratch + vagrantfile.parent.mkdir(exist_ok=True, parents=True) + vagrantfile.write_text( + textwrap.dedent( + f"""# generated file, do not change. + + Vagrant.configure("2") do |config| + config.vm.box = "buildserver" + config.vm.synced_folder ".", "/vagrant", disabled: true + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = {cpus} + libvirt.memory = {memory} + end + end + """ + ) + ) + + v = vagrant.Vagrant(vagrantfile.parent) + + if not any((b for b in v.box_list() if b.name == 'buildserver')): + raise BuildException("'buildserver' box not added to vagrant") + + v.up() + + +def up_wrapper(appid, vercode, virt_container_type, cpus=None, memory=None): + if virt_container_type == 'vagrant': + run_vagrant(appid, vercode, cpus, memory) + elif virt_container_type == 'podman': + run_podman(appid, vercode) + + +def main(): + parser = ArgumentParser( + description="Create dedicated VM/container to run single build." + ) + common.setup_global_opts(parser) + common.setup_virt_container_type_opts(parser) + parser.add_argument( + "APPID:VERCODE", + help="Application ID with Version Code in the form APPID:VERCODE", + ) + parser.add_argument( + "--cpus", + default=None, + type=int, + help="How many CPUs the Vagrant VM should be allocated.", + ) + parser.add_argument( + "--memory", + default=None, + type=int, + help="How many MB of RAM the Vagrant VM should be allocated.", + ) + options = common.parse_args(parser) + common.set_console_logging(options.verbose) + + try: + appid, vercode = common.split_pkg_arg(options.__dict__['APPID:VERCODE']) + up_wrapper( + appid, + vercode, + common.get_virt_container_type(options), + ) + except Exception as e: + if options.verbose: + logging.error(traceback.format_exc()) + else: + logging.error(e) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 5fa1c9b4..51ac523c 100755 --- a/setup.py +++ b/setup.py @@ -122,6 +122,7 @@ setup( 'pycountry', 'python-magic', ], + 'podman': ['podman'], 'test': ['pyjks', 'html5print', 'testcontainers[minio]'], 'docs': [ 'sphinx', diff --git a/tests/shared_test_code.py b/tests/shared_test_code.py index 3e34900b..adb7763c 100644 --- a/tests/shared_test_code.py +++ b/tests/shared_test_code.py @@ -29,6 +29,11 @@ class VerboseFalseOptions: verbose = False +APPID = 'com.example' +VERCODE = 123 +APPID_VERCODE = f'{APPID}:{VERCODE}' + + class TmpCwd: """Context-manager for temporarily changing the current working directory.""" diff --git a/tests/test_up.py b/tests/test_up.py new file mode 100755 index 00000000..67783296 --- /dev/null +++ b/tests/test_up.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +import importlib +import os +import shutil +import unittest + +from pathlib import Path +from unittest import mock, skipIf, skipUnless + +from fdroidserver import common, exception, up +from .shared_test_code import mkdtemp, APPID, VERCODE, APPID_VERCODE + + +class UpTest(unittest.TestCase): + basedir = Path(__file__).resolve().parent + + def setUp(self): + self._td = mkdtemp() + self.testdir = self._td.name + os.chdir(self.testdir) + + def tearDown(self): + self._td.cleanup() + common.config = None + + +class Up_main(UpTest): + def setUp(self): + super().setUp() + common.config = dict() + + @skipIf( + importlib.util.find_spec("podman") is None or not shutil.which('podman'), + 'Requires podman and podman-py to run.', + ) + @mock.patch('sys.argv', ['fdroid up', APPID_VERCODE]) + @mock.patch('fdroidserver.common.get_default_cachedir') + @mock.patch('fdroidserver.up.run_podman') + def test_podman(self, run_podman, get_default_cachedir): + get_default_cachedir.return_value = self.testdir + common.config['virt_container_type'] = 'podman' + up.main() + run_podman.assert_called_once() + + @mock.patch('sys.argv', ['fdroid up', APPID_VERCODE]) + @mock.patch('fdroidserver.common.get_default_cachedir') + @mock.patch('fdroidserver.up.run_vagrant') + def test_vagrant(self, run_vagrant, get_default_cachedir): + get_default_cachedir.return_value = self.testdir + common.config['virt_container_type'] = 'vagrant' + up.main() + run_vagrant.assert_called_once() + + +@skipIf( + importlib.util.find_spec("podman") is None or not shutil.which('podman'), + 'Requires podman and podman-py to run.', +) +class Up_run_podman(UpTest): + @skipUnless( + os.path.exists(f'/run/user/{os.getuid()}/podman/podman.sock'), + 'Requires systemd podman.socket to run.', + ) + def test_up_with_systemd_socket(self): + common.get_podman_client() + + @skipIf( + os.path.exists(f'/run/user/{os.getuid()}/podman/podman.sock'), + 'Requires the systemd podman.socket is not present.', + ) + def test_up_with_podman_system_service_start(self): + common.get_podman_client() + + def test_recreate_existing(self): + try: + common.get_podman_container(APPID, VERCODE) + except exception.BuildException as e: + # To run these tests, first do: `./fdroid up com.example:123` + self.skipTest(f'Requires Podman container {APPID_VERCODE} to run: {e}') + + short_id = common.get_podman_container(APPID, VERCODE).short_id + up.run_podman(APPID, VERCODE) + self.assertNotEqual( + short_id, + common.get_podman_container(APPID, VERCODE).short_id, + "This should never reuse an existing container.", + ) + + +@skipIf(importlib.util.find_spec("podman") is None, 'Requires podman-py to run.') +class Up_run_fake_podman(UpTest): + @skipIf( + os.path.exists(f'/run/user/{os.getuid()}/podman/podman.sock'), + 'Requires the systemd podman.socket is not present.', + ) + @mock.patch.dict(os.environ, clear=True) + @mock.patch("podman.PodmanClient.ping") + def test_up_with_podman_system_service_start(self, mock_client_ping): + """Test that the system service gets started if no socket is present.""" + os.environ['PATH'] = os.path.join(self.testdir, 'bin') + os.mkdir('bin') + podman = Path('bin/podman') + podman.write_text('#!/bin/sh\nprintf "$1 $2" > args\n') + os.chmod(podman, 0o700) + common.get_podman_client() + self.assertEqual('system service', Path('args').read_text()) + mock_client_ping.assert_called_once() + + +@skipIf(importlib.util.find_spec("vagrant") is None, 'Requires python-vagrant to run.') +class Up_run_vagrant(UpTest): + def setUp(self): + super().setUp() + b = mock.Mock() + b.name = 'buildserver' + self.box_list_return = [b] + name = common.get_container_name(APPID, VERCODE) + self.vagrantdir = Path('tmp/buildserver') / name + + @mock.patch('vagrant.Vagrant.up') + @mock.patch('vagrant.Vagrant.box_list') + def test_no_existing(self, box_list, vagrant_up): + box_list.return_value = self.box_list_return + up.run_vagrant(APPID, VERCODE, 1, 1) + vagrant_up.assert_called_once() + self.assertTrue((Path(self.testdir) / self.vagrantdir).exists())