new subcommand "up" for vagrant up and `podman run

.gitlab-ci.yml

@@ -852,3 +852,24 @@ PUBLISH:
     - fdroid gpgsign --verbose
     - fdroid signindex --verbose
     - rsync --stats repo/* $serverwebroot/
+
+
+# This tests the `podman system service` auto-starting and other general podman things.
+podman system service:
+  rules:
+    - changes:
+        - .gitlab-ci.yml
+        - fdroidserver/up.py
+        - tests/test_up.py
+  image: debian:trixie-slim
+  <<: *apt-template
+  script:
+    - apt-get install
+        python3-asn1crypto
+        python3-defusedxml
+        python3-git
+        python3-pillow
+        python3-podman
+        python3-ruamel.yaml
+        python3-yaml
+    - python3 -m unittest tests/test_up.py --verbose

fdroidserver/__main__.py

@@ -65,6 +65,7 @@ COMMANDS = OrderedDict([
 # interactively because they rely on the presense of a VM/container,
 # modify the local environment, or even run things as root.
 COMMANDS_INTERNAL = [
+    "up",
 ]
 
 

fdroidserver/common.py

@@ -82,7 +82,7 @@ from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from queue import Queue
 from typing import List
-from urllib.parse import urlparse, urlsplit, urlunparse
+from urllib.parse import urlparse, urlsplit, urlunparse, unquote
 from zipfile import ZipFile
 
 import defusedxml.ElementTree as XMLElementTree
@@ -1303,6 +1303,16 @@ def get_source_date_epoch(build_dir):
             return repo.git.log('-n1', '--pretty=%ct', '--', metadatapath)
 
 
+def get_container_name(appid, vercode):
+    """Return unique name for associating a build with a container or VM."""
+    return f'{appid}_{vercode}'
+
+
+def get_pod_name(appid, vercode):
+    """Return unique name for associating a build with a Podman "pod"."""
+    return f'{get_container_name(appid, vercode)}_pod'
+
+
 def get_build_dir(app):
     """Get the dir that this app will be built in."""
     if app.RepoType == 'srclib':
@@ -5056,3 +5066,74 @@ FDROIDORG_MIRRORS = [
 FDROIDORG_FINGERPRINT = (
     '43238D512C1E5EB2D6569F4A3AFBF5523418B82E0A3ED1552770ABB9A9C9CCAB'
 )
+
+
+def get_podman_client():
+    """Return an instance of podman-py to work with Podman.
+
+    This assumes that it will use the default local UNIX Domain Socket
+    to connect to the Podman service.  The preferred setup is to first
+    do `systemctl --user start podman.socket` to keep the Podman UNIX
+    socket running all the time.  If the socket is not present, this
+    will automatically start the service, which has a default five
+    second timeout.  After the timeout, it shuts itself down.  This is
+    done outside of systemd so it will work where systemd is not
+    installed.  The systemd socket calls the same command anyway.
+
+    https://docs.podman.io/en/latest/markdown/podman-system-service.1.html
+
+    """
+    import podman
+
+    client = podman.PodmanClient()
+    url = client.api.base_url
+    socket_path = unquote(url.netloc)
+    if not os.path.exists(socket_path):
+        logging.info(f'Starting podman system service at {socket_path}')
+        subprocess.Popen(
+            ['podman', 'system', 'service'],
+            close_fds=True,
+        )
+        retried = 0
+        while not os.path.exists(socket_path):
+            time.sleep(0.1)
+            retried += 1
+            if retried > 100:  # wait for ten seconds
+                break
+    if not client.ping():
+        path = f'{url.scheme}://{unquote(url.netloc)}'
+        logging.error(f'No Podman service found at {path}!')
+        sys.exit(1)
+    return client
+
+
+PODMAN_BUILDSERVER_IMAGE = 'registry.gitlab.com/fdroid/fdroidserver:buildserver'
+
+
+def get_podman_container(appid, vercode):
+    """Singleton getter, since podman-py is just an interface to the podman daemon singleton."""
+    container_name = get_container_name(appid, vercode)
+    client = get_podman_client()
+    ret = None
+    for c in client.containers.list(all=True):
+        if c.name == container_name:
+            ret = c
+            break
+    client.close()
+    if ret is None:
+        raise BuildException(f'Container for {appid}:{vercode} not found!')
+    if PODMAN_BUILDSERVER_IMAGE not in ret.image.tags:
+        raise BuildException(
+            f'Container for {appid}:{vercode} has wrong image: {ret.image}'
+        )
+    return ret
+
+
+def get_vagrantfile_path(appid, vercode):
+    """Return the path for the unique VM for a given build.
+
+    Vagrant doesn't support ':' in VM names, so this uses '_' instead.
+    Plus filesystems are often grumpy about using `:` in paths.
+
+    """
+    return Path('tmp/buildserver', get_container_name(appid, vercode), 'Vagrantfile')

fdroidserver/up.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+#
+# up.py - part of the FDroid server tools
+# Copyright (C) 2024-2025, Hans-Christoph Steiner <hans@eds.org>
+# Copyright (C) 2024-2025, Michael Pöhn <michael@poehn.at>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+"""Create dedicated VM/container to run single build, destroying any existing ones.
+
+The ApplicationID:versionCode argument from the command line should be
+used as the unique identifier.  This is necessary so that the other
+related processes (push, pull, destroy) can find the dedicated
+container/VM without there being any other database or file lookup.
+
+Since this is an internal command, the strings are not localized.
+
+"""
+
+import logging
+import sys
+import textwrap
+import traceback
+from argparse import ArgumentParser
+
+from . import common
+from .exception import BuildException
+
+
+def run_podman(appid, vercode):
+    """Create a Podman container env isolated for a single app build.
+
+    This creates a Podman "pod", which is like an isolated box to
+    create containers in.  Then it creates a container in that pod to
+    run the actual processes.  Using the "pod" seems to be a
+    requirement of Podman.  It also further isolates each app build,
+    so seems fine to use.  It is confusing because these containers
+    won't show up by default when listing containers using defaults.
+
+    The container is set up with an interactive bash process to keep
+    the container running.
+
+    """
+    container_name = common.get_container_name(appid, vercode)
+    pod_name = common.get_pod_name(appid, vercode)
+    client = common.get_podman_client()
+
+    logging.debug(f'Pulling {common.PODMAN_BUILDSERVER_IMAGE}...')
+    image = client.images.pull(common.PODMAN_BUILDSERVER_IMAGE)
+
+    for c in client.containers.list():
+        if c.name == container_name:
+            logging.warning(f'Container {container_name} exists, removing!')
+            c.remove(force=True)
+
+    for p in client.pods.list():
+        if p.name == pod_name:
+            logging.warning(f'Pod {pod_name} exists, removing!')
+            p.remove(force=True)
+
+    pod = client.pods.create(pod_name)
+    container = client.containers.create(
+        image,
+        command=['/bin/bash', '-e', '-i', '-l'],
+        pod=pod,
+        name=container_name,
+        detach=True,
+        remove=True,
+        stdin_open=True,
+    )
+    pod.start()
+    pod.reload()
+    if container.status != 'created':
+        raise BuildException(
+            f'Container {container_name} failed to start ({container.status})!'
+        )
+
+
+def run_vagrant(appid, vercode, cpus, memory):
+    import vagrant
+
+    if cpus is None or not isinstance(cpus, int) or not cpus > 0:
+        raise BuildException(
+            f"vagrant cpu setting required, '{cpus}' not a valid value!"
+        )
+    if memory is None or not isinstance(memory, int) or not memory > 0:
+        raise BuildException(
+            f"vagrant memory setting required, '{memory}' not a valid value!"
+        )
+
+    vagrantfile = common.get_vagrantfile_path(appid, vercode)
+
+    # start new dedicated buildserver vagrant vm from scratch
+    vagrantfile.parent.mkdir(exist_ok=True, parents=True)
+    vagrantfile.write_text(
+        textwrap.dedent(
+            f"""# generated file, do not change.
+
+                Vagrant.configure("2") do |config|
+                  config.vm.box = "buildserver"
+                  config.vm.synced_folder ".", "/vagrant", disabled: true
+
+                  config.vm.provider :libvirt do |libvirt|
+                    libvirt.cpus = {cpus}
+                    libvirt.memory = {memory}
+                  end
+                end
+            """
+        )
+    )
+
+    v = vagrant.Vagrant(vagrantfile.parent)
+
+    if not any((b for b in v.box_list() if b.name == 'buildserver')):
+        raise BuildException("'buildserver' box not added to vagrant")
+
+    v.up()
+
+
+def up_wrapper(appid, vercode, virt_container_type, cpus=None, memory=None):
+    if virt_container_type == 'vagrant':
+        run_vagrant(appid, vercode, cpus, memory)
+    elif virt_container_type == 'podman':
+        run_podman(appid, vercode)
+
+
+def main():
+    parser = ArgumentParser(
+        description="Create dedicated VM/container to run single build."
+    )
+    common.setup_global_opts(parser)
+    common.setup_virt_container_type_opts(parser)
+    parser.add_argument(
+        "APPID:VERCODE",
+        help="Application ID with Version Code in the form APPID:VERCODE",
+    )
+    parser.add_argument(
+        "--cpus",
+        default=None,
+        type=int,
+        help="How many CPUs the Vagrant VM should be allocated.",
+    )
+    parser.add_argument(
+        "--memory",
+        default=None,
+        type=int,
+        help="How many MB of RAM the Vagrant VM should be allocated.",
+    )
+    options = common.parse_args(parser)
+    common.set_console_logging(options.verbose)
+
+    try:
+        appid, vercode = common.split_pkg_arg(options.__dict__['APPID:VERCODE'])
+        up_wrapper(
+            appid,
+            vercode,
+            common.get_virt_container_type(options),
+        )
+    except Exception as e:
+        if options.verbose:
+            logging.error(traceback.format_exc())
+        else:
+            logging.error(e)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

setup.py

@@ -122,6 +122,7 @@ setup(
             'pycountry',
             'python-magic',
         ],
+        'podman': ['podman'],
         'test': ['pyjks', 'html5print', 'testcontainers[minio]'],
         'docs': [
             'sphinx',

tests/shared_test_code.py

@@ -29,6 +29,11 @@ class VerboseFalseOptions:
     verbose = False
 
 
+APPID = 'com.example'
+VERCODE = 123
+APPID_VERCODE = f'{APPID}:{VERCODE}'
+
+
 class TmpCwd:
     """Context-manager for temporarily changing the current working directory."""
 

tests/test_up.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+
+import importlib
+import os
+import shutil
+import unittest
+
+from pathlib import Path
+from unittest import mock, skipIf, skipUnless
+
+from fdroidserver import common, exception, up
+from .shared_test_code import mkdtemp, APPID, VERCODE, APPID_VERCODE
+
+
+class UpTest(unittest.TestCase):
+    basedir = Path(__file__).resolve().parent
+
+    def setUp(self):
+        self._td = mkdtemp()
+        self.testdir = self._td.name
+        os.chdir(self.testdir)
+
+    def tearDown(self):
+        self._td.cleanup()
+        common.config = None
+
+
+class Up_main(UpTest):
+    def setUp(self):
+        super().setUp()
+        common.config = dict()
+
+    @skipIf(
+        importlib.util.find_spec("podman") is None or not shutil.which('podman'),
+        'Requires podman and podman-py to run.',
+    )
+    @mock.patch('sys.argv', ['fdroid up', APPID_VERCODE])
+    @mock.patch('fdroidserver.common.get_default_cachedir')
+    @mock.patch('fdroidserver.up.run_podman')
+    def test_podman(self, run_podman, get_default_cachedir):
+        get_default_cachedir.return_value = self.testdir
+        common.config['virt_container_type'] = 'podman'
+        up.main()
+        run_podman.assert_called_once()
+
+    @mock.patch('sys.argv', ['fdroid up', APPID_VERCODE])
+    @mock.patch('fdroidserver.common.get_default_cachedir')
+    @mock.patch('fdroidserver.up.run_vagrant')
+    def test_vagrant(self, run_vagrant, get_default_cachedir):
+        get_default_cachedir.return_value = self.testdir
+        common.config['virt_container_type'] = 'vagrant'
+        up.main()
+        run_vagrant.assert_called_once()
+
+
+@skipIf(
+    importlib.util.find_spec("podman") is None or not shutil.which('podman'),
+    'Requires podman and podman-py to run.',
+)
+class Up_run_podman(UpTest):
+    @skipUnless(
+        os.path.exists(f'/run/user/{os.getuid()}/podman/podman.sock'),
+        'Requires systemd podman.socket to run.',
+    )
+    def test_up_with_systemd_socket(self):
+        common.get_podman_client()
+
+    @skipIf(
+        os.path.exists(f'/run/user/{os.getuid()}/podman/podman.sock'),
+        'Requires the systemd podman.socket is not present.',
+    )
+    def test_up_with_podman_system_service_start(self):
+        common.get_podman_client()
+
+    def test_recreate_existing(self):
+        try:
+            common.get_podman_container(APPID, VERCODE)
+        except exception.BuildException as e:
+            # To run these tests, first do: `./fdroid up com.example:123`
+            self.skipTest(f'Requires Podman container {APPID_VERCODE} to run: {e}')
+
+        short_id = common.get_podman_container(APPID, VERCODE).short_id
+        up.run_podman(APPID, VERCODE)
+        self.assertNotEqual(
+            short_id,
+            common.get_podman_container(APPID, VERCODE).short_id,
+            "This should never reuse an existing container.",
+        )
+
+
+@skipIf(importlib.util.find_spec("podman") is None, 'Requires podman-py to run.')
+class Up_run_fake_podman(UpTest):
+    @skipIf(
+        os.path.exists(f'/run/user/{os.getuid()}/podman/podman.sock'),
+        'Requires the systemd podman.socket is not present.',
+    )
+    @mock.patch.dict(os.environ, clear=True)
+    @mock.patch("podman.PodmanClient.ping")
+    def test_up_with_podman_system_service_start(self, mock_client_ping):
+        """Test that the system service gets started if no socket is present."""
+        os.environ['PATH'] = os.path.join(self.testdir, 'bin')
+        os.mkdir('bin')
+        podman = Path('bin/podman')
+        podman.write_text('#!/bin/sh\nprintf "$1 $2" > args\n')
+        os.chmod(podman, 0o700)
+        common.get_podman_client()
+        self.assertEqual('system service', Path('args').read_text())
+        mock_client_ping.assert_called_once()
+
+
+@skipIf(importlib.util.find_spec("vagrant") is None, 'Requires python-vagrant to run.')
+class Up_run_vagrant(UpTest):
+    def setUp(self):
+        super().setUp()
+        b = mock.Mock()
+        b.name = 'buildserver'
+        self.box_list_return = [b]
+        name = common.get_container_name(APPID, VERCODE)
+        self.vagrantdir = Path('tmp/buildserver') / name
+
+    @mock.patch('vagrant.Vagrant.up')
+    @mock.patch('vagrant.Vagrant.box_list')
+    def test_no_existing(self, box_list, vagrant_up):
+        box_list.return_value = self.box_list_return
+        up.run_vagrant(APPID, VERCODE, 1, 1)
+        vagrant_up.assert_called_once()
+        self.assertTrue((Path(self.testdir) / self.vagrantdir).exists())