Merge branch 'buildbot-subcommands-up-destroy-exec-push-pull' into 'master'

set up new mechanism for unlisted subcommands; implement up/destroy/exec/push/pull

See merge request fdroid/fdroidserver!1709

.gitlab-ci.yml

@@ -852,3 +852,24 @@ PUBLISH:
     - fdroid gpgsign --verbose
     - fdroid signindex --verbose
     - rsync --stats repo/* $serverwebroot/
+
+
+# This tests the `podman system service` auto-starting and other general podman things.
+podman system service:
+  rules:
+    - changes:
+        - .gitlab-ci.yml
+        - fdroidserver/up.py
+        - tests/test_up.py
+  image: debian:trixie-slim
+  <<: *apt-template
+  script:
+    - apt-get install
+        python3-asn1crypto
+        python3-defusedxml
+        python3-git
+        python3-pillow
+        python3-podman
+        python3-ruamel.yaml
+        python3-yaml
+    - python3 -m unittest tests/test_up.py --verbose

fdroidserver/__main__.py

@@ -55,6 +55,23 @@ COMMANDS = OrderedDict([
     ("mirror", _("Download complete mirrors of small repos")),
 ])
 
+# The list of subcommands that are not advertised as public api,
+# intended for the use-case of breaking down builds into atomic steps.
+#
+# For the build automation, there will be lots of subcommands that are
+# meant to run in scripted environments.  They are not intended for
+# interactive use.  Although they sometimes might be useful
+# interactively, they might also sometimes be dangerous to run
+# interactively because they rely on the presense of a VM/container,
+# modify the local environment, or even run things as root.
+COMMANDS_INTERNAL = [
+    "destroy",
+    "exec",
+    "pull",
+    "push",
+    "up",
+]
+
 
 def print_help(available_plugins=None):
     print(_("usage: ") + _("fdroid [<command>] [-h|--help|--version|<args>]"))
@@ -136,7 +153,12 @@ def main():
         sys.exit(0)
 
     command = sys.argv[1]
-    if command not in COMMANDS and command not in available_plugins:
+    command_not_found = (
+        command not in COMMANDS
+        and command not in COMMANDS_INTERNAL
+        and command not in available_plugins
+    )
+    if command_not_found:
         if command in ('-h', '--help'):
             print_help(available_plugins=available_plugins)
             sys.exit(0)
@@ -186,7 +208,7 @@ def main():
     sys.argv[0] += ' ' + command
 
     del sys.argv[1]
-    if command in COMMANDS.keys():
+    if command in COMMANDS.keys() or command in COMMANDS_INTERNAL:
         # import is named import_subcommand internally b/c import is reserved by Python
         command = 'import_subcommand' if command == 'import' else command
         mod = __import__('fdroidserver.' + command, None, None, [command])

fdroidserver/common.py

@@ -4,10 +4,10 @@
 #
 # Copyright (C) 2010-2016, Ciaran Gultnieks, ciaran@ciarang.com
 # Copyright (C) 2013-2017, Daniel Martí <mvdan@mvdan.cc>
-# Copyright (C) 2013-2021, Hans-Christoph Steiner <hans@eds.org>
+# Copyright (C) 2013-2025, Hans-Christoph Steiner <hans@eds.org>
 # Copyright (C) 2017-2018, Torsten Grote <t@grobox.de>
 # Copyright (C) 2017, tobiasKaminsky <tobias@kaminsky.me>
-# Copyright (C) 2017-2021, Michael Pöhn <michael.poehn@fsfe.org>
+# Copyright (C) 2017-2025, Michael Pöhn <michael.poehn@fsfe.org>
 # Copyright (C) 2017,2021, mimi89999 <michel@lebihan.pl>
 # Copyright (C) 2019-2021, Jochen Sprickerhof <git@jochen.sprickerhof.de>
 # Copyright (C) 2021, Felix C. Stegerman <flx@obfusk.net>
@@ -67,6 +67,7 @@ import logging
 import operator
 import os
 import re
+import shlex
 import shutil
 import socket
 import stat
@@ -82,7 +83,7 @@ from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from queue import Queue
 from typing import List
-from urllib.parse import urlparse, urlsplit, urlunparse
+from urllib.parse import urlparse, urlsplit, urlunparse, unquote
 from zipfile import ZipFile
 
 import defusedxml.ElementTree as XMLElementTree
@@ -108,6 +109,13 @@ from .looseversion import LooseVersion
 # The path to this fdroidserver distribution
 FDROID_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))
 
+SUPPORTED_VIRT_CONTAINER_TYPES = ('podman', 'vagrant')
+
+# The path to the homedir where apps are built in the container/VM.
+BUILD_HOME = '/home/vagrant'
+# Username inside of build containers/VMs used for runing builds.
+BUILD_USER = 'vagrant'
+
 # There needs to be a default, and this is the most common for software.
 DEFAULT_LOCALE = 'en-US'
 
@@ -280,6 +288,27 @@ def setup_global_opts(parser):
     )
 
 
+def setup_virt_container_type_opts(parser):
+    parser.add_argument(
+        "--virt-container-type",
+        choices=SUPPORTED_VIRT_CONTAINER_TYPES,
+        help="Set the VM/container type used by the build process.",
+    )
+
+
+def get_virt_container_type(options):
+    if options.virt_container_type:
+        return options.virt_container_type
+    vct = get_config().get('virt_container_type')
+    if vct not in SUPPORTED_VIRT_CONTAINER_TYPES:
+        supported = ', '.join(sorted(SUPPORTED_VIRT_CONTAINER_TYPES))
+        logging.error(
+            f"'virt_container_type: {vct}' not supported, try: {supported}"
+        )
+        sys.exit(1)
+    return vct
+
+
 class ColorFormatter(logging.Formatter):
 
     def __init__(self, msg):
@@ -1046,6 +1075,27 @@ def get_local_metadata_files():
     return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
 
 
+def split_pkg_arg(appid_versionCode_pair):
+    """Split 'appid:versionCode' pair into 2 separate values safely.
+
+    :raises ValueError: if argument is not parseable
+    :return: (appid, versionCode) tuple with the 2 parsed values
+    """
+    tokens = appid_versionCode_pair.split(":")
+    if len(tokens) != 2:
+        raise ValueError(
+            _("'{}' is not a valid pair of the form appId:versionCode pair").format(
+                appid_versionCode_pair
+            )
+        )
+    if not is_valid_package_name(tokens[0]):
+        raise ValueError(
+            _("'{}' does not start with a valid appId").format(appid_versionCode_pair)
+        )
+    versionCode = version_code_string_to_int(tokens[1])
+    return tokens[0], versionCode
+
+
 def read_pkg_args(appid_versionCode_pairs, allow_version_codes=False):
     """No summary.
 
@@ -1259,6 +1309,16 @@ def get_source_date_epoch(build_dir):
             return repo.git.log('-n1', '--pretty=%ct', '--', metadatapath)
 
 
+def get_container_name(appid, vercode):
+    """Return unique name for associating a build with a container or VM."""
+    return f'{appid}_{vercode}'
+
+
+def get_pod_name(appid, vercode):
+    """Return unique name for associating a build with a Podman "pod"."""
+    return f'{get_container_name(appid, vercode)}_pod'
+
+
 def get_build_dir(app):
     """Get the dir that this app will be built in."""
     if app.RepoType == 'srclib':
@@ -5012,3 +5072,184 @@ FDROIDORG_MIRRORS = [
 FDROIDORG_FINGERPRINT = (
     '43238D512C1E5EB2D6569F4A3AFBF5523418B82E0A3ED1552770ABB9A9C9CCAB'
 )
+
+
+def get_podman_client():
+    """Return an instance of podman-py to work with Podman.
+
+    This assumes that it will use the default local UNIX Domain Socket
+    to connect to the Podman service.  The preferred setup is to first
+    do `systemctl --user start podman.socket` to keep the Podman UNIX
+    socket running all the time.  If the socket is not present, this
+    will automatically start the service, which has a default five
+    second timeout.  After the timeout, it shuts itself down.  This is
+    done outside of systemd so it will work where systemd is not
+    installed.  The systemd socket calls the same command anyway.
+
+    https://docs.podman.io/en/latest/markdown/podman-system-service.1.html
+
+    """
+    import podman
+
+    client = podman.PodmanClient()
+    url = client.api.base_url
+    socket_path = unquote(url.netloc)
+    if not os.path.exists(socket_path):
+        logging.info(f'Starting podman system service at {socket_path}')
+        subprocess.Popen(
+            ['podman', 'system', 'service'],
+            close_fds=True,
+        )
+        retried = 0
+        while not os.path.exists(socket_path):
+            time.sleep(0.1)
+            retried += 1
+            if retried > 100:  # wait for ten seconds
+                break
+    if not client.ping():
+        path = f'{url.scheme}://{unquote(url.netloc)}'
+        logging.error(f'No Podman service found at {path}!')
+        sys.exit(1)
+    return client
+
+
+PODMAN_BUILDSERVER_IMAGE = 'registry.gitlab.com/fdroid/fdroidserver:buildserver'
+
+
+def get_podman_container(appid, vercode):
+    """Singleton getter, since podman-py is just an interface to the podman daemon singleton."""
+    container_name = get_container_name(appid, vercode)
+    client = get_podman_client()
+    ret = None
+    for c in client.containers.list(all=True):
+        if c.name == container_name:
+            ret = c
+            break
+    client.close()
+    if ret is None:
+        raise BuildException(f'Container for {appid}:{vercode} not found!')
+    if PODMAN_BUILDSERVER_IMAGE not in ret.image.tags:
+        raise BuildException(
+            f'Container for {appid}:{vercode} has wrong image: {ret.image}'
+        )
+    return ret
+
+
+def inside_exec(appid, vercode, command, virt_container_type, as_root=False):
+    """Execute the command inside of the VM for the build."""
+    if virt_container_type == 'vagrant':
+        return vagrant_exec(appid, vercode, command, as_root)
+    elif virt_container_type == 'podman':
+        return podman_exec(appid, vercode, command, as_root)
+    else:
+        raise Exception(
+            f"'{virt_container_type}' not supported, currently supported: vagrant, podman"
+        )
+
+
+def podman_exec(appid, vercode, command, as_root=False):
+    """Execute the command inside of a podman container for the build."""
+    container_name = get_container_name(appid, vercode)
+    to_stdin = shlex.join(command)
+    user = 'root' if as_root else BUILD_USER
+    p = subprocess.run(
+        [
+            'podman',
+            'exec',
+            '--interactive',
+            f'--user={user}',
+            f'--workdir={BUILD_HOME}',
+            container_name,
+        ]
+        + ['/bin/bash', '-e', '-l', '-x'],  # the shell that runs the command
+        input=to_stdin,
+        text=True,
+        stderr=subprocess.STDOUT,
+    )
+    if p.returncode != 0:
+        raise subprocess.CalledProcessError(
+            p.returncode, f"{to_stdin} | {' '.join(p.args)}"
+        )
+
+
+def get_vagrantfile_path(appid, vercode):
+    """Return the path for the unique VM for a given build.
+
+    Vagrant doesn't support ':' in VM names, so this uses '_' instead.
+    Plus filesystems are often grumpy about using `:` in paths.
+
+    """
+    return Path('tmp/buildserver', get_container_name(appid, vercode), 'Vagrantfile')
+
+
+def vagrant_exec(appid, vercode, command, as_root=False):
+    """Execute a command in the Vagrant VM via ssh."""
+    vagrantfile = get_vagrantfile_path(appid, vercode)
+    to_stdin = shlex.join(command)
+    p = subprocess.run(
+        [
+            'vagrant',
+            'ssh',
+            '-c',
+            'sudo bash' if as_root else 'bash',
+        ],
+        input=to_stdin,
+        text=True,
+        cwd=vagrantfile.parent,
+    )
+    if p.returncode != 0:
+        raise subprocess.CalledProcessError(
+            p.returncode, f"{to_stdin} | {' '.join(p.args)}"
+        )
+
+
+def vagrant_destroy(appid, vercode):
+    import vagrant
+
+    vagrantfile = get_vagrantfile_path(appid, vercode)
+    if vagrantfile.is_file():
+        logging.info(f"Destroying Vagrant buildserver VM ({appid}:{vercode})")
+        v = vagrant.Vagrant(vagrantfile.parent)
+        v.destroy()
+    if vagrantfile.parent.exists():
+        shutil.rmtree(vagrantfile.parent, ignore_errors=True)
+
+
+def get_vagrant_bin_path():
+    p = shutil.which("vagrant")
+    if p is None:
+        raise Exception(
+            "'vagrant' not found, make sure it's installed and added to your path"
+        )
+    return p
+
+
+def get_rsync_bin_path():
+    p = shutil.which("rsync")
+    if p is None:
+        raise Exception(
+            "'rsync' not found, make sure it's installed and added to your path"
+        )
+    return p
+
+
+class TmpVagrantSshConf:
+    """Context manager for getting access to a ssh config of a vagrant VM in form of a temp file."""
+
+    def __init__(self, vagrant_bin_path, vagrant_vm_dir):
+        self.vagrant_bin_path = vagrant_bin_path
+        self.vagrant_vm_dir = vagrant_vm_dir
+
+    def __enter__(self):
+        self.ssh_config_tf = tempfile.NamedTemporaryFile('wt', encoding="utf8")
+        self.ssh_config_tf.write(
+            subprocess.check_output(
+                [self.vagrant_bin_path, 'ssh-config'],
+                cwd=self.vagrant_vm_dir,
+            ).decode('utf8')
+        )
+        self.ssh_config_tf.flush()
+        return self.ssh_config_tf.name
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.ssh_config_tf.close()

fdroidserver/destroy.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+#
+# destroy.py - part of the FDroid server tools
+# Copyright (C) 2024, Hans-Christoph Steiner <hans@eds.org>
+# Copyright (C) 2024, Michael Pöhn <michael@poehn.at>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Destroy any existing per-build container/VM structures.
+
+After this runs, there should be no trace of the given
+ApplicationID:versionCode left in the container/VM system.
+
+Since this is an internal command, the strings are not localized.
+
+"""
+
+import logging
+import sys
+import traceback
+from argparse import ArgumentParser
+
+from . import common
+
+# TODO should this track whether it actually removed something?
+# What do `podman rm` and `vagrant destroy` do?
+
+
+def podman_rm(appid, vercode):
+    """Remove a Podman pod and all its containers."""
+    pod_name = common.get_pod_name(appid, vercode)
+    for p in common.get_podman_client().pods.list():
+        if p.name == pod_name:
+            logging.debug(f'Removing {pod_name}.')
+            p.remove(force=True)
+
+
+def destroy_wrapper(appid, vercode, virt_container_type):
+    if virt_container_type == 'vagrant':
+        common.vagrant_destroy(appid, vercode)
+    elif virt_container_type == 'podman':
+        podman_rm(appid, vercode)
+
+
+def main():
+    parser = ArgumentParser(
+        description="Push files into the buildserver container/box."
+    )
+    common.setup_global_opts(parser)
+    common.setup_virt_container_type_opts(parser)
+    parser.add_argument(
+        "APPID:VERCODE",
+        help="Application ID with Version Code in the form APPID:VERCODE",
+    )
+    options = common.parse_args(parser)
+    common.set_console_logging(options.verbose)
+
+    try:
+        appid, vercode = common.split_pkg_arg(options.__dict__['APPID:VERCODE'])
+        destroy_wrapper(
+            appid,
+            vercode,
+            common.get_virt_container_type(options),
+        )
+    except Exception as e:
+        if options.verbose:
+            logging.error(traceback.format_exc())
+        else:
+            logging.error(e)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

fdroidserver/exec.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+#
+# exec.py - part of the FDroid server tools
+# Copyright (C) 2024-2025, Hans-Christoph Steiner <hans@eds.org>
+# Copyright (C) 2024-2025, Michael Pöhn <michael@poehn.at>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Run an fdroidserver subcommand inside of the VM for the build.
+
+Since this is an internal command, the strings are not localized.
+
+"""
+
+import sys
+import logging
+import traceback
+from argparse import ArgumentParser
+
+from . import common
+
+
+def main():
+    parser = ArgumentParser(
+        description="Run a subcommand in the buildserver container/box."
+    )
+    parser.add_argument(
+        '--as-root',
+        default=False,
+        action='store_true',
+        help="run command inside of container/VM as root user",
+    )
+    common.setup_global_opts(parser)
+    common.setup_virt_container_type_opts(parser)
+    parser.add_argument(
+        "APPID:VERCODE",
+        help="Application ID with Version Code in the form APPID:VERCODE",
+    )
+    parser.add_argument(
+        "COMMAND", nargs="*", help="Command to run inside the container/box."
+    )
+    options = common.parse_args(parser)
+    common.set_console_logging(options.verbose)
+
+    try:
+        appid, vercode = common.split_pkg_arg(options.__dict__['APPID:VERCODE'])
+        common.inside_exec(
+            appid,
+            vercode,
+            options.COMMAND,
+            common.get_virt_container_type(options),
+            options.as_root,
+        )
+    except Exception as e:
+        if options.verbose:
+            logging.error(traceback.format_exc())
+        else:
+            logging.error(e)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

fdroidserver/pull.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+#
+# pull.py - part of the FDroid server tools
+# Copyright (C) 2024-2025, Hans-Christoph Steiner <hans@eds.org>
+# Copyright (C) 2024-2025, Michael Pöhn <michael@poehn.at>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Pull a single file from the build setup for a given ApplicationID:versionCode.
+
+This is a carefully constructed method to copy files from inside the
+buildserver VM/container to the host filesystem for further processing
+and publishing.  The source path is forced to the within the app's
+build dir, and the destination is forced to the repo's unsigned/ dir.
+This is not meant as a generic method for getting files, that is already
+provided by each VM/container system (e.g. `podman cp`).
+
+Since this is an internal command, the strings are not localized.
+
+"""
+
+import os
+import sys
+import logging
+import subprocess
+import tarfile
+import tempfile
+import traceback
+from argparse import ArgumentParser
+
+from . import common, metadata
+
+
+def podman_pull(appid, vercode, path):
+    """Implement `fdroid pull` for Podman (e.g. not `podman pull`)."""
+    path_in_container = os.path.join(common.BUILD_HOME, path)
+    container = common.get_podman_container(appid, vercode)
+    stream, stat = container.get_archive(path_in_container)
+    if not stat['linkTarget'].endswith(path) or stat['name'] != os.path.basename(path):
+        logging.warning(f'{path} not found!')
+        return
+    with tempfile.NamedTemporaryFile(prefix=".fdroidserver_pull_", suffix=".tar") as tf:
+        for i in stream:
+            tf.write(i)
+        tf.seek(0)
+        with tarfile.TarFile(fileobj=tf, mode='r') as tar:
+            tar.extract(stat['name'], 'unsigned', set_attrs=False)
+
+
+def vagrant_pull(appid, vercode, path):
+    """Pull the path from the Vagrant VM."""
+    vagrantfile = common.get_vagrantfile_path(appid, vercode)
+    path_in_vm = os.path.join(common.BUILD_HOME, path)
+
+    vagrantbin = common.get_vagrant_bin_path()
+    rsyncbin = common.get_rsync_bin_path()
+
+    with common.TmpVagrantSshConf(vagrantbin, vagrantfile.parent) as ssh_config_file:
+        cmd = [
+            rsyncbin,
+            '-av',
+            '-e',
+            f'ssh -F {ssh_config_file}',
+            f'default:{path_in_vm}',
+            './unsigned',
+        ]
+        subprocess.check_call(cmd)
+
+
+def make_file_list(appid, vercode):
+    app, build = metadata.get_single_build(appid, vercode)
+    ext = common.get_output_extension(build)
+    return [
+        os.path.join('unsigned', common.get_release_filename(app, build, ext)),
+        os.path.join(
+            'unsigned', common.get_src_tarball_name(app.id, build.versionCode)
+        ),
+    ]
+
+
+def pull_wrapper(appid, vercode, virt_container_type):
+    os.makedirs('unsigned', exist_ok=True)
+    files = make_file_list(appid, vercode)
+    for f in files:
+        logging.info(f"""Pulling {f} from {appid}:{vercode}""")
+        if virt_container_type == 'vagrant':
+            vagrant_pull(appid, vercode, f)
+        elif virt_container_type == 'podman':
+            podman_pull(appid, vercode, f)
+
+
+def main():
+    parser = ArgumentParser(
+        description="Pull build products from the buildserver container/box."
+    )
+    common.setup_global_opts(parser)
+    common.setup_virt_container_type_opts(parser)
+    parser.add_argument(
+        "APPID:VERCODE",
+        help="Application ID with Version Code in the form APPID:VERCODE",
+    )
+    options = common.parse_args(parser)
+    common.set_console_logging(options.verbose)
+
+    try:
+        appid, vercode = common.split_pkg_arg(options.__dict__['APPID:VERCODE'])
+        pull_wrapper(
+            appid,
+            vercode,
+            common.get_virt_container_type(options),
+        )
+    except Exception as e:
+        if options.verbose:
+            logging.error(traceback.format_exc())
+        else:
+            logging.error(e)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

fdroidserver/push.py

@@ -0,0 +1,226 @@
+#!/usr/bin/env python3
+#
+# push.py - part of the FDroid server tools
+# Copyright (C) 2024-2025, Hans-Christoph Steiner <hans@eds.org>
+# Copyright (C) 2024-2025, Michael Pöhn <michael@poehn.at>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Push files into the build setup for a given ApplicationID:versionCode.
+
+This is a carefully constructed method to copy files from the host
+filesystem to inside the buildserver VM/container to run the build.  The
+source paths are generated based on the build metadata, and the
+destination is forced to build home.  This is not meant as a generic
+method for copying files, that is already provided by each VM/container
+system (e.g. `podman cp`).
+
+Since this is an internal command, the strings are not localized.
+
+"""
+
+import logging
+import os
+import subprocess
+import sys
+import tarfile
+import tempfile
+import traceback
+from argparse import ArgumentParser
+from pathlib import Path
+
+from . import common, metadata
+from .exception import BuildException
+
+
+def podman_push(paths, appid, vercode, as_root=False):
+    """Push relative paths into the podman container using the tar method.
+
+    This builds up a tar file from the supplied paths to send into
+    container via put_archive(). This assumes it is running in the
+    base of fdroiddata and it will push into common.BUILD_HOME in the
+    container.
+
+    This is a version of podman.api.create_tar() that builds up via
+    adding paths rather than giving a base dir and an exclude list.
+
+    """
+
+    def _tar_perms_filter(tarinfo):
+        """Force perms to something safe."""
+        if '__pycache__' in tarinfo.name:
+            return
+
+        if as_root:
+            tarinfo.uid = tarinfo.gid = 0
+            tarinfo.uname = tarinfo.gname = 'root'
+        else:
+            tarinfo.uid = tarinfo.gid = 1000
+            tarinfo.uname = tarinfo.gname = common.BUILD_USER
+
+        if tarinfo.isdir():
+            tarinfo.mode = 0o0755
+        elif tarinfo.isfile():
+            if tarinfo.mode & 0o111:
+                tarinfo.mode = 0o0755
+            else:
+                tarinfo.mode = 0o0644
+        elif not tarinfo.issym():  # symlinks shouldn't need perms set
+            raise BuildException(f'{tarinfo.name} is not a file or directory!')
+        return tarinfo
+
+    if isinstance(paths, Path):
+        paths = [str(paths)]
+    if isinstance(paths, str):
+        paths = [paths]
+
+    container = common.get_podman_container(appid, vercode)
+    with tempfile.TemporaryFile() as tf:
+        with tarfile.TarFile(fileobj=tf, mode='w') as tar:
+            for f in paths:
+                if Path(f).is_absolute():
+                    raise BuildException(f'{f} must be relative to {Path.cwd()}')
+                # throw ValueError on bad path
+                f = (Path.cwd() / f).resolve().relative_to(Path.cwd())
+                tar.add(f, filter=_tar_perms_filter)
+        tf.seek(0)
+        container.put_archive(common.BUILD_HOME, tf.read())
+
+
+def vagrant_push(paths, appid, vercode):
+    """Push files into a build specific vagrant VM."""
+    vagrantbin = common.get_vagrant_bin_path()
+    rsyncbin = common.get_rsync_bin_path()
+
+    vagrantfile = common.get_vagrantfile_path(appid, vercode)
+    with common.TmpVagrantSshConf(vagrantbin, vagrantfile.parent) as ssh_config_file:
+        for path in paths:
+            cmd = [
+                rsyncbin,
+                '-av',
+                '--relative',
+                '-e',
+                f'ssh -F {ssh_config_file}',
+                path,
+                'default:{}'.format(common.BUILD_HOME),
+            ]
+            subprocess.check_call(
+                cmd,
+            )
+
+
+# TODO split out shared code into _podman.py, _vagrant.py, etc
+
+
+def push_wrapper(paths, appid, vercode, virt_container_type):
+    """Push standard set of files into VM/container."""
+    if virt_container_type == 'vagrant':
+        vagrant_push(paths, appid, vercode)
+    elif virt_container_type == 'podman':
+        podman_push(paths, appid, vercode)
+
+
+def make_file_list(appid, vercode):
+    """Assemble list of files/folders that go into this specific build."""
+    files = [f'build/{appid}', f'metadata/{appid}.yml']
+    app_dir = f'metadata/{appid}'
+    if Path(app_dir).exists():
+        files.append(app_dir)
+    app, build = metadata.get_single_build(appid, vercode)
+
+    for lib in build.srclibs:
+        srclib = common.getsrclib(lib, 'build/srclib', basepath=True, prepare=False)
+        if srclib:
+            # srclib metadata file
+            files.append(f"srclibs/{srclib[0]}.yml")
+            # srclib sourcecode
+            files.append(srclib[2])
+
+    extlib_paths = [f"build/extlib/{lib}" for lib in build.extlibs]
+    missing_extlibs = [p for p in extlib_paths if not os.path.exists(p)]
+    if any(missing_extlibs):
+        raise Exception(
+            "error: requested missing extlibs: {}".format(" ".join(missing_extlibs))
+        )
+    files.extend(extlib_paths)
+
+    return files
+
+
+def create_build_dirs(appid, vercode, virt_container_type):
+    """Create directories required for running builds."""
+    dirs = ('build', 'build/extlib', 'build/srclib', 'metadata', 'fdroidserver')
+    for dir_name in dirs:
+        cmd = ['mkdir', '--parents', f'{common.BUILD_HOME}/{dir_name}']
+        common.inside_exec(
+            appid,
+            vercode,
+            cmd,
+            virt_container_type,
+        )
+
+
+def full_push_sequence(appid, vercode, virt_container_type):
+    """Push all files into vm required for specified build."""
+    create_build_dirs(appid, vercode, virt_container_type)
+    push_wrapper(make_file_list(appid, vercode), appid, vercode, virt_container_type)
+
+    # fdroidserver is pushed in every build
+    cwd = Path('.').absolute()
+    try:
+        os.chdir(Path(__file__).parent.parent.parent)
+        push_wrapper(
+            [
+                'fdroidserver/fdroidserver',
+                'fdroidserver/fdroid',
+            ],
+            appid,
+            vercode,
+            virt_container_type,
+        )
+    finally:
+        os.chdir(cwd)
+
+
+def main():
+    """CLI main method for this subcommand."""
+    parser = ArgumentParser(
+        description="Push full build setup into the buildserver container/box."
+    )
+    common.setup_global_opts(parser)
+    common.setup_virt_container_type_opts(parser)
+    parser.add_argument(
+        "APPID:VERCODE",
+        help="Application ID with Version Code in the form APPID:VERCODE",
+    )
+    options = common.parse_args(parser)
+    common.set_console_logging(options.verbose)
+
+    try:
+        appid, vercode = common.split_pkg_arg(options.__dict__['APPID:VERCODE'])
+        full_push_sequence(
+            appid,
+            vercode,
+            common.get_virt_container_type(options),
+        )
+    except Exception as e:
+        if options.verbose:
+            logging.error(traceback.format_exc())
+        else:
+            logging.error(e)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

fdroidserver/up.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+#
+# up.py - part of the FDroid server tools
+# Copyright (C) 2024-2025, Hans-Christoph Steiner <hans@eds.org>
+# Copyright (C) 2024-2025, Michael Pöhn <michael@poehn.at>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+"""Create dedicated VM/container to run single build, destroying any existing ones.
+
+The ApplicationID:versionCode argument from the command line should be
+used as the unique identifier.  This is necessary so that the other
+related processes (push, pull, destroy) can find the dedicated
+container/VM without there being any other database or file lookup.
+
+Since this is an internal command, the strings are not localized.
+
+"""
+
+import logging
+import sys
+import textwrap
+import traceback
+from argparse import ArgumentParser
+
+from . import common
+from .exception import BuildException
+
+
+def run_podman(appid, vercode):
+    """Create a Podman container env isolated for a single app build.
+
+    This creates a Podman "pod", which is like an isolated box to
+    create containers in.  Then it creates a container in that pod to
+    run the actual processes.  Using the "pod" seems to be a
+    requirement of Podman.  It also further isolates each app build,
+    so seems fine to use.  It is confusing because these containers
+    won't show up by default when listing containers using defaults.
+
+    The container is set up with an interactive bash process to keep
+    the container running.
+
+    """
+    container_name = common.get_container_name(appid, vercode)
+    pod_name = common.get_pod_name(appid, vercode)
+    client = common.get_podman_client()
+
+    logging.debug(f'Pulling {common.PODMAN_BUILDSERVER_IMAGE}...')
+    image = client.images.pull(common.PODMAN_BUILDSERVER_IMAGE)
+
+    for c in client.containers.list():
+        if c.name == container_name:
+            logging.warning(f'Container {container_name} exists, removing!')
+            c.remove(force=True)
+
+    for p in client.pods.list():
+        if p.name == pod_name:
+            logging.warning(f'Pod {pod_name} exists, removing!')
+            p.remove(force=True)
+
+    pod = client.pods.create(pod_name)
+    container = client.containers.create(
+        image,
+        command=['/bin/bash', '-e', '-i', '-l'],
+        pod=pod,
+        name=container_name,
+        detach=True,
+        remove=True,
+        stdin_open=True,
+    )
+    pod.start()
+    pod.reload()
+    if container.status != 'created':
+        raise BuildException(
+            f'Container {container_name} failed to start ({container.status})!'
+        )
+
+
+def run_vagrant(appid, vercode, cpus, memory):
+    import vagrant
+
+    if cpus is None or not isinstance(cpus, int) or not cpus > 0:
+        raise BuildException(
+            f"vagrant cpu setting required, '{cpus}' not a valid value!"
+        )
+    if memory is None or not isinstance(memory, int) or not memory > 0:
+        raise BuildException(
+            f"vagrant memory setting required, '{memory}' not a valid value!"
+        )
+
+    vagrantfile = common.get_vagrantfile_path(appid, vercode)
+
+    # cleanup potentially still existsing vagrant VMs/dirs
+    common.vagrant_destroy(appid, vercode)
+
+    # start new dedicated buildserver vagrant vm from scratch
+    vagrantfile.parent.mkdir(exist_ok=True, parents=True)
+    vagrantfile.write_text(
+        textwrap.dedent(
+            f"""# generated file, do not change.
+
+                Vagrant.configure("2") do |config|
+                  config.vm.box = "buildserver"
+                  config.vm.synced_folder ".", "/vagrant", disabled: true
+
+                  config.vm.provider :libvirt do |libvirt|
+                    libvirt.cpus = {cpus}
+                    libvirt.memory = {memory}
+                  end
+                end
+            """
+        )
+    )
+
+    v = vagrant.Vagrant(vagrantfile.parent)
+
+    if not any((b for b in v.box_list() if b.name == 'buildserver')):
+        raise BuildException("'buildserver' box not added to vagrant")
+
+    v.up()
+
+
+def up_wrapper(appid, vercode, virt_container_type, cpus=None, memory=None):
+    if virt_container_type == 'vagrant':
+        run_vagrant(appid, vercode, cpus, memory)
+    elif virt_container_type == 'podman':
+        run_podman(appid, vercode)
+
+
+def main():
+    parser = ArgumentParser(
+        description="Create dedicated VM/container to run single build."
+    )
+    common.setup_global_opts(parser)
+    common.setup_virt_container_type_opts(parser)
+    parser.add_argument(
+        "APPID:VERCODE",
+        help="Application ID with Version Code in the form APPID:VERCODE",
+    )
+    parser.add_argument(
+        "--cpus",
+        default=None,
+        type=int,
+        help="How many CPUs the Vagrant VM should be allocated.",
+    )
+    parser.add_argument(
+        "--memory",
+        default=None,
+        type=int,
+        help="How many MB of RAM the Vagrant VM should be allocated.",
+    )
+    options = common.parse_args(parser)
+    common.set_console_logging(options.verbose)
+
+    try:
+        appid, vercode = common.split_pkg_arg(options.__dict__['APPID:VERCODE'])
+        up_wrapper(
+            appid,
+            vercode,
+            common.get_virt_container_type(options),
+        )
+    except Exception as e:
+        if options.verbose:
+            logging.error(traceback.format_exc())
+        else:
+            logging.error(e)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

setup.py

@@ -122,6 +122,7 @@ setup(
             'pycountry',
             'python-magic',
         ],
+        'podman': ['podman'],
         'test': ['pyjks', 'html5print', 'testcontainers[minio]'],
         'docs': [
             'sphinx',

tests/shared_test_code.py

@@ -29,6 +29,11 @@ class VerboseFalseOptions:
     verbose = False
 
 
+APPID = 'com.example'
+VERCODE = 123
+APPID_VERCODE = f'{APPID}:{VERCODE}'
+
+
 class TmpCwd:
     """Context-manager for temporarily changing the current working directory."""
 

tests/test_common.py

@@ -3651,3 +3651,57 @@ class GetHeadCommitIdTest(unittest.TestCase):
         git_repo.git.add(all=True)
         git_repo.index.commit("add code")
         self.assertIsNotNone(fdroidserver.common.get_head_commit_id(git_repo))
+
+
+class VirtContainerTypeTest(unittest.TestCase):
+    """Test the logic for choosing which VM/container system to use."""
+
+    def setUp(self):
+        self._td = mkdtemp()
+        self.testdir = self._td.name
+        os.chdir(self.testdir)
+        fdroidserver.common.config = None
+        fdroidserver.common.options = None
+        # self.options represents the output of argparse.parser()
+        self.options = mock.Mock()
+        self.options.virt_container_type = None
+
+    def tearDown(self):
+        os.chdir(basedir)
+        self._td.cleanup()
+
+    def test_get_virt_container_type_unset(self):
+        with self.assertLogs(level=logging.ERROR) as logs:
+            with self.assertRaises(SystemExit):
+                fdroidserver.common.get_virt_container_type(self.options)
+            self.assertIn('virt_container_type', logs.output[0])
+
+    def test_get_virt_container_type_config(self):
+        testvalue = 'podman'
+        Path('config.yml').write_text(f'virt_container_type: {testvalue}\n')
+        self.assertEqual(
+            testvalue, fdroidserver.common.get_virt_container_type(self.options)
+        )
+
+    def test_get_virt_container_type_options(self):
+        testvalue = 'podman'
+        self.options.virt_container_type = testvalue
+        self.assertEqual(
+            testvalue, fdroidserver.common.get_virt_container_type(self.options)
+        )
+
+    def test_get_virt_container_type_options_override_config(self):
+        testvalue = 'podman'
+        self.options.virt_container_type = testvalue
+        Path('config.yml').write_text('virt_container_type: vagrant\n')
+        self.assertEqual(
+            testvalue, fdroidserver.common.get_virt_container_type(self.options)
+        )
+
+    def test_get_virt_container_type_config_bad_value(self):
+        testvalue = 'doesnotexist'
+        Path('config.yml').write_text(f'virt_container_type: {testvalue}\n')
+        with self.assertLogs(level=logging.ERROR) as logs:
+            with self.assertRaises(SystemExit):
+                fdroidserver.common.get_virt_container_type(self.options)
+            self.assertIn(testvalue, logs.output[0])

tests/test_exec.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+import unittest
+
+from pathlib import Path
+from unittest import mock, skipUnless
+
+from fdroidserver import common, exception, exec
+from .shared_test_code import mkdtemp, APPID, VERCODE, APPID_VERCODE
+
+
+class ExecTest(unittest.TestCase):
+    def setUp(self):
+        self._td = mkdtemp()
+        self.testdir = self._td.name
+        os.chdir(self.testdir)
+        common.config = dict()
+
+    def tearDown(self):
+        self._td.cleanup()
+        common.config = None
+
+
+class Exec_main(ExecTest):
+    @mock.patch('sys.argv', ['fdroid exec', APPID_VERCODE])
+    @mock.patch('fdroidserver.common.get_default_cachedir')
+    @mock.patch('fdroidserver.common.podman_exec')
+    def test_podman(self, podman_exec, get_default_cachedir):
+        get_default_cachedir.return_value = self.testdir
+        common.config['virt_container_type'] = 'podman'
+        exec.main()
+        podman_exec.assert_called_once()
+
+
+@skipUnless(os.path.isdir('/run/podman'), 'Requires Podman to run.')
+class Exec_podman_exec(ExecTest):
+    def _only_run_if_container_exists(self):
+        try:
+            common.get_podman_container(APPID, VERCODE)
+        except exception.BuildException as e:
+            # To run these tests, first do: `./fdroid up com.example:123`
+            self.skipTest(f'Requires Podman container {APPID_VERCODE} to run: {e}')
+
+    def test_no_existing_container(self):
+        appid = 'should.never.exist'
+        f = Path(f'metadata/{appid}.yml')
+        f.parent.mkdir()
+        f.write_text(f.name)
+        with self.assertRaises(subprocess.CalledProcessError) as e:
+            common.podman_exec(appid, 9999999999, ['ls'])
+            self.assertEqual(e.exception.returncode, 1)
+
+    def test_clean_run(self):
+        self._only_run_if_container_exists()
+        common.podman_exec(APPID, VERCODE, ['ls'])
+
+    def test_error_run(self):
+        self._only_run_if_container_exists()
+        with self.assertRaises(subprocess.CalledProcessError) as e:
+            common.podman_exec(APPID, VERCODE, ['/bin/false'])
+            self.assertEqual(e.exception.returncode, 1)

tests/test_pull.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+
+import importlib
+import os
+import unittest
+
+from pathlib import Path
+from unittest import mock, skipIf
+
+from fdroidserver import common, exception, pull
+from .shared_test_code import mkdtemp, APPID, VERCODE, APPID_VERCODE
+
+
+class PullTest(unittest.TestCase):
+    basedir = Path(__file__).resolve().parent
+
+    def setUp(self):
+        self._td = mkdtemp()
+        self.testdir = self._td.name
+        os.chdir(self.testdir)
+        common.config = dict()
+
+    def tearDown(self):
+        self._td.cleanup()
+        common.config = None
+
+
+class Pull_main(PullTest):
+    def setUp(self):
+        super().setUp()
+        metadatapath = Path(common.get_metadatapath(APPID))
+        metadatapath.parent.mkdir()
+        metadatapath.write_text(f'Name: Test\nBuilds:\n - versionCode: {VERCODE}\n')
+
+    @mock.patch('sys.argv', ['fdroid pull', APPID_VERCODE])
+    @mock.patch('fdroidserver.pull.podman_pull')
+    def test_podman(self, podman_pull):
+        common.config['virt_container_type'] = 'podman'
+        common.options = mock.Mock()
+        pull.main()
+        podman_pull.assert_called()
+
+    @mock.patch('sys.argv', ['fdroid pull', APPID_VERCODE])
+    @mock.patch('fdroidserver.pull.vagrant_pull')
+    def test_vagrant(self, vagrant_pull):
+        common.config['virt_container_type'] = 'vagrant'
+        pull.main()
+        vagrant_pull.assert_called()
+
+
+@skipIf(importlib.util.find_spec("podman") is None, 'Requires podman-py to run.')
+class Pull_podman_pull(PullTest):
+    def setUp(self):
+        try:
+            common.get_podman_container(APPID, VERCODE)
+        except exception.BuildException as e:
+            self.skipTest(f'Requires Podman container {APPID_VERCODE} to run: {e}')
+        super().setUp()
+
+    def test_no_existing(self):
+        appid = 'should.never.exist'
+        with self.assertRaises(exception.BuildException) as e:
+            pull.podman_pull(appid, 9999, 'unsigned/foo.apk')
+        self.assertIn(appid, e.exception.value)
+
+    def test_existing(self):
+        """Check files get deposited in unsigned/."""
+        filename = 'buildserverid'
+        f = Path('unsigned') / filename
+        self.assertFalse(f.exists())
+        pull.podman_pull(APPID, VERCODE, filename)
+        self.assertTrue(f.exists())
+
+
+class Pull_make_file_list(PullTest):
+    def setUp(self):
+        super().setUp()
+        self.metadatapath = Path(common.get_metadatapath(APPID))
+        self.metadatapath.parent.mkdir()
+
+    def test_implied(self):
+        self.metadatapath.write_text(f"""Builds: [versionCode: {VERCODE}]""")
+        self.assertEqual(
+            [
+                f'unsigned/{APPID}_{VERCODE}.apk',
+                f'unsigned/{APPID}_{VERCODE}_src.tar.gz',
+            ],
+            pull.make_file_list(APPID, VERCODE),
+        )
+
+    def test_gradle(self):
+        self.metadatapath.write_text(
+            f"""Builds:
+                      - versionCode: {VERCODE}
+                        gradle: fdroid
+            """
+        )
+        self.assertEqual(
+            [
+                f'unsigned/{APPID}_{VERCODE}.apk',
+                f'unsigned/{APPID}_{VERCODE}_src.tar.gz',
+            ],
+            pull.make_file_list(APPID, VERCODE),
+        )
+
+    def test_raw(self):
+        ext = 'foo'
+        Path(common.get_metadatapath(APPID)).write_text(
+            f"""Builds:
+              - versionCode: {VERCODE}
+                versionName: 1.0
+                commit: cafebabe123
+                output: path/to/output.{ext}
+            """
+        )
+        self.assertEqual(
+            [
+                f'unsigned/{APPID}_{VERCODE}.{ext}',
+                f'unsigned/{APPID}_{VERCODE}_src.tar.gz',
+            ],
+            pull.make_file_list(APPID, VERCODE),
+        )

tests/test_push.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+
+import importlib
+import os
+import unittest
+
+from pathlib import Path
+from unittest import mock, skipIf
+
+from fdroidserver import common, exception, push
+from .shared_test_code import mkdtemp, APPID, VERCODE, APPID_VERCODE
+
+
+class PushTest(unittest.TestCase):
+    basedir = Path(__file__).resolve().parent
+
+    def setUp(self):
+        self._td = mkdtemp()
+        self.testdir = self._td.name
+        os.chdir(self.testdir)
+        common.config = dict()
+
+    def tearDown(self):
+        self._td.cleanup()
+        common.config = None
+
+
+class Push_main(PushTest):
+    def setUp(self):
+        super().setUp()
+        metadatapath = Path(common.get_metadatapath(APPID))
+        metadatapath.parent.mkdir()
+        metadatapath.write_text(f'Name: Test\nBuilds:\n - versionCode: {VERCODE}\n')
+
+    @mock.patch('sys.argv', ['fdroid push', APPID_VERCODE])
+    @mock.patch('fdroidserver.push.create_build_dirs')
+    @mock.patch('fdroidserver.push.podman_push')
+    def test_podman(self, podman_push, create_build_dirs):
+        common.config['virt_container_type'] = 'podman'
+        push.main()
+        create_build_dirs.assert_called_once()
+        podman_push.assert_called()
+
+    @mock.patch('sys.argv', ['fdroid push', APPID_VERCODE])
+    @mock.patch('fdroidserver.push.create_build_dirs')
+    @mock.patch('fdroidserver.push.vagrant_push')
+    def test_vagrant(self, vagrant_push, create_build_dirs):
+        common.config['virt_container_type'] = 'vagrant'
+        push.main()
+        create_build_dirs.assert_called_once()
+        vagrant_push.assert_called()
+
+
+@skipIf(importlib.util.find_spec("podman") is None, 'Requires podman-py to run.')
+class Push_podman_push(PushTest):
+    def _only_run_if_container_exists(self):
+        try:
+            common.get_podman_container(APPID, VERCODE)
+        except exception.BuildException as e:
+            # To run these tests, first do: `./fdroid up com.example:123`
+            self.skipTest(f'Requires Podman container {APPID_VERCODE} to run: {e}')
+
+    def test_no_existing_container(self):
+        appid = 'should.never.exist'
+        f = Path(f'metadata/{appid}.yml')
+        f.parent.mkdir()
+        f.write_text(f.name)
+        with self.assertRaises(exception.BuildException):
+            push.podman_push(f, appid, 9999)
+
+    def test_bad_absolute_path(self):
+        self._only_run_if_container_exists()
+        with self.assertRaises(exception.BuildException):
+            push.podman_push('/etc/passwd', APPID, VERCODE)
+
+    def test_bad_relative_path(self):
+        self._only_run_if_container_exists()
+        with self.assertRaises(ValueError):
+            push.podman_push('../../etc/passwd', APPID, VERCODE)
+
+    def test_existing(self):
+        self._only_run_if_container_exists()
+        f = Path(f'metadata/{APPID}.yml')
+        f.parent.mkdir()
+        f.write_text(f.name)
+        push.podman_push(f, APPID, VERCODE)
+        common.inside_exec(APPID, VERCODE, ['test', '-e', str(f)], 'podman')

tests/test_up.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+
+import importlib
+import os
+import shutil
+import time
+import unittest
+
+from pathlib import Path
+from unittest import mock, skipIf, skipUnless
+
+from fdroidserver import common, exception, up
+from .shared_test_code import mkdtemp, APPID, VERCODE, APPID_VERCODE
+
+
+class UpTest(unittest.TestCase):
+    basedir = Path(__file__).resolve().parent
+
+    def setUp(self):
+        self._td = mkdtemp()
+        self.testdir = self._td.name
+        os.chdir(self.testdir)
+
+    def tearDown(self):
+        self._td.cleanup()
+        common.config = None
+
+
+class Up_main(UpTest):
+    def setUp(self):
+        super().setUp()
+        common.config = dict()
+
+    @skipIf(
+        importlib.util.find_spec("podman") is None or not shutil.which('podman'),
+        'Requires podman and podman-py to run.',
+    )
+    @mock.patch('sys.argv', ['fdroid up', APPID_VERCODE])
+    @mock.patch('fdroidserver.common.get_default_cachedir')
+    @mock.patch('fdroidserver.up.run_podman')
+    def test_podman(self, run_podman, get_default_cachedir):
+        get_default_cachedir.return_value = self.testdir
+        common.config['virt_container_type'] = 'podman'
+        up.main()
+        run_podman.assert_called_once()
+
+    @mock.patch('sys.argv', ['fdroid up', APPID_VERCODE])
+    @mock.patch('fdroidserver.common.get_default_cachedir')
+    @mock.patch('fdroidserver.up.run_vagrant')
+    def test_vagrant(self, run_vagrant, get_default_cachedir):
+        get_default_cachedir.return_value = self.testdir
+        common.config['virt_container_type'] = 'vagrant'
+        up.main()
+        run_vagrant.assert_called_once()
+
+
+@skipIf(
+    importlib.util.find_spec("podman") is None or not shutil.which('podman'),
+    'Requires podman and podman-py to run.',
+)
+class Up_run_podman(UpTest):
+    @skipUnless(
+        os.path.exists(f'/run/user/{os.getuid()}/podman/podman.sock'),
+        'Requires systemd podman.socket to run.',
+    )
+    def test_up_with_systemd_socket(self):
+        common.get_podman_client()
+
+    @skipIf(
+        os.path.exists(f'/run/user/{os.getuid()}/podman/podman.sock'),
+        'Requires the systemd podman.socket is not present.',
+    )
+    def test_up_with_podman_system_service_start(self):
+        common.get_podman_client()
+
+    def test_recreate_existing(self):
+        try:
+            common.get_podman_container(APPID, VERCODE)
+        except exception.BuildException as e:
+            # To run these tests, first do: `./fdroid up com.example:123`
+            self.skipTest(f'Requires Podman container {APPID_VERCODE} to run: {e}')
+
+        short_id = common.get_podman_container(APPID, VERCODE).short_id
+        up.run_podman(APPID, VERCODE)
+        self.assertNotEqual(
+            short_id,
+            common.get_podman_container(APPID, VERCODE).short_id,
+            "This should never reuse an existing container.",
+        )
+
+
+@skipIf(importlib.util.find_spec("podman") is None, 'Requires podman-py to run.')
+class Up_run_fake_podman(UpTest):
+    @skipIf(
+        os.path.exists(f'/run/user/{os.getuid()}/podman/podman.sock'),
+        'Requires the systemd podman.socket is not present.',
+    )
+    @mock.patch.dict(os.environ, clear=True)
+    @mock.patch("podman.PodmanClient.ping")
+    def test_up_with_podman_system_service_start(self, mock_client_ping):
+        """Test that the system service gets started if no socket is present."""
+        os.environ['PATH'] = os.path.join(self.testdir, 'bin')
+        os.mkdir('bin')
+        podman = Path('bin/podman')
+        podman.write_text('#!/bin/sh\nprintf "$1 $2" > args\n')
+        os.chmod(podman, 0o700)
+        common.get_podman_client()
+        self.assertEqual('system service', Path('args').read_text())
+        mock_client_ping.assert_called_once()
+
+
+@skipIf(importlib.util.find_spec("vagrant") is None, 'Requires python-vagrant to run.')
+class Up_run_vagrant(UpTest):
+    def setUp(self):
+        super().setUp()
+        b = mock.Mock()
+        b.name = 'buildserver'
+        self.box_list_return = [b]
+        name = common.get_container_name(APPID, VERCODE)
+        self.vagrantdir = Path('tmp/buildserver') / name
+
+    @mock.patch('vagrant.Vagrant.up')
+    @mock.patch('vagrant.Vagrant.box_list')
+    def test_no_existing(self, box_list, vagrant_up):
+        box_list.return_value = self.box_list_return
+        up.run_vagrant(APPID, VERCODE, 1, 1)
+        vagrant_up.assert_called_once()
+        self.assertTrue((Path(self.testdir) / self.vagrantdir).exists())
+
+    @mock.patch('vagrant.Vagrant.up')
+    @mock.patch('vagrant.Vagrant.destroy')
+    @mock.patch('vagrant.Vagrant.box_list')
+    def test_existing(self, box_list, vagrant_destroy, vagrant_up):
+        "This should never reuse an existing VM."
+        box_list.return_value = self.box_list_return
+        up.run_vagrant(APPID, VERCODE, 1, 1)
+        vagrantfile = self.vagrantdir / 'Vagrantfile'
+        ctime = os.path.getctime(vagrantfile)
+        vagrant_up.assert_called_once()
+
+        time.sleep(0.01)  # ensure reliable failure when testing ctime
+        up.run_vagrant(APPID, VERCODE, 1, 1)
+        vagrant_destroy.assert_called_once()
+        self.assertNotEqual(ctime, os.path.getctime(vagrantfile))