new subcommand "push": podman/vagrant implementation

fdroidserver/__main__.py

@@ -67,6 +67,7 @@ COMMANDS = OrderedDict([
 COMMANDS_INTERNAL = [
     "destroy",
     "exec",
+    "push",
     "up",
 ]
 

fdroidserver/common.py

@@ -4,10 +4,10 @@
 #
 # Copyright (C) 2010-2016, Ciaran Gultnieks, ciaran@ciarang.com
 # Copyright (C) 2013-2017, Daniel Martí <mvdan@mvdan.cc>
-# Copyright (C) 2013-2021, Hans-Christoph Steiner <hans@eds.org>
+# Copyright (C) 2013-2025, Hans-Christoph Steiner <hans@eds.org>
 # Copyright (C) 2017-2018, Torsten Grote <t@grobox.de>
 # Copyright (C) 2017, tobiasKaminsky <tobias@kaminsky.me>
-# Copyright (C) 2017-2021, Michael Pöhn <michael.poehn@fsfe.org>
+# Copyright (C) 2017-2025, Michael Pöhn <michael.poehn@fsfe.org>
 # Copyright (C) 2017,2021, mimi89999 <michel@lebihan.pl>
 # Copyright (C) 2019-2021, Jochen Sprickerhof <git@jochen.sprickerhof.de>
 # Copyright (C) 2021, Felix C. Stegerman <flx@obfusk.net>

fdroidserver/push.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+#
+# push.py - part of the FDroid server tools
+# Copyright (C) 2024-2025, Hans-Christoph Steiner <hans@eds.org>
+# Copyright (C) 2024-2025, Michael Pöhn <michael@poehn.at>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Push files into the build setup for a given ApplicationID:versionCode.
+
+This is a carefully constructed method to copy files from the host
+filesystem to inside the buildserver VM/container to run the build.  The
+source paths are generated based on the build metadata, and the
+destination is forced to build home.  This is not meant as a generic
+method for copying files, that is already provided by each VM/container
+system (e.g. `podman cp`).
+
+Since this is an internal command, the strings are not localized.
+
+"""
+
+import logging
+import os
+import sys
+import tarfile
+import tempfile
+import traceback
+from argparse import ArgumentParser
+from pathlib import Path
+
+from . import common, metadata
+from .exception import BuildException
+
+
+def podman_push(paths, appid, vercode, as_root=False):
+    """Push relative paths into the podman container using the tar method.
+
+    This builds up a tar file from the supplied paths to send into
+    container via put_archive(). This assumes it is running in the
+    base of fdroiddata and it will push into common.BUILD_HOME in the
+    container.
+
+    This is a version of podman.api.create_tar() that builds up via
+    adding paths rather than giving a base dir and an exclude list.
+
+    """
+
+    def _tar_perms_filter(tarinfo):
+        """Force perms to something safe."""
+        if '__pycache__' in tarinfo.name:
+            return
+
+        if as_root:
+            tarinfo.uid = tarinfo.gid = 0
+            tarinfo.uname = tarinfo.gname = 'root'
+        else:
+            tarinfo.uid = tarinfo.gid = 1000
+            tarinfo.uname = tarinfo.gname = common.BUILD_USER
+
+        if tarinfo.isdir():
+            tarinfo.mode = 0o0755
+        elif tarinfo.isfile():
+            if tarinfo.mode & 0o111:
+                tarinfo.mode = 0o0755
+            else:
+                tarinfo.mode = 0o0644
+        elif not tarinfo.issym():  # symlinks shouldn't need perms set
+            raise BuildException(f'{tarinfo.name} is not a file or directory!')
+        return tarinfo
+
+    if isinstance(paths, Path):
+        paths = [str(paths)]
+    if isinstance(paths, str):
+        paths = [paths]
+
+    container = common.get_podman_container(appid, vercode)
+    with tempfile.TemporaryFile() as tf:
+        with tarfile.TarFile(fileobj=tf, mode='w') as tar:
+            for f in paths:
+                if Path(f).is_absolute():
+                    raise BuildException(f'{f} must be relative to {Path.cwd()}')
+                # throw ValueError on bad path
+                f = (Path.cwd() / f).resolve().relative_to(Path.cwd())
+                tar.add(f, filter=_tar_perms_filter)
+        tf.seek(0)
+        container.put_archive(common.BUILD_HOME, tf.read())
+
+
+def vagrant_push(paths, appid, vercode):
+    """Push files into a build specific vagrant VM."""
+    # TODO implement with `vagrant ssh` and rsync?
+
+
+# TODO split out shared code into _podman.py, _vagrant.py, etc
+
+
+def push_wrapper(paths, appid, vercode, virt_container_type):
+    """Push standard set of files into VM/container."""
+    if virt_container_type == 'vagrant':
+        vagrant_push(paths, appid, vercode)
+    elif virt_container_type == 'podman':
+        podman_push(paths, appid, vercode)
+
+
+def make_file_list(appid, vercode):
+    """Assemble list of files/folders that go into this specific build."""
+    files = [f'build/{appid}', f'metadata/{appid}.yml']
+    app_dir = f'metadata/{appid}'
+    if Path(app_dir).exists():
+        files.append(app_dir)
+    app, build = metadata.get_single_build(appid, vercode)
+
+    for lib in build.srclibs:
+        srclib = common.getsrclib(lib, 'build/srclib', basepath=True, prepare=False)
+        if srclib:
+            # srclib metadata file
+            files.append(f"srclibs/{srclib[0]}.yml")
+            # srclib sourcecode
+            files.append(srclib[2])
+
+    extlib_paths = [f"build/extlib/{lib}" for lib in build.extlibs]
+    missing_extlibs = [p for p in extlib_paths if not os.path.exists(p)]
+    if any(missing_extlibs):
+        raise Exception(
+            "error: requested missing extlibs: {}".format(" ".join(missing_extlibs))
+        )
+    files.extend(extlib_paths)
+
+    return files
+
+
+def create_build_dirs(appid, vercode, virt_container_type):
+    """Create directories required for running builds."""
+    dirs = ('build', 'build/extlib', 'build/srclib', 'metadata', 'fdroidserver')
+    for dir_name in dirs:
+        cmd = ['mkdir', '--parents', f'{common.BUILD_HOME}/{dir_name}']
+        common.inside_exec(
+            appid,
+            vercode,
+            cmd,
+            virt_container_type,
+        )
+
+
+def full_push_sequence(appid, vercode, virt_container_type):
+    """Push all files into vm required for specified build."""
+    create_build_dirs(appid, vercode, virt_container_type)
+    push_wrapper(make_file_list(appid, vercode), appid, vercode, virt_container_type)
+
+    # fdroidserver is pushed in every build
+    cwd = Path('.').absolute()
+    try:
+        os.chdir(Path(__file__).parent.parent.parent)
+        push_wrapper(
+            [
+                'fdroidserver/fdroidserver',
+                'fdroidserver/fdroid',
+            ],
+            appid,
+            vercode,
+            virt_container_type,
+        )
+    finally:
+        os.chdir(cwd)
+
+
+def main():
+    """CLI main method for this subcommand."""
+    parser = ArgumentParser(
+        description="Push full build setup into the buildserver container/box."
+    )
+    common.setup_global_opts(parser)
+    common.setup_virt_container_type_opts(parser)
+    parser.add_argument(
+        "APPID:VERCODE",
+        help="Application ID with Version Code in the form APPID:VERCODE",
+    )
+    options = common.parse_args(parser)
+    common.set_console_logging(options.verbose)
+
+    try:
+        appid, vercode = common.split_pkg_arg(options.__dict__['APPID:VERCODE'])
+        full_push_sequence(
+            appid,
+            vercode,
+            common.get_virt_container_type(options),
+        )
+    except Exception as e:
+        if options.verbose:
+            logging.error(traceback.format_exc())
+        else:
+            logging.error(e)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

tests/test_push.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+
+import importlib
+import os
+import unittest
+
+from pathlib import Path
+from unittest import mock, skipIf
+
+from fdroidserver import common, exception, push
+from .shared_test_code import mkdtemp, APPID, VERCODE, APPID_VERCODE
+
+
+class PushTest(unittest.TestCase):
+    basedir = Path(__file__).resolve().parent
+
+    def setUp(self):
+        self._td = mkdtemp()
+        self.testdir = self._td.name
+        os.chdir(self.testdir)
+        common.config = dict()
+
+    def tearDown(self):
+        self._td.cleanup()
+        common.config = None
+
+
+class Push_main(PushTest):
+    def setUp(self):
+        super().setUp()
+        metadatapath = Path(common.get_metadatapath(APPID))
+        metadatapath.parent.mkdir()
+        metadatapath.write_text(f'Name: Test\nBuilds:\n - versionCode: {VERCODE}\n')
+
+    @mock.patch('sys.argv', ['fdroid push', APPID_VERCODE])
+    @mock.patch('fdroidserver.push.create_build_dirs')
+    @mock.patch('fdroidserver.push.podman_push')
+    def test_podman(self, podman_push, create_build_dirs):
+        common.config['virt_container_type'] = 'podman'
+        push.main()
+        create_build_dirs.assert_called_once()
+        podman_push.assert_called()
+
+    @mock.patch('sys.argv', ['fdroid push', APPID_VERCODE])
+    @mock.patch('fdroidserver.push.create_build_dirs')
+    @mock.patch('fdroidserver.push.vagrant_push')
+    def test_vagrant(self, vagrant_push, create_build_dirs):
+        common.config['virt_container_type'] = 'vagrant'
+        push.main()
+        create_build_dirs.assert_called_once()
+        vagrant_push.assert_called()
+
+
+@skipIf(importlib.util.find_spec("podman") is None, 'Requires podman-py to run.')
+class Push_podman_push(PushTest):
+    def _only_run_if_container_exists(self):
+        try:
+            common.get_podman_container(APPID, VERCODE)
+        except exception.BuildException as e:
+            # To run these tests, first do: `./fdroid up com.example:123`
+            self.skipTest(f'Requires Podman container {APPID_VERCODE} to run: {e}')
+
+    def test_no_existing_container(self):
+        appid = 'should.never.exist'
+        f = Path(f'metadata/{appid}.yml')
+        f.parent.mkdir()
+        f.write_text(f.name)
+        with self.assertRaises(exception.BuildException):
+            push.podman_push(f, appid, 9999)
+
+    def test_bad_absolute_path(self):
+        self._only_run_if_container_exists()
+        with self.assertRaises(exception.BuildException):
+            push.podman_push('/etc/passwd', APPID, VERCODE)
+
+    def test_bad_relative_path(self):
+        self._only_run_if_container_exists()
+        with self.assertRaises(ValueError):
+            push.podman_push('../../etc/passwd', APPID, VERCODE)
+
+    def test_existing(self):
+        self._only_run_if_container_exists()
+        f = Path(f'metadata/{APPID}.yml')
+        f.parent.mkdir()
+        f.write_text(f.name)
+        push.podman_push(f, APPID, VERCODE)
+        common.inside_exec(APPID, VERCODE, ['test', '-e', str(f)], 'podman')