From da032d517a5aa908ded509481bba49917b65e131 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 8 Oct 2025 11:13:16 +0200 Subject: [PATCH] new subcommand "pull": podman/vagrant implementation --- fdroidserver/__main__.py | 1 + fdroidserver/pull.py | 115 ++++++++++++++++++++++++++++++++++++ tests/test_pull.py | 122 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 238 insertions(+) create mode 100644 fdroidserver/pull.py create mode 100755 tests/test_pull.py diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index 73d32b97..0555c498 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -67,6 +67,7 @@ COMMANDS = OrderedDict([ COMMANDS_INTERNAL = [ "destroy", "exec", + "pull", "push", "up", ] diff --git a/fdroidserver/pull.py b/fdroidserver/pull.py new file mode 100644 index 00000000..8144470f --- /dev/null +++ b/fdroidserver/pull.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +# +# pull.py - part of the FDroid server tools +# Copyright (C) 2024-2025, Hans-Christoph Steiner +# Copyright (C) 2024-2025, Michael Pöhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Pull a single file from the build setup for a given ApplicationID:versionCode. + +This is a carefully constructed method to copy files from inside the +buildserver VM/container to the host filesystem for further processing +and publishing. The source path is forced to the within the app's +build dir, and the destination is forced to the repo's unsigned/ dir. +This is not meant as a generic method for getting files, that is already +provided by each VM/container system (e.g. `podman cp`). + +Since this is an internal command, the strings are not localized. + +""" + +import os +import sys +import logging +import tarfile +import tempfile +import traceback +from argparse import ArgumentParser + +from . import common, metadata + + +def podman_pull(appid, vercode, path): + """Implement `fdroid pull` for Podman (e.g. not `podman pull`).""" + path_in_container = os.path.join(common.BUILD_HOME, path) + container = common.get_podman_container(appid, vercode) + stream, stat = container.get_archive(path_in_container) + if not stat['linkTarget'].endswith(path) or stat['name'] != os.path.basename(path): + logging.warning(f'{path} not found!') + return + with tempfile.NamedTemporaryFile(prefix=".fdroidserver_pull_", suffix=".tar") as tf: + for i in stream: + tf.write(i) + tf.seek(0) + with tarfile.TarFile(fileobj=tf, mode='r') as tar: + tar.extract(stat['name'], 'unsigned', set_attrs=False) + + +def vagrant_pull(appid, vercode, path): + """Pull the path from the Vagrant VM.""" + + +def make_file_list(appid, vercode): + app, build = metadata.get_single_build(appid, vercode) + ext = common.get_output_extension(build) + return [ + os.path.join('unsigned', common.get_release_filename(app, build, ext)), + os.path.join( + 'unsigned', common.get_src_tarball_name(app.id, build.versionCode) + ), + ] + + +def pull_wrapper(appid, vercode, virt_container_type): + os.makedirs('unsigned', exist_ok=True) + files = make_file_list(appid, vercode) + for f in files: + logging.info(f"""Pulling {f} from {appid}:{vercode}""") + if virt_container_type == 'vagrant': + vagrant_pull(appid, vercode, f) + elif virt_container_type == 'podman': + podman_pull(appid, vercode, f) + + +def main(): + parser = ArgumentParser( + description="Pull build products from the buildserver container/box." + ) + common.setup_global_opts(parser) + common.setup_virt_container_type_opts(parser) + parser.add_argument( + "APPID:VERCODE", + help="Application ID with Version Code in the form APPID:VERCODE", + ) + options = common.parse_args(parser) + common.set_console_logging(options.verbose) + + try: + appid, vercode = common.split_pkg_arg(options.__dict__['APPID:VERCODE']) + pull_wrapper( + appid, + vercode, + common.get_virt_container_type(options), + ) + except Exception as e: + if options.verbose: + logging.error(traceback.format_exc()) + else: + logging.error(e) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/tests/test_pull.py b/tests/test_pull.py new file mode 100755 index 00000000..6d520818 --- /dev/null +++ b/tests/test_pull.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 + +import importlib +import os +import unittest + +from pathlib import Path +from unittest import mock, skipIf + +from fdroidserver import common, exception, pull +from .shared_test_code import mkdtemp, APPID, VERCODE, APPID_VERCODE + + +class PullTest(unittest.TestCase): + basedir = Path(__file__).resolve().parent + + def setUp(self): + self._td = mkdtemp() + self.testdir = self._td.name + os.chdir(self.testdir) + common.config = dict() + + def tearDown(self): + self._td.cleanup() + common.config = None + + +class Pull_main(PullTest): + def setUp(self): + super().setUp() + metadatapath = Path(common.get_metadatapath(APPID)) + metadatapath.parent.mkdir() + metadatapath.write_text(f'Name: Test\nBuilds:\n - versionCode: {VERCODE}\n') + + @mock.patch('sys.argv', ['fdroid pull', APPID_VERCODE]) + @mock.patch('fdroidserver.pull.podman_pull') + def test_podman(self, podman_pull): + common.config['virt_container_type'] = 'podman' + common.options = mock.Mock() + pull.main() + podman_pull.assert_called() + + @mock.patch('sys.argv', ['fdroid pull', APPID_VERCODE]) + @mock.patch('fdroidserver.pull.vagrant_pull') + def test_vagrant(self, vagrant_pull): + common.config['virt_container_type'] = 'vagrant' + pull.main() + vagrant_pull.assert_called() + + +@skipIf(importlib.util.find_spec("podman") is None, 'Requires podman-py to run.') +class Pull_podman_pull(PullTest): + def setUp(self): + try: + common.get_podman_container(APPID, VERCODE) + except exception.BuildException as e: + self.skipTest(f'Requires Podman container {APPID_VERCODE} to run: {e}') + super().setUp() + + def test_no_existing(self): + appid = 'should.never.exist' + with self.assertRaises(exception.BuildException) as e: + pull.podman_pull(appid, 9999, 'unsigned/foo.apk') + self.assertIn(appid, e.exception.value) + + def test_existing(self): + """Check files get deposited in unsigned/.""" + filename = 'buildserverid' + f = Path('unsigned') / filename + self.assertFalse(f.exists()) + pull.podman_pull(APPID, VERCODE, filename) + self.assertTrue(f.exists()) + + +class Pull_make_file_list(PullTest): + def setUp(self): + super().setUp() + self.metadatapath = Path(common.get_metadatapath(APPID)) + self.metadatapath.parent.mkdir() + + def test_implied(self): + self.metadatapath.write_text(f"""Builds: [versionCode: {VERCODE}]""") + self.assertEqual( + [ + f'unsigned/{APPID}_{VERCODE}.apk', + f'unsigned/{APPID}_{VERCODE}_src.tar.gz', + ], + pull.make_file_list(APPID, VERCODE), + ) + + def test_gradle(self): + self.metadatapath.write_text( + f"""Builds: + - versionCode: {VERCODE} + gradle: fdroid + """ + ) + self.assertEqual( + [ + f'unsigned/{APPID}_{VERCODE}.apk', + f'unsigned/{APPID}_{VERCODE}_src.tar.gz', + ], + pull.make_file_list(APPID, VERCODE), + ) + + def test_raw(self): + ext = 'foo' + Path(common.get_metadatapath(APPID)).write_text( + f"""Builds: + - versionCode: {VERCODE} + versionName: 1.0 + commit: cafebabe123 + output: path/to/output.{ext} + """ + ) + self.assertEqual( + [ + f'unsigned/{APPID}_{VERCODE}.{ext}', + f'unsigned/{APPID}_{VERCODE}_src.tar.gz', + ], + pull.make_file_list(APPID, VERCODE), + )