From 0a5f15dad765a3238fb8c5482e3e8fdfd37cb038 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 17 Jun 2025 11:28:06 +0200 Subject: [PATCH 01/14] nightly: GitLab URLs end in '.git' to avoid warning and redirects warning: redirecting to https://gitlab.com/fdroid/fdroidclient-nightly.git/ --- fdroidserver/nightly.py | 11 +++++------ tests/test_nightly.py | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 372390ea..5a422f47 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -325,14 +325,13 @@ def main(): if 'CI_PROJECT_PATH' in os.environ and 'CI_PROJECT_URL' in os.environ: # we are in GitLab CI repo_git_base = os.getenv('CI_PROJECT_PATH') + NIGHTLY - clone_url = os.getenv('CI_PROJECT_URL') + NIGHTLY + base_url = f"{os.getenv('CI_PROJECT_URL')}{NIGHTLY}" + clone_url = f'{base_url}.git' # avoid redirects while cloning repo_base = get_repo_base_url( - clone_url, repo_git_base, force_type='gitlab.com' - ) - servergitmirror = 'git@' + urlparse(clone_url).netloc + ':' + repo_git_base - deploy_key_url = ( - f'{clone_url}/-/settings/repository#js-deploy-keys-settings' + base_url, repo_git_base, force_type='gitlab.com' ) + servergitmirror = f'git@{urlparse(base_url).netloc}:{repo_git_base}.git' + deploy_key_url = f'{base_url}/-/settings/repository#js-deploy-keys-settings' git_user_name = os.getenv('GITLAB_USER_NAME') git_user_email = os.getenv('GITLAB_USER_EMAIL') elif 'TRAVIS_REPO_SLUG' in os.environ: diff --git a/tests/test_nightly.py b/tests/test_nightly.py index fb1614b7..e7c6991c 100755 --- a/tests/test_nightly.py +++ b/tests/test_nightly.py @@ -359,7 +359,7 @@ class NightlyTest(unittest.TestCase): 'repo_keyalias': 'androiddebugkey', 'repo_name': 'fdroid/test-nightly', 'repo_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/repo', - 'servergitmirrors': [{"url": 'git@gitlab.com:fdroid/test-nightly'}], + 'servergitmirrors': [{"url": 'git@gitlab.com:fdroid/test-nightly.git'}], } with open(common.CONFIG_FILE) as fp: config = yaml.safe_load(fp) From 86be5be09d6a27e1f7fa31854ac4a6a4a3469787 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 15 Sep 2025 14:45:26 +0200 Subject: [PATCH 02/14] import GitPython exceptions using public module to placate pylint ************* Module fdroidserver.nightly fdroidserver/nightly.py:239:11: E1101: Instance of 'GitError' has no 'GitCommandError' member (no-member) fdroidserver/nightly.py:239:11: E1101: Instance of 'Exception' has no 'GitCommandError' member (no-member) --- fdroidserver/__main__.py | 2 +- fdroidserver/nightly.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index 71c39b2c..83a0c81c 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -156,7 +156,7 @@ def main(): ).git.describe(always=True, tags=True) ) sys.exit(0) - except git.exc.InvalidGitRepositoryError: + except git.InvalidGitRepositoryError: print(_('No version information could be found.')) sys.exit(1) else: diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 5a422f47..88386029 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -236,7 +236,7 @@ def clone_git_repo(clone_url, git_mirror_path): 'GIT_TERMINAL_PROMPT': '0', }, ) - except git.exc.GitCommandError as e: + except git.GitCommandError as e: logging.warning(_('WARNING: only public git repos are supported!')) raise VCSException(f'git clone {clone_url} failed:', str(e)) from e From 61842d626d6032ffcdbd2c697fb574f78c74a7b9 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 18 Jun 2025 22:46:50 +0200 Subject: [PATCH 03/14] nightly: print error if env var is missing This should help with debugging a lot. Before this, it showed an obtuse stacktrace when an env var was not set: ``` Traceback (most recent call last): File "/usr/bin/fdroid", line 33, in sys.exit(load_entry_point('fdroidserver==2.4.0', 'console_scripts', 'fdroid')()) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3/dist-packages/fdroidserver/__main__.py", line 222, in main raise e File "/usr/lib/python3/dist-packages/fdroidserver/__main__.py", line 203, in main mod.main() File "/usr/lib/python3/dist-packages/fdroidserver/nightly.py", line 352, in main writer.set_value('user', 'name', git_user_name) File "/usr/lib/python3/dist-packages/git/config.py", line 122, in assure_data_present return func(self, *args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3/dist-packages/git/config.py", line 134, in flush_changes rval = non_const_func(self, *args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3/dist-packages/git/config.py", line 855, in set_value self.set(section, option, self._value_to_string(value)) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3/dist-packages/git/config.py", line 838, in _value_to_string return force_text(value) ^^^^^^^^^^^^^^^^^ File "/usr/lib/python3/dist-packages/gitdb/utils/encoding.py", line 18, in force_text return str(data, encoding) ^^^^^^^^^^^^^^^^^^^ TypeError: decoding to str: need a bytes-like object, NoneType found ``` --- fdroidserver/nightly.py | 42 +++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 88386029..0c2fbb9c 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -241,6 +241,16 @@ def clone_git_repo(clone_url, git_mirror_path): raise VCSException(f'git clone {clone_url} failed:', str(e)) from e +def _getenv(name): + """Return the value of an environment variable, printing an error if None.""" + value = os.getenv(name) + if not value: + logging.error( + _('Environment variable "{name}" has an empty value!').format(name=name) + ) + return value + + def main(): """Deploy to F-Droid repository or generate SSH private key from keystore. @@ -303,7 +313,7 @@ def main(): umask = os.umask(0o077) if 'CI' in os.environ: - v = os.getenv('DEBUG_KEYSTORE') + v = _getenv('DEBUG_KEYSTORE') debug_keystore = None if v: debug_keystore = base64.b64decode(v) @@ -324,19 +334,19 @@ def main(): # the 'master' branch is hardcoded in fdroidserver/deploy.py if 'CI_PROJECT_PATH' in os.environ and 'CI_PROJECT_URL' in os.environ: # we are in GitLab CI - repo_git_base = os.getenv('CI_PROJECT_PATH') + NIGHTLY - base_url = f"{os.getenv('CI_PROJECT_URL')}{NIGHTLY}" + repo_git_base = f"{_getenv('CI_PROJECT_PATH')}{NIGHTLY}" + base_url = f"{_getenv('CI_PROJECT_URL')}{NIGHTLY}" clone_url = f'{base_url}.git' # avoid redirects while cloning repo_base = get_repo_base_url( base_url, repo_git_base, force_type='gitlab.com' ) servergitmirror = f'git@{urlparse(base_url).netloc}:{repo_git_base}.git' deploy_key_url = f'{base_url}/-/settings/repository#js-deploy-keys-settings' - git_user_name = os.getenv('GITLAB_USER_NAME') - git_user_email = os.getenv('GITLAB_USER_EMAIL') + git_user_name = _getenv('GITLAB_USER_NAME') + git_user_email = _getenv('GITLAB_USER_EMAIL') elif 'TRAVIS_REPO_SLUG' in os.environ: # we are in Travis CI - repo_git_base = os.getenv('TRAVIS_REPO_SLUG') + NIGHTLY + repo_git_base = _getenv('TRAVIS_REPO_SLUG') + NIGHTLY clone_url = 'https://github.com/' + repo_git_base repo_base = get_repo_base_url( clone_url, repo_git_base, force_type='github.com' @@ -347,7 +357,7 @@ def main(): + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys' ) git_user_name = repo_git_base - git_user_email = os.getenv('USER') + '@' + platform.node() + git_user_email = _getenv('USER') + '@' + platform.node() elif ( 'CIRCLE_REPOSITORY_URL' in os.environ and 'CIRCLE_PROJECT_USERNAME' in os.environ @@ -355,12 +365,12 @@ def main(): ): # we are in Circle CI repo_git_base = ( - os.getenv('CIRCLE_PROJECT_USERNAME') + _getenv('CIRCLE_PROJECT_USERNAME') + '/' - + os.getenv('CIRCLE_PROJECT_REPONAME') + + _getenv('CIRCLE_PROJECT_REPONAME') + NIGHTLY ) - clone_url = os.getenv('CIRCLE_REPOSITORY_URL') + NIGHTLY + clone_url = _getenv('CIRCLE_REPOSITORY_URL') + NIGHTLY repo_base = get_repo_base_url( clone_url, repo_git_base, force_type='github.com' ) @@ -369,12 +379,12 @@ def main(): f'https://github.com/{repo_git_base}/settings/keys' + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys' ) - git_user_name = os.getenv('CIRCLE_USERNAME') + git_user_name = _getenv('CIRCLE_USERNAME') git_user_email = git_user_name + '@' + platform.node() elif 'GITHUB_ACTIONS' in os.environ: # we are in Github actions - repo_git_base = os.getenv('GITHUB_REPOSITORY') + NIGHTLY - clone_url = os.getenv('GITHUB_SERVER_URL') + '/' + repo_git_base + repo_git_base = _getenv('GITHUB_REPOSITORY') + NIGHTLY + clone_url = _getenv('GITHUB_SERVER_URL') + '/' + repo_git_base repo_base = get_repo_base_url( clone_url, repo_git_base, force_type='github.com' ) @@ -383,7 +393,7 @@ def main(): f'https://github.com/{repo_git_base}/settings/keys' + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys' ) - git_user_name = os.getenv('GITHUB_ACTOR') + git_user_name = _getenv('GITHUB_ACTOR') git_user_email = git_user_name + '@' + platform.node() else: print(_('ERROR: unsupported CI type, patches welcome!')) @@ -441,7 +451,7 @@ Last updated: {date}'''.format( ssh_private_key_file = _ssh_key_from_debug_keystore() # this is needed for GitPython to find the SSH key - ssh_dir = os.path.join(os.getenv('HOME'), '.ssh') + ssh_dir = os.path.join(_getenv('HOME'), '.ssh') os.makedirs(ssh_dir, exist_ok=True) ssh_config = os.path.join(ssh_dir, 'config') logging.debug(_('adding IdentityFile to {path}').format(path=ssh_config)) @@ -586,7 +596,7 @@ Last updated: {date}'''.format( + '\n -dname "CN=Android Debug,O=Android,C=US"' ) sys.exit(1) - ssh_dir = os.path.join(os.getenv('HOME'), '.ssh') + ssh_dir = os.path.join(_getenv('HOME'), '.ssh') privkey = _ssh_key_from_debug_keystore(options.keystore) if os.path.exists(ssh_dir): ssh_private_key_file = os.path.join(ssh_dir, os.path.basename(privkey)) From b6e73345e2d4e9f8722d35cc94a1591576bfabcc Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 23 Jul 2025 16:31:18 +0200 Subject: [PATCH 04/14] deploy: convert to common.get_config() 717df09be020e3a3ac613a0f5e5625263c7f08b1 --- fdroidserver/deploy.py | 6 ++-- tests/test_deploy.py | 72 +++++++++++++++------------------------ tests/test_integration.py | 14 ++++---- 3 files changed, 38 insertions(+), 54 deletions(-) diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index f1dcce21..4651cd76 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -40,7 +40,6 @@ import fdroidserver.github from . import _, common, index from .exception import FDroidException -config = None start_timestamp = time.gmtime() GIT_BRANCH = 'master' @@ -144,6 +143,7 @@ def update_remote_storage_with_rclone( """ logging.debug(_('Using rclone to sync to "{name}"').format(name=awsbucket)) + config = common.get_config() rclone_config = config.get('rclone_config', []) if rclone_config and isinstance(rclone_config, str): rclone_config = [rclone_config] @@ -271,6 +271,7 @@ def update_serverwebroot(serverwebroot, repo_section): has a low resolution timestamp """ + config = common.get_config() try: subprocess.run(['rsync', '--version'], capture_output=True, check=True) except Exception as e: @@ -431,6 +432,7 @@ def update_servergitmirrors(servergitmirrors, repo_section): """ from clint.textui import progress + config = common.get_config() if config.get('local_copy_dir') and not config.get('sync_from_local_copy_dir'): logging.debug( _('Offline machine, skipping git mirror generation until `fdroid deploy`') @@ -1029,8 +1031,6 @@ def upload_to_github_releases_repo(repo_conf, release_infos, global_gh_token): def main(): - global config - parser = ArgumentParser() common.setup_global_opts(parser) parser.add_argument( diff --git a/tests/test_deploy.py b/tests/test_deploy.py index d7de7545..bbf69621 100755 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -32,9 +32,10 @@ class DeployTest(unittest.TestCase): self.testdir = self._td.name fdroidserver.common.options = mock.Mock() - fdroidserver.deploy.config = {} + fdroidserver.common.get_config() def tearDown(self): + fdroidserver.common.config = None self._td.cleanup() def test_update_serverwebroots_bad_None(self): @@ -64,7 +65,6 @@ class DeployTest(unittest.TestCase): # setup parameters for this test run fdroidserver.common.options.identity_file = None - fdroidserver.deploy.config['make_current_version_link'] = False dest_apk0 = url0 / fake_apk dest_apk1 = url1 / fake_apk @@ -120,9 +120,9 @@ class DeployTest(unittest.TestCase): # setup parameters for this test run awsbucket = 'test_bucket_folder' - fdroidserver.deploy.config['awsbucket'] = awsbucket - fdroidserver.deploy.config['rclone_config'] = 'test-local-config' - fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file) + fdroidserver.common.config['awsbucket'] = awsbucket + fdroidserver.common.config['rclone_config'] = 'test-local-config' + fdroidserver.common.config['path_to_custom_rclone_config'] = str(rclone_file) fdroidserver.common.options = VerboseFalseOptions # write out destination path @@ -163,9 +163,9 @@ class DeployTest(unittest.TestCase): # setup parameters for this test run awsbucket = 'test_bucket_folder' - fdroidserver.deploy.config['awsbucket'] = awsbucket - fdroidserver.deploy.config['rclone_config'] = 'test-local-config' - fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file) + fdroidserver.common.config['awsbucket'] = awsbucket + fdroidserver.common.config['rclone_config'] = 'test-local-config' + fdroidserver.common.config['path_to_custom_rclone_config'] = str(rclone_file) fdroidserver.common.options = VerboseFalseOptions # write out destination path @@ -224,7 +224,7 @@ class DeployTest(unittest.TestCase): return 0 mock_call.side_effect = _mock_subprocess_call - fdroidserver.deploy.config = {'awsbucket': awsbucket} + fdroidserver.common.config = {'awsbucket': awsbucket} fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket) mock_call.assert_called() @@ -243,7 +243,7 @@ class DeployTest(unittest.TestCase): mock_call.side_effect = _mock_subprocess_call - fdroidserver.deploy.config = {'awsbucket': awsbucket} + fdroidserver.common.config = {'awsbucket': awsbucket} fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket) self.maxDiff = None self.assertEqual( @@ -262,7 +262,6 @@ class DeployTest(unittest.TestCase): @mock.patch('subprocess.check_output', _mock_rclone_config_file) @mock.patch('subprocess.call') def test_update_remote_storage_with_rclone_mock_rclone_config(self, mock_call): - awsbucket = 'test_bucket_folder' self.last_cmd = None def _mock_subprocess_call(cmd): @@ -271,10 +270,9 @@ class DeployTest(unittest.TestCase): mock_call.side_effect = _mock_subprocess_call - fdroidserver.deploy.config = { - 'awsbucket': awsbucket, - 'rclone_config': 'test_local_config', - } + awsbucket = 'test_bucket_folder' + fdroidserver.common.config['awsbucket'] = awsbucket + fdroidserver.common.config['rclone_config'] = 'test_local_config' fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket) self.maxDiff = None self.assertEqual( @@ -304,8 +302,8 @@ class DeployTest(unittest.TestCase): Path('rclone.conf').write_text('placeholder, contents ignored') awsbucket = 'test_bucket_folder' - fdroidserver.deploy.config['awsbucket'] = awsbucket - fdroidserver.deploy.config['rclone_config'] = config_name + fdroidserver.common.config['awsbucket'] = awsbucket + fdroidserver.common.config['rclone_config'] = config_name fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket) self.maxDiff = None self.assertEqual( @@ -339,7 +337,6 @@ class DeployTest(unittest.TestCase): fdroidserver.common.options = mock.Mock() fdroidserver.common.options.identity_file = None fdroidserver.common.options.identity_file = None - fdroidserver.deploy.config['make_current_version_link'] = False dest_apk = Path(url) / fake_apk dest_index = Path(url) / fake_index @@ -366,7 +363,6 @@ class DeployTest(unittest.TestCase): # setup parameters for this test run fdroidserver.common.options = mock.Mock() fdroidserver.common.options.identity_file = None - fdroidserver.deploy.config['make_current_version_link'] = False dest_apk = Path(url) / fake_apk dest_index = Path(url) / fake_index @@ -396,7 +392,7 @@ class DeployTest(unittest.TestCase): fdroidserver.common.options.verbose = False fdroidserver.common.options.quiet = True fdroidserver.common.options.index_only = False - fdroidserver.deploy.config = {'make_current_version_link': True} + fdroidserver.common.config['make_current_version_link'] = True url = "example.com:/var/www/fdroid" repo_section = 'repo' @@ -504,7 +500,7 @@ class DeployTest(unittest.TestCase): fdroidserver.common.options.verbose = False fdroidserver.common.options.quiet = True fdroidserver.common.options.identity_file = None - fdroidserver.deploy.config['make_current_version_link'] = True + fdroidserver.common.config['make_current_version_link'] = True url = "example.com:/var/www/fdroid" repo_section = 'repo' @@ -603,7 +599,7 @@ class DeployTest(unittest.TestCase): fdroidserver.common.options.quiet = False fdroidserver.common.options.identity_file = None fdroidserver.common.options.index_only = False - fdroidserver.deploy.config = {'identity_file': './id_rsa'} + fdroidserver.common.config = {'identity_file': './id_rsa'} url = "example.com:/var/www/fdroid" repo_section = 'archive' @@ -623,7 +619,7 @@ class DeployTest(unittest.TestCase): '--verbose', '-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' - + fdroidserver.deploy.config['identity_file'], + + fdroidserver.common.config['identity_file'], '--exclude', 'archive/altstore-index.json', '--exclude', @@ -669,7 +665,7 @@ class DeployTest(unittest.TestCase): '--verbose', '-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' - + fdroidserver.deploy.config['identity_file'], + + fdroidserver.common.config['identity_file'], 'archive', url, ], @@ -690,8 +686,7 @@ class DeployTest(unittest.TestCase): fdroidserver.common.options.verbose = True fdroidserver.common.options.quiet = False fdroidserver.common.options.identity_file = None - fdroidserver.deploy.config['identity_file'] = './id_rsa' - fdroidserver.deploy.config['make_current_version_link'] = False + fdroidserver.common.config['identity_file'] = './id_rsa' url = "example.com:/var/www/fdroid" repo_section = 'archive' @@ -711,7 +706,7 @@ class DeployTest(unittest.TestCase): '--verbose', '-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' - + fdroidserver.deploy.config['identity_file'], + + fdroidserver.common.config['identity_file'], 'archive/altstore-index.json', 'archive/altstore-index.json.asc', 'archive/entry.jar', @@ -741,7 +736,7 @@ class DeployTest(unittest.TestCase): '--verbose', '-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' - + fdroidserver.deploy.config['identity_file'], + + fdroidserver.common.config['identity_file'], "example.com:/var/www/fdroid/archive/", ], ) @@ -793,10 +788,6 @@ class DeployTest(unittest.TestCase): fdroidserver.common.options.verbose = False fdroidserver.common.options.quiet = True - config = {} - fdroidserver.common.fill_config_defaults(config) - fdroidserver.deploy.config = config - os.chdir(self.testdir) repo_section = 'repo' @@ -807,7 +798,8 @@ class DeployTest(unittest.TestCase): remote_git_repo = git.Repo.init( remote_repo, initial_branch=initial_branch, bare=True ) - fdroidserver.deploy.config["servergitmirrors"] = [{"url": str(remote_repo)}] + fdroidserver.common.get_config() + fdroidserver.common.config["servergitmirrors"] = [{"url": str(remote_repo)}] os.chdir(self.testdir) repo = Path('repo') @@ -820,7 +812,7 @@ class DeployTest(unittest.TestCase): fp.write('not a real one, but has the right filename') fdroidserver.deploy.update_servergitmirrors( - fdroidserver.deploy.config["servergitmirrors"], repo_section + fdroidserver.common.config["servergitmirrors"], repo_section ) verify_repo = remote_git_repo.clone( @@ -844,10 +836,6 @@ class DeployTest(unittest.TestCase): fdroidserver.common.options.verbose = False fdroidserver.common.options.quiet = True - config = {} - fdroidserver.common.fill_config_defaults(config) - fdroidserver.deploy.config = config - os.chdir(self.testdir) repo_section = 'repo' @@ -858,7 +846,7 @@ class DeployTest(unittest.TestCase): remote_git_repo = git.Repo.init( remote_repo, initial_branch=initial_branch, bare=True ) - fdroidserver.deploy.config["servergitmirrors"] = [ + fdroidserver.common.config["servergitmirrors"] = [ {"url": str(remote_repo), "index_only": True} ] @@ -873,7 +861,7 @@ class DeployTest(unittest.TestCase): fp.write('not a real one, but has the right filename') fdroidserver.deploy.update_servergitmirrors( - fdroidserver.deploy.config["servergitmirrors"], repo_section + fdroidserver.common.config["servergitmirrors"], repo_section ) verify_repo = remote_git_repo.clone( @@ -905,10 +893,6 @@ class DeployTest(unittest.TestCase): fdroidserver.common.options.quiet = True fdroidserver.common.options.identity_file = None - config = {} - fdroidserver.common.fill_config_defaults(config) - fdroidserver.deploy.config = config - repo_section = 'repo' initial_branch = fdroidserver.deploy.GIT_BRANCH diff --git a/tests/test_integration.py b/tests/test_integration.py index 2cdf19d9..d051275b 100755 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -75,7 +75,6 @@ class IntegrationTest(unittest.TestCase): self.testdir = mkdir_testfiles(WORKSPACE, self) self.tmp_repo_root = self.testdir / "fdroid" self.tmp_repo_root.mkdir(parents=True) - deploy.config = {} os.chdir(self.tmp_repo_root) def tearDown(self): @@ -1611,10 +1610,11 @@ class IntegrationTest(unittest.TestCase): rclone_config.write(configfile) # set up config for run + common.get_config() awsbucket = "test-bucket" - deploy.config['awsbucket'] = awsbucket - deploy.config['rclone_config'] = "test-minio-config" - deploy.config['path_to_custom_rclone_config'] = str(rclone_file) + common.config['awsbucket'] = awsbucket + common.config['rclone_config'] = "test-minio-config" + common.config['path_to_custom_rclone_config'] = str(rclone_file) common.options = VerboseFalseOptions # call function @@ -1667,9 +1667,9 @@ class IntegrationTest(unittest.TestCase): # set up config for run awsbucket = "test-bucket" - deploy.config['awsbucket'] = awsbucket - deploy.config['rclone_config'] = "test-minio-config" - deploy.config['path_to_custom_rclone_config'] = str(rclone_file) + common.config['awsbucket'] = awsbucket + common.config['rclone_config'] = "test-minio-config" + common.config['path_to_custom_rclone_config'] = str(rclone_file) common.options = VerboseFalseOptions # call function From a96e647b2f9b2f1df0473558f73735349871af43 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 18 Jun 2025 13:56:39 +0200 Subject: [PATCH 05/14] deploy: common test setup for all tests of servergitmirrors This refactors the servergitmirrors tests into their own class. --- tests/test_deploy.py | 80 +++++++++++++------------------------------- 1 file changed, 23 insertions(+), 57 deletions(-) diff --git a/tests/test_deploy.py b/tests/test_deploy.py index bbf69621..baead67c 100755 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -780,7 +780,11 @@ class DeployTest(unittest.TestCase): name, fdroidserver.deploy.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url) ) - def test_update_servergitmirrors(self): + +class TestServerGitMirrors(unittest.TestCase): + def setUp(self): + fdroidserver.deploy.USER_RCLONE_CONF = False + # setup parameters for this test run fdroidserver.common.options = mock.Mock() fdroidserver.common.options.identity_file = None @@ -788,20 +792,26 @@ class DeployTest(unittest.TestCase): fdroidserver.common.options.verbose = False fdroidserver.common.options.quiet = True + self._td = mkdtemp() + self.testdir = self._td.name os.chdir(self.testdir) - repo_section = 'repo' - initial_branch = fdroidserver.deploy.GIT_BRANCH - remote_repo = Path(self.testdir) / 'remote' remote_repo.mkdir(parents=True) - remote_git_repo = git.Repo.init( - remote_repo, initial_branch=initial_branch, bare=True + self.remote_git_repo = git.Repo.init( + remote_repo, initial_branch=fdroidserver.deploy.GIT_BRANCH, bare=True ) + fdroidserver.common.get_config() fdroidserver.common.config["servergitmirrors"] = [{"url": str(remote_repo)}] - os.chdir(self.testdir) + def tearDown(self): + fdroidserver.common.config = None + fdroidserver.common.options = None + self._td.cleanup() + + def test_update_servergitmirrors(self): + repo_section = 'repo' repo = Path('repo') repo.mkdir(parents=True) fake_apk = 'Sym.apk' @@ -815,9 +825,7 @@ class DeployTest(unittest.TestCase): fdroidserver.common.config["servergitmirrors"], repo_section ) - verify_repo = remote_git_repo.clone( - Path(self.testdir) / 'verify', - ) + verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify') for filename in fake_files: remote_file = f"fdroid/{repo_section}/{filename}" @@ -829,28 +837,8 @@ class DeployTest(unittest.TestCase): ) def test_update_servergitmirrors_in_index_only_mode(self): - # setup parameters for this test run - fdroidserver.common.options = mock.Mock() - fdroidserver.common.options.identity_file = None - fdroidserver.common.options.no_keep_git_mirror_archive = False - fdroidserver.common.options.verbose = False - fdroidserver.common.options.quiet = True - - os.chdir(self.testdir) - + fdroidserver.common.config["servergitmirrors"][0]["index_only"] = True repo_section = 'repo' - initial_branch = fdroidserver.deploy.GIT_BRANCH - - remote_repo = Path(self.testdir) / 'remote' - remote_repo.mkdir(parents=True) - remote_git_repo = git.Repo.init( - remote_repo, initial_branch=initial_branch, bare=True - ) - fdroidserver.common.config["servergitmirrors"] = [ - {"url": str(remote_repo), "index_only": True} - ] - - os.chdir(self.testdir) repo = Path('repo') repo.mkdir(parents=True) fake_apk = 'Sym.apk' @@ -864,9 +852,7 @@ class DeployTest(unittest.TestCase): fdroidserver.common.config["servergitmirrors"], repo_section ) - verify_repo = remote_git_repo.clone( - Path(self.testdir) / 'verify', - ) + verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify') for filename in fdroidserver.common.INDEX_FILES: remote_file = f"fdroid/{repo_section}/{filename}" @@ -885,22 +871,10 @@ class DeployTest(unittest.TestCase): ) def test_upload_to_servergitmirror_in_index_only_mode(self): - # setup parameters for this test run - fdroidserver.common.options = mock.Mock() - fdroidserver.common.options.identity_file = None - fdroidserver.common.options.no_keep_git_mirror_archive = False - fdroidserver.common.options.verbose = False - fdroidserver.common.options.quiet = True - fdroidserver.common.options.identity_file = None - repo_section = 'repo' - initial_branch = fdroidserver.deploy.GIT_BRANCH - - os.chdir(self.testdir) - local_git_repo_path = Path(self.testdir) / 'local' local_git_repo = git.Repo.init( - local_git_repo_path, initial_branch=initial_branch + local_git_repo_path, initial_branch=fdroidserver.deploy.GIT_BRANCH ) fdroid_dir = local_git_repo_path / 'fdroid' @@ -913,13 +887,7 @@ class DeployTest(unittest.TestCase): with fake_file.open('w') as fp: fp.write('not a real one, but has the right filename') - # The remote repo must be a bare repo to allow being pushed to - remote_git_repo_dir = Path(self.testdir) / 'remote' - remote_git_repo = git.Repo.init( - remote_git_repo_dir, initial_branch=initial_branch, bare=True - ) - - mirror_config = {"url": str(remote_git_repo_dir), "index_only": True} + mirror_config = {"url": str(self.remote_git_repo.git_dir), "index_only": True} enabled_remotes = [] ssh_cmd = 'ssh -oBatchMode=yes' fdroidserver.deploy.upload_to_servergitmirror( @@ -934,9 +902,7 @@ class DeployTest(unittest.TestCase): progress=git.RemoteProgress(), ) - verify_repo = remote_git_repo.clone( - Path(self.testdir) / 'verify', - ) + verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify') for filename in fdroidserver.common.INDEX_FILES: remote_file = f"fdroid/{repo_section}/{filename}" From 5fcc5b12521ee6f4b6005b70051b35c1e3c97fe8 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 18 Jun 2025 14:02:50 +0200 Subject: [PATCH 06/14] deploy: simplify asserts in servergitmirrors tests --- tests/test_deploy.py | 37 ++++++++----------------------------- 1 file changed, 8 insertions(+), 29 deletions(-) diff --git a/tests/test_deploy.py b/tests/test_deploy.py index baead67c..a4b93b41 100755 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -826,15 +826,10 @@ class TestServerGitMirrors(unittest.TestCase): ) verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify') - + self.assertIsNotNone(verify_repo.working_tree_dir) for filename in fake_files: remote_file = f"fdroid/{repo_section}/{filename}" - - self.assertIsNotNone(verify_repo.working_tree_dir) - if verify_repo.working_tree_dir is not None: - self.assertTrue( - (Path(verify_repo.working_tree_dir) / remote_file).exists() - ) + self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists()) def test_update_servergitmirrors_in_index_only_mode(self): fdroidserver.common.config["servergitmirrors"][0]["index_only"] = True @@ -853,22 +848,14 @@ class TestServerGitMirrors(unittest.TestCase): ) verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify') - + self.assertIsNotNone(verify_repo.working_tree_dir) for filename in fdroidserver.common.INDEX_FILES: remote_file = f"fdroid/{repo_section}/{filename}" - - self.assertIsNotNone(verify_repo.working_tree_dir) - if verify_repo.working_tree_dir is not None: - self.assertTrue( - (Path(verify_repo.working_tree_dir) / remote_file).exists() - ) + self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists()) # Should not have the APK file remote_file = f"fdroid/{repo_section}/{fake_apk}" - if verify_repo.working_tree_dir is not None: - self.assertFalse( - (Path(verify_repo.working_tree_dir) / remote_file).exists() - ) + self.assertFalse((Path(verify_repo.working_tree_dir) / remote_file).exists()) def test_upload_to_servergitmirror_in_index_only_mode(self): repo_section = 'repo' @@ -903,22 +890,14 @@ class TestServerGitMirrors(unittest.TestCase): ) verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify') - + self.assertIsNotNone(verify_repo.working_tree_dir) for filename in fdroidserver.common.INDEX_FILES: remote_file = f"fdroid/{repo_section}/{filename}" - - self.assertIsNotNone(verify_repo.working_tree_dir) - if verify_repo.working_tree_dir is not None: - self.assertTrue( - (Path(verify_repo.working_tree_dir) / remote_file).exists() - ) + self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists()) # Should not have the APK file remote_file = f"fdroid/{repo_section}/{fake_apk}" - if verify_repo.working_tree_dir is not None: - self.assertFalse( - (Path(verify_repo.working_tree_dir) / remote_file).exists() - ) + self.assertFalse((Path(verify_repo.working_tree_dir) / remote_file).exists()) class GitHubReleasesTest(unittest.TestCase): From 04ec16d68a0720151cac8c22ab33df3e8170d08e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 18 Jun 2025 19:15:53 +0200 Subject: [PATCH 07/14] deploy: move remote repo to common test setup in TestServerGitMirrors --- tests/test_deploy.py | 43 ++++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/tests/test_deploy.py b/tests/test_deploy.py index a4b93b41..fcc90bd3 100755 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -805,59 +805,52 @@ class TestServerGitMirrors(unittest.TestCase): fdroidserver.common.get_config() fdroidserver.common.config["servergitmirrors"] = [{"url": str(remote_repo)}] + self.repo_section = 'repo' + repo = Path(self.repo_section) + repo.mkdir() + self.fake_apk = 'Sym.apk' + self.fake_files = fdroidserver.common.INDEX_FILES + [self.fake_apk] + for filename in self.fake_files: + fake_file = repo / filename + with fake_file.open('w') as fp: + fp.write('not a real one, but has the right filename') + def tearDown(self): fdroidserver.common.config = None fdroidserver.common.options = None self._td.cleanup() def test_update_servergitmirrors(self): - repo_section = 'repo' - repo = Path('repo') - repo.mkdir(parents=True) - fake_apk = 'Sym.apk' - fake_files = fdroidserver.common.INDEX_FILES + [fake_apk] - for filename in fake_files: - fake_file = repo / filename - with fake_file.open('w') as fp: - fp.write('not a real one, but has the right filename') - fdroidserver.deploy.update_servergitmirrors( - fdroidserver.common.config["servergitmirrors"], repo_section + fdroidserver.common.config["servergitmirrors"], self.repo_section ) verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify') self.assertIsNotNone(verify_repo.working_tree_dir) - for filename in fake_files: - remote_file = f"fdroid/{repo_section}/{filename}" + for filename in self.fake_files: + remote_file = f"fdroid/{self.repo_section}/{filename}" self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists()) def test_update_servergitmirrors_in_index_only_mode(self): fdroidserver.common.config["servergitmirrors"][0]["index_only"] = True - repo_section = 'repo' - repo = Path('repo') - repo.mkdir(parents=True) - fake_apk = 'Sym.apk' - fake_files = fdroidserver.common.INDEX_FILES + [fake_apk] - for filename in fake_files: - fake_file = repo / filename - with fake_file.open('w') as fp: - fp.write('not a real one, but has the right filename') fdroidserver.deploy.update_servergitmirrors( - fdroidserver.common.config["servergitmirrors"], repo_section + fdroidserver.common.config["servergitmirrors"], self.repo_section ) verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify') self.assertIsNotNone(verify_repo.working_tree_dir) for filename in fdroidserver.common.INDEX_FILES: - remote_file = f"fdroid/{repo_section}/{filename}" + remote_file = f"fdroid/{self.repo_section}/{filename}" self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists()) # Should not have the APK file - remote_file = f"fdroid/{repo_section}/{fake_apk}" + remote_file = f"fdroid/{self.repo_section}/{self.fake_apk}" self.assertFalse((Path(verify_repo.working_tree_dir) / remote_file).exists()) def test_upload_to_servergitmirror_in_index_only_mode(self): + shutil.rmtree('repo') # the class-wide test files are not used here + repo_section = 'repo' local_git_repo_path = Path(self.testdir) / 'local' local_git_repo = git.Repo.init( From e8f7771c1fe0feaa40682b63696b19d40973d610 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 18 Jun 2025 14:26:07 +0200 Subject: [PATCH 08/14] tests: suppress "WARNING:root:unsafe permissions on 'config.yml' (should be 0600)!" --- tests/test_deploy.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_deploy.py b/tests/test_deploy.py index fcc90bd3..f9371c68 100755 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -26,6 +26,11 @@ def _mock_rclone_config_file(cmd, text): # pylint: disable=unused-argument class DeployTest(unittest.TestCase): '''fdroidserver/deploy.py''' + @classmethod + def setUpClass(cls): + # suppress "WARNING:root:unsafe permissions on 'config.yml' (should be 0600)!" + os.chmod(os.path.join(basedir, fdroidserver.common.CONFIG_FILE), 0o600) + def setUp(self): os.chdir(basedir) self._td = mkdtemp() From 8601749734a44bfa56afc9b6aaa5188823fb7ded Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 18 Jun 2025 14:36:22 +0200 Subject: [PATCH 09/14] deploy: assert logs when error messges should be thrown --- tests/test_deploy.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_deploy.py b/tests/test_deploy.py index f9371c68..1464883a 100755 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import configparser +import logging import os import shutil import tempfile @@ -86,17 +87,17 @@ class DeployTest(unittest.TestCase): self.assertTrue(dest_apk1.is_file()) def test_update_serverwebroots_url_does_not_end_with_fdroid(self): - with self.assertRaises(SystemExit): + with self.assertRaises(SystemExit), self.assertLogs(level=logging.ERROR): fdroidserver.deploy.update_serverwebroots([{'url': 'url'}], 'repo') def test_update_serverwebroots_bad_ssh_url(self): - with self.assertRaises(SystemExit): + with self.assertRaises(SystemExit), self.assertLogs(level=logging.ERROR): fdroidserver.deploy.update_serverwebroots( [{'url': 'f@b.ar::/path/to/fdroid'}], 'repo' ) def test_update_serverwebroots_unsupported_ssh_url(self): - with self.assertRaises(SystemExit): + with self.assertRaises(SystemExit), self.assertLogs(level=logging.ERROR): fdroidserver.deploy.update_serverwebroots([{'url': 'ssh://nope'}], 'repo') @unittest.skipUnless(shutil.which('rclone'), 'requires rclone') From 473068f31102886b7f4addd3f327056fde9afb77 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 18 Jun 2025 19:16:30 +0200 Subject: [PATCH 10/14] deploy: test case for !1666 This is included here and not there because it relies on lots of stuff that was refactored. !1666 was broken out to get the fix out for fdroidclient as soon as possible. --- tests/test_deploy.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_deploy.py b/tests/test_deploy.py index 1464883a..db9cb50b 100755 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -837,6 +837,30 @@ class TestServerGitMirrors(unittest.TestCase): remote_file = f"fdroid/{self.repo_section}/{filename}" self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists()) + def test_update_servergitmirrors_with_existing_git_repo(self): + """Confirm it works with clones done manually or with nightly.""" + fdroidserver.deploy.update_servergitmirrors( + fdroidserver.common.config["servergitmirrors"], self.repo_section + ) + + # now delete the local setup, clone the remote, and add a new APK + git_mirror = os.path.join(self.testdir, 'git-mirror') + shutil.rmtree(git_mirror) + self.remote_git_repo.clone(git_mirror) + new_fake_apk = 'Sym2.apk' + self.fake_files.append(new_fake_apk) + (Path(self.repo_section) / new_fake_apk).write_text('a new fake APK') + + fdroidserver.deploy.update_servergitmirrors( + fdroidserver.common.config["servergitmirrors"], self.repo_section + ) + + verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify') + self.assertIsNotNone(verify_repo.working_tree_dir) + for filename in self.fake_files: + remote_file = f"fdroid/{self.repo_section}/{filename}" + self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists()) + def test_update_servergitmirrors_in_index_only_mode(self): fdroidserver.common.config["servergitmirrors"][0]["index_only"] = True From 9e3291302774f0b0e2ca189a623d7a698096cd05 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 24 Jun 2025 12:05:37 +0200 Subject: [PATCH 11/14] deploy: separate git-mirror commit message in index-only mode --- fdroidserver/deploy.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 4651cd76..2c255514 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -559,18 +559,18 @@ def upload_to_servergitmirror( logging.info('Mirroring to: ' + remote_url) if is_index_only: + logging.debug(_('Committing index files to git mirror')) files_to_upload = _get_index_file_paths( os.path.join(local_repo.working_tree_dir, 'fdroid', repo_section) ) files_to_upload = _remove_missing_files(files_to_upload) local_repo.index.add(files_to_upload) + local_repo.index.commit("servergitmirrors: index-only in git-mirror") else: # sadly index.add don't allow the --all parameter - logging.debug('Adding all files to git mirror') + logging.debug(_('Adding all files to git mirror')) local_repo.git.add(all=True) - - logging.debug('Committing files into git mirror') - local_repo.index.commit("fdroidserver git-mirror") + local_repo.index.commit("servergitmirrors: in git-mirror") # only deploy to GitLab Artifacts if too big for GitLab Pages if ( From 9a6148c5b4dfcb4eb837017c30bb9bfd45ea60f7 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 25 Jun 2025 14:02:18 +0200 Subject: [PATCH 12/14] deploy: do not leak username/hostname from machine pushing repo Git will use the username/hostname to set the Author and Committer fields if the config items user.name and user.email are not set. This might inadvertently leak info about the machine that is hosting the deploy process. So this changes it to be a hardcoded value, unless the repo environment has explicitly set these values either in the Git config or in environment variables. --- fdroidserver/deploy.py | 23 +++++++++++++++++++-- tests/test_deploy.py | 47 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 2c255514..01a83de0 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -532,6 +532,20 @@ def update_servergitmirrors(servergitmirrors, repo_section): progressbar.done() +def _get_commit_author(git_repo): + """If the author is set locally, use it, otherwise use static info.""" + ret = {'name': 'servergitmirrors', 'email': 'fdroid@deploy'} + with git_repo.config_reader() as cr: + for option in ('name', 'email'): + try: + value = cr.get_value('user', option) + except (configparser.NoSectionError, configparser.NoOptionError): + value = os.getenv(f'GITLAB_USER_{option.upper()}') + if value: + ret[option] = value + return git.Actor(ret['name'], ret['email']) + + def upload_to_servergitmirror( mirror_config: Dict[str, str], local_repo: Repo, @@ -565,12 +579,17 @@ def upload_to_servergitmirror( ) files_to_upload = _remove_missing_files(files_to_upload) local_repo.index.add(files_to_upload) - local_repo.index.commit("servergitmirrors: index-only in git-mirror") + local_repo.index.commit( + "servergitmirrors: index-only in git-mirror", + author=_get_commit_author(local_repo), + ) else: # sadly index.add don't allow the --all parameter logging.debug(_('Adding all files to git mirror')) local_repo.git.add(all=True) - local_repo.index.commit("servergitmirrors: in git-mirror") + local_repo.index.commit( + "servergitmirrors: in git-mirror", author=_get_commit_author(local_repo) + ) # only deploy to GitLab Artifacts if too big for GitLab Pages if ( diff --git a/tests/test_deploy.py b/tests/test_deploy.py index db9cb50b..91798e51 100755 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -786,6 +786,53 @@ class DeployTest(unittest.TestCase): name, fdroidserver.deploy.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url) ) + @mock.patch.dict(os.environ, clear=True) + def test_get_commit_author_no_config(self): + os.environ['HOME'] = self.testdir + git_repo = git.Repo.init(self.testdir) + self.assertEqual( + git.Actor('servergitmirrors', 'fdroid@deploy'), + fdroidserver.deploy._get_commit_author(git_repo), + ) + + @mock.patch.dict(os.environ, clear=True) + def test_get_commit_author_repo_config(self): + os.environ['HOME'] = self.testdir + git_repo = git.Repo.init(self.testdir) + user_name = 'Foo Bar' + user_email = 'foo@bar.com' + with git_repo.config_writer() as cw: + cw.set_value('user', 'name', user_name) + cw.set_value('user', 'email', user_email) + self.assertEqual( + git.Actor(user_name, user_email), + fdroidserver.deploy._get_commit_author(git_repo), + ) + + @mock.patch.dict(os.environ, clear=True) + def test_get_commit_author_repo_config_name_only(self): + os.environ['HOME'] = self.testdir + git_repo = git.Repo.init(self.testdir) + user_name = 'Foo Bar' + with git_repo.config_writer() as cw: + cw.set_value('user', 'name', user_name) + self.assertEqual( + git.Actor(user_name, 'fdroid@deploy'), + fdroidserver.deploy._get_commit_author(git_repo), + ) + + @mock.patch.dict(os.environ, clear=True) + def test_get_commit_author_repo_config_email_only(self): + os.environ['HOME'] = self.testdir + git_repo = git.Repo.init(self.testdir) + user_email = 'foo@bar.com' + with git_repo.config_writer() as cw: + cw.set_value('user', 'email', user_email) + self.assertEqual( + git.Actor('servergitmirrors', user_email), + fdroidserver.deploy._get_commit_author(git_repo), + ) + class TestServerGitMirrors(unittest.TestCase): def setUp(self): From b03fe31ae4ae1022d65b1c57f0f2497735db328e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 24 Jun 2025 12:08:56 +0200 Subject: [PATCH 13/14] nightly: include existing APKs in index --- fdroidserver/nightly.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 0c2fbb9c..9565b8d8 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -19,6 +19,7 @@ import base64 import datetime +import glob import hashlib import inspect import logging @@ -406,6 +407,8 @@ def main(): git_mirror_metadatadir = os.path.join(git_mirror_fdroiddir, 'metadata') if not os.path.isdir(git_mirror_repodir): clone_git_repo(clone_url, git_mirror_path) + for f in glob.glob(f'{git_mirror_repodir}/*.apk'): + shutil.copy2(f, repodir) if not os.path.isdir(git_mirror_repodir): os.makedirs(git_mirror_repodir, mode=0o755) if os.path.exists('LICENSE'): From fb499db9ebc16931fbce41714287c95932ae3b53 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 26 Jun 2025 09:43:02 +0200 Subject: [PATCH 14/14] update: document "added" dates --- fdroidserver/update.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 75c4d907..887e59d3 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -2466,16 +2466,18 @@ def create_metadata_from_template(apk): def read_added_date_from_all_apks(apps, apks): - """No summary. + """Read the "added" date from all packages. - Added dates come from the repo/index-v2.json file but are - read when scanning apks and thus need to be applied form apk - level to app level for _all_ apps and not only from non-archived - ones + Added dates come from index-v2.json file but are read when scanning + APKs and thus need to be applied from package-level to app-level for + _all_ apps and not only from non-archived ones. TODO: read the added dates directly from index-v2.json instead of going through apks that way it also works for for repos that - don't keep an archive of apks. + don't keep an archive of APKs. + + https://gitlab.com/fdroid/wiki/-/wikis/PackageDateHandling + """ for appid, app in apps.items(): for apk in apks: