From ce018158ee33f330cf0b3db84896fcc236d7832f Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 20 Nov 2024 16:39:01 +0100 Subject: [PATCH 1/3] nightly: set up test for git clone function --- fdroidserver/nightly.py | 16 ++++++++++------ tests/test_nightly.py | 23 +++++++++++++++++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 3eb4333c..30161c1d 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -203,6 +203,15 @@ def get_repo_base_url(clone_url: str, repo_git_base: str, force_type: Optional[s sys.exit(1) +def clone_git_repo(clone_url, git_mirror_path): + logging.debug(_('cloning {url}').format(url=clone_url)) + vcs = common.getvcs('git', clone_url, git_mirror_path) + p = vcs.git(['clone', '--', vcs.remote, str(vcs.local)]) + if p.returncode != 0: + print('WARNING: only public git repos are supported!') + raise VCSException('git clone %s failed:' % clone_url, p.output) + + def main(): """Deploy to F-Droid repository or generate SSH private key from keystore. @@ -338,12 +347,7 @@ def main(): git_mirror_repodir = os.path.join(git_mirror_fdroiddir, 'repo') git_mirror_metadatadir = os.path.join(git_mirror_fdroiddir, 'metadata') if not os.path.isdir(git_mirror_repodir): - logging.debug(_('cloning {url}').format(url=clone_url)) - vcs = common.getvcs('git', clone_url, git_mirror_path) - p = vcs.git(['clone', '--', vcs.remote, str(vcs.local)]) - if p.returncode != 0: - print('WARNING: only public git repos are supported!') - raise VCSException('git clone %s failed:' % clone_url, p.output) + clone_git_repo(clone_url, git_mirror_repodir) if not os.path.isdir(git_mirror_repodir): os.makedirs(git_mirror_repodir, mode=0o755) diff --git a/tests/test_nightly.py b/tests/test_nightly.py index 750a22fc..681df96d 100755 --- a/tests/test_nightly.py +++ b/tests/test_nightly.py @@ -192,6 +192,29 @@ class NightlyTest(unittest.TestCase): with self.assertRaises(exception.VCSException): nightly.main() + def test_clone_git_repo(self): + os.chdir(self.testdir) + common.options = Options + d = 'fakeappid' + nightly.clone_git_repo('https://gitlab.com/fdroid/ci-test-tiny-repo.git', d) + self.assertTrue(os.path.isdir(Path(d) / '.git')) + + def test_clone_git_repo_fails_on_gitlab_password_prompt(self): + os.chdir(self.testdir) + common.options = Options + d = 'shouldnotbecreated' + with self.assertRaises(exception.VCSException): + nightly.clone_git_repo(f'https://gitlab.com/{d}/{d}.git', d) + self.assertFalse(os.path.isdir(Path(d))) + + def test_clone_git_repo_fails_on_github_password_prompt(self): + os.chdir(self.testdir) + common.options = Options + d = 'shouldnotbecreated' + with self.assertRaises(exception.VCSException): + nightly.clone_git_repo(f'https://github.com/{d}/{d}.git', d) + self.assertFalse(os.path.isdir(Path(d))) + def _put_fdroid_in_args(self, args): """Find fdroid command that belongs to this source code tree""" fdroid = os.path.join(basedir.parent, 'fdroid') From b2057a1ce0065ba17ddde38a2ab24f4817d55bc4 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 20 Nov 2024 17:13:46 +0100 Subject: [PATCH 2/3] nightly: switch dep from vcs_git to GitPython This code already depends on GitPython, and hopefully the common.vcs* stuff can eventually go away entirely. GitPython should provide those bits already, and they are maintained by someone else. https://github.com/gitpython-developers/GitPython/pull/2029 --- fdroidserver/nightly.py | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 30161c1d..5b656464 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -21,6 +21,7 @@ import base64 import datetime import git import hashlib +import inspect import logging import os import paramiko @@ -204,12 +205,38 @@ def get_repo_base_url(clone_url: str, repo_git_base: str, force_type: Optional[s def clone_git_repo(clone_url, git_mirror_path): + """Clone a git repo into the given path, failing if a password is required. + + If GitPython's safe mode is present, this will use that. Otherwise, + this includes a very limited version of the safe mode just to ensure + this won't hang on password prompts. + + https://github.com/gitpython-developers/GitPython/pull/2029 + + """ logging.debug(_('cloning {url}').format(url=clone_url)) - vcs = common.getvcs('git', clone_url, git_mirror_path) - p = vcs.git(['clone', '--', vcs.remote, str(vcs.local)]) - if p.returncode != 0: - print('WARNING: only public git repos are supported!') - raise VCSException('git clone %s failed:' % clone_url, p.output) + try: + sig = inspect.signature(git.Repo.clone_from) + if 'safe' in sig.parameters: + git.Repo.clone_from(clone_url, git_mirror_path, safe=True) + else: + git.Repo.clone_from( + clone_url, + git_mirror_path, + env={ + 'GIT_ASKPASS': '/bin/true', + 'SSH_ASKPASS': '/bin/true', + 'GIT_USERNAME': 'u', + 'GIT_PASSWORD': 'p', + 'GIT_HTTP_USERNAME': 'u', + 'GIT_HTTP_PASSWORD': 'p', + 'GIT_SSH': '/bin/false', # for git < 2.3 + 'GIT_TERMINAL_PROMPT': '0', + }, + ) + except git.exc.GitCommandError as e: + logging.warning(_('WARNING: only public git repos are supported!')) + raise VCSException(f'git clone {clone_url} failed:', str(e)) from e def main(): From 76d711ba3c890b2352a4d020200d4861b6e43d14 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 18 Jun 2025 18:19:38 +0200 Subject: [PATCH 3/3] nightly: convert to black format --- fdroidserver/nightly.py | 85 ++++++++++++++++++++++++++++------------- pyproject.toml | 1 - 2 files changed, 58 insertions(+), 28 deletions(-) diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 5b656464..5156fb9a 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -177,7 +177,9 @@ def _ssh_key_from_debug_keystore(keystore: Optional[str] = None) -> str: return ssh_private_key_file -def get_repo_base_url(clone_url: str, repo_git_base: str, force_type: Optional[str] = None) -> str: +def get_repo_base_url( + clone_url: str, repo_git_base: str, force_type: Optional[str] = None +) -> str: """Generate the base URL for the F-Droid repository. Parameters @@ -324,19 +326,27 @@ def main(): # we are in GitLab CI repo_git_base = os.getenv('CI_PROJECT_PATH') + NIGHTLY clone_url = os.getenv('CI_PROJECT_URL') + NIGHTLY - repo_base = get_repo_base_url(clone_url, repo_git_base, force_type='gitlab.com') + repo_base = get_repo_base_url( + clone_url, repo_git_base, force_type='gitlab.com' + ) servergitmirror = 'git@' + urlparse(clone_url).netloc + ':' + repo_git_base - deploy_key_url = clone_url + '/-/settings/repository#js-deploy-keys-settings' + deploy_key_url = ( + f'{clone_url}/-/settings/repository#js-deploy-keys-settings' + ) git_user_name = os.getenv('GITLAB_USER_NAME') git_user_email = os.getenv('GITLAB_USER_EMAIL') elif 'TRAVIS_REPO_SLUG' in os.environ: # we are in Travis CI repo_git_base = os.getenv('TRAVIS_REPO_SLUG') + NIGHTLY clone_url = 'https://github.com/' + repo_git_base - repo_base = get_repo_base_url(clone_url, repo_git_base, force_type='github.com') + repo_base = get_repo_base_url( + clone_url, repo_git_base, force_type='github.com' + ) servergitmirror = 'git@github.com:' + repo_git_base - deploy_key_url = ('https://github.com/' + repo_git_base + '/settings/keys' - + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys') + deploy_key_url = ( + f'https://github.com/{repo_git_base}/settings/keys' + + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys' + ) git_user_name = repo_git_base git_user_email = os.getenv('USER') + '@' + platform.node() elif ( @@ -345,23 +355,35 @@ def main(): and 'CIRCLE_PROJECT_REPONAME' in os.environ ): # we are in Circle CI - repo_git_base = (os.getenv('CIRCLE_PROJECT_USERNAME') - + '/' + os.getenv('CIRCLE_PROJECT_REPONAME') + NIGHTLY) + repo_git_base = ( + os.getenv('CIRCLE_PROJECT_USERNAME') + + '/' + + os.getenv('CIRCLE_PROJECT_REPONAME') + + NIGHTLY + ) clone_url = os.getenv('CIRCLE_REPOSITORY_URL') + NIGHTLY - repo_base = get_repo_base_url(clone_url, repo_git_base, force_type='github.com') + repo_base = get_repo_base_url( + clone_url, repo_git_base, force_type='github.com' + ) servergitmirror = 'git@' + urlparse(clone_url).netloc + ':' + repo_git_base - deploy_key_url = ('https://github.com/' + repo_git_base + '/settings/keys' - + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys') + deploy_key_url = ( + f'https://github.com/{repo_git_base}/settings/keys' + + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys' + ) git_user_name = os.getenv('CIRCLE_USERNAME') git_user_email = git_user_name + '@' + platform.node() elif 'GITHUB_ACTIONS' in os.environ: # we are in Github actions - repo_git_base = (os.getenv('GITHUB_REPOSITORY') + NIGHTLY) - clone_url = (os.getenv('GITHUB_SERVER_URL') + '/' + repo_git_base) - repo_base = get_repo_base_url(clone_url, repo_git_base, force_type='github.com') + repo_git_base = os.getenv('GITHUB_REPOSITORY') + NIGHTLY + clone_url = os.getenv('GITHUB_SERVER_URL') + '/' + repo_git_base + repo_base = get_repo_base_url( + clone_url, repo_git_base, force_type='github.com' + ) servergitmirror = 'git@' + urlparse(clone_url).netloc + ':' + repo_git_base - deploy_key_url = ('https://github.com/' + repo_git_base + '/settings/keys' - + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys') + deploy_key_url = ( + f'https://github.com/{repo_git_base}/settings/keys' + + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys' + ) git_user_name = os.getenv('GITHUB_ACTOR') git_user_email = git_user_name + '@' + platform.node() else: @@ -395,9 +417,13 @@ You can use it with the [F-Droid](https://f-droid.org/) Android app. [![{repo_url}]({repo_url}/icons/icon.png)](https://fdroid.link/#{repo_url}) -Last updated: {date}'''.format(repo_git_base=repo_git_base, - repo_url=repo_url, - date=datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')) +Last updated: {date}'''.format( + repo_git_base=repo_git_base, + repo_url=repo_url, + date=datetime.datetime.now(datetime.timezone.utc).strftime( + '%Y-%m-%d %H:%M:%S UTC' + ), + ) with open(readme_path, 'w') as fp: fp.write(readme) mirror_git_repo.git.add(all=True) @@ -458,10 +484,9 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, common.assert_config_keystore(config) logging.debug( - _('Run over {cibase} to find -debug.apk. and skip repo_basedir {repo_basedir}').format( - cibase=cibase, - repo_basedir=repo_basedir - ) + _( + 'Run over {cibase} to find -debug.apk. and skip repo_basedir {repo_basedir}' + ).format(cibase=cibase, repo_basedir=repo_basedir) ) for root, dirs, files in os.walk(cibase): @@ -549,10 +574,16 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, if not os.path.exists(androiddir): os.mkdir(androiddir) logging.info(_('created {path}').format(path=androiddir)) - logging.error(_('{path} does not exist! Create it by running:').format(path=options.keystore) - + '\n keytool -genkey -v -keystore ' + options.keystore + ' -storepass android \\' - + '\n -alias androiddebugkey -keypass android -keyalg RSA -keysize 2048 -validity 10000 \\' - + '\n -dname "CN=Android Debug,O=Android,C=US"') + logging.error( + _('{path} does not exist! Create it by running:').format( + path=options.keystore + ) + + '\n keytool -genkey -v -keystore ' + + options.keystore + + ' -storepass android \\' + + '\n -alias androiddebugkey -keypass android -keyalg RSA -keysize 2048 -validity 10000 \\' + + '\n -dname "CN=Android Debug,O=Android,C=US"' + ) sys.exit(1) ssh_dir = os.path.join(os.getenv('HOME'), '.ssh') privkey = _ssh_key_from_debug_keystore(options.keystore) diff --git a/pyproject.toml b/pyproject.toml index 0199f037..a1f8a99b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,6 @@ force-exclude = '''( | fdroidserver/common\.py | fdroidserver/index\.py | fdroidserver/metadata\.py - | fdroidserver/nightly\.py | fdroidserver/update\.py | fdroidserver/vmtools\.py | tests/config\.py