diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index 71c39b2c..83a0c81c 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -156,7 +156,7 @@ def main(): ).git.describe(always=True, tags=True) ) sys.exit(0) - except git.exc.InvalidGitRepositoryError: + except git.InvalidGitRepositoryError: print(_('No version information could be found.')) sys.exit(1) else: diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index f1dcce21..01a83de0 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -40,7 +40,6 @@ import fdroidserver.github from . import _, common, index from .exception import FDroidException -config = None start_timestamp = time.gmtime() GIT_BRANCH = 'master' @@ -144,6 +143,7 @@ def update_remote_storage_with_rclone( """ logging.debug(_('Using rclone to sync to "{name}"').format(name=awsbucket)) + config = common.get_config() rclone_config = config.get('rclone_config', []) if rclone_config and isinstance(rclone_config, str): rclone_config = [rclone_config] @@ -271,6 +271,7 @@ def update_serverwebroot(serverwebroot, repo_section): has a low resolution timestamp """ + config = common.get_config() try: subprocess.run(['rsync', '--version'], capture_output=True, check=True) except Exception as e: @@ -431,6 +432,7 @@ def update_servergitmirrors(servergitmirrors, repo_section): """ from clint.textui import progress + config = common.get_config() if config.get('local_copy_dir') and not config.get('sync_from_local_copy_dir'): logging.debug( _('Offline machine, skipping git mirror generation until `fdroid deploy`') @@ -530,6 +532,20 @@ def update_servergitmirrors(servergitmirrors, repo_section): progressbar.done() +def _get_commit_author(git_repo): + """If the author is set locally, use it, otherwise use static info.""" + ret = {'name': 'servergitmirrors', 'email': 'fdroid@deploy'} + with git_repo.config_reader() as cr: + for option in ('name', 'email'): + try: + value = cr.get_value('user', option) + except (configparser.NoSectionError, configparser.NoOptionError): + value = os.getenv(f'GITLAB_USER_{option.upper()}') + if value: + ret[option] = value + return git.Actor(ret['name'], ret['email']) + + def upload_to_servergitmirror( mirror_config: Dict[str, str], local_repo: Repo, @@ -557,18 +573,23 @@ def upload_to_servergitmirror( logging.info('Mirroring to: ' + remote_url) if is_index_only: + logging.debug(_('Committing index files to git mirror')) files_to_upload = _get_index_file_paths( os.path.join(local_repo.working_tree_dir, 'fdroid', repo_section) ) files_to_upload = _remove_missing_files(files_to_upload) local_repo.index.add(files_to_upload) + local_repo.index.commit( + "servergitmirrors: index-only in git-mirror", + author=_get_commit_author(local_repo), + ) else: # sadly index.add don't allow the --all parameter - logging.debug('Adding all files to git mirror') + logging.debug(_('Adding all files to git mirror')) local_repo.git.add(all=True) - - logging.debug('Committing files into git mirror') - local_repo.index.commit("fdroidserver git-mirror") + local_repo.index.commit( + "servergitmirrors: in git-mirror", author=_get_commit_author(local_repo) + ) # only deploy to GitLab Artifacts if too big for GitLab Pages if ( @@ -1029,8 +1050,6 @@ def upload_to_github_releases_repo(repo_conf, release_infos, global_gh_token): def main(): - global config - parser = ArgumentParser() common.setup_global_opts(parser) parser.add_argument( diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 372390ea..9565b8d8 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -19,6 +19,7 @@ import base64 import datetime +import glob import hashlib import inspect import logging @@ -236,11 +237,21 @@ def clone_git_repo(clone_url, git_mirror_path): 'GIT_TERMINAL_PROMPT': '0', }, ) - except git.exc.GitCommandError as e: + except git.GitCommandError as e: logging.warning(_('WARNING: only public git repos are supported!')) raise VCSException(f'git clone {clone_url} failed:', str(e)) from e +def _getenv(name): + """Return the value of an environment variable, printing an error if None.""" + value = os.getenv(name) + if not value: + logging.error( + _('Environment variable "{name}" has an empty value!').format(name=name) + ) + return value + + def main(): """Deploy to F-Droid repository or generate SSH private key from keystore. @@ -303,7 +314,7 @@ def main(): umask = os.umask(0o077) if 'CI' in os.environ: - v = os.getenv('DEBUG_KEYSTORE') + v = _getenv('DEBUG_KEYSTORE') debug_keystore = None if v: debug_keystore = base64.b64decode(v) @@ -324,20 +335,19 @@ def main(): # the 'master' branch is hardcoded in fdroidserver/deploy.py if 'CI_PROJECT_PATH' in os.environ and 'CI_PROJECT_URL' in os.environ: # we are in GitLab CI - repo_git_base = os.getenv('CI_PROJECT_PATH') + NIGHTLY - clone_url = os.getenv('CI_PROJECT_URL') + NIGHTLY + repo_git_base = f"{_getenv('CI_PROJECT_PATH')}{NIGHTLY}" + base_url = f"{_getenv('CI_PROJECT_URL')}{NIGHTLY}" + clone_url = f'{base_url}.git' # avoid redirects while cloning repo_base = get_repo_base_url( - clone_url, repo_git_base, force_type='gitlab.com' + base_url, repo_git_base, force_type='gitlab.com' ) - servergitmirror = 'git@' + urlparse(clone_url).netloc + ':' + repo_git_base - deploy_key_url = ( - f'{clone_url}/-/settings/repository#js-deploy-keys-settings' - ) - git_user_name = os.getenv('GITLAB_USER_NAME') - git_user_email = os.getenv('GITLAB_USER_EMAIL') + servergitmirror = f'git@{urlparse(base_url).netloc}:{repo_git_base}.git' + deploy_key_url = f'{base_url}/-/settings/repository#js-deploy-keys-settings' + git_user_name = _getenv('GITLAB_USER_NAME') + git_user_email = _getenv('GITLAB_USER_EMAIL') elif 'TRAVIS_REPO_SLUG' in os.environ: # we are in Travis CI - repo_git_base = os.getenv('TRAVIS_REPO_SLUG') + NIGHTLY + repo_git_base = _getenv('TRAVIS_REPO_SLUG') + NIGHTLY clone_url = 'https://github.com/' + repo_git_base repo_base = get_repo_base_url( clone_url, repo_git_base, force_type='github.com' @@ -348,7 +358,7 @@ def main(): + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys' ) git_user_name = repo_git_base - git_user_email = os.getenv('USER') + '@' + platform.node() + git_user_email = _getenv('USER') + '@' + platform.node() elif ( 'CIRCLE_REPOSITORY_URL' in os.environ and 'CIRCLE_PROJECT_USERNAME' in os.environ @@ -356,12 +366,12 @@ def main(): ): # we are in Circle CI repo_git_base = ( - os.getenv('CIRCLE_PROJECT_USERNAME') + _getenv('CIRCLE_PROJECT_USERNAME') + '/' - + os.getenv('CIRCLE_PROJECT_REPONAME') + + _getenv('CIRCLE_PROJECT_REPONAME') + NIGHTLY ) - clone_url = os.getenv('CIRCLE_REPOSITORY_URL') + NIGHTLY + clone_url = _getenv('CIRCLE_REPOSITORY_URL') + NIGHTLY repo_base = get_repo_base_url( clone_url, repo_git_base, force_type='github.com' ) @@ -370,12 +380,12 @@ def main(): f'https://github.com/{repo_git_base}/settings/keys' + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys' ) - git_user_name = os.getenv('CIRCLE_USERNAME') + git_user_name = _getenv('CIRCLE_USERNAME') git_user_email = git_user_name + '@' + platform.node() elif 'GITHUB_ACTIONS' in os.environ: # we are in Github actions - repo_git_base = os.getenv('GITHUB_REPOSITORY') + NIGHTLY - clone_url = os.getenv('GITHUB_SERVER_URL') + '/' + repo_git_base + repo_git_base = _getenv('GITHUB_REPOSITORY') + NIGHTLY + clone_url = _getenv('GITHUB_SERVER_URL') + '/' + repo_git_base repo_base = get_repo_base_url( clone_url, repo_git_base, force_type='github.com' ) @@ -384,7 +394,7 @@ def main(): f'https://github.com/{repo_git_base}/settings/keys' + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys' ) - git_user_name = os.getenv('GITHUB_ACTOR') + git_user_name = _getenv('GITHUB_ACTOR') git_user_email = git_user_name + '@' + platform.node() else: print(_('ERROR: unsupported CI type, patches welcome!')) @@ -397,6 +407,8 @@ def main(): git_mirror_metadatadir = os.path.join(git_mirror_fdroiddir, 'metadata') if not os.path.isdir(git_mirror_repodir): clone_git_repo(clone_url, git_mirror_path) + for f in glob.glob(f'{git_mirror_repodir}/*.apk'): + shutil.copy2(f, repodir) if not os.path.isdir(git_mirror_repodir): os.makedirs(git_mirror_repodir, mode=0o755) if os.path.exists('LICENSE'): @@ -442,7 +454,7 @@ Last updated: {date}'''.format( ssh_private_key_file = _ssh_key_from_debug_keystore() # this is needed for GitPython to find the SSH key - ssh_dir = os.path.join(os.getenv('HOME'), '.ssh') + ssh_dir = os.path.join(_getenv('HOME'), '.ssh') os.makedirs(ssh_dir, exist_ok=True) ssh_config = os.path.join(ssh_dir, 'config') logging.debug(_('adding IdentityFile to {path}').format(path=ssh_config)) @@ -587,7 +599,7 @@ Last updated: {date}'''.format( + '\n -dname "CN=Android Debug,O=Android,C=US"' ) sys.exit(1) - ssh_dir = os.path.join(os.getenv('HOME'), '.ssh') + ssh_dir = os.path.join(_getenv('HOME'), '.ssh') privkey = _ssh_key_from_debug_keystore(options.keystore) if os.path.exists(ssh_dir): ssh_private_key_file = os.path.join(ssh_dir, os.path.basename(privkey)) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 75c4d907..887e59d3 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -2466,16 +2466,18 @@ def create_metadata_from_template(apk): def read_added_date_from_all_apks(apps, apks): - """No summary. + """Read the "added" date from all packages. - Added dates come from the repo/index-v2.json file but are - read when scanning apks and thus need to be applied form apk - level to app level for _all_ apps and not only from non-archived - ones + Added dates come from index-v2.json file but are read when scanning + APKs and thus need to be applied from package-level to app-level for + _all_ apps and not only from non-archived ones. TODO: read the added dates directly from index-v2.json instead of going through apks that way it also works for for repos that - don't keep an archive of apks. + don't keep an archive of APKs. + + https://gitlab.com/fdroid/wiki/-/wikis/PackageDateHandling + """ for appid, app in apps.items(): for apk in apks: diff --git a/tests/test_deploy.py b/tests/test_deploy.py index d7de7545..91798e51 100755 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import configparser +import logging import os import shutil import tempfile @@ -26,15 +27,21 @@ def _mock_rclone_config_file(cmd, text): # pylint: disable=unused-argument class DeployTest(unittest.TestCase): '''fdroidserver/deploy.py''' + @classmethod + def setUpClass(cls): + # suppress "WARNING:root:unsafe permissions on 'config.yml' (should be 0600)!" + os.chmod(os.path.join(basedir, fdroidserver.common.CONFIG_FILE), 0o600) + def setUp(self): os.chdir(basedir) self._td = mkdtemp() self.testdir = self._td.name fdroidserver.common.options = mock.Mock() - fdroidserver.deploy.config = {} + fdroidserver.common.get_config() def tearDown(self): + fdroidserver.common.config = None self._td.cleanup() def test_update_serverwebroots_bad_None(self): @@ -64,7 +71,6 @@ class DeployTest(unittest.TestCase): # setup parameters for this test run fdroidserver.common.options.identity_file = None - fdroidserver.deploy.config['make_current_version_link'] = False dest_apk0 = url0 / fake_apk dest_apk1 = url1 / fake_apk @@ -81,17 +87,17 @@ class DeployTest(unittest.TestCase): self.assertTrue(dest_apk1.is_file()) def test_update_serverwebroots_url_does_not_end_with_fdroid(self): - with self.assertRaises(SystemExit): + with self.assertRaises(SystemExit), self.assertLogs(level=logging.ERROR): fdroidserver.deploy.update_serverwebroots([{'url': 'url'}], 'repo') def test_update_serverwebroots_bad_ssh_url(self): - with self.assertRaises(SystemExit): + with self.assertRaises(SystemExit), self.assertLogs(level=logging.ERROR): fdroidserver.deploy.update_serverwebroots( [{'url': 'f@b.ar::/path/to/fdroid'}], 'repo' ) def test_update_serverwebroots_unsupported_ssh_url(self): - with self.assertRaises(SystemExit): + with self.assertRaises(SystemExit), self.assertLogs(level=logging.ERROR): fdroidserver.deploy.update_serverwebroots([{'url': 'ssh://nope'}], 'repo') @unittest.skipUnless(shutil.which('rclone'), 'requires rclone') @@ -120,9 +126,9 @@ class DeployTest(unittest.TestCase): # setup parameters for this test run awsbucket = 'test_bucket_folder' - fdroidserver.deploy.config['awsbucket'] = awsbucket - fdroidserver.deploy.config['rclone_config'] = 'test-local-config' - fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file) + fdroidserver.common.config['awsbucket'] = awsbucket + fdroidserver.common.config['rclone_config'] = 'test-local-config' + fdroidserver.common.config['path_to_custom_rclone_config'] = str(rclone_file) fdroidserver.common.options = VerboseFalseOptions # write out destination path @@ -163,9 +169,9 @@ class DeployTest(unittest.TestCase): # setup parameters for this test run awsbucket = 'test_bucket_folder' - fdroidserver.deploy.config['awsbucket'] = awsbucket - fdroidserver.deploy.config['rclone_config'] = 'test-local-config' - fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file) + fdroidserver.common.config['awsbucket'] = awsbucket + fdroidserver.common.config['rclone_config'] = 'test-local-config' + fdroidserver.common.config['path_to_custom_rclone_config'] = str(rclone_file) fdroidserver.common.options = VerboseFalseOptions # write out destination path @@ -224,7 +230,7 @@ class DeployTest(unittest.TestCase): return 0 mock_call.side_effect = _mock_subprocess_call - fdroidserver.deploy.config = {'awsbucket': awsbucket} + fdroidserver.common.config = {'awsbucket': awsbucket} fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket) mock_call.assert_called() @@ -243,7 +249,7 @@ class DeployTest(unittest.TestCase): mock_call.side_effect = _mock_subprocess_call - fdroidserver.deploy.config = {'awsbucket': awsbucket} + fdroidserver.common.config = {'awsbucket': awsbucket} fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket) self.maxDiff = None self.assertEqual( @@ -262,7 +268,6 @@ class DeployTest(unittest.TestCase): @mock.patch('subprocess.check_output', _mock_rclone_config_file) @mock.patch('subprocess.call') def test_update_remote_storage_with_rclone_mock_rclone_config(self, mock_call): - awsbucket = 'test_bucket_folder' self.last_cmd = None def _mock_subprocess_call(cmd): @@ -271,10 +276,9 @@ class DeployTest(unittest.TestCase): mock_call.side_effect = _mock_subprocess_call - fdroidserver.deploy.config = { - 'awsbucket': awsbucket, - 'rclone_config': 'test_local_config', - } + awsbucket = 'test_bucket_folder' + fdroidserver.common.config['awsbucket'] = awsbucket + fdroidserver.common.config['rclone_config'] = 'test_local_config' fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket) self.maxDiff = None self.assertEqual( @@ -304,8 +308,8 @@ class DeployTest(unittest.TestCase): Path('rclone.conf').write_text('placeholder, contents ignored') awsbucket = 'test_bucket_folder' - fdroidserver.deploy.config['awsbucket'] = awsbucket - fdroidserver.deploy.config['rclone_config'] = config_name + fdroidserver.common.config['awsbucket'] = awsbucket + fdroidserver.common.config['rclone_config'] = config_name fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket) self.maxDiff = None self.assertEqual( @@ -339,7 +343,6 @@ class DeployTest(unittest.TestCase): fdroidserver.common.options = mock.Mock() fdroidserver.common.options.identity_file = None fdroidserver.common.options.identity_file = None - fdroidserver.deploy.config['make_current_version_link'] = False dest_apk = Path(url) / fake_apk dest_index = Path(url) / fake_index @@ -366,7 +369,6 @@ class DeployTest(unittest.TestCase): # setup parameters for this test run fdroidserver.common.options = mock.Mock() fdroidserver.common.options.identity_file = None - fdroidserver.deploy.config['make_current_version_link'] = False dest_apk = Path(url) / fake_apk dest_index = Path(url) / fake_index @@ -396,7 +398,7 @@ class DeployTest(unittest.TestCase): fdroidserver.common.options.verbose = False fdroidserver.common.options.quiet = True fdroidserver.common.options.index_only = False - fdroidserver.deploy.config = {'make_current_version_link': True} + fdroidserver.common.config['make_current_version_link'] = True url = "example.com:/var/www/fdroid" repo_section = 'repo' @@ -504,7 +506,7 @@ class DeployTest(unittest.TestCase): fdroidserver.common.options.verbose = False fdroidserver.common.options.quiet = True fdroidserver.common.options.identity_file = None - fdroidserver.deploy.config['make_current_version_link'] = True + fdroidserver.common.config['make_current_version_link'] = True url = "example.com:/var/www/fdroid" repo_section = 'repo' @@ -603,7 +605,7 @@ class DeployTest(unittest.TestCase): fdroidserver.common.options.quiet = False fdroidserver.common.options.identity_file = None fdroidserver.common.options.index_only = False - fdroidserver.deploy.config = {'identity_file': './id_rsa'} + fdroidserver.common.config = {'identity_file': './id_rsa'} url = "example.com:/var/www/fdroid" repo_section = 'archive' @@ -623,7 +625,7 @@ class DeployTest(unittest.TestCase): '--verbose', '-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' - + fdroidserver.deploy.config['identity_file'], + + fdroidserver.common.config['identity_file'], '--exclude', 'archive/altstore-index.json', '--exclude', @@ -669,7 +671,7 @@ class DeployTest(unittest.TestCase): '--verbose', '-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' - + fdroidserver.deploy.config['identity_file'], + + fdroidserver.common.config['identity_file'], 'archive', url, ], @@ -690,8 +692,7 @@ class DeployTest(unittest.TestCase): fdroidserver.common.options.verbose = True fdroidserver.common.options.quiet = False fdroidserver.common.options.identity_file = None - fdroidserver.deploy.config['identity_file'] = './id_rsa' - fdroidserver.deploy.config['make_current_version_link'] = False + fdroidserver.common.config['identity_file'] = './id_rsa' url = "example.com:/var/www/fdroid" repo_section = 'archive' @@ -711,7 +712,7 @@ class DeployTest(unittest.TestCase): '--verbose', '-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' - + fdroidserver.deploy.config['identity_file'], + + fdroidserver.common.config['identity_file'], 'archive/altstore-index.json', 'archive/altstore-index.json.asc', 'archive/entry.jar', @@ -741,7 +742,7 @@ class DeployTest(unittest.TestCase): '--verbose', '-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' - + fdroidserver.deploy.config['identity_file'], + + fdroidserver.common.config['identity_file'], "example.com:/var/www/fdroid/archive/", ], ) @@ -785,7 +786,58 @@ class DeployTest(unittest.TestCase): name, fdroidserver.deploy.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url) ) - def test_update_servergitmirrors(self): + @mock.patch.dict(os.environ, clear=True) + def test_get_commit_author_no_config(self): + os.environ['HOME'] = self.testdir + git_repo = git.Repo.init(self.testdir) + self.assertEqual( + git.Actor('servergitmirrors', 'fdroid@deploy'), + fdroidserver.deploy._get_commit_author(git_repo), + ) + + @mock.patch.dict(os.environ, clear=True) + def test_get_commit_author_repo_config(self): + os.environ['HOME'] = self.testdir + git_repo = git.Repo.init(self.testdir) + user_name = 'Foo Bar' + user_email = 'foo@bar.com' + with git_repo.config_writer() as cw: + cw.set_value('user', 'name', user_name) + cw.set_value('user', 'email', user_email) + self.assertEqual( + git.Actor(user_name, user_email), + fdroidserver.deploy._get_commit_author(git_repo), + ) + + @mock.patch.dict(os.environ, clear=True) + def test_get_commit_author_repo_config_name_only(self): + os.environ['HOME'] = self.testdir + git_repo = git.Repo.init(self.testdir) + user_name = 'Foo Bar' + with git_repo.config_writer() as cw: + cw.set_value('user', 'name', user_name) + self.assertEqual( + git.Actor(user_name, 'fdroid@deploy'), + fdroidserver.deploy._get_commit_author(git_repo), + ) + + @mock.patch.dict(os.environ, clear=True) + def test_get_commit_author_repo_config_email_only(self): + os.environ['HOME'] = self.testdir + git_repo = git.Repo.init(self.testdir) + user_email = 'foo@bar.com' + with git_repo.config_writer() as cw: + cw.set_value('user', 'email', user_email) + self.assertEqual( + git.Actor('servergitmirrors', user_email), + fdroidserver.deploy._get_commit_author(git_repo), + ) + + +class TestServerGitMirrors(unittest.TestCase): + def setUp(self): + fdroidserver.deploy.USER_RCLONE_CONF = False + # setup parameters for this test run fdroidserver.common.options = mock.Mock() fdroidserver.common.options.identity_file = None @@ -793,130 +845,93 @@ class DeployTest(unittest.TestCase): fdroidserver.common.options.verbose = False fdroidserver.common.options.quiet = True - config = {} - fdroidserver.common.fill_config_defaults(config) - fdroidserver.deploy.config = config - + self._td = mkdtemp() + self.testdir = self._td.name os.chdir(self.testdir) - repo_section = 'repo' - initial_branch = fdroidserver.deploy.GIT_BRANCH - remote_repo = Path(self.testdir) / 'remote' remote_repo.mkdir(parents=True) - remote_git_repo = git.Repo.init( - remote_repo, initial_branch=initial_branch, bare=True + self.remote_git_repo = git.Repo.init( + remote_repo, initial_branch=fdroidserver.deploy.GIT_BRANCH, bare=True ) - fdroidserver.deploy.config["servergitmirrors"] = [{"url": str(remote_repo)}] - os.chdir(self.testdir) - repo = Path('repo') - repo.mkdir(parents=True) - fake_apk = 'Sym.apk' - fake_files = fdroidserver.common.INDEX_FILES + [fake_apk] - for filename in fake_files: + fdroidserver.common.get_config() + fdroidserver.common.config["servergitmirrors"] = [{"url": str(remote_repo)}] + + self.repo_section = 'repo' + repo = Path(self.repo_section) + repo.mkdir() + self.fake_apk = 'Sym.apk' + self.fake_files = fdroidserver.common.INDEX_FILES + [self.fake_apk] + for filename in self.fake_files: fake_file = repo / filename with fake_file.open('w') as fp: fp.write('not a real one, but has the right filename') + def tearDown(self): + fdroidserver.common.config = None + fdroidserver.common.options = None + self._td.cleanup() + + def test_update_servergitmirrors(self): fdroidserver.deploy.update_servergitmirrors( - fdroidserver.deploy.config["servergitmirrors"], repo_section + fdroidserver.common.config["servergitmirrors"], self.repo_section ) - verify_repo = remote_git_repo.clone( - Path(self.testdir) / 'verify', + verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify') + self.assertIsNotNone(verify_repo.working_tree_dir) + for filename in self.fake_files: + remote_file = f"fdroid/{self.repo_section}/{filename}" + self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists()) + + def test_update_servergitmirrors_with_existing_git_repo(self): + """Confirm it works with clones done manually or with nightly.""" + fdroidserver.deploy.update_servergitmirrors( + fdroidserver.common.config["servergitmirrors"], self.repo_section ) - for filename in fake_files: - remote_file = f"fdroid/{repo_section}/{filename}" + # now delete the local setup, clone the remote, and add a new APK + git_mirror = os.path.join(self.testdir, 'git-mirror') + shutil.rmtree(git_mirror) + self.remote_git_repo.clone(git_mirror) + new_fake_apk = 'Sym2.apk' + self.fake_files.append(new_fake_apk) + (Path(self.repo_section) / new_fake_apk).write_text('a new fake APK') - self.assertIsNotNone(verify_repo.working_tree_dir) - if verify_repo.working_tree_dir is not None: - self.assertTrue( - (Path(verify_repo.working_tree_dir) / remote_file).exists() - ) + fdroidserver.deploy.update_servergitmirrors( + fdroidserver.common.config["servergitmirrors"], self.repo_section + ) + + verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify') + self.assertIsNotNone(verify_repo.working_tree_dir) + for filename in self.fake_files: + remote_file = f"fdroid/{self.repo_section}/{filename}" + self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists()) def test_update_servergitmirrors_in_index_only_mode(self): - # setup parameters for this test run - fdroidserver.common.options = mock.Mock() - fdroidserver.common.options.identity_file = None - fdroidserver.common.options.no_keep_git_mirror_archive = False - fdroidserver.common.options.verbose = False - fdroidserver.common.options.quiet = True - - config = {} - fdroidserver.common.fill_config_defaults(config) - fdroidserver.deploy.config = config - - os.chdir(self.testdir) - - repo_section = 'repo' - initial_branch = fdroidserver.deploy.GIT_BRANCH - - remote_repo = Path(self.testdir) / 'remote' - remote_repo.mkdir(parents=True) - remote_git_repo = git.Repo.init( - remote_repo, initial_branch=initial_branch, bare=True - ) - fdroidserver.deploy.config["servergitmirrors"] = [ - {"url": str(remote_repo), "index_only": True} - ] - - os.chdir(self.testdir) - repo = Path('repo') - repo.mkdir(parents=True) - fake_apk = 'Sym.apk' - fake_files = fdroidserver.common.INDEX_FILES + [fake_apk] - for filename in fake_files: - fake_file = repo / filename - with fake_file.open('w') as fp: - fp.write('not a real one, but has the right filename') + fdroidserver.common.config["servergitmirrors"][0]["index_only"] = True fdroidserver.deploy.update_servergitmirrors( - fdroidserver.deploy.config["servergitmirrors"], repo_section - ) - - verify_repo = remote_git_repo.clone( - Path(self.testdir) / 'verify', + fdroidserver.common.config["servergitmirrors"], self.repo_section ) + verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify') + self.assertIsNotNone(verify_repo.working_tree_dir) for filename in fdroidserver.common.INDEX_FILES: - remote_file = f"fdroid/{repo_section}/{filename}" - - self.assertIsNotNone(verify_repo.working_tree_dir) - if verify_repo.working_tree_dir is not None: - self.assertTrue( - (Path(verify_repo.working_tree_dir) / remote_file).exists() - ) + remote_file = f"fdroid/{self.repo_section}/{filename}" + self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists()) # Should not have the APK file - remote_file = f"fdroid/{repo_section}/{fake_apk}" - if verify_repo.working_tree_dir is not None: - self.assertFalse( - (Path(verify_repo.working_tree_dir) / remote_file).exists() - ) + remote_file = f"fdroid/{self.repo_section}/{self.fake_apk}" + self.assertFalse((Path(verify_repo.working_tree_dir) / remote_file).exists()) def test_upload_to_servergitmirror_in_index_only_mode(self): - # setup parameters for this test run - fdroidserver.common.options = mock.Mock() - fdroidserver.common.options.identity_file = None - fdroidserver.common.options.no_keep_git_mirror_archive = False - fdroidserver.common.options.verbose = False - fdroidserver.common.options.quiet = True - fdroidserver.common.options.identity_file = None - - config = {} - fdroidserver.common.fill_config_defaults(config) - fdroidserver.deploy.config = config + shutil.rmtree('repo') # the class-wide test files are not used here repo_section = 'repo' - initial_branch = fdroidserver.deploy.GIT_BRANCH - - os.chdir(self.testdir) - local_git_repo_path = Path(self.testdir) / 'local' local_git_repo = git.Repo.init( - local_git_repo_path, initial_branch=initial_branch + local_git_repo_path, initial_branch=fdroidserver.deploy.GIT_BRANCH ) fdroid_dir = local_git_repo_path / 'fdroid' @@ -929,13 +944,7 @@ class DeployTest(unittest.TestCase): with fake_file.open('w') as fp: fp.write('not a real one, but has the right filename') - # The remote repo must be a bare repo to allow being pushed to - remote_git_repo_dir = Path(self.testdir) / 'remote' - remote_git_repo = git.Repo.init( - remote_git_repo_dir, initial_branch=initial_branch, bare=True - ) - - mirror_config = {"url": str(remote_git_repo_dir), "index_only": True} + mirror_config = {"url": str(self.remote_git_repo.git_dir), "index_only": True} enabled_remotes = [] ssh_cmd = 'ssh -oBatchMode=yes' fdroidserver.deploy.upload_to_servergitmirror( @@ -950,25 +959,15 @@ class DeployTest(unittest.TestCase): progress=git.RemoteProgress(), ) - verify_repo = remote_git_repo.clone( - Path(self.testdir) / 'verify', - ) - + verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify') + self.assertIsNotNone(verify_repo.working_tree_dir) for filename in fdroidserver.common.INDEX_FILES: remote_file = f"fdroid/{repo_section}/{filename}" - - self.assertIsNotNone(verify_repo.working_tree_dir) - if verify_repo.working_tree_dir is not None: - self.assertTrue( - (Path(verify_repo.working_tree_dir) / remote_file).exists() - ) + self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists()) # Should not have the APK file remote_file = f"fdroid/{repo_section}/{fake_apk}" - if verify_repo.working_tree_dir is not None: - self.assertFalse( - (Path(verify_repo.working_tree_dir) / remote_file).exists() - ) + self.assertFalse((Path(verify_repo.working_tree_dir) / remote_file).exists()) class GitHubReleasesTest(unittest.TestCase): diff --git a/tests/test_integration.py b/tests/test_integration.py index 2cdf19d9..d051275b 100755 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -75,7 +75,6 @@ class IntegrationTest(unittest.TestCase): self.testdir = mkdir_testfiles(WORKSPACE, self) self.tmp_repo_root = self.testdir / "fdroid" self.tmp_repo_root.mkdir(parents=True) - deploy.config = {} os.chdir(self.tmp_repo_root) def tearDown(self): @@ -1611,10 +1610,11 @@ class IntegrationTest(unittest.TestCase): rclone_config.write(configfile) # set up config for run + common.get_config() awsbucket = "test-bucket" - deploy.config['awsbucket'] = awsbucket - deploy.config['rclone_config'] = "test-minio-config" - deploy.config['path_to_custom_rclone_config'] = str(rclone_file) + common.config['awsbucket'] = awsbucket + common.config['rclone_config'] = "test-minio-config" + common.config['path_to_custom_rclone_config'] = str(rclone_file) common.options = VerboseFalseOptions # call function @@ -1667,9 +1667,9 @@ class IntegrationTest(unittest.TestCase): # set up config for run awsbucket = "test-bucket" - deploy.config['awsbucket'] = awsbucket - deploy.config['rclone_config'] = "test-minio-config" - deploy.config['path_to_custom_rclone_config'] = str(rclone_file) + common.config['awsbucket'] = awsbucket + common.config['rclone_config'] = "test-minio-config" + common.config['path_to_custom_rclone_config'] = str(rclone_file) common.options = VerboseFalseOptions # call function diff --git a/tests/test_nightly.py b/tests/test_nightly.py index fb1614b7..e7c6991c 100755 --- a/tests/test_nightly.py +++ b/tests/test_nightly.py @@ -359,7 +359,7 @@ class NightlyTest(unittest.TestCase): 'repo_keyalias': 'androiddebugkey', 'repo_name': 'fdroid/test-nightly', 'repo_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/repo', - 'servergitmirrors': [{"url": 'git@gitlab.com:fdroid/test-nightly'}], + 'servergitmirrors': [{"url": 'git@gitlab.com:fdroid/test-nightly.git'}], } with open(common.CONFIG_FILE) as fp: config = yaml.safe_load(fp)