Merge branch 'nightly-fixes' into 'master'

fixes to deploy/nightly made while working on !1666 and reviewing !1650

See merge request fdroid/fdroidserver!1668

fdroidserver/__main__.py

@@ -156,7 +156,7 @@ def main():
                     ).git.describe(always=True, tags=True)
                 )
                 sys.exit(0)
-            except git.exc.InvalidGitRepositoryError:
+            except git.InvalidGitRepositoryError:
                 print(_('No version information could be found.'))
                 sys.exit(1)
         else:

fdroidserver/deploy.py

@@ -40,7 +40,6 @@ import fdroidserver.github
 from . import _, common, index
 from .exception import FDroidException
 
-config = None
 start_timestamp = time.gmtime()
 
 GIT_BRANCH = 'master'
@@ -144,6 +143,7 @@ def update_remote_storage_with_rclone(
     """
     logging.debug(_('Using rclone to sync to "{name}"').format(name=awsbucket))
 
+    config = common.get_config()
     rclone_config = config.get('rclone_config', [])
     if rclone_config and isinstance(rclone_config, str):
         rclone_config = [rclone_config]
@@ -271,6 +271,7 @@ def update_serverwebroot(serverwebroot, repo_section):
     has a low resolution timestamp
 
     """
+    config = common.get_config()
     try:
         subprocess.run(['rsync', '--version'], capture_output=True, check=True)
     except Exception as e:
@@ -431,6 +432,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
     """
     from clint.textui import progress
 
+    config = common.get_config()
     if config.get('local_copy_dir') and not config.get('sync_from_local_copy_dir'):
         logging.debug(
             _('Offline machine, skipping git mirror generation until `fdroid deploy`')
@@ -530,6 +532,20 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             progressbar.done()
 
 
+def _get_commit_author(git_repo):
+    """If the author is set locally, use it, otherwise use static info."""
+    ret = {'name': 'servergitmirrors', 'email': 'fdroid@deploy'}
+    with git_repo.config_reader() as cr:
+        for option in ('name', 'email'):
+            try:
+                value = cr.get_value('user', option)
+            except (configparser.NoSectionError, configparser.NoOptionError):
+                value = os.getenv(f'GITLAB_USER_{option.upper()}')
+            if value:
+                ret[option] = value
+    return git.Actor(ret['name'], ret['email'])
+
+
 def upload_to_servergitmirror(
     mirror_config: Dict[str, str],
     local_repo: Repo,
@@ -557,18 +573,23 @@ def upload_to_servergitmirror(
     logging.info('Mirroring to: ' + remote_url)
 
     if is_index_only:
+        logging.debug(_('Committing index files to git mirror'))
         files_to_upload = _get_index_file_paths(
             os.path.join(local_repo.working_tree_dir, 'fdroid', repo_section)
         )
         files_to_upload = _remove_missing_files(files_to_upload)
         local_repo.index.add(files_to_upload)
+        local_repo.index.commit(
+            "servergitmirrors: index-only in git-mirror",
+            author=_get_commit_author(local_repo),
+        )
     else:
         # sadly index.add don't allow the --all parameter
-        logging.debug('Adding all files to git mirror')
+        logging.debug(_('Adding all files to git mirror'))
         local_repo.git.add(all=True)
-
-    logging.debug('Committing files into git mirror')
-    local_repo.index.commit("fdroidserver git-mirror")
+        local_repo.index.commit(
+            "servergitmirrors: in git-mirror", author=_get_commit_author(local_repo)
+        )
 
     # only deploy to GitLab Artifacts if too big for GitLab Pages
     if (
@@ -1029,8 +1050,6 @@ def upload_to_github_releases_repo(repo_conf, release_infos, global_gh_token):
 
 
 def main():
-    global config
-
     parser = ArgumentParser()
     common.setup_global_opts(parser)
     parser.add_argument(

fdroidserver/nightly.py

@@ -19,6 +19,7 @@
 
 import base64
 import datetime
+import glob
 import hashlib
 import inspect
 import logging
@@ -236,11 +237,21 @@ def clone_git_repo(clone_url, git_mirror_path):
                     'GIT_TERMINAL_PROMPT': '0',
                 },
             )
-    except git.exc.GitCommandError as e:
+    except git.GitCommandError as e:
         logging.warning(_('WARNING: only public git repos are supported!'))
         raise VCSException(f'git clone {clone_url} failed:', str(e)) from e
 
 
+def _getenv(name):
+    """Return the value of an environment variable, printing an error if None."""
+    value = os.getenv(name)
+    if not value:
+        logging.error(
+            _('Environment variable "{name}" has an empty value!').format(name=name)
+        )
+    return value
+
+
 def main():
     """Deploy to F-Droid repository or generate SSH private key from keystore.
 
@@ -303,7 +314,7 @@ def main():
     umask = os.umask(0o077)
 
     if 'CI' in os.environ:
-        v = os.getenv('DEBUG_KEYSTORE')
+        v = _getenv('DEBUG_KEYSTORE')
         debug_keystore = None
         if v:
             debug_keystore = base64.b64decode(v)
@@ -324,20 +335,19 @@ def main():
         # the 'master' branch is hardcoded in fdroidserver/deploy.py
         if 'CI_PROJECT_PATH' in os.environ and 'CI_PROJECT_URL' in os.environ:
             # we are in GitLab CI
-            repo_git_base = os.getenv('CI_PROJECT_PATH') + NIGHTLY
-            clone_url = os.getenv('CI_PROJECT_URL') + NIGHTLY
+            repo_git_base = f"{_getenv('CI_PROJECT_PATH')}{NIGHTLY}"
+            base_url = f"{_getenv('CI_PROJECT_URL')}{NIGHTLY}"
+            clone_url = f'{base_url}.git'  # avoid redirects while cloning
             repo_base = get_repo_base_url(
-                clone_url, repo_git_base, force_type='gitlab.com'
+                base_url, repo_git_base, force_type='gitlab.com'
             )
-            servergitmirror = 'git@' + urlparse(clone_url).netloc + ':' + repo_git_base
-            deploy_key_url = (
-                f'{clone_url}/-/settings/repository#js-deploy-keys-settings'
-            )
-            git_user_name = os.getenv('GITLAB_USER_NAME')
-            git_user_email = os.getenv('GITLAB_USER_EMAIL')
+            servergitmirror = f'git@{urlparse(base_url).netloc}:{repo_git_base}.git'
+            deploy_key_url = f'{base_url}/-/settings/repository#js-deploy-keys-settings'
+            git_user_name = _getenv('GITLAB_USER_NAME')
+            git_user_email = _getenv('GITLAB_USER_EMAIL')
         elif 'TRAVIS_REPO_SLUG' in os.environ:
             # we are in Travis CI
-            repo_git_base = os.getenv('TRAVIS_REPO_SLUG') + NIGHTLY
+            repo_git_base = _getenv('TRAVIS_REPO_SLUG') + NIGHTLY
             clone_url = 'https://github.com/' + repo_git_base
             repo_base = get_repo_base_url(
                 clone_url, repo_git_base, force_type='github.com'
@@ -348,7 +358,7 @@ def main():
                 + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys'
             )
             git_user_name = repo_git_base
-            git_user_email = os.getenv('USER') + '@' + platform.node()
+            git_user_email = _getenv('USER') + '@' + platform.node()
         elif (
             'CIRCLE_REPOSITORY_URL' in os.environ
             and 'CIRCLE_PROJECT_USERNAME' in os.environ
@@ -356,12 +366,12 @@ def main():
         ):
             # we are in Circle CI
             repo_git_base = (
-                os.getenv('CIRCLE_PROJECT_USERNAME')
+                _getenv('CIRCLE_PROJECT_USERNAME')
                 + '/'
-                + os.getenv('CIRCLE_PROJECT_REPONAME')
+                + _getenv('CIRCLE_PROJECT_REPONAME')
                 + NIGHTLY
             )
-            clone_url = os.getenv('CIRCLE_REPOSITORY_URL') + NIGHTLY
+            clone_url = _getenv('CIRCLE_REPOSITORY_URL') + NIGHTLY
             repo_base = get_repo_base_url(
                 clone_url, repo_git_base, force_type='github.com'
             )
@@ -370,12 +380,12 @@ def main():
                 f'https://github.com/{repo_git_base}/settings/keys'
                 + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys'
             )
-            git_user_name = os.getenv('CIRCLE_USERNAME')
+            git_user_name = _getenv('CIRCLE_USERNAME')
             git_user_email = git_user_name + '@' + platform.node()
         elif 'GITHUB_ACTIONS' in os.environ:
             # we are in Github actions
-            repo_git_base = os.getenv('GITHUB_REPOSITORY') + NIGHTLY
-            clone_url = os.getenv('GITHUB_SERVER_URL') + '/' + repo_git_base
+            repo_git_base = _getenv('GITHUB_REPOSITORY') + NIGHTLY
+            clone_url = _getenv('GITHUB_SERVER_URL') + '/' + repo_git_base
             repo_base = get_repo_base_url(
                 clone_url, repo_git_base, force_type='github.com'
             )
@@ -384,7 +394,7 @@ def main():
                 f'https://github.com/{repo_git_base}/settings/keys'
                 + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys'
             )
-            git_user_name = os.getenv('GITHUB_ACTOR')
+            git_user_name = _getenv('GITHUB_ACTOR')
             git_user_email = git_user_name + '@' + platform.node()
         else:
             print(_('ERROR: unsupported CI type, patches welcome!'))
@@ -397,6 +407,8 @@ def main():
         git_mirror_metadatadir = os.path.join(git_mirror_fdroiddir, 'metadata')
         if not os.path.isdir(git_mirror_repodir):
             clone_git_repo(clone_url, git_mirror_path)
+            for f in glob.glob(f'{git_mirror_repodir}/*.apk'):
+                shutil.copy2(f, repodir)
         if not os.path.isdir(git_mirror_repodir):
             os.makedirs(git_mirror_repodir, mode=0o755)
         if os.path.exists('LICENSE'):
@@ -442,7 +454,7 @@ Last updated: {date}'''.format(
 
         ssh_private_key_file = _ssh_key_from_debug_keystore()
         # this is needed for GitPython to find the SSH key
-        ssh_dir = os.path.join(os.getenv('HOME'), '.ssh')
+        ssh_dir = os.path.join(_getenv('HOME'), '.ssh')
         os.makedirs(ssh_dir, exist_ok=True)
         ssh_config = os.path.join(ssh_dir, 'config')
         logging.debug(_('adding IdentityFile to {path}').format(path=ssh_config))
@@ -587,7 +599,7 @@ Last updated: {date}'''.format(
                 + '\n     -dname "CN=Android Debug,O=Android,C=US"'
             )
             sys.exit(1)
-        ssh_dir = os.path.join(os.getenv('HOME'), '.ssh')
+        ssh_dir = os.path.join(_getenv('HOME'), '.ssh')
         privkey = _ssh_key_from_debug_keystore(options.keystore)
         if os.path.exists(ssh_dir):
             ssh_private_key_file = os.path.join(ssh_dir, os.path.basename(privkey))

fdroidserver/update.py

@@ -2466,16 +2466,18 @@ def create_metadata_from_template(apk):
 
 
 def read_added_date_from_all_apks(apps, apks):
-    """No summary.
+    """Read the "added" date from all packages.
 
-    Added dates come from the repo/index-v2.json file but are
-    read when scanning apks and thus need to be applied form apk
-    level to app level for _all_ apps and not only from non-archived
-    ones
+    Added dates come from index-v2.json file but are read when scanning
+    APKs and thus need to be applied from package-level to app-level for
+    _all_ apps and not only from non-archived ones.
 
     TODO: read the added dates directly from index-v2.json instead of
           going through apks that way it also works for for repos that
-          don't keep an archive of apks.
+          don't keep an archive of APKs.
+
+    https://gitlab.com/fdroid/wiki/-/wikis/PackageDateHandling
+
     """
     for appid, app in apps.items():
         for apk in apks:

tests/test_deploy.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import configparser
+import logging
 import os
 import shutil
 import tempfile
@@ -26,15 +27,21 @@ def _mock_rclone_config_file(cmd, text):  # pylint: disable=unused-argument
 class DeployTest(unittest.TestCase):
     '''fdroidserver/deploy.py'''
 
+    @classmethod
+    def setUpClass(cls):
+        # suppress "WARNING:root:unsafe permissions on 'config.yml' (should be 0600)!"
+        os.chmod(os.path.join(basedir, fdroidserver.common.CONFIG_FILE), 0o600)
+
     def setUp(self):
         os.chdir(basedir)
         self._td = mkdtemp()
         self.testdir = self._td.name
 
         fdroidserver.common.options = mock.Mock()
-        fdroidserver.deploy.config = {}
+        fdroidserver.common.get_config()
 
     def tearDown(self):
+        fdroidserver.common.config = None
         self._td.cleanup()
 
     def test_update_serverwebroots_bad_None(self):
@@ -64,7 +71,6 @@ class DeployTest(unittest.TestCase):
 
         # setup parameters for this test run
         fdroidserver.common.options.identity_file = None
-        fdroidserver.deploy.config['make_current_version_link'] = False
 
         dest_apk0 = url0 / fake_apk
         dest_apk1 = url1 / fake_apk
@@ -81,17 +87,17 @@ class DeployTest(unittest.TestCase):
         self.assertTrue(dest_apk1.is_file())
 
     def test_update_serverwebroots_url_does_not_end_with_fdroid(self):
-        with self.assertRaises(SystemExit):
+        with self.assertRaises(SystemExit), self.assertLogs(level=logging.ERROR):
             fdroidserver.deploy.update_serverwebroots([{'url': 'url'}], 'repo')
 
     def test_update_serverwebroots_bad_ssh_url(self):
-        with self.assertRaises(SystemExit):
+        with self.assertRaises(SystemExit), self.assertLogs(level=logging.ERROR):
             fdroidserver.deploy.update_serverwebroots(
                 [{'url': 'f@b.ar::/path/to/fdroid'}], 'repo'
             )
 
     def test_update_serverwebroots_unsupported_ssh_url(self):
-        with self.assertRaises(SystemExit):
+        with self.assertRaises(SystemExit), self.assertLogs(level=logging.ERROR):
             fdroidserver.deploy.update_serverwebroots([{'url': 'ssh://nope'}], 'repo')
 
     @unittest.skipUnless(shutil.which('rclone'), 'requires rclone')
@@ -120,9 +126,9 @@ class DeployTest(unittest.TestCase):
 
         # setup parameters for this test run
         awsbucket = 'test_bucket_folder'
-        fdroidserver.deploy.config['awsbucket'] = awsbucket
-        fdroidserver.deploy.config['rclone_config'] = 'test-local-config'
-        fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
+        fdroidserver.common.config['awsbucket'] = awsbucket
+        fdroidserver.common.config['rclone_config'] = 'test-local-config'
+        fdroidserver.common.config['path_to_custom_rclone_config'] = str(rclone_file)
         fdroidserver.common.options = VerboseFalseOptions
 
         # write out destination path
@@ -163,9 +169,9 @@ class DeployTest(unittest.TestCase):
 
         # setup parameters for this test run
         awsbucket = 'test_bucket_folder'
-        fdroidserver.deploy.config['awsbucket'] = awsbucket
-        fdroidserver.deploy.config['rclone_config'] = 'test-local-config'
-        fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
+        fdroidserver.common.config['awsbucket'] = awsbucket
+        fdroidserver.common.config['rclone_config'] = 'test-local-config'
+        fdroidserver.common.config['path_to_custom_rclone_config'] = str(rclone_file)
         fdroidserver.common.options = VerboseFalseOptions
 
         # write out destination path
@@ -224,7 +230,7 @@ class DeployTest(unittest.TestCase):
             return 0
 
         mock_call.side_effect = _mock_subprocess_call
-        fdroidserver.deploy.config = {'awsbucket': awsbucket}
+        fdroidserver.common.config = {'awsbucket': awsbucket}
         fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket)
         mock_call.assert_called()
 
@@ -243,7 +249,7 @@ class DeployTest(unittest.TestCase):
 
         mock_call.side_effect = _mock_subprocess_call
 
-        fdroidserver.deploy.config = {'awsbucket': awsbucket}
+        fdroidserver.common.config = {'awsbucket': awsbucket}
         fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket)
         self.maxDiff = None
         self.assertEqual(
@@ -262,7 +268,6 @@ class DeployTest(unittest.TestCase):
     @mock.patch('subprocess.check_output', _mock_rclone_config_file)
     @mock.patch('subprocess.call')
     def test_update_remote_storage_with_rclone_mock_rclone_config(self, mock_call):
-        awsbucket = 'test_bucket_folder'
         self.last_cmd = None
 
         def _mock_subprocess_call(cmd):
@@ -271,10 +276,9 @@ class DeployTest(unittest.TestCase):
 
         mock_call.side_effect = _mock_subprocess_call
 
-        fdroidserver.deploy.config = {
-            'awsbucket': awsbucket,
-            'rclone_config': 'test_local_config',
-        }
+        awsbucket = 'test_bucket_folder'
+        fdroidserver.common.config['awsbucket'] = awsbucket
+        fdroidserver.common.config['rclone_config'] = 'test_local_config'
         fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket)
         self.maxDiff = None
         self.assertEqual(
@@ -304,8 +308,8 @@ class DeployTest(unittest.TestCase):
         Path('rclone.conf').write_text('placeholder, contents ignored')
 
         awsbucket = 'test_bucket_folder'
-        fdroidserver.deploy.config['awsbucket'] = awsbucket
-        fdroidserver.deploy.config['rclone_config'] = config_name
+        fdroidserver.common.config['awsbucket'] = awsbucket
+        fdroidserver.common.config['rclone_config'] = config_name
         fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket)
         self.maxDiff = None
         self.assertEqual(
@@ -339,7 +343,6 @@ class DeployTest(unittest.TestCase):
         fdroidserver.common.options = mock.Mock()
         fdroidserver.common.options.identity_file = None
         fdroidserver.common.options.identity_file = None
-        fdroidserver.deploy.config['make_current_version_link'] = False
 
         dest_apk = Path(url) / fake_apk
         dest_index = Path(url) / fake_index
@@ -366,7 +369,6 @@ class DeployTest(unittest.TestCase):
         # setup parameters for this test run
         fdroidserver.common.options = mock.Mock()
         fdroidserver.common.options.identity_file = None
-        fdroidserver.deploy.config['make_current_version_link'] = False
 
         dest_apk = Path(url) / fake_apk
         dest_index = Path(url) / fake_index
@@ -396,7 +398,7 @@ class DeployTest(unittest.TestCase):
         fdroidserver.common.options.verbose = False
         fdroidserver.common.options.quiet = True
         fdroidserver.common.options.index_only = False
-        fdroidserver.deploy.config = {'make_current_version_link': True}
+        fdroidserver.common.config['make_current_version_link'] = True
         url = "example.com:/var/www/fdroid"
         repo_section = 'repo'
 
@@ -504,7 +506,7 @@ class DeployTest(unittest.TestCase):
         fdroidserver.common.options.verbose = False
         fdroidserver.common.options.quiet = True
         fdroidserver.common.options.identity_file = None
-        fdroidserver.deploy.config['make_current_version_link'] = True
+        fdroidserver.common.config['make_current_version_link'] = True
         url = "example.com:/var/www/fdroid"
         repo_section = 'repo'
 
@@ -603,7 +605,7 @@ class DeployTest(unittest.TestCase):
         fdroidserver.common.options.quiet = False
         fdroidserver.common.options.identity_file = None
         fdroidserver.common.options.index_only = False
-        fdroidserver.deploy.config = {'identity_file': './id_rsa'}
+        fdroidserver.common.config = {'identity_file': './id_rsa'}
         url = "example.com:/var/www/fdroid"
         repo_section = 'archive'
 
@@ -623,7 +625,7 @@ class DeployTest(unittest.TestCase):
                         '--verbose',
                         '-e',
                         'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
-                        + fdroidserver.deploy.config['identity_file'],
+                        + fdroidserver.common.config['identity_file'],
                         '--exclude',
                         'archive/altstore-index.json',
                         '--exclude',
@@ -669,7 +671,7 @@ class DeployTest(unittest.TestCase):
                         '--verbose',
                         '-e',
                         'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
-                        + fdroidserver.deploy.config['identity_file'],
+                        + fdroidserver.common.config['identity_file'],
                         'archive',
                         url,
                     ],
@@ -690,8 +692,7 @@ class DeployTest(unittest.TestCase):
         fdroidserver.common.options.verbose = True
         fdroidserver.common.options.quiet = False
         fdroidserver.common.options.identity_file = None
-        fdroidserver.deploy.config['identity_file'] = './id_rsa'
-        fdroidserver.deploy.config['make_current_version_link'] = False
+        fdroidserver.common.config['identity_file'] = './id_rsa'
         url = "example.com:/var/www/fdroid"
         repo_section = 'archive'
 
@@ -711,7 +712,7 @@ class DeployTest(unittest.TestCase):
                         '--verbose',
                         '-e',
                         'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
-                        + fdroidserver.deploy.config['identity_file'],
+                        + fdroidserver.common.config['identity_file'],
                         'archive/altstore-index.json',
                         'archive/altstore-index.json.asc',
                         'archive/entry.jar',
@@ -741,7 +742,7 @@ class DeployTest(unittest.TestCase):
                         '--verbose',
                         '-e',
                         'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
-                        + fdroidserver.deploy.config['identity_file'],
+                        + fdroidserver.common.config['identity_file'],
                         "example.com:/var/www/fdroid/archive/",
                     ],
                 )
@@ -785,7 +786,58 @@ class DeployTest(unittest.TestCase):
                 name, fdroidserver.deploy.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
             )
 
-    def test_update_servergitmirrors(self):
+    @mock.patch.dict(os.environ, clear=True)
+    def test_get_commit_author_no_config(self):
+        os.environ['HOME'] = self.testdir
+        git_repo = git.Repo.init(self.testdir)
+        self.assertEqual(
+            git.Actor('servergitmirrors', 'fdroid@deploy'),
+            fdroidserver.deploy._get_commit_author(git_repo),
+        )
+
+    @mock.patch.dict(os.environ, clear=True)
+    def test_get_commit_author_repo_config(self):
+        os.environ['HOME'] = self.testdir
+        git_repo = git.Repo.init(self.testdir)
+        user_name = 'Foo Bar'
+        user_email = 'foo@bar.com'
+        with git_repo.config_writer() as cw:
+            cw.set_value('user', 'name', user_name)
+            cw.set_value('user', 'email', user_email)
+        self.assertEqual(
+            git.Actor(user_name, user_email),
+            fdroidserver.deploy._get_commit_author(git_repo),
+        )
+
+    @mock.patch.dict(os.environ, clear=True)
+    def test_get_commit_author_repo_config_name_only(self):
+        os.environ['HOME'] = self.testdir
+        git_repo = git.Repo.init(self.testdir)
+        user_name = 'Foo Bar'
+        with git_repo.config_writer() as cw:
+            cw.set_value('user', 'name', user_name)
+        self.assertEqual(
+            git.Actor(user_name, 'fdroid@deploy'),
+            fdroidserver.deploy._get_commit_author(git_repo),
+        )
+
+    @mock.patch.dict(os.environ, clear=True)
+    def test_get_commit_author_repo_config_email_only(self):
+        os.environ['HOME'] = self.testdir
+        git_repo = git.Repo.init(self.testdir)
+        user_email = 'foo@bar.com'
+        with git_repo.config_writer() as cw:
+            cw.set_value('user', 'email', user_email)
+        self.assertEqual(
+            git.Actor('servergitmirrors', user_email),
+            fdroidserver.deploy._get_commit_author(git_repo),
+        )
+
+
+class TestServerGitMirrors(unittest.TestCase):
+    def setUp(self):
+        fdroidserver.deploy.USER_RCLONE_CONF = False
+
         # setup parameters for this test run
         fdroidserver.common.options = mock.Mock()
         fdroidserver.common.options.identity_file = None
@@ -793,130 +845,93 @@ class DeployTest(unittest.TestCase):
         fdroidserver.common.options.verbose = False
         fdroidserver.common.options.quiet = True
 
-        config = {}
-        fdroidserver.common.fill_config_defaults(config)
-        fdroidserver.deploy.config = config
-
+        self._td = mkdtemp()
+        self.testdir = self._td.name
         os.chdir(self.testdir)
 
-        repo_section = 'repo'
-        initial_branch = fdroidserver.deploy.GIT_BRANCH
-
         remote_repo = Path(self.testdir) / 'remote'
         remote_repo.mkdir(parents=True)
-        remote_git_repo = git.Repo.init(
-            remote_repo, initial_branch=initial_branch, bare=True
+        self.remote_git_repo = git.Repo.init(
+            remote_repo, initial_branch=fdroidserver.deploy.GIT_BRANCH, bare=True
         )
-        fdroidserver.deploy.config["servergitmirrors"] = [{"url": str(remote_repo)}]
 
-        os.chdir(self.testdir)
-        repo = Path('repo')
-        repo.mkdir(parents=True)
-        fake_apk = 'Sym.apk'
-        fake_files = fdroidserver.common.INDEX_FILES + [fake_apk]
-        for filename in fake_files:
+        fdroidserver.common.get_config()
+        fdroidserver.common.config["servergitmirrors"] = [{"url": str(remote_repo)}]
+
+        self.repo_section = 'repo'
+        repo = Path(self.repo_section)
+        repo.mkdir()
+        self.fake_apk = 'Sym.apk'
+        self.fake_files = fdroidserver.common.INDEX_FILES + [self.fake_apk]
+        for filename in self.fake_files:
             fake_file = repo / filename
             with fake_file.open('w') as fp:
                 fp.write('not a real one, but has the right filename')
 
+    def tearDown(self):
+        fdroidserver.common.config = None
+        fdroidserver.common.options = None
+        self._td.cleanup()
+
+    def test_update_servergitmirrors(self):
         fdroidserver.deploy.update_servergitmirrors(
-            fdroidserver.deploy.config["servergitmirrors"], repo_section
+            fdroidserver.common.config["servergitmirrors"], self.repo_section
         )
 
-        verify_repo = remote_git_repo.clone(
-            Path(self.testdir) / 'verify',
-        )
-
-        for filename in fake_files:
-            remote_file = f"fdroid/{repo_section}/{filename}"
-
+        verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify')
         self.assertIsNotNone(verify_repo.working_tree_dir)
-            if verify_repo.working_tree_dir is not None:
-                self.assertTrue(
-                    (Path(verify_repo.working_tree_dir) / remote_file).exists()
+        for filename in self.fake_files:
+            remote_file = f"fdroid/{self.repo_section}/{filename}"
+            self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists())
+
+    def test_update_servergitmirrors_with_existing_git_repo(self):
+        """Confirm it works with clones done manually or with nightly."""
+        fdroidserver.deploy.update_servergitmirrors(
+            fdroidserver.common.config["servergitmirrors"], self.repo_section
         )
 
+        # now delete the local setup, clone the remote, and add a new APK
+        git_mirror = os.path.join(self.testdir, 'git-mirror')
+        shutil.rmtree(git_mirror)
+        self.remote_git_repo.clone(git_mirror)
+        new_fake_apk = 'Sym2.apk'
+        self.fake_files.append(new_fake_apk)
+        (Path(self.repo_section) / new_fake_apk).write_text('a new fake APK')
+
+        fdroidserver.deploy.update_servergitmirrors(
+            fdroidserver.common.config["servergitmirrors"], self.repo_section
+        )
+
+        verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify')
+        self.assertIsNotNone(verify_repo.working_tree_dir)
+        for filename in self.fake_files:
+            remote_file = f"fdroid/{self.repo_section}/{filename}"
+            self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists())
+
     def test_update_servergitmirrors_in_index_only_mode(self):
-        # setup parameters for this test run
-        fdroidserver.common.options = mock.Mock()
-        fdroidserver.common.options.identity_file = None
-        fdroidserver.common.options.no_keep_git_mirror_archive = False
-        fdroidserver.common.options.verbose = False
-        fdroidserver.common.options.quiet = True
-
-        config = {}
-        fdroidserver.common.fill_config_defaults(config)
-        fdroidserver.deploy.config = config
-
-        os.chdir(self.testdir)
-
-        repo_section = 'repo'
-        initial_branch = fdroidserver.deploy.GIT_BRANCH
-
-        remote_repo = Path(self.testdir) / 'remote'
-        remote_repo.mkdir(parents=True)
-        remote_git_repo = git.Repo.init(
-            remote_repo, initial_branch=initial_branch, bare=True
-        )
-        fdroidserver.deploy.config["servergitmirrors"] = [
-            {"url": str(remote_repo), "index_only": True}
-        ]
-
-        os.chdir(self.testdir)
-        repo = Path('repo')
-        repo.mkdir(parents=True)
-        fake_apk = 'Sym.apk'
-        fake_files = fdroidserver.common.INDEX_FILES + [fake_apk]
-        for filename in fake_files:
-            fake_file = repo / filename
-            with fake_file.open('w') as fp:
-                fp.write('not a real one, but has the right filename')
+        fdroidserver.common.config["servergitmirrors"][0]["index_only"] = True
 
         fdroidserver.deploy.update_servergitmirrors(
-            fdroidserver.deploy.config["servergitmirrors"], repo_section
+            fdroidserver.common.config["servergitmirrors"], self.repo_section
         )
 
-        verify_repo = remote_git_repo.clone(
-            Path(self.testdir) / 'verify',
-        )
-
-        for filename in fdroidserver.common.INDEX_FILES:
-            remote_file = f"fdroid/{repo_section}/{filename}"
-
+        verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify')
         self.assertIsNotNone(verify_repo.working_tree_dir)
-            if verify_repo.working_tree_dir is not None:
-                self.assertTrue(
-                    (Path(verify_repo.working_tree_dir) / remote_file).exists()
-                )
+        for filename in fdroidserver.common.INDEX_FILES:
+            remote_file = f"fdroid/{self.repo_section}/{filename}"
+            self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists())
 
         # Should not have the APK file
-        remote_file = f"fdroid/{repo_section}/{fake_apk}"
-        if verify_repo.working_tree_dir is not None:
-            self.assertFalse(
-                (Path(verify_repo.working_tree_dir) / remote_file).exists()
-            )
+        remote_file = f"fdroid/{self.repo_section}/{self.fake_apk}"
+        self.assertFalse((Path(verify_repo.working_tree_dir) / remote_file).exists())
 
     def test_upload_to_servergitmirror_in_index_only_mode(self):
-        # setup parameters for this test run
-        fdroidserver.common.options = mock.Mock()
-        fdroidserver.common.options.identity_file = None
-        fdroidserver.common.options.no_keep_git_mirror_archive = False
-        fdroidserver.common.options.verbose = False
-        fdroidserver.common.options.quiet = True
-        fdroidserver.common.options.identity_file = None
-
-        config = {}
-        fdroidserver.common.fill_config_defaults(config)
-        fdroidserver.deploy.config = config
+        shutil.rmtree('repo')  # the class-wide test files are not used here
 
         repo_section = 'repo'
-        initial_branch = fdroidserver.deploy.GIT_BRANCH
-
-        os.chdir(self.testdir)
-
         local_git_repo_path = Path(self.testdir) / 'local'
         local_git_repo = git.Repo.init(
-            local_git_repo_path, initial_branch=initial_branch
+            local_git_repo_path, initial_branch=fdroidserver.deploy.GIT_BRANCH
         )
 
         fdroid_dir = local_git_repo_path / 'fdroid'
@@ -929,13 +944,7 @@ class DeployTest(unittest.TestCase):
             with fake_file.open('w') as fp:
                 fp.write('not a real one, but has the right filename')
 
-        # The remote repo must be a bare repo to allow being pushed to
-        remote_git_repo_dir = Path(self.testdir) / 'remote'
-        remote_git_repo = git.Repo.init(
-            remote_git_repo_dir, initial_branch=initial_branch, bare=True
-        )
-
-        mirror_config = {"url": str(remote_git_repo_dir), "index_only": True}
+        mirror_config = {"url": str(self.remote_git_repo.git_dir), "index_only": True}
         enabled_remotes = []
         ssh_cmd = 'ssh -oBatchMode=yes'
         fdroidserver.deploy.upload_to_servergitmirror(
@@ -950,25 +959,15 @@ class DeployTest(unittest.TestCase):
             progress=git.RemoteProgress(),
         )
 
-        verify_repo = remote_git_repo.clone(
-            Path(self.testdir) / 'verify',
-        )
-
+        verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify')
+        self.assertIsNotNone(verify_repo.working_tree_dir)
         for filename in fdroidserver.common.INDEX_FILES:
             remote_file = f"fdroid/{repo_section}/{filename}"
-
-            self.assertIsNotNone(verify_repo.working_tree_dir)
-            if verify_repo.working_tree_dir is not None:
-                self.assertTrue(
-                    (Path(verify_repo.working_tree_dir) / remote_file).exists()
-                )
+            self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists())
 
         # Should not have the APK file
         remote_file = f"fdroid/{repo_section}/{fake_apk}"
-        if verify_repo.working_tree_dir is not None:
-            self.assertFalse(
-                (Path(verify_repo.working_tree_dir) / remote_file).exists()
-            )
+        self.assertFalse((Path(verify_repo.working_tree_dir) / remote_file).exists())
 
 
 class GitHubReleasesTest(unittest.TestCase):

tests/test_integration.py

@@ -75,7 +75,6 @@ class IntegrationTest(unittest.TestCase):
         self.testdir = mkdir_testfiles(WORKSPACE, self)
         self.tmp_repo_root = self.testdir / "fdroid"
         self.tmp_repo_root.mkdir(parents=True)
-        deploy.config = {}
         os.chdir(self.tmp_repo_root)
 
     def tearDown(self):
@@ -1611,10 +1610,11 @@ class IntegrationTest(unittest.TestCase):
                 rclone_config.write(configfile)
 
             # set up config for run
+            common.get_config()
             awsbucket = "test-bucket"
-            deploy.config['awsbucket'] = awsbucket
-            deploy.config['rclone_config'] = "test-minio-config"
-            deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
+            common.config['awsbucket'] = awsbucket
+            common.config['rclone_config'] = "test-minio-config"
+            common.config['path_to_custom_rclone_config'] = str(rclone_file)
             common.options = VerboseFalseOptions
 
             # call function
@@ -1667,9 +1667,9 @@ class IntegrationTest(unittest.TestCase):
 
             # set up config for run
             awsbucket = "test-bucket"
-            deploy.config['awsbucket'] = awsbucket
-            deploy.config['rclone_config'] = "test-minio-config"
-            deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
+            common.config['awsbucket'] = awsbucket
+            common.config['rclone_config'] = "test-minio-config"
+            common.config['path_to_custom_rclone_config'] = str(rclone_file)
             common.options = VerboseFalseOptions
 
             # call function

tests/test_nightly.py

@@ -359,7 +359,7 @@ class NightlyTest(unittest.TestCase):
             'repo_keyalias': 'androiddebugkey',
             'repo_name': 'fdroid/test-nightly',
             'repo_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/repo',
-            'servergitmirrors': [{"url": 'git@gitlab.com:fdroid/test-nightly'}],
+            'servergitmirrors': [{"url": 'git@gitlab.com:fdroid/test-nightly.git'}],
         }
         with open(common.CONFIG_FILE) as fp:
             config = yaml.safe_load(fp)