Merge branch 'serverwebroot-as-dict' into 'master'

config: convert serverwebroot: to list-of-dicts format

See merge request fdroid/fdroidserver!1430

examples/config.yml

@@ -178,6 +178,12 @@
 # serverwebroot:
 #   - foo.com:/usr/share/nginx/www/fdroid
 #   - bar.info:/var/www/fdroid
+#
+# There is a special mode to only deploy the index file:
+#
+# serverwebroot:
+#   - url: 'me@b.az:/srv/fdroid'
+#     indexOnly: true
 
 
 # When running fdroid processes on a remote server, it is possible to

fdroidserver/__init__.py

@@ -55,7 +55,9 @@ scan_apk  # NOQA: B101
 scan_repo_files  # NOQA: B101
 from fdroidserver.deploy import (update_awsbucket,
                                  update_servergitmirrors,
+                                 update_serverwebroots,
                                  update_serverwebroot)  # NOQA: E402
 update_awsbucket  # NOQA: B101
 update_servergitmirrors  # NOQA: B101
+update_serverwebroots  # NOQA: B101
 update_serverwebroot  # NOQA: B101

fdroidserver/common.py

@@ -462,18 +462,22 @@ def read_config(opts=None):
 
     if 'serverwebroot' in config:
         if isinstance(config['serverwebroot'], str):
-            roots = [config['serverwebroot']]
+            roots = [{'url': config['serverwebroot']}]
         elif all(isinstance(item, str) for item in config['serverwebroot']):
+            roots = [{'url': i} for i in config['serverwebroot']]
+        elif all(isinstance(item, dict) for item in config['serverwebroot']):
             roots = config['serverwebroot']
         else:
             raise TypeError(_('only accepts strings, lists, and tuples'))
         rootlist = []
-        for rootstr in roots:
+        for d in roots:
             # since this is used with rsync, where trailing slashes have
             # meaning, ensure there is always a trailing slash
+            rootstr = d['url']
             if rootstr[-1] != '/':
                 rootstr += '/'
-            rootlist.append(rootstr.replace('//', '/'))
+            d['url'] = rootstr.replace('//', '/')
+            rootlist.append(d)
         config['serverwebroot'] = rootlist
 
     if 'servergitmirrors' in config:
@@ -4052,7 +4056,8 @@ def rsync_status_file_to_repo(path, repo_subdir=None):
         logging.debug(_('skip deploying full build logs: not enabled in config'))
         return
 
-    for webroot in config.get('serverwebroot', []):
+    for d in config.get('serverwebroot', []):
+        webroot = d['url']
         cmd = ['rsync',
                '--archive',
                '--delete-after',

fdroidserver/deploy.py

@@ -284,34 +284,63 @@ def update_serverwebroot(serverwebroot, repo_section):
             _('rsync is missing or broken: {error}').format(error=e)
         ) from e
     rsyncargs = ['rsync', '--archive', '--delete-after', '--safe-links']
-    if not options.no_checksum:
+    if not options or not options.no_checksum:
         rsyncargs.append('--checksum')
-    if options.verbose:
+    if options and options.verbose:
         rsyncargs += ['--verbose']
-    if options.quiet:
+    if options and options.quiet:
         rsyncargs += ['--quiet']
-    if options.identity_file is not None:
+    if options and options.identity_file:
         rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + options.identity_file]
-    elif 'identity_file' in config:
+    elif config and config.get('identity_file'):
         rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']]
-    logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot)
+    url = serverwebroot['url']
+    logging.info('rsyncing ' + repo_section + ' to ' + url)
     excludes = _get_index_excludes(repo_section)
-    if subprocess.call(rsyncargs + excludes + [repo_section, serverwebroot]) != 0:
+    if subprocess.call(rsyncargs + excludes + [repo_section, url]) != 0:
         raise FDroidException()
-    if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0:
+    if subprocess.call(rsyncargs + [repo_section, url]) != 0:
         raise FDroidException()
     # upload "current version" symlinks if requested
-    if config['make_current_version_link'] and repo_section == 'repo':
+    if config and config.get('make_current_version_link') and repo_section == 'repo':
         links_to_upload = []
         for f in glob.glob('*.apk') \
                 + glob.glob('*.apk.asc') + glob.glob('*.apk.sig'):
             if os.path.islink(f):
                 links_to_upload.append(f)
         if len(links_to_upload) > 0:
-            if subprocess.call(rsyncargs + links_to_upload + [serverwebroot]) != 0:
+            if subprocess.call(rsyncargs + links_to_upload + [url]) != 0:
                 raise FDroidException()
 
 
+def update_serverwebroots(serverwebroots, repo_section, standardwebroot=True):
+    for d in serverwebroots:
+        # this supports both an ssh host:path and just a path
+        serverwebroot = d['url']
+        s = serverwebroot.rstrip('/').split(':')
+        if len(s) == 1:
+            fdroiddir = s[0]
+        elif len(s) == 2:
+            host, fdroiddir = s
+        else:
+            logging.error(_('Malformed serverwebroot line:') + ' ' + serverwebroot)
+            sys.exit(1)
+        repobase = os.path.basename(fdroiddir)
+        if standardwebroot and repobase != 'fdroid':
+            logging.error(
+                _(
+                    'serverwebroot: path does not end with "fdroid", perhaps you meant one of these:'
+                )
+                + '\n\t'
+                + serverwebroot.rstrip('/')
+                + '/fdroid\n\t'
+                + serverwebroot.rstrip('/').rstrip(repobase)
+                + 'fdroid'
+            )
+            sys.exit(1)
+        update_serverwebroot(d, repo_section)
+
+
 def sync_from_localcopy(repo_section, local_copy_dir):
     """Sync the repo from "local copy dir" filesystem to this box.
 
@@ -747,24 +776,6 @@ def main():
     else:
         standardwebroot = True
 
-    for serverwebroot in config.get('serverwebroot', []):
-        # this supports both an ssh host:path and just a path
-        s = serverwebroot.rstrip('/').split(':')
-        if len(s) == 1:
-            fdroiddir = s[0]
-        elif len(s) == 2:
-            host, fdroiddir = s
-        else:
-            logging.error(_('Malformed serverwebroot line:') + ' ' + serverwebroot)
-            sys.exit(1)
-        repobase = os.path.basename(fdroiddir)
-        if standardwebroot and repobase != 'fdroid':
-            logging.error('serverwebroot path does not end with "fdroid", '
-                          + 'perhaps you meant one of these:\n\t'
-                          + serverwebroot.rstrip('/') + '/fdroid\n\t'
-                          + serverwebroot.rstrip('/').rstrip(repobase) + 'fdroid')
-            sys.exit(1)
-
     if options.local_copy_dir is not None:
         local_copy_dir = options.local_copy_dir
     elif config.get('local_copy_dir'):
@@ -825,8 +836,10 @@ def main():
                 sync_from_localcopy(repo_section, local_copy_dir)
             else:
                 update_localcopy(repo_section, local_copy_dir)
-        for serverwebroot in config.get('serverwebroot', []):
+        if config.get('serverwebroot'):
-            update_serverwebroot(serverwebroot, repo_section)
+            update_serverwebroots(
+                config['serverwebroot'], repo_section, standardwebroot
+            )
         if config.get('servergitmirrors', []):
             # update_servergitmirrors will take care of multiple mirrors so don't need a foreach
             servergitmirrors = config.get('servergitmirrors', [])

tests/common.TestCase

@@ -1655,8 +1655,8 @@ class CommonTest(unittest.TestCase):
         fdroidserver.common.options.quiet = False
         fdroidserver.common.config = {}
         fdroidserver.common.config['serverwebroot'] = [
-            'example.com:/var/www/fdroid/',
+            {'url': 'example.com:/var/www/fdroid/'},
-            'example.com:/var/www/fbot/',
+            {'url': 'example.com:/var/www/fbot/'},
         ]
         fdroidserver.common.config['deploy_process_logs'] = True
         fdroidserver.common.config['identity_file'] = 'ssh/id_rsa'
@@ -1718,7 +1718,7 @@ class CommonTest(unittest.TestCase):
 
         fdroidserver.common.options = mock.Mock()
         fdroidserver.common.config = {}
-        fdroidserver.common.config['serverwebroot'] = [fakeserver]
+        fdroidserver.common.config['serverwebroot'] = [{'url': fakeserver}]
         fdroidserver.common.config['identity_file'] = 'ssh/id_rsa'
 
         def assert_subprocess_call(cmd):
@@ -2868,6 +2868,30 @@ class CommonTest(unittest.TestCase):
         with self.assertRaises(TypeError):
             fdroidserver.common.config_type_check('config/mirrors.yml', str())
 
+    def test_config_serverwebroot_str(self):
+        os.chdir(self.testdir)
+        Path('config.yml').write_text("""serverwebroot: 'foo@example.com:/var/www'""")
+        self.assertEqual(
+            [{'url': 'foo@example.com:/var/www/'}],
+            fdroidserver.common.read_config()['serverwebroot'],
+        )
+
+    def test_config_serverwebroot_list(self):
+        os.chdir(self.testdir)
+        Path('config.yml').write_text("""serverwebroot:\n  - foo@example.com:/var/www""")
+        self.assertEqual(
+            [{'url': 'foo@example.com:/var/www/'}],
+            fdroidserver.common.read_config()['serverwebroot'],
+        )
+
+    def test_config_serverwebroot_dict(self):
+        os.chdir(self.testdir)
+        Path('config.yml').write_text("""serverwebroot:\n  - url: 'foo@example.com:/var/www'""")
+        self.assertEqual(
+            [{'url': 'foo@example.com:/var/www/'}],
+            fdroidserver.common.read_config()['serverwebroot'],
+        )
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

tests/deploy.TestCase

@@ -32,29 +32,76 @@ class DeployTest(unittest.TestCase):
         self._td = mkdtemp()
         self.testdir = self._td.name
 
-        fdroidserver.deploy.options = mock.Mock()
-        fdroidserver.deploy.config = {}
-
     def tearDown(self):
         self._td.cleanup()
 
+    def test_update_serverwebroots_bad_None(self):
+        with self.assertRaises(TypeError):
+            fdroidserver.deploy.update_serverwebroots(None, 'repo')
+
+    def test_update_serverwebroots_bad_int(self):
+        with self.assertRaises(TypeError):
+            fdroidserver.deploy.update_serverwebroots(9, 'repo')
+
+    def test_update_serverwebroots_bad_float(self):
+        with self.assertRaises(TypeError):
+            fdroidserver.deploy.update_serverwebroots(1.0, 'repo')
+
+    def test_update_serverwebroots(self):
+        """rsync works with file paths, so this test uses paths for the URLs"""
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir()
+        fake_apk = repo / 'fake.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+        url0 = Path('url0/fdroid')
+        url0.mkdir(parents=True)
+        url1 = Path('url1/fdroid')
+        url1.mkdir(parents=True)
+
+        dest_apk0 = url0 / fake_apk
+        dest_apk1 = url1 / fake_apk
+        self.assertFalse(dest_apk0.is_file())
+        self.assertFalse(dest_apk1.is_file())
+        fdroidserver.deploy.update_serverwebroots(
+            [
+                {'url': str(url0)},
+                {'url': str(url1)},
+            ],
+            str(repo),
+        )
+        self.assertTrue(dest_apk0.is_file())
+        self.assertTrue(dest_apk1.is_file())
+
+    def test_update_serverwebroots_url_does_not_end_with_fdroid(self):
+        with self.assertRaises(SystemExit):
+            fdroidserver.deploy.update_serverwebroots([{'url': 'url'}], 'repo')
+
+    def test_update_serverwebroots_bad_ssh_url(self):
+        with self.assertRaises(SystemExit):
+            fdroidserver.deploy.update_serverwebroots(
+                [{'url': 'f@b.ar::/path/to/fdroid'}], 'repo'
+            )
+
+    def test_update_serverwebroots_unsupported_ssh_url(self):
+        with self.assertRaises(SystemExit):
+            fdroidserver.deploy.update_serverwebroots([{'url': 'ssh://nope'}], 'repo')
+
     def test_update_serverwebroot(self):
+        """rsync works with file paths, so this test uses paths for the URLs"""
         os.chdir(self.testdir)
         repo = Path('repo')
         repo.mkdir(parents=True)
         fake_apk = repo / 'fake.apk'
         with fake_apk.open('w') as fp:
             fp.write('not an APK, but has the right filename')
-        serverwebroot = Path('serverwebroot')
+        url = Path('url')
-        serverwebroot.mkdir()
+        url.mkdir()
 
-        # setup parameters for this test run
+        dest_apk = url / fake_apk
-        fdroidserver.deploy.options.identity_file = None
-        fdroidserver.deploy.config['make_current_version_link'] = False
-
-        dest_apk = Path(serverwebroot) / fake_apk
         self.assertFalse(dest_apk.is_file())
-        fdroidserver.deploy.update_serverwebroot(str(serverwebroot), 'repo')
+        fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo')
         self.assertTrue(dest_apk.is_file())
 
     @mock.patch.dict(os.environ, clear=True)
@@ -66,13 +113,13 @@ class DeployTest(unittest.TestCase):
 
     def test_update_serverwebroot_make_cur_version_link(self):
         # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
         fdroidserver.deploy.options.no_checksum = True
         fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.options.verbose = False
         fdroidserver.deploy.options.quiet = True
-        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.config = {'make_current_version_link': True}
-        fdroidserver.deploy.config['make_current_version_link'] = True
+        url = "example.com:/var/www/fdroid"
-        serverwebroot = "example.com:/var/www/fdroid"
         repo_section = 'repo'
 
         # setup function for asserting subprocess.call invocations
@@ -123,7 +170,7 @@ class DeployTest(unittest.TestCase):
                         '--safe-links',
                         '--quiet',
                         'repo',
-                        serverwebroot,
+                        url,
                     ],
                 )
             elif call_iteration == 2:
@@ -152,18 +199,19 @@ class DeployTest(unittest.TestCase):
             os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
             os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
             with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
-                fdroidserver.deploy.update_serverwebroot(serverwebroot, repo_section)
+                fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
         self.assertEqual(call_iteration, 3, 'expected 3 invocations of subprocess.call')
 
     def test_update_serverwebroot_with_id_file(self):
         # setup parameters for this test run
-        fdroidserver.deploy.options.no_chcksum = False
+        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.no_checksum = True
         fdroidserver.deploy.options.verbose = True
         fdroidserver.deploy.options.quiet = False
         fdroidserver.deploy.options.identity_file = None
-        fdroidserver.deploy.config['identity_file'] = './id_rsa'
+        fdroidserver.deploy.config = {'identity_file': './id_rsa'}
-        fdroidserver.deploy.config['make_current_version_link'] = False
+        url = "example.com:/var/www/fdroid"
-        serverwebroot = "example.com:/var/www/fdroid"
         repo_section = 'archive'
 
         # setup function for asserting subprocess.call invocations
@@ -204,7 +252,7 @@ class DeployTest(unittest.TestCase):
                         '--exclude',
                         'archive/index.xml',
                         'archive',
-                        serverwebroot,
+                        url,
                     ],
                 )
             elif call_iteration == 1:
@@ -220,7 +268,7 @@ class DeployTest(unittest.TestCase):
                         'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
                         + fdroidserver.deploy.config['identity_file'],
                         'archive',
-                        serverwebroot,
+                        url,
                     ],
                 )
             else:
@@ -229,7 +277,7 @@ class DeployTest(unittest.TestCase):
             return 0
 
         with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
-            fdroidserver.deploy.update_serverwebroot(serverwebroot, repo_section)
+            fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
         self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
 
     @unittest.skipIf(