diff --git a/examples/config.yml b/examples/config.yml index 12f7d138..59453376 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -178,6 +178,12 @@ # serverwebroot: # - foo.com:/usr/share/nginx/www/fdroid # - bar.info:/var/www/fdroid +# +# There is a special mode to only deploy the index file: +# +# serverwebroot: +# - url: 'me@b.az:/srv/fdroid' +# indexOnly: true # When running fdroid processes on a remote server, it is possible to diff --git a/fdroidserver/__init__.py b/fdroidserver/__init__.py index ab9ab1bc..9e4c197f 100644 --- a/fdroidserver/__init__.py +++ b/fdroidserver/__init__.py @@ -55,7 +55,9 @@ scan_apk # NOQA: B101 scan_repo_files # NOQA: B101 from fdroidserver.deploy import (update_awsbucket, update_servergitmirrors, + update_serverwebroots, update_serverwebroot) # NOQA: E402 update_awsbucket # NOQA: B101 update_servergitmirrors # NOQA: B101 +update_serverwebroots # NOQA: B101 update_serverwebroot # NOQA: B101 diff --git a/fdroidserver/common.py b/fdroidserver/common.py index e97cedf5..3da0a193 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -462,18 +462,22 @@ def read_config(opts=None): if 'serverwebroot' in config: if isinstance(config['serverwebroot'], str): - roots = [config['serverwebroot']] + roots = [{'url': config['serverwebroot']}] elif all(isinstance(item, str) for item in config['serverwebroot']): + roots = [{'url': i} for i in config['serverwebroot']] + elif all(isinstance(item, dict) for item in config['serverwebroot']): roots = config['serverwebroot'] else: raise TypeError(_('only accepts strings, lists, and tuples')) rootlist = [] - for rootstr in roots: + for d in roots: # since this is used with rsync, where trailing slashes have # meaning, ensure there is always a trailing slash + rootstr = d['url'] if rootstr[-1] != '/': rootstr += '/' - rootlist.append(rootstr.replace('//', '/')) + d['url'] = rootstr.replace('//', '/') + rootlist.append(d) config['serverwebroot'] = rootlist if 'servergitmirrors' in config: @@ -4052,7 +4056,8 @@ def rsync_status_file_to_repo(path, repo_subdir=None): logging.debug(_('skip deploying full build logs: not enabled in config')) return - for webroot in config.get('serverwebroot', []): + for d in config.get('serverwebroot', []): + webroot = d['url'] cmd = ['rsync', '--archive', '--delete-after', diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 7cc709d6..92287f1b 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -284,34 +284,63 @@ def update_serverwebroot(serverwebroot, repo_section): _('rsync is missing or broken: {error}').format(error=e) ) from e rsyncargs = ['rsync', '--archive', '--delete-after', '--safe-links'] - if not options.no_checksum: + if not options or not options.no_checksum: rsyncargs.append('--checksum') - if options.verbose: + if options and options.verbose: rsyncargs += ['--verbose'] - if options.quiet: + if options and options.quiet: rsyncargs += ['--quiet'] - if options.identity_file is not None: + if options and options.identity_file: rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + options.identity_file] - elif 'identity_file' in config: + elif config and config.get('identity_file'): rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']] - logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot) + url = serverwebroot['url'] + logging.info('rsyncing ' + repo_section + ' to ' + url) excludes = _get_index_excludes(repo_section) - if subprocess.call(rsyncargs + excludes + [repo_section, serverwebroot]) != 0: + if subprocess.call(rsyncargs + excludes + [repo_section, url]) != 0: raise FDroidException() - if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0: + if subprocess.call(rsyncargs + [repo_section, url]) != 0: raise FDroidException() # upload "current version" symlinks if requested - if config['make_current_version_link'] and repo_section == 'repo': + if config and config.get('make_current_version_link') and repo_section == 'repo': links_to_upload = [] for f in glob.glob('*.apk') \ + glob.glob('*.apk.asc') + glob.glob('*.apk.sig'): if os.path.islink(f): links_to_upload.append(f) if len(links_to_upload) > 0: - if subprocess.call(rsyncargs + links_to_upload + [serverwebroot]) != 0: + if subprocess.call(rsyncargs + links_to_upload + [url]) != 0: raise FDroidException() +def update_serverwebroots(serverwebroots, repo_section, standardwebroot=True): + for d in serverwebroots: + # this supports both an ssh host:path and just a path + serverwebroot = d['url'] + s = serverwebroot.rstrip('/').split(':') + if len(s) == 1: + fdroiddir = s[0] + elif len(s) == 2: + host, fdroiddir = s + else: + logging.error(_('Malformed serverwebroot line:') + ' ' + serverwebroot) + sys.exit(1) + repobase = os.path.basename(fdroiddir) + if standardwebroot and repobase != 'fdroid': + logging.error( + _( + 'serverwebroot: path does not end with "fdroid", perhaps you meant one of these:' + ) + + '\n\t' + + serverwebroot.rstrip('/') + + '/fdroid\n\t' + + serverwebroot.rstrip('/').rstrip(repobase) + + 'fdroid' + ) + sys.exit(1) + update_serverwebroot(d, repo_section) + + def sync_from_localcopy(repo_section, local_copy_dir): """Sync the repo from "local copy dir" filesystem to this box. @@ -747,24 +776,6 @@ def main(): else: standardwebroot = True - for serverwebroot in config.get('serverwebroot', []): - # this supports both an ssh host:path and just a path - s = serverwebroot.rstrip('/').split(':') - if len(s) == 1: - fdroiddir = s[0] - elif len(s) == 2: - host, fdroiddir = s - else: - logging.error(_('Malformed serverwebroot line:') + ' ' + serverwebroot) - sys.exit(1) - repobase = os.path.basename(fdroiddir) - if standardwebroot and repobase != 'fdroid': - logging.error('serverwebroot path does not end with "fdroid", ' - + 'perhaps you meant one of these:\n\t' - + serverwebroot.rstrip('/') + '/fdroid\n\t' - + serverwebroot.rstrip('/').rstrip(repobase) + 'fdroid') - sys.exit(1) - if options.local_copy_dir is not None: local_copy_dir = options.local_copy_dir elif config.get('local_copy_dir'): @@ -825,8 +836,10 @@ def main(): sync_from_localcopy(repo_section, local_copy_dir) else: update_localcopy(repo_section, local_copy_dir) - for serverwebroot in config.get('serverwebroot', []): - update_serverwebroot(serverwebroot, repo_section) + if config.get('serverwebroot'): + update_serverwebroots( + config['serverwebroot'], repo_section, standardwebroot + ) if config.get('servergitmirrors', []): # update_servergitmirrors will take care of multiple mirrors so don't need a foreach servergitmirrors = config.get('servergitmirrors', []) diff --git a/tests/common.TestCase b/tests/common.TestCase index 4d9ce009..55202dcc 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -1655,8 +1655,8 @@ class CommonTest(unittest.TestCase): fdroidserver.common.options.quiet = False fdroidserver.common.config = {} fdroidserver.common.config['serverwebroot'] = [ - 'example.com:/var/www/fdroid/', - 'example.com:/var/www/fbot/', + {'url': 'example.com:/var/www/fdroid/'}, + {'url': 'example.com:/var/www/fbot/'}, ] fdroidserver.common.config['deploy_process_logs'] = True fdroidserver.common.config['identity_file'] = 'ssh/id_rsa' @@ -1718,7 +1718,7 @@ class CommonTest(unittest.TestCase): fdroidserver.common.options = mock.Mock() fdroidserver.common.config = {} - fdroidserver.common.config['serverwebroot'] = [fakeserver] + fdroidserver.common.config['serverwebroot'] = [{'url': fakeserver}] fdroidserver.common.config['identity_file'] = 'ssh/id_rsa' def assert_subprocess_call(cmd): @@ -2868,6 +2868,30 @@ class CommonTest(unittest.TestCase): with self.assertRaises(TypeError): fdroidserver.common.config_type_check('config/mirrors.yml', str()) + def test_config_serverwebroot_str(self): + os.chdir(self.testdir) + Path('config.yml').write_text("""serverwebroot: 'foo@example.com:/var/www'""") + self.assertEqual( + [{'url': 'foo@example.com:/var/www/'}], + fdroidserver.common.read_config()['serverwebroot'], + ) + + def test_config_serverwebroot_list(self): + os.chdir(self.testdir) + Path('config.yml').write_text("""serverwebroot:\n - foo@example.com:/var/www""") + self.assertEqual( + [{'url': 'foo@example.com:/var/www/'}], + fdroidserver.common.read_config()['serverwebroot'], + ) + + def test_config_serverwebroot_dict(self): + os.chdir(self.testdir) + Path('config.yml').write_text("""serverwebroot:\n - url: 'foo@example.com:/var/www'""") + self.assertEqual( + [{'url': 'foo@example.com:/var/www/'}], + fdroidserver.common.read_config()['serverwebroot'], + ) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase index 5539af4c..e4334725 100755 --- a/tests/deploy.TestCase +++ b/tests/deploy.TestCase @@ -32,29 +32,76 @@ class DeployTest(unittest.TestCase): self._td = mkdtemp() self.testdir = self._td.name - fdroidserver.deploy.options = mock.Mock() - fdroidserver.deploy.config = {} - def tearDown(self): self._td.cleanup() + def test_update_serverwebroots_bad_None(self): + with self.assertRaises(TypeError): + fdroidserver.deploy.update_serverwebroots(None, 'repo') + + def test_update_serverwebroots_bad_int(self): + with self.assertRaises(TypeError): + fdroidserver.deploy.update_serverwebroots(9, 'repo') + + def test_update_serverwebroots_bad_float(self): + with self.assertRaises(TypeError): + fdroidserver.deploy.update_serverwebroots(1.0, 'repo') + + def test_update_serverwebroots(self): + """rsync works with file paths, so this test uses paths for the URLs""" + os.chdir(self.testdir) + repo = Path('repo') + repo.mkdir() + fake_apk = repo / 'fake.apk' + with fake_apk.open('w') as fp: + fp.write('not an APK, but has the right filename') + url0 = Path('url0/fdroid') + url0.mkdir(parents=True) + url1 = Path('url1/fdroid') + url1.mkdir(parents=True) + + dest_apk0 = url0 / fake_apk + dest_apk1 = url1 / fake_apk + self.assertFalse(dest_apk0.is_file()) + self.assertFalse(dest_apk1.is_file()) + fdroidserver.deploy.update_serverwebroots( + [ + {'url': str(url0)}, + {'url': str(url1)}, + ], + str(repo), + ) + self.assertTrue(dest_apk0.is_file()) + self.assertTrue(dest_apk1.is_file()) + + def test_update_serverwebroots_url_does_not_end_with_fdroid(self): + with self.assertRaises(SystemExit): + fdroidserver.deploy.update_serverwebroots([{'url': 'url'}], 'repo') + + def test_update_serverwebroots_bad_ssh_url(self): + with self.assertRaises(SystemExit): + fdroidserver.deploy.update_serverwebroots( + [{'url': 'f@b.ar::/path/to/fdroid'}], 'repo' + ) + + def test_update_serverwebroots_unsupported_ssh_url(self): + with self.assertRaises(SystemExit): + fdroidserver.deploy.update_serverwebroots([{'url': 'ssh://nope'}], 'repo') + def test_update_serverwebroot(self): + """rsync works with file paths, so this test uses paths for the URLs""" os.chdir(self.testdir) repo = Path('repo') repo.mkdir(parents=True) fake_apk = repo / 'fake.apk' with fake_apk.open('w') as fp: fp.write('not an APK, but has the right filename') - serverwebroot = Path('serverwebroot') - serverwebroot.mkdir() + url = Path('url') + url.mkdir() - # setup parameters for this test run - fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.config['make_current_version_link'] = False - - dest_apk = Path(serverwebroot) / fake_apk + dest_apk = url / fake_apk self.assertFalse(dest_apk.is_file()) - fdroidserver.deploy.update_serverwebroot(str(serverwebroot), 'repo') + fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo') self.assertTrue(dest_apk.is_file()) @mock.patch.dict(os.environ, clear=True) @@ -66,13 +113,13 @@ class DeployTest(unittest.TestCase): def test_update_serverwebroot_make_cur_version_link(self): # setup parameters for this test run + fdroidserver.deploy.options = mock.Mock() fdroidserver.deploy.options.no_checksum = True fdroidserver.deploy.options.identity_file = None fdroidserver.deploy.options.verbose = False fdroidserver.deploy.options.quiet = True - fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.config['make_current_version_link'] = True - serverwebroot = "example.com:/var/www/fdroid" + fdroidserver.deploy.config = {'make_current_version_link': True} + url = "example.com:/var/www/fdroid" repo_section = 'repo' # setup function for asserting subprocess.call invocations @@ -123,7 +170,7 @@ class DeployTest(unittest.TestCase): '--safe-links', '--quiet', 'repo', - serverwebroot, + url, ], ) elif call_iteration == 2: @@ -152,18 +199,19 @@ class DeployTest(unittest.TestCase): os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc') os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig') with mock.patch('subprocess.call', side_effect=update_server_webroot_call): - fdroidserver.deploy.update_serverwebroot(serverwebroot, repo_section) + fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section) self.assertEqual(call_iteration, 3, 'expected 3 invocations of subprocess.call') def test_update_serverwebroot_with_id_file(self): # setup parameters for this test run - fdroidserver.deploy.options.no_chcksum = False + fdroidserver.deploy.options = mock.Mock() + fdroidserver.deploy.options.identity_file = None + fdroidserver.deploy.options.no_checksum = True fdroidserver.deploy.options.verbose = True fdroidserver.deploy.options.quiet = False fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.config['identity_file'] = './id_rsa' - fdroidserver.deploy.config['make_current_version_link'] = False - serverwebroot = "example.com:/var/www/fdroid" + fdroidserver.deploy.config = {'identity_file': './id_rsa'} + url = "example.com:/var/www/fdroid" repo_section = 'archive' # setup function for asserting subprocess.call invocations @@ -204,7 +252,7 @@ class DeployTest(unittest.TestCase): '--exclude', 'archive/index.xml', 'archive', - serverwebroot, + url, ], ) elif call_iteration == 1: @@ -220,7 +268,7 @@ class DeployTest(unittest.TestCase): 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + fdroidserver.deploy.config['identity_file'], 'archive', - serverwebroot, + url, ], ) else: @@ -229,7 +277,7 @@ class DeployTest(unittest.TestCase): return 0 with mock.patch('subprocess.call', side_effect=update_server_webroot_call): - fdroidserver.deploy.update_serverwebroot(serverwebroot, repo_section) + fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section) self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call') @unittest.skipIf(