diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index a19bdeda..953c6f71 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -46,6 +46,35 @@ USER_S3CFG = 's3cfg' REMOTE_HOSTNAME_REGEX = re.compile(r'\W*\w+\W+(\w+).*') +def _get_index_excludes(repo_section): + """Return the list of files to be synced last, since they finalize the deploy. + + The process of pushing all the new packages to the various + services can take a while. So the index files should be updated + last. That ensures that the package files are available when the + client learns about them from the new index files. + + """ + indexes = [ + os.path.join(repo_section, 'entry.jar'), + os.path.join(repo_section, 'entry.json'), + os.path.join(repo_section, 'entry.json.asc'), + os.path.join(repo_section, 'index-v1.jar'), + os.path.join(repo_section, 'index-v1.json'), + os.path.join(repo_section, 'index-v1.json.asc'), + os.path.join(repo_section, 'index-v2.jar'), + os.path.join(repo_section, 'index-v2.json'), + os.path.join(repo_section, 'index-v2.json.asc'), + os.path.join(repo_section, 'index.jar'), + os.path.join(repo_section, 'index.xml'), + ] + index_excludes = [] + for f in indexes: + index_excludes.append('--exclude') + index_excludes.append(f) + return index_excludes + + def update_awsbucket(repo_section): """Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket". @@ -104,33 +133,23 @@ def update_awsbucket_s3cmd(repo_section): s3cmd_sync += ['--verbose'] if options.quiet: s3cmd_sync += ['--quiet'] - indexxml = os.path.join(repo_section, 'index.xml') - indexjar = os.path.join(repo_section, 'index.jar') - indexv1jar = os.path.join(repo_section, 'index-v1.jar') - indexv1json = os.path.join(repo_section, 'index-v1.json') - indexv1jsonasc = os.path.join(repo_section, 'index-v1.json.asc') s3url = s3bucketurl + '/fdroid/' logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url) logging.debug(_('Running first pass with MD5 checking disabled')) - if subprocess.call(s3cmd_sync - + ['--no-check-md5', '--skip-existing', - '--exclude', indexxml, - '--exclude', indexjar, - '--exclude', indexv1jar, - '--exclude', indexv1json, - '--exclude', indexv1jsonasc, - repo_section, s3url]) != 0: + excludes = _get_index_excludes(repo_section) + returncode = subprocess.call( + s3cmd_sync + + excludes + + ['--no-check-md5', '--skip-existing', repo_section, s3url] + ) + if returncode != 0: raise FDroidException() logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url) - if subprocess.call(s3cmd_sync - + ['--no-check-md5', - '--exclude', indexxml, - '--exclude', indexjar, - '--exclude', indexv1jar, - '--exclude', indexv1json, - '--exclude', indexv1jsonasc, - repo_section, s3url]) != 0: + returncode = subprocess.call( + s3cmd_sync + excludes + ['--no-check-md5', repo_section, s3url] + ) + if returncode != 0: raise FDroidException() logging.debug(_('s3cmd sync indexes {path} to {url} and delete') @@ -241,8 +260,22 @@ def update_awsbucket_libcloud(repo_section): def update_serverwebroot(serverwebroot, repo_section): - # use a checksum comparison for accurate comparisons on different - # filesystems, for example, FAT has a low resolution timestamp + """Deploy the index files to the serverwebroot using rsync. + + Upload the first time without the index files and delay the + deletion as much as possible. That keeps the repo functional + while this update is running. Then once it is complete, rerun the + command again to upload the index files. Always using the same + target with rsync allows for very strict settings on the receiving + server, you can literally specify the one rsync command that is + allowed to run in ~/.ssh/authorized_keys. (serverwebroot is + guaranteed to have a trailing slash in common.py) + + It is possible to optionally use a checksum comparison for + accurate comparisons on different filesystems, for example, FAT + has a low resolution timestamp + + """ rsyncargs = ['rsync', '--archive', '--delete-after', '--safe-links'] if not options.no_checksum: rsyncargs.append('--checksum') @@ -254,26 +287,9 @@ def update_serverwebroot(serverwebroot, repo_section): rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + options.identity_file] elif 'identity_file' in config: rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']] - indexxml = os.path.join(repo_section, 'index.xml') - indexjar = os.path.join(repo_section, 'index.jar') - indexv1jar = os.path.join(repo_section, 'index-v1.jar') - indexv1json = os.path.join(repo_section, 'index-v1.json') - indexv1jsonasc = os.path.join(repo_section, 'index-v1.json.asc') - # Upload the first time without the index files and delay the deletion as - # much as possible, that keeps the repo functional while this update is - # running. Then once it is complete, rerun the command again to upload - # the index files. Always using the same target with rsync allows for - # very strict settings on the receiving server, you can literally specify - # the one rsync command that is allowed to run in ~/.ssh/authorized_keys. - # (serverwebroot is guaranteed to have a trailing slash in common.py) logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot) - if subprocess.call(rsyncargs - + ['--exclude', indexxml, - '--exclude', indexjar, - '--exclude', indexv1jar, - '--exclude', indexv1json, - '--exclude', indexv1jsonasc, - repo_section, serverwebroot]) != 0: + excludes = _get_index_excludes(repo_section) + if subprocess.call(rsyncargs + excludes + [repo_section, serverwebroot]) != 0: raise FDroidException() if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0: raise FDroidException() diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase index f493d7ea..046dcc2b 100755 --- a/tests/deploy.TestCase +++ b/tests/deploy.TestCase @@ -57,15 +57,27 @@ class DeployTest(unittest.TestCase): '--safe-links', '--quiet', '--exclude', - 'repo/index.xml', + 'repo/entry.jar', '--exclude', - 'repo/index.jar', + 'repo/entry.json', + '--exclude', + 'repo/entry.json.asc', '--exclude', 'repo/index-v1.jar', '--exclude', 'repo/index-v1.json', '--exclude', 'repo/index-v1.json.asc', + '--exclude', + 'repo/index-v2.jar', + '--exclude', + 'repo/index-v2.json', + '--exclude', + 'repo/index-v2.json.asc', + '--exclude', + 'repo/index.jar', + '--exclude', + 'repo/index.xml', 'repo', 'example.com:/var/www/fdroid', ], @@ -142,15 +154,27 @@ class DeployTest(unittest.TestCase): 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + fdroidserver.deploy.config['identity_file'], '--exclude', - 'archive/index.xml', + 'archive/entry.jar', '--exclude', - 'archive/index.jar', + 'archive/entry.json', + '--exclude', + 'archive/entry.json.asc', '--exclude', 'archive/index-v1.jar', '--exclude', 'archive/index-v1.json', '--exclude', 'archive/index-v1.json.asc', + '--exclude', + 'archive/index-v2.jar', + '--exclude', + 'archive/index-v2.json', + '--exclude', + 'archive/index-v2.json.asc', + '--exclude', + 'archive/index.jar', + '--exclude', + 'archive/index.xml', 'archive', serverwebroot, ],