Merge branch 'deploy-index-v2' into 'master'

deploy: handle index-v2 files on two pass sync methods

See merge request fdroid/fdroidserver!1132

fdroidserver/deploy.py

@@ -46,6 +46,35 @@ USER_S3CFG = 's3cfg'
 REMOTE_HOSTNAME_REGEX = re.compile(r'\W*\w+\W+(\w+).*')
 
 
+def _get_index_excludes(repo_section):
+    """Return the list of files to be synced last, since they finalize the deploy.
+
+    The process of pushing all the new packages to the various
+    services can take a while.  So the index files should be updated
+    last.  That ensures that the package files are available when the
+    client learns about them from the new index files.
+
+    """
+    indexes = [
+        os.path.join(repo_section, 'entry.jar'),
+        os.path.join(repo_section, 'entry.json'),
+        os.path.join(repo_section, 'entry.json.asc'),
+        os.path.join(repo_section, 'index-v1.jar'),
+        os.path.join(repo_section, 'index-v1.json'),
+        os.path.join(repo_section, 'index-v1.json.asc'),
+        os.path.join(repo_section, 'index-v2.jar'),
+        os.path.join(repo_section, 'index-v2.json'),
+        os.path.join(repo_section, 'index-v2.json.asc'),
+        os.path.join(repo_section, 'index.jar'),
+        os.path.join(repo_section, 'index.xml'),
+    ]
+    index_excludes = []
+    for f in indexes:
+        index_excludes.append('--exclude')
+        index_excludes.append(f)
+    return index_excludes
+
+
 def update_awsbucket(repo_section):
     """Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
 
@@ -104,33 +133,23 @@ def update_awsbucket_s3cmd(repo_section):
         s3cmd_sync += ['--verbose']
     if options.quiet:
         s3cmd_sync += ['--quiet']
-    indexxml = os.path.join(repo_section, 'index.xml')
-    indexjar = os.path.join(repo_section, 'index.jar')
-    indexv1jar = os.path.join(repo_section, 'index-v1.jar')
-    indexv1json = os.path.join(repo_section, 'index-v1.json')
-    indexv1jsonasc = os.path.join(repo_section, 'index-v1.json.asc')
 
     s3url = s3bucketurl + '/fdroid/'
     logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url)
     logging.debug(_('Running first pass with MD5 checking disabled'))
-    if subprocess.call(s3cmd_sync
+    excludes = _get_index_excludes(repo_section)
-                       + ['--no-check-md5', '--skip-existing',
+    returncode = subprocess.call(
-                          '--exclude', indexxml,
+        s3cmd_sync
-                          '--exclude', indexjar,
+        + excludes
-                          '--exclude', indexv1jar,
+        + ['--no-check-md5', '--skip-existing', repo_section, s3url]
-                          '--exclude', indexv1json,
+    )
-                          '--exclude', indexv1jsonasc,
+    if returncode != 0:
-                          repo_section, s3url]) != 0:
         raise FDroidException()
     logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
-    if subprocess.call(s3cmd_sync
+    returncode = subprocess.call(
-                       + ['--no-check-md5',
+        s3cmd_sync + excludes + ['--no-check-md5', repo_section, s3url]
-                          '--exclude', indexxml,
+    )
-                          '--exclude', indexjar,
+    if returncode != 0:
-                          '--exclude', indexv1jar,
-                          '--exclude', indexv1json,
-                          '--exclude', indexv1jsonasc,
-                          repo_section, s3url]) != 0:
         raise FDroidException()
 
     logging.debug(_('s3cmd sync indexes {path} to {url} and delete')
@@ -241,8 +260,22 @@ def update_awsbucket_libcloud(repo_section):
 
 
 def update_serverwebroot(serverwebroot, repo_section):
-    # use a checksum comparison for accurate comparisons on different
+    """Deploy the index files to the serverwebroot using rsync.
-    # filesystems, for example, FAT has a low resolution timestamp
+
+    Upload the first time without the index files and delay the
+    deletion as much as possible.  That keeps the repo functional
+    while this update is running.  Then once it is complete, rerun the
+    command again to upload the index files.  Always using the same
+    target with rsync allows for very strict settings on the receiving
+    server, you can literally specify the one rsync command that is
+    allowed to run in ~/.ssh/authorized_keys.  (serverwebroot is
+    guaranteed to have a trailing slash in common.py)
+
+    It is possible to optionally use a checksum comparison for
+    accurate comparisons on different filesystems, for example, FAT
+    has a low resolution timestamp
+
+    """
     rsyncargs = ['rsync', '--archive', '--delete-after', '--safe-links']
     if not options.no_checksum:
         rsyncargs.append('--checksum')
@@ -254,26 +287,9 @@ def update_serverwebroot(serverwebroot, repo_section):
         rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + options.identity_file]
     elif 'identity_file' in config:
         rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']]
-    indexxml = os.path.join(repo_section, 'index.xml')
-    indexjar = os.path.join(repo_section, 'index.jar')
-    indexv1jar = os.path.join(repo_section, 'index-v1.jar')
-    indexv1json = os.path.join(repo_section, 'index-v1.json')
-    indexv1jsonasc = os.path.join(repo_section, 'index-v1.json.asc')
-    # Upload the first time without the index files and delay the deletion as
-    # much as possible, that keeps the repo functional while this update is
-    # running.  Then once it is complete, rerun the command again to upload
-    # the index files.  Always using the same target with rsync allows for
-    # very strict settings on the receiving server, you can literally specify
-    # the one rsync command that is allowed to run in ~/.ssh/authorized_keys.
-    # (serverwebroot is guaranteed to have a trailing slash in common.py)
     logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot)
-    if subprocess.call(rsyncargs
+    excludes = _get_index_excludes(repo_section)
-                       + ['--exclude', indexxml,
+    if subprocess.call(rsyncargs + excludes + [repo_section, serverwebroot]) != 0:
-                          '--exclude', indexjar,
-                          '--exclude', indexv1jar,
-                          '--exclude', indexv1json,
-                          '--exclude', indexv1jsonasc,
-                          repo_section, serverwebroot]) != 0:
         raise FDroidException()
     if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0:
         raise FDroidException()

tests/deploy.TestCase

@@ -57,15 +57,27 @@ class DeployTest(unittest.TestCase):
                         '--safe-links',
                         '--quiet',
                         '--exclude',
-                        'repo/index.xml',
+                        'repo/entry.jar',
                         '--exclude',
-                        'repo/index.jar',
+                        'repo/entry.json',
+                        '--exclude',
+                        'repo/entry.json.asc',
                         '--exclude',
                         'repo/index-v1.jar',
                         '--exclude',
                         'repo/index-v1.json',
                         '--exclude',
                         'repo/index-v1.json.asc',
+                        '--exclude',
+                        'repo/index-v2.jar',
+                        '--exclude',
+                        'repo/index-v2.json',
+                        '--exclude',
+                        'repo/index-v2.json.asc',
+                        '--exclude',
+                        'repo/index.jar',
+                        '--exclude',
+                        'repo/index.xml',
                         'repo',
                         'example.com:/var/www/fdroid',
                     ],
@@ -142,15 +154,27 @@ class DeployTest(unittest.TestCase):
                         'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
                         + fdroidserver.deploy.config['identity_file'],
                         '--exclude',
-                        'archive/index.xml',
+                        'archive/entry.jar',
                         '--exclude',
-                        'archive/index.jar',
+                        'archive/entry.json',
+                        '--exclude',
+                        'archive/entry.json.asc',
                         '--exclude',
                         'archive/index-v1.jar',
                         '--exclude',
                         'archive/index-v1.json',
                         '--exclude',
                         'archive/index-v1.json.asc',
+                        '--exclude',
+                        'archive/index-v2.jar',
+                        '--exclude',
+                        'archive/index-v2.json',
+                        '--exclude',
+                        'archive/index-v2.json.asc',
+                        '--exclude',
+                        'archive/index.jar',
+                        '--exclude',
+                        'archive/index.xml',
                         'archive',
                         serverwebroot,
                     ],