publish: remove last use of stats/ dir

This file can be treated like the other index files in repo/. This also has
the advantage that it will automatically get synced by @CiaranG's existing
sync scripts.

fdroidserver/common.py

@@ -4227,7 +4227,7 @@ def load_publish_signer_fingerprints():
     dict
         containing the signing-key fingerprints.
     """
-    jar_file = os.path.join('stats', 'publishsigkeys.jar')
+    jar_file = os.path.join('repo', 'signer-index.jar')
     if not os.path.isfile(jar_file):
         return {}
     try:
@@ -4247,7 +4247,7 @@ def load_publish_signer_fingerprints():
         write_to_config(config, 'repo_key_sha256')
 
     with zipfile.ZipFile(jar_file, 'r') as f:
-        return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
+        return json.loads(str(f.read('signer-index.json'), 'utf-8'))
 
 
 def write_config_file(config):
@@ -4478,6 +4478,7 @@ NO_GPG_INDEX_FILES = [
     "index.jar",
     "index.png",
     "index.xml",
+    "signer-index.jar",
 ]
 
 # list of index files that are signed by gpgsign.py to make a .asc file
@@ -4486,6 +4487,7 @@ GPG_INDEX_FILES = [
     "entry.json",
     "index-v1.json",
     "index-v2.json",
+    "signer-index.json",
 ]
 
 

fdroidserver/deploy.py

@@ -60,8 +60,15 @@ def _get_index_file_paths(base_dir):
     services can take a while.  So the index files should be updated
     last.  That ensures that the package files are available when the
     client learns about them from the new index files.
+
+    signer-index.* are only published in the repo/ section.
+
     """
-    return [os.path.join(base_dir, filename) for filename in common.INDEX_FILES]
+    return [
+        os.path.join(base_dir, filename)
+        for filename in common.INDEX_FILES
+        if not (filename.startswith('signer-index.') and base_dir.endswith('archive'))
+    ]
 
 
 def _get_index_excludes(base_dir):

fdroidserver/nightly.py

@@ -337,7 +337,6 @@ def main():
         git_mirror_fdroiddir = os.path.join(git_mirror_path, 'fdroid')
         git_mirror_repodir = os.path.join(git_mirror_fdroiddir, 'repo')
         git_mirror_metadatadir = os.path.join(git_mirror_fdroiddir, 'metadata')
-        git_mirror_statsdir = os.path.join(git_mirror_fdroiddir, 'stats')
         if not os.path.isdir(git_mirror_repodir):
             logging.debug(_('cloning {url}').format(url=clone_url))
             vcs = common.getvcs('git', clone_url, git_mirror_path)
@@ -381,8 +380,6 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
             common.local_rsync(options, [git_mirror_repodir + '/'], 'repo/')
         if os.path.isdir(git_mirror_metadatadir):
             common.local_rsync(options, [git_mirror_metadatadir + '/'], 'metadata/')
-        if os.path.isdir(git_mirror_statsdir):
-            common.local_rsync(options, [git_mirror_statsdir + '/'], 'stats/')
 
         ssh_private_key_file = _ssh_key_from_debug_keystore()
         # this is needed for GitPython to find the SSH key
@@ -495,9 +492,6 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
         common.local_rsync(
             options, [repo_basedir + '/metadata/'], git_mirror_metadatadir + '/'
         )
-        stats = repo_basedir + '/stats/'
-        if os.path.exists(stats):
-            common.local_rsync(options, [stats], git_mirror_statsdir + '/')
         mirror_git_repo.git.add(all=True)
         mirror_git_repo.index.commit("update app metadata")
 

fdroidserver/publish.py

@@ -144,8 +144,8 @@ def store_publish_signer_fingerprints(appids, indent=None):
 
     This list will later on be needed by fdroid update.
     """
-    if not os.path.exists('stats'):
-        os.makedirs('stats')
+    if not os.path.exists('repo'):
+        os.makedirs('repo')
     data = OrderedDict()
     fps = read_fingerprints_from_keystore()
     for appid in sorted(appids):
@@ -153,9 +153,12 @@ def store_publish_signer_fingerprints(appids, indent=None):
         if alias in fps:
             data[appid] = {'signer': fps[key_alias(appid)]}
 
-    jar_file = os.path.join('stats', 'publishsigkeys.jar')
+    jar_file = os.path.join('repo', 'signer-index.jar')
+    output = json.dumps(data, indent=indent)
     with zipfile.ZipFile(jar_file, 'w', zipfile.ZIP_DEFLATED) as jar:
-        jar.writestr('publishsigkeys.json', json.dumps(data, indent=indent))
+        jar.writestr('signer-index.json', output)
+    with open(os.path.join('repo', 'signer-index.json'), 'w') as fp:
+        fp.write(output)
     sign_sig_key_fingerprint_list(jar_file)
 
 

tests/test_deploy.py

@@ -326,6 +326,12 @@ class DeployTest(unittest.TestCase):
                         'repo/index.png',
                         '--exclude',
                         'repo/index.xml',
+                        '--exclude',
+                        'repo/signer-index.jar',
+                        '--exclude',
+                        'repo/signer-index.json',
+                        '--exclude',
+                        'repo/signer-index.json.asc',
                         'repo',
                         'example.com:/var/www/fdroid',
                     ],
@@ -413,6 +419,9 @@ class DeployTest(unittest.TestCase):
                         'repo/index.jar',
                         'repo/index.png',
                         'repo/index.xml',
+                        'repo/signer-index.jar',
+                        'repo/signer-index.json',
+                        'repo/signer-index.json.asc',
                         'example.com:/var/www/fdroid/repo/',
                     ],
                 )
@@ -729,6 +738,12 @@ class DeployTest(unittest.TestCase):
                         'repo/index.png',
                         '--exclude',
                         'repo/index.xml',
+                        '--exclude',
+                        'repo/signer-index.jar',
+                        '--exclude',
+                        'repo/signer-index.json',
+                        '--exclude',
+                        'repo/signer-index.json.asc',
                         '--no-check-md5',
                         '--skip-existing',
                         repo_section,
@@ -774,6 +789,12 @@ class DeployTest(unittest.TestCase):
                         'repo/index.png',
                         '--exclude',
                         'repo/index.xml',
+                        '--exclude',
+                        'repo/signer-index.jar',
+                        '--exclude',
+                        'repo/signer-index.json',
+                        '--exclude',
+                        'repo/signer-index.json.asc',
                         '--no-check-md5',
                         repo_section,
                         f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
@@ -880,6 +901,12 @@ class DeployTest(unittest.TestCase):
                         'repo/index.png',
                         '--include',
                         'repo/index.xml',
+                        '--include',
+                        'repo/signer-index.jar',
+                        '--include',
+                        'repo/signer-index.json',
+                        '--include',
+                        'repo/signer-index.json.asc',
                         '--delete-removed',
                         '--delete-after',
                         '--no-check-md5',

tests/test_index.py

@@ -375,10 +375,10 @@ class IndexTest(unittest.TestCase):
                     "signer": "b33a601a9da97c82e6eb121eb6b90adab561f396602ec4dc8b0019fb587e2af6"
                 }
             }
-            os.makedirs('stats')
-            jarfile = 'stats/publishsigkeys.jar'
+            os.mkdir('repo')
+            jarfile = 'repo/signer-index.jar'
             with zipfile.ZipFile(jarfile, 'w', zipfile.ZIP_DEFLATED) as jar:
-                jar.writestr('publishsigkeys.json', json.dumps(sigkeyfps))
+                jar.writestr('signer-index.json', json.dumps(sigkeyfps))
             publish.sign_sig_key_fingerprint_list(jarfile)
             common.write_config_file('')
 
@@ -387,7 +387,7 @@ class IndexTest(unittest.TestCase):
             self.assertEqual(json.dumps(i, indent=2), json.dumps(o, indent=2))
 
             # and test it still works with get_first_signer_certificate
-            outdir = os.path.join(self.testdir, 'publishsigkeys')
+            outdir = os.path.join(self.testdir, 'index-signer-fingerprints')
             os.mkdir(outdir)
             common.apk_extract_signatures(jarfile, outdir)
             certs = glob.glob(os.path.join(outdir, '*.RSA'))

tests/test_integration.py

@@ -747,7 +747,6 @@ class IntegrationTest(unittest.TestCase):
         self.update_yaml(common.CONFIG_FILE, {"archive_older": 3})
         Path("metadata").mkdir()
         Path("archive").mkdir()
-        Path("stats").mkdir()
         shutil.copy(FILES / "repo/com.politedroid_6.apk", "repo")
         shutil.copy(FILES / "repo/index-v2.json", "repo")
         shutil.copy(FILES / "repo/com.politedroid_5.apk", "archive")