Merge branch 'no-more-stats-dir' into 'master'

remove last uses of stats/ dir

See merge request fdroid/fdroidserver!1620

fdroidserver/common.py

@@ -4219,7 +4219,7 @@ def get_jar_signer_certificate(pkcs7obj: cms.ContentInfo, signature_file: bytes)
         return first_verified_signer_info_signing_certificate
 
 
-def load_stats_fdroid_signing_key_fingerprints():
+def load_publish_signer_fingerprints():
     """Load signing-key fingerprints stored in file generated by fdroid publish.
 
     Returns
@@ -4227,7 +4227,7 @@ def load_stats_fdroid_signing_key_fingerprints():
     dict
         containing the signing-key fingerprints.
     """
-    jar_file = os.path.join('stats', 'publishsigkeys.jar')
+    jar_file = os.path.join('repo', 'signer-index.jar')
     if not os.path.isfile(jar_file):
         return {}
     try:
@@ -4247,7 +4247,7 @@ def load_stats_fdroid_signing_key_fingerprints():
         write_to_config(config, 'repo_key_sha256')
 
     with zipfile.ZipFile(jar_file, 'r') as f:
-        return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
+        return json.loads(str(f.read('signer-index.json'), 'utf-8'))
 
 
 def write_config_file(config):
@@ -4478,6 +4478,7 @@ NO_GPG_INDEX_FILES = [
     "index.jar",
     "index.png",
     "index.xml",
+    "signer-index.jar",
 ]
 
 # list of index files that are signed by gpgsign.py to make a .asc file
@@ -4486,6 +4487,7 @@ GPG_INDEX_FILES = [
     "entry.json",
     "index-v1.json",
     "index-v2.json",
+    "signer-index.json",
 ]
 
 

fdroidserver/deploy.py

@@ -60,8 +60,15 @@ def _get_index_file_paths(base_dir):
     services can take a while.  So the index files should be updated
     last.  That ensures that the package files are available when the
     client learns about them from the new index files.
+
+    signer-index.* are only published in the repo/ section.
+
     """
-    return [os.path.join(base_dir, filename) for filename in common.INDEX_FILES]
+    return [
+        os.path.join(base_dir, filename)
+        for filename in common.INDEX_FILES
+        if not (filename.startswith('signer-index.') and base_dir.endswith('archive'))
+    ]
 
 
 def _get_index_excludes(base_dir):

fdroidserver/index.py

@@ -51,7 +51,7 @@ from . import _
 from . import common
 from . import metadata
 from . import signindex
-from fdroidserver.common import ANTIFEATURES_CONFIG_NAME, CATEGORIES_CONFIG_NAME, CONFIG_CONFIG_NAME, MIRRORS_CONFIG_NAME, RELEASECHANNELS_CONFIG_NAME, DEFAULT_LOCALE, FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints
+from fdroidserver.common import ANTIFEATURES_CONFIG_NAME, CATEGORIES_CONFIG_NAME, CONFIG_CONFIG_NAME, MIRRORS_CONFIG_NAME, RELEASECHANNELS_CONFIG_NAME, DEFAULT_LOCALE, FDroidPopen, FDroidPopenBytes, load_publish_signer_fingerprints
 from fdroidserver._yaml import yaml
 from fdroidserver.exception import FDroidException, VerificationException
 
@@ -125,14 +125,13 @@ def make(apps, apks, repodir, archive):
                 raise TypeError(_('only accepts strings, lists, and tuples'))
         requestsdict[command] = packageNames
 
-    fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints()
+    signer_fingerprints = load_publish_signer_fingerprints()
 
-    make_v0(sortedapps, apks, repodir, repodict, requestsdict,
+    make_v0(sortedapps, apks, repodir, repodict, requestsdict, signer_fingerprints)
-            fdroid_signing_key_fingerprints)
+    make_v1(sortedapps, apks, repodir, repodict, requestsdict, signer_fingerprints)
-    make_v1(sortedapps, apks, repodir, repodict, requestsdict,
+    make_v2(
-            fdroid_signing_key_fingerprints)
+        sortedapps, apks, repodir, repodict, requestsdict, signer_fingerprints, archive
-    make_v2(sortedapps, apks, repodir, repodict, requestsdict,
+    )
-            fdroid_signing_key_fingerprints, archive)
     make_website(sortedapps, repodir, repodict)
     make_altstore(
         sortedapps,
@@ -709,7 +708,7 @@ def v2_repo(repodict, repodir, archive):
     return repo
 
 
-def make_v2(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints, archive):
+def make_v2(apps, packages, repodir, repodict, requestsdict, signer_fingerprints, archive):
 
     def _index_encoder_default(obj):
         if isinstance(obj, set):
@@ -731,7 +730,7 @@ def make_v2(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_
         output["repo"]["requests"] = requestsdict
 
     # establish sort order of the index
-    v1_sort_packages(packages, fdroid_signing_key_fingerprints)
+    sort_package_versions(packages, signer_fingerprints)
 
     output_packages = collections.OrderedDict()
     output['packages'] = output_packages
@@ -846,7 +845,7 @@ def make_v2(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_
         signindex.sign_index(repodir, json_name)
 
 
-def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
+def make_v1(apps, packages, repodir, repodict, requestsdict, signer_fingerprints):
 
     def _index_encoder_default(obj):
         if isinstance(obj, set):
@@ -876,7 +875,7 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_
         output['repo']['mirrors'] = mirrors
 
     # establish sort order of the index
-    v1_sort_packages(packages, fdroid_signing_key_fingerprints)
+    sort_package_versions(packages, signer_fingerprints)
 
     appslist = []
     output['apps'] = appslist
@@ -984,8 +983,8 @@ def _copy_to_local_copy_dir(repodir, f):
                               .format(path=local_copy_dir))
 
 
-def v1_sort_packages(packages, fdroid_signing_key_fingerprints):
+def sort_package_versions(packages, signer_fingerprints):
-    """Sort the supplied list to ensure a deterministic sort order for package entries in the index file.
+    """Sort to ensure a deterministic order for package versions in the index file.
 
     This sort-order also expresses
     installation preference to the clients.
@@ -1010,7 +1009,7 @@ def v1_sort_packages(packages, fdroid_signing_key_fingerprints):
         if dev_signer and dev_signer == signer:
             group = GROUP_DEV_SIGNED
         else:
-            fdroid_signer = fdroid_signing_key_fingerprints.get(packageName, {}).get('signer')
+            fdroid_signer = signer_fingerprints.get(packageName, {}).get('signer')
             if fdroid_signer and fdroid_signer == signer:
                 group = GROUP_FDROID_SIGNED
 
@@ -1023,7 +1022,7 @@ def v1_sort_packages(packages, fdroid_signing_key_fingerprints):
     packages.sort(key=v1_sort_keys)
 
 
-def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
+def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints):
     """Aka index.jar aka index.xml."""
     doc = Document()
 
@@ -1122,7 +1121,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing
                 if name_from_apk is None:
                     name_from_apk = apk.get('name')
         for versionCode, apksforver in apksbyversion.items():
-            fdroid_signer = fdroid_signing_key_fingerprints.get(appid, {}).get('signer')
+            fdroid_signer = signer_fingerprints.get(appid, {}).get('signer')
             fdroid_signed_apk = None
             name_match_apk = None
             for x in apksforver:

fdroidserver/nightly.py

@@ -337,7 +337,6 @@ def main():
         git_mirror_fdroiddir = os.path.join(git_mirror_path, 'fdroid')
         git_mirror_repodir = os.path.join(git_mirror_fdroiddir, 'repo')
         git_mirror_metadatadir = os.path.join(git_mirror_fdroiddir, 'metadata')
-        git_mirror_statsdir = os.path.join(git_mirror_fdroiddir, 'stats')
         if not os.path.isdir(git_mirror_repodir):
             logging.debug(_('cloning {url}').format(url=clone_url))
             vcs = common.getvcs('git', clone_url, git_mirror_path)
@@ -381,8 +380,6 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
             common.local_rsync(options, [git_mirror_repodir + '/'], 'repo/')
         if os.path.isdir(git_mirror_metadatadir):
             common.local_rsync(options, [git_mirror_metadatadir + '/'], 'metadata/')
-        if os.path.isdir(git_mirror_statsdir):
-            common.local_rsync(options, [git_mirror_statsdir + '/'], 'stats/')
 
         ssh_private_key_file = _ssh_key_from_debug_keystore()
         # this is needed for GitPython to find the SSH key
@@ -495,9 +492,6 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
         common.local_rsync(
             options, [repo_basedir + '/metadata/'], git_mirror_metadatadir + '/'
         )
-        stats = repo_basedir + '/stats/'
-        if os.path.exists(stats):
-            common.local_rsync(options, [stats], git_mirror_statsdir + '/')
         mirror_git_repo.git.add(all=True)
         mirror_git_repo.index.commit("update app metadata")
 

fdroidserver/publish.py

@@ -139,13 +139,13 @@ def sign_sig_key_fingerprint_list(jar_file):
         raise FDroidException("Failed to sign '{}'!".format(jar_file))
 
 
-def store_stats_fdroid_signing_key_fingerprints(appids, indent=None):
+def store_publish_signer_fingerprints(appids, indent=None):
     """Store list of all signing-key fingerprints for given appids to HD.
 
     This list will later on be needed by fdroid update.
     """
-    if not os.path.exists('stats'):
+    if not os.path.exists('repo'):
-        os.makedirs('stats')
+        os.makedirs('repo')
     data = OrderedDict()
     fps = read_fingerprints_from_keystore()
     for appid in sorted(appids):
@@ -153,9 +153,12 @@ def store_stats_fdroid_signing_key_fingerprints(appids, indent=None):
         if alias in fps:
             data[appid] = {'signer': fps[key_alias(appid)]}
 
-    jar_file = os.path.join('stats', 'publishsigkeys.jar')
+    jar_file = os.path.join('repo', 'signer-index.jar')
+    output = json.dumps(data, indent=indent)
     with zipfile.ZipFile(jar_file, 'w', zipfile.ZIP_DEFLATED) as jar:
-        jar.writestr('publishsigkeys.json', json.dumps(data, indent=indent))
+        jar.writestr('signer-index.json', output)
+    with open(os.path.join('repo', 'signer-index.json'), 'w') as fp:
+        fp.write(output)
     sign_sig_key_fingerprint_list(jar_file)
 
 
@@ -460,7 +463,7 @@ def main():
                 publish_source_tarball(apkfilename, unsigned_dir, output_dir)
                 logging.info('Published ' + apkfilename)
 
-    store_stats_fdroid_signing_key_fingerprints(allapps.keys())
+    store_publish_signer_fingerprints(allapps.keys())
     status_update_json(generated_keys, signed_apks)
     logging.info('published list signing-key fingerprints')
 

tests/test_deploy.py

@@ -326,6 +326,12 @@ class DeployTest(unittest.TestCase):
                         'repo/index.png',
                         '--exclude',
                         'repo/index.xml',
+                        '--exclude',
+                        'repo/signer-index.jar',
+                        '--exclude',
+                        'repo/signer-index.json',
+                        '--exclude',
+                        'repo/signer-index.json.asc',
                         'repo',
                         'example.com:/var/www/fdroid',
                     ],
@@ -413,6 +419,9 @@ class DeployTest(unittest.TestCase):
                         'repo/index.jar',
                         'repo/index.png',
                         'repo/index.xml',
+                        'repo/signer-index.jar',
+                        'repo/signer-index.json',
+                        'repo/signer-index.json.asc',
                         'example.com:/var/www/fdroid/repo/',
                     ],
                 )
@@ -729,6 +738,12 @@ class DeployTest(unittest.TestCase):
                         'repo/index.png',
                         '--exclude',
                         'repo/index.xml',
+                        '--exclude',
+                        'repo/signer-index.jar',
+                        '--exclude',
+                        'repo/signer-index.json',
+                        '--exclude',
+                        'repo/signer-index.json.asc',
                         '--no-check-md5',
                         '--skip-existing',
                         repo_section,
@@ -774,6 +789,12 @@ class DeployTest(unittest.TestCase):
                         'repo/index.png',
                         '--exclude',
                         'repo/index.xml',
+                        '--exclude',
+                        'repo/signer-index.jar',
+                        '--exclude',
+                        'repo/signer-index.json',
+                        '--exclude',
+                        'repo/signer-index.json.asc',
                         '--no-check-md5',
                         repo_section,
                         f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
@@ -880,6 +901,12 @@ class DeployTest(unittest.TestCase):
                         'repo/index.png',
                         '--include',
                         'repo/index.xml',
+                        '--include',
+                        'repo/signer-index.jar',
+                        '--include',
+                        'repo/signer-index.json',
+                        '--include',
+                        'repo/signer-index.json.asc',
                         '--delete-removed',
                         '--delete-after',
                         '--no-check-md5',

tests/test_index.py

@@ -240,7 +240,7 @@ class IndexTest(unittest.TestCase):
         with self.assertRaises(fdroidserver.exception.VerificationException):
             data, _ignored = index.download_repo_index_v2(url)
 
-    def test_v1_sort_packages(self):
+    def test_sort_package_versions(self):
         i = [
             {
                 'packageName': 'org.smssecure.smssecure',
@@ -375,21 +375,19 @@ class IndexTest(unittest.TestCase):
                     "signer": "b33a601a9da97c82e6eb121eb6b90adab561f396602ec4dc8b0019fb587e2af6"
                 }
             }
-            os.makedirs('stats')
+            os.mkdir('repo')
-            jarfile = 'stats/publishsigkeys.jar'
+            jarfile = 'repo/signer-index.jar'
             with zipfile.ZipFile(jarfile, 'w', zipfile.ZIP_DEFLATED) as jar:
-                jar.writestr('publishsigkeys.json', json.dumps(sigkeyfps))
+                jar.writestr('signer-index.json', json.dumps(sigkeyfps))
             publish.sign_sig_key_fingerprint_list(jarfile)
             common.write_config_file('')
 
-            index.v1_sort_packages(
+            index.sort_package_versions(i, common.load_publish_signer_fingerprints())
-                i, common.load_stats_fdroid_signing_key_fingerprints()
-            )
             self.maxDiff = None
             self.assertEqual(json.dumps(i, indent=2), json.dumps(o, indent=2))
 
             # and test it still works with get_first_signer_certificate
-            outdir = os.path.join(self.testdir, 'publishsigkeys')
+            outdir = os.path.join(self.testdir, 'index-signer-fingerprints')
             os.mkdir(outdir)
             common.apk_extract_signatures(jarfile, outdir)
             certs = glob.glob(os.path.join(outdir, '*.RSA'))
@@ -646,7 +644,7 @@ class IndexTest(unittest.TestCase):
             self.maxDiff = None
             self.assertEqual(css, pretty_css)
 
-    def test_v1_sort_packages_with_invalid(self):
+    def test_sort_package_versions_with_invalid(self):
         i = [
             {
                 'packageName': 'org.smssecure.smssecure',
@@ -656,7 +654,7 @@ class IndexTest(unittest.TestCase):
             }
         ]
 
-        index.v1_sort_packages(i, common.load_stats_fdroid_signing_key_fingerprints())
+        index.sort_package_versions(i, common.load_publish_signer_fingerprints())
 
     def test_package_metadata(self):
         """A smoke check and format check of index.package_metadata()"""

tests/test_integration.py

@@ -747,7 +747,6 @@ class IntegrationTest(unittest.TestCase):
         self.update_yaml(common.CONFIG_FILE, {"archive_older": 3})
         Path("metadata").mkdir()
         Path("archive").mkdir()
-        Path("stats").mkdir()
         shutil.copy(FILES / "repo/com.politedroid_6.apk", "repo")
         shutil.copy(FILES / "repo/index-v2.json", "repo")
         shutil.copy(FILES / "repo/com.politedroid_5.apk", "archive")

tests/test_publish.py

@@ -77,7 +77,7 @@ class PublishTest(unittest.TestCase):
         self.maxDiff = None
         self.assertEqual(expected, result)
 
-    def test_store_and_load_fdroid_signing_key_fingerprints(self):
+    def test_store_and_load_signer_fingerprints(self):
         common.config = {}
         common.fill_config_defaults(common.config)
         publish.config = common.config
@@ -97,7 +97,7 @@ class PublishTest(unittest.TestCase):
         os.chdir(self.testdir)
         common.write_config_file('')
 
-        publish.store_stats_fdroid_signing_key_fingerprints(appids, indent=2)
+        publish.store_publish_signer_fingerprints(appids, indent=2)
 
         self.maxDiff = None
         expected = {
@@ -114,7 +114,7 @@ class PublishTest(unittest.TestCase):
                 "signer": "6ae5355157a47ddcc3834a71f57f6fb5a8c2621c8e0dc739e9ddf59f865e497c"
             },
         }
-        self.assertEqual(expected, common.load_stats_fdroid_signing_key_fingerprints())
+        self.assertEqual(expected, common.load_publish_signer_fingerprints())
 
         with open(common.CONFIG_FILE) as fp:
             config = yaml.load(fp)
@@ -123,7 +123,7 @@ class PublishTest(unittest.TestCase):
             config['repo_key_sha256'],
         )
 
-    def test_store_and_load_fdroid_signing_key_fingerprints_with_missmatch(self):
+    def test_store_and_load_signer_fingerprints_with_missmatch(self):
         common.config = {}
         common.fill_config_defaults(common.config)
         publish.config = common.config
@@ -134,9 +134,9 @@ class PublishTest(unittest.TestCase):
         publish.config['repo_key_sha256'] = 'bad bad bad bad bad bad bad bad bad bad bad bad'
 
         os.chdir(self.testdir)
-        publish.store_stats_fdroid_signing_key_fingerprints({}, indent=2)
+        publish.store_publish_signer_fingerprints({}, indent=2)
         with self.assertRaises(FDroidException):
-            common.load_stats_fdroid_signing_key_fingerprints()
+            common.load_publish_signer_fingerprints()
 
     def test_reproducible_binaries_process(self):
         common.config = {}