mirrors/fdroidserver
1
0
Fork 0
mirror of https://github.com/f-droid/fdroidserver.git synced 2025-11-04 06:30:27 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'no-more-stats-dir' into 'master'

Browse source 
remove last uses of stats/ dir

See merge request fdroid/fdroidserver!1620
This commit is contained in:
Hans-Christoph Steiner 2025-03-20 14:42:09 +00:00
commit 88f4a36f38
9 changed files with 79 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

8
fdroidserver/common.py
View file

@ -4219,7 +4219,7 @@ def get_jar_signer_certificate(pkcs7obj: cms.ContentInfo, signature_file: bytes)
return first_verified_signer_info_signing_certificate
def load_stats_fdroid_signing_key_fingerprints():
def load_publish_signer_fingerprints():
"""Load signing-key fingerprints stored in file generated by fdroid publish.
Returns
@ -4227,7 +4227,7 @@ def load_stats_fdroid_signing_key_fingerprints():
dict
containing the signing-key fingerprints.
"""
jar_file = os.path.join('stats', 'publishsigkeys.jar')
jar_file = os.path.join('repo', 'signer-index.jar')
if not os.path.isfile(jar_file):
return {}
try:
@ -4247,7 +4247,7 @@ def load_stats_fdroid_signing_key_fingerprints():
write_to_config(config, 'repo_key_sha256')
with zipfile.ZipFile(jar_file, 'r') as f:
return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
return json.loads(str(f.read('signer-index.json'), 'utf-8'))
def write_config_file(config):
@ -4478,6 +4478,7 @@ NO_GPG_INDEX_FILES = [
"index.jar",
"index.png",
"index.xml",
"signer-index.jar",
]
# list of index files that are signed by gpgsign.py to make a .asc file
@ -4486,6 +4487,7 @@ GPG_INDEX_FILES = [
"entry.json",
"index-v1.json",
"index-v2.json",
"signer-index.json",
]

9
fdroidserver/deploy.py
View file

@ -60,8 +60,15 @@ def _get_index_file_paths(base_dir):
services can take a while. So the index files should be updated
last. That ensures that the package files are available when the
client learns about them from the new index files.
signer-index.* are only published in the repo/ section.
"""
return [os.path.join(base_dir, filename) for filename in common.INDEX_FILES]
return [
os.path.join(base_dir, filename)
for filename in common.INDEX_FILES
if not (filename.startswith('signer-index.') and base_dir.endswith('archive'))
]
def _get_index_excludes(base_dir):

33
fdroidserver/index.py
View file

@ -51,7 +51,7 @@ from . import _
from . import common
from . import metadata
from . import signindex
from fdroidserver.common import ANTIFEATURES_CONFIG_NAME, CATEGORIES_CONFIG_NAME, CONFIG_CONFIG_NAME, MIRRORS_CONFIG_NAME, RELEASECHANNELS_CONFIG_NAME, DEFAULT_LOCALE, FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints
from fdroidserver.common import ANTIFEATURES_CONFIG_NAME, CATEGORIES_CONFIG_NAME, CONFIG_CONFIG_NAME, MIRRORS_CONFIG_NAME, RELEASECHANNELS_CONFIG_NAME, DEFAULT_LOCALE, FDroidPopen, FDroidPopenBytes, load_publish_signer_fingerprints
from fdroidserver._yaml import yaml
from fdroidserver.exception import FDroidException, VerificationException
@ -125,14 +125,13 @@ def make(apps, apks, repodir, archive):
raise TypeError(_('only accepts strings, lists, and tuples'))
requestsdict[command] = packageNames
fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints()
signer_fingerprints = load_publish_signer_fingerprints()
make_v0(sortedapps, apks, repodir, repodict, requestsdict,
fdroid_signing_key_fingerprints)
make_v1(sortedapps, apks, repodir, repodict, requestsdict,
fdroid_signing_key_fingerprints)
make_v2(sortedapps, apks, repodir, repodict, requestsdict,
fdroid_signing_key_fingerprints, archive)
make_v0(sortedapps, apks, repodir, repodict, requestsdict, signer_fingerprints)
make_v1(sortedapps, apks, repodir, repodict, requestsdict, signer_fingerprints)
make_v2(
sortedapps, apks, repodir, repodict, requestsdict, signer_fingerprints, archive
)
make_website(sortedapps, repodir, repodict)
make_altstore(
sortedapps,
@ -709,7 +708,7 @@ def v2_repo(repodict, repodir, archive):
return repo
def make_v2(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints, archive):
def make_v2(apps, packages, repodir, repodict, requestsdict, signer_fingerprints, archive):
def _index_encoder_default(obj):
if isinstance(obj, set):
@ -731,7 +730,7 @@ def make_v2(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_
output["repo"]["requests"] = requestsdict
# establish sort order of the index
v1_sort_packages(packages, fdroid_signing_key_fingerprints)
sort_package_versions(packages, signer_fingerprints)
output_packages = collections.OrderedDict()
output['packages'] = output_packages
@ -846,7 +845,7 @@ def make_v2(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_
signindex.sign_index(repodir, json_name)
def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
def make_v1(apps, packages, repodir, repodict, requestsdict, signer_fingerprints):
def _index_encoder_default(obj):
if isinstance(obj, set):
@ -876,7 +875,7 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_
output['repo']['mirrors'] = mirrors
# establish sort order of the index
v1_sort_packages(packages, fdroid_signing_key_fingerprints)
sort_package_versions(packages, signer_fingerprints)
appslist = []
output['apps'] = appslist
@ -984,8 +983,8 @@ def _copy_to_local_copy_dir(repodir, f):
.format(path=local_copy_dir))
def v1_sort_packages(packages, fdroid_signing_key_fingerprints):
"""Sort the supplied list to ensure a deterministic sort order for package entries in the index file.
def sort_package_versions(packages, signer_fingerprints):
"""Sort to ensure a deterministic order for package versions in the index file.
This sort-order also expresses
installation preference to the clients.
@ -1010,7 +1009,7 @@ def v1_sort_packages(packages, fdroid_signing_key_fingerprints):
if dev_signer and dev_signer == signer:
group = GROUP_DEV_SIGNED
else:
fdroid_signer = fdroid_signing_key_fingerprints.get(packageName, {}).get('signer')
fdroid_signer = signer_fingerprints.get(packageName, {}).get('signer')
if fdroid_signer and fdroid_signer == signer:
group = GROUP_FDROID_SIGNED
@ -1023,7 +1022,7 @@ def v1_sort_packages(packages, fdroid_signing_key_fingerprints):
packages.sort(key=v1_sort_keys)
def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints):
"""Aka index.jar aka index.xml."""
doc = Document()
@ -1122,7 +1121,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing
if name_from_apk is None:
name_from_apk = apk.get('name')
for versionCode, apksforver in apksbyversion.items():
fdroid_signer = fdroid_signing_key_fingerprints.get(appid, {}).get('signer')
fdroid_signer = signer_fingerprints.get(appid, {}).get('signer')
fdroid_signed_apk = None
name_match_apk = None
for x in apksforver:

6
fdroidserver/nightly.py
View file

@ -337,7 +337,6 @@ def main():
git_mirror_fdroiddir = os.path.join(git_mirror_path, 'fdroid')
git_mirror_repodir = os.path.join(git_mirror_fdroiddir, 'repo')
git_mirror_metadatadir = os.path.join(git_mirror_fdroiddir, 'metadata')
git_mirror_statsdir = os.path.join(git_mirror_fdroiddir, 'stats')
if not os.path.isdir(git_mirror_repodir):
logging.debug(_('cloning {url}').format(url=clone_url))
vcs = common.getvcs('git', clone_url, git_mirror_path)
@ -381,8 +380,6 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
common.local_rsync(options, [git_mirror_repodir + '/'], 'repo/')
if os.path.isdir(git_mirror_metadatadir):
common.local_rsync(options, [git_mirror_metadatadir + '/'], 'metadata/')
if os.path.isdir(git_mirror_statsdir):
common.local_rsync(options, [git_mirror_statsdir + '/'], 'stats/')
ssh_private_key_file = _ssh_key_from_debug_keystore()
# this is needed for GitPython to find the SSH key
@ -495,9 +492,6 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
common.local_rsync(
options, [repo_basedir + '/metadata/'], git_mirror_metadatadir + '/'
)
stats = repo_basedir + '/stats/'
if os.path.exists(stats):
common.local_rsync(options, [stats], git_mirror_statsdir + '/')
mirror_git_repo.git.add(all=True)
mirror_git_repo.index.commit("update app metadata")

15
fdroidserver/publish.py
View file

@ -139,13 +139,13 @@ def sign_sig_key_fingerprint_list(jar_file):
raise FDroidException("Failed to sign '{}'!".format(jar_file))
def store_stats_fdroid_signing_key_fingerprints(appids, indent=None):
def store_publish_signer_fingerprints(appids, indent=None):
"""Store list of all signing-key fingerprints for given appids to HD.
This list will later on be needed by fdroid update.
"""
if not os.path.exists('stats'):
os.makedirs('stats')
if not os.path.exists('repo'):
os.makedirs('repo')
data = OrderedDict()
fps = read_fingerprints_from_keystore()
for appid in sorted(appids):
@ -153,9 +153,12 @@ def store_stats_fdroid_signing_key_fingerprints(appids, indent=None):
if alias in fps:
data[appid] = {'signer': fps[key_alias(appid)]}
jar_file = os.path.join('stats', 'publishsigkeys.jar')
jar_file = os.path.join('repo', 'signer-index.jar')
output = json.dumps(data, indent=indent)
with zipfile.ZipFile(jar_file, 'w', zipfile.ZIP_DEFLATED) as jar:
jar.writestr('publishsigkeys.json', json.dumps(data, indent=indent))
jar.writestr('signer-index.json', output)
with open(os.path.join('repo', 'signer-index.json'), 'w') as fp:
fp.write(output)
sign_sig_key_fingerprint_list(jar_file)
@ -460,7 +463,7 @@ def main():
publish_source_tarball(apkfilename, unsigned_dir, output_dir)
logging.info('Published ' + apkfilename)
store_stats_fdroid_signing_key_fingerprints(allapps.keys())
store_publish_signer_fingerprints(allapps.keys())
status_update_json(generated_keys, signed_apks)
logging.info('published list signing-key fingerprints')

27
tests/test_deploy.py
View file

@ -326,6 +326,12 @@ class DeployTest(unittest.TestCase):
'repo/index.png',
'--exclude',
'repo/index.xml',
'--exclude',
'repo/signer-index.jar',
'--exclude',
'repo/signer-index.json',
'--exclude',
'repo/signer-index.json.asc',
'repo',
'example.com:/var/www/fdroid',
],
@ -413,6 +419,9 @@ class DeployTest(unittest.TestCase):
'repo/index.jar',
'repo/index.png',
'repo/index.xml',
'repo/signer-index.jar',
'repo/signer-index.json',
'repo/signer-index.json.asc',
'example.com:/var/www/fdroid/repo/',
],
)
@ -729,6 +738,12 @@ class DeployTest(unittest.TestCase):
'repo/index.png',
'--exclude',
'repo/index.xml',
'--exclude',
'repo/signer-index.jar',
'--exclude',
'repo/signer-index.json',
'--exclude',
'repo/signer-index.json.asc',
'--no-check-md5',
'--skip-existing',
repo_section,
@ -774,6 +789,12 @@ class DeployTest(unittest.TestCase):
'repo/index.png',
'--exclude',
'repo/index.xml',
'--exclude',
'repo/signer-index.jar',
'--exclude',
'repo/signer-index.json',
'--exclude',
'repo/signer-index.json.asc',
'--no-check-md5',
repo_section,
f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
@ -880,6 +901,12 @@ class DeployTest(unittest.TestCase):
'repo/index.png',
'--include',
'repo/index.xml',
'--include',
'repo/signer-index.jar',
'--include',
'repo/signer-index.json',
'--include',
'repo/signer-index.json.asc',
'--delete-removed',
'--delete-after',
'--no-check-md5',

18
tests/test_index.py
View file

@ -240,7 +240,7 @@ class IndexTest(unittest.TestCase):
with self.assertRaises(fdroidserver.exception.VerificationException):
data, _ignored = index.download_repo_index_v2(url)
def test_v1_sort_packages(self):
def test_sort_package_versions(self):
i = [
{
'packageName': 'org.smssecure.smssecure',
@ -375,21 +375,19 @@ class IndexTest(unittest.TestCase):
"signer": "b33a601a9da97c82e6eb121eb6b90adab561f396602ec4dc8b0019fb587e2af6"
}
}
os.makedirs('stats')
jarfile = 'stats/publishsigkeys.jar'
os.mkdir('repo')
jarfile = 'repo/signer-index.jar'
with zipfile.ZipFile(jarfile, 'w', zipfile.ZIP_DEFLATED) as jar:
jar.writestr('publishsigkeys.json', json.dumps(sigkeyfps))
jar.writestr('signer-index.json', json.dumps(sigkeyfps))
publish.sign_sig_key_fingerprint_list(jarfile)
common.write_config_file('')
index.v1_sort_packages(
i, common.load_stats_fdroid_signing_key_fingerprints()
)
index.sort_package_versions(i, common.load_publish_signer_fingerprints())
self.maxDiff = None
self.assertEqual(json.dumps(i, indent=2), json.dumps(o, indent=2))
# and test it still works with get_first_signer_certificate
outdir = os.path.join(self.testdir, 'publishsigkeys')
outdir = os.path.join(self.testdir, 'index-signer-fingerprints')
os.mkdir(outdir)
common.apk_extract_signatures(jarfile, outdir)
certs = glob.glob(os.path.join(outdir, '*.RSA'))
@ -646,7 +644,7 @@ class IndexTest(unittest.TestCase):
self.maxDiff = None
self.assertEqual(css, pretty_css)
def test_v1_sort_packages_with_invalid(self):
def test_sort_package_versions_with_invalid(self):
i = [
{
'packageName': 'org.smssecure.smssecure',
@ -656,7 +654,7 @@ class IndexTest(unittest.TestCase):
}
]
index.v1_sort_packages(i, common.load_stats_fdroid_signing_key_fingerprints())
index.sort_package_versions(i, common.load_publish_signer_fingerprints())
def test_package_metadata(self):
"""A smoke check and format check of index.package_metadata()"""

1
tests/test_integration.py
View file

@ -747,7 +747,6 @@ class IntegrationTest(unittest.TestCase):
self.update_yaml(common.CONFIG_FILE, {"archive_older": 3})
Path("metadata").mkdir()
Path("archive").mkdir()
Path("stats").mkdir()
shutil.copy(FILES / "repo/com.politedroid_6.apk", "repo")
shutil.copy(FILES / "repo/index-v2.json", "repo")
shutil.copy(FILES / "repo/com.politedroid_5.apk", "archive")

12
tests/test_publish.py
View file

@ -77,7 +77,7 @@ class PublishTest(unittest.TestCase):
self.maxDiff = None
self.assertEqual(expected, result)
def test_store_and_load_fdroid_signing_key_fingerprints(self):
def test_store_and_load_signer_fingerprints(self):
common.config = {}
common.fill_config_defaults(common.config)
publish.config = common.config
@ -97,7 +97,7 @@ class PublishTest(unittest.TestCase):
os.chdir(self.testdir)
common.write_config_file('')
publish.store_stats_fdroid_signing_key_fingerprints(appids, indent=2)
publish.store_publish_signer_fingerprints(appids, indent=2)
self.maxDiff = None
expected = {
@ -114,7 +114,7 @@ class PublishTest(unittest.TestCase):
"signer": "6ae5355157a47ddcc3834a71f57f6fb5a8c2621c8e0dc739e9ddf59f865e497c"
},
}
self.assertEqual(expected, common.load_stats_fdroid_signing_key_fingerprints())
self.assertEqual(expected, common.load_publish_signer_fingerprints())
with open(common.CONFIG_FILE) as fp:
config = yaml.load(fp)
@ -123,7 +123,7 @@ class PublishTest(unittest.TestCase):
config['repo_key_sha256'],
)
def test_store_and_load_fdroid_signing_key_fingerprints_with_missmatch(self):
def test_store_and_load_signer_fingerprints_with_missmatch(self):
common.config = {}
common.fill_config_defaults(common.config)
publish.config = common.config
@ -134,9 +134,9 @@ class PublishTest(unittest.TestCase):
publish.config['repo_key_sha256'] = 'bad bad bad bad bad bad bad bad bad bad bad bad'
os.chdir(self.testdir)
publish.store_stats_fdroid_signing_key_fingerprints({}, indent=2)
publish.store_publish_signer_fingerprints({}, indent=2)
with self.assertRaises(FDroidException):
common.load_stats_fdroid_signing_key_fingerprints()
common.load_publish_signer_fingerprints()
def test_reproducible_binaries_process(self):
common.config = {}