mirrors/fdroidserver
1
0
Fork 0
mirror of https://github.com/f-droid/fdroidserver.git synced 2025-11-04 06:30:27 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

update: add execution stages to status JSON

Browse source 
This should help us profile what takes so long in `fdroid update`.  It also
gives feedback so that people can see how close to done it is, or where it
failed.

This is based on how incremental status JSON works for `fdroid build`.
This commit is contained in:
Hans-Christoph Steiner 2025-03-14 11:30:07 +01:00
parent 51487192b9
commit acbab69722
2 changed files with 34 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

15
fdroidserver/common.py
View file

@ -1216,6 +1216,19 @@ class Encoder(json.JSONEncoder):
return super().default(obj)
def epoch_millis_now():
"""Get the current time in epoch milliseconds.
This is the format returned by Java's System.currentTimeMillis().
Parameters
----------
millis
Java-style integer time since UNIX epoch in milliseconds
"""
return int(datetime.now(timezone.utc).timestamp() * 1000)
def setup_status_output(start_timestamp):
"""Create the common output dictionary for public status updates."""
output = {
@ -1258,7 +1271,7 @@ def write_status_json(output, pretty=False, name=None):
if not os.path.exists(status_dir):
os.makedirs(status_dir)
if not name:
output['endTimestamp'] = int(datetime.now(timezone.utc).timestamp() * 1000)
output['endTimestamp'] = epoch_millis_now()
names = ['running', sys.argv[0].split()[1]] # fdroid subcommand
else:
names = [name]

21
fdroidserver/update.py
View file

@ -202,6 +202,13 @@ def status_update_json(apps, apks):
common.write_status_json(output, options.pretty)
def output_status_stage(output, stage):
if 'stages' not in output:
output['stages'] = dict()
output['stages'][stage] = common.epoch_millis_now()
common.write_running_status_json(output)
def delete_disabled_builds(apps, apkcache, repodirs):
"""Delete disabled build outputs.
@ -2588,7 +2595,7 @@ def main():
metadata.warnings_action = options.W
config = common.read_config()
common.setup_status_output(start_timestamp)
status_output = common.setup_status_output(start_timestamp)
if not (('jarsigner' in config or 'apksigner' in config)
and 'keytool' in config):
@ -2651,12 +2658,15 @@ def main():
cache_timestamp = get_cache_mtime()
# Delete builds for disabled apps
output_status_stage(status_output, 'delete_disabled_builds')
delete_disabled_builds(apps, apkcache, repodirs)
# Scan all apks in the main repo
output_status_stage(status_output, 'process_apks')
apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks,
options.use_date_from_apk, apps, cache_timestamp)
output_status_stage(status_output, 'scan_repo_files')
files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
options.use_date_from_apk)
cachechanged = cachechanged or fcachechanged
@ -2666,6 +2676,7 @@ def main():
cachechanged = cachechanged or icachechanged
apks += ipas
output_status_stage(status_output, 'remove_apks')
appid_has_apks = set()
appid_has_repo_files = set()
sha256_has_files = collections.defaultdict(list)
@ -2748,18 +2759,25 @@ def main():
if cachechanged:
write_cache(apkcache)
output_status_stage(status_output, 'read_added_date_from_all_apks')
# The added date currently comes from the oldest apk which might be in the archive.
# So we need this populated at app level before continuing with only processing /repo
# or /archive
read_added_date_from_all_apks(apps, apks + archapks)
if len(repodirs) > 1:
output_status_stage(status_output, 'archive_old_apks archive')
archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
output_status_stage(status_output, 'prepare_apps archive')
archived_apps = prepare_apps(apps, archapks, repodirs[1])
output_status_stage(status_output, 'index.make archive')
fdroidserver.index.make(archived_apps, archapks, repodirs[1], True)
output_status_stage(status_output, 'prepare_apps repo')
repoapps = prepare_apps(apps, apks, repodirs[0])
output_status_stage(status_output, 'index.make repo')
# APKs are placed into multiple repos based on the app package, providing
# per-app subscription feeds for nightly builds and things like it
if config['per_app_repos']:
@ -2780,6 +2798,7 @@ def main():
git_remote = config.get('binary_transparency_remote')
if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
from . import btlog
output_status_stage(status_output, 'make_binary_transparency_log')
btlog.make_binary_transparency_log(repodirs)
status_update_json(apps, apks + archapks)