mirror of
https://github.com/f-droid/fdroidserver.git
synced 2025-11-11 09:40:28 +03:00
Drop stats/known_apks.txt
Use repo/index-v2.json instead. Also fix deprecated datetime.utcnow().
This commit is contained in:
Jochen Sprickerhof
Hans-Christoph Steiner
parent
83a3227235
commit
81c0c9d4aa
14 changed files with 33 additions and 120 deletions
|
|
@ -162,7 +162,6 @@ default_config = {
|
|||
'make_current_version_link': False,
|
||||
'current_version_name_source': 'Name',
|
||||
'deploy_process_logs': False,
|
||||
'update_stats': False,
|
||||
'repo_maxage': 0,
|
||||
'build_server_always': False,
|
||||
'keystore': 'keystore.p12',
|
||||
|
|
@ -2555,40 +2554,17 @@ class KnownApks:
|
|||
this is parsed as a list from the end to allow the filename to
|
||||
have any combo of spaces.
|
||||
"""
|
||||
self.path = os.path.join('stats', 'known_apks.txt')
|
||||
self.apks = {}
|
||||
if os.path.isfile(self.path):
|
||||
with open(self.path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
t = line.rstrip().split(' ')
|
||||
if len(t) == 2:
|
||||
self.apks[t[0]] = (t[1], None)
|
||||
else:
|
||||
appid = t[-2]
|
||||
date = datetime.strptime(t[-1], '%Y-%m-%d')
|
||||
filename = line[0:line.rfind(appid) - 1]
|
||||
self.apks[filename] = (appid, date)
|
||||
check_system_clock(date, self.path)
|
||||
self.changed = False
|
||||
|
||||
def writeifchanged(self):
|
||||
if not self.changed:
|
||||
return
|
||||
|
||||
if not os.path.exists('stats'):
|
||||
os.mkdir('stats')
|
||||
|
||||
lst = []
|
||||
for apk, app in self.apks.items():
|
||||
appid, added = app
|
||||
line = apk + ' ' + appid
|
||||
if added:
|
||||
line += ' ' + added.strftime('%Y-%m-%d')
|
||||
lst.append(line)
|
||||
|
||||
with open(self.path, 'w') as f:
|
||||
for line in sorted(lst, key=natural_key):
|
||||
f.write(line + '\n')
|
||||
for part in ('repo', 'archive'):
|
||||
path = os.path.join(part, 'index-v2.json')
|
||||
if os.path.isfile(path):
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
index = json.load(f)
|
||||
for appid, data in index["packages"].items():
|
||||
for version in data["versions"].values():
|
||||
filename = version["file"]["name"][1:]
|
||||
date = datetime.fromtimestamp(version["added"] // 1000, tz=timezone.utc)
|
||||
self.apks[filename] = (appid, date)
|
||||
|
||||
def recordapk(self, apkName, app, default_date=None):
|
||||
"""
|
||||
|
|
@ -2601,38 +2577,12 @@ class KnownApks:
|
|||
"""
|
||||
if apkName not in self.apks:
|
||||
if default_date is None:
|
||||
default_date = datetime.utcnow()
|
||||
default_date = datetime.now(timezone.utc)
|
||||
self.apks[apkName] = (app, default_date)
|
||||
self.changed = True
|
||||
_ignored, added = self.apks[apkName]
|
||||
return added
|
||||
|
||||
def getapp(self, apkname):
|
||||
"""Look up information - given the 'apkname'.
|
||||
|
||||
Returns (app id, date added/None).
|
||||
Or returns None for an unknown apk.
|
||||
"""
|
||||
if apkname in self.apks:
|
||||
return self.apks[apkname]
|
||||
return None
|
||||
|
||||
def getlatest(self, num):
|
||||
"""Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first."""
|
||||
apps = {}
|
||||
for apk, app in self.apks.items():
|
||||
appid, added = app
|
||||
if added:
|
||||
if appid in apps:
|
||||
if apps[appid] > added:
|
||||
apps[appid] = added
|
||||
else:
|
||||
apps[appid] = added
|
||||
sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
|
||||
lst = [app for app, _ignored in sortedapps]
|
||||
lst.reverse()
|
||||
return lst
|
||||
|
||||
|
||||
def get_file_extension(filename):
|
||||
"""Get the normalized file extension, can be blank string but never None."""
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ def make(apps, apks, repodir, archive):
|
|||
sortedapps[appid] = apps[appid]
|
||||
|
||||
repodict = collections.OrderedDict()
|
||||
repodict['timestamp'] = datetime.utcnow().replace(tzinfo=timezone.utc)
|
||||
repodict['timestamp'] = datetime.now(timezone.utc)
|
||||
repodict['version'] = METADATA_VERSION
|
||||
|
||||
if common.config['repo_maxage'] != 0:
|
||||
|
|
|
|||
|
|
@ -367,7 +367,7 @@ You can use it with the [F-Droid](https://f-droid.org/) Android app.
|
|||
|
||||
Last updated: {date}'''.format(repo_git_base=repo_git_base,
|
||||
repo_url=repo_url,
|
||||
date=datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC'))
|
||||
date=datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC'))
|
||||
with open(readme_path, 'w') as fp:
|
||||
fp.write(readme)
|
||||
mirror_git_repo.git.add(all=True)
|
||||
|
|
@ -422,7 +422,6 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
|
|||
'keypass': PASSWORD,
|
||||
'keydname': DISTINGUISHED_NAME,
|
||||
'make_current_version_link': False,
|
||||
'update_stats': True,
|
||||
}
|
||||
with open('config.yml', 'w') as fp:
|
||||
yaml.dump(config, fp, default_flow_style=False)
|
||||
|
|
@ -496,7 +495,9 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
|
|||
common.local_rsync(
|
||||
options, [repo_basedir + '/metadata/'], git_mirror_metadatadir + '/'
|
||||
)
|
||||
common.local_rsync(options, [repo_basedir + '/stats/'], git_mirror_statsdir + '/')
|
||||
stats = repo_basedir + '/stats/'
|
||||
if os.path.exists(stats):
|
||||
common.local_rsync(options, [stats], git_mirror_statsdir + '/')
|
||||
mirror_git_repo.git.add(all=True)
|
||||
mirror_git_repo.index.commit("update app metadata")
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ import urllib.request
|
|||
import zipfile
|
||||
from argparse import ArgumentParser
|
||||
from dataclasses import dataclass, field, fields
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from enum import IntEnum
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
|
|
@ -330,7 +330,7 @@ def get_embedded_classes(apkfile, depth=0):
|
|||
|
||||
def _datetime_now():
|
||||
"""Get datetime.now(), using this funciton allows mocking it for testing."""
|
||||
return datetime.utcnow()
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _scanner_cachedir():
|
||||
|
|
@ -389,7 +389,7 @@ class SignatureDataController:
|
|||
last_updated = self.data.get("last_updated", None)
|
||||
if last_updated:
|
||||
try:
|
||||
last_updated = datetime.fromtimestamp(last_updated)
|
||||
last_updated = datetime.fromtimestamp(last_updated, timezone.utc)
|
||||
except ValueError as e:
|
||||
raise SignatureDataMalformedException() from e
|
||||
except TypeError as e:
|
||||
|
|
|
|||
|
|
@ -362,7 +362,7 @@ def get_cache():
|
|||
if not isinstance(v['antiFeatures'], dict):
|
||||
v['antiFeatures'] = {k: {} for k in sorted(v['antiFeatures'])}
|
||||
if 'added' in v:
|
||||
v['added'] = datetime.fromtimestamp(v['added'])
|
||||
v['added'] = datetime.fromtimestamp(v['added'], tz=timezone.utc)
|
||||
|
||||
return apkcache
|
||||
|
||||
|
|
@ -2001,7 +2001,7 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal
|
|||
fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
|
||||
|
||||
if use_date_from_apk:
|
||||
default_date_param = datetime.fromtimestamp(os.stat(apkfile).st_mtime)
|
||||
default_date_param = datetime.fromtimestamp(os.stat(apkfile).st_mtime, tz=timezone.utc)
|
||||
else:
|
||||
default_date_param = None
|
||||
|
||||
|
|
@ -2420,12 +2420,12 @@ def create_metadata_from_template(apk):
|
|||
def read_added_date_from_all_apks(apps, apks):
|
||||
"""No summary.
|
||||
|
||||
Added dates come from the stats/known_apks.txt file but are
|
||||
Added dates come from the repo/index-v2.json file but are
|
||||
read when scanning apks and thus need to be applied form apk
|
||||
level to app level for _all_ apps and not only from non-archived
|
||||
ones
|
||||
|
||||
TODO: read the added dates directly from known_apks.txt instead of
|
||||
TODO: read the added dates directly from index-v2.json instead of
|
||||
going through apks that way it also works for for repos that
|
||||
don't keep an archive of apks.
|
||||
"""
|
||||
|
|
@ -2772,10 +2772,6 @@ def main():
|
|||
from . import btlog
|
||||
btlog.make_binary_transparency_log(repodirs)
|
||||
|
||||
if config['update_stats']:
|
||||
# Update known apks info...
|
||||
knownapks.writeifchanged()
|
||||
|
||||
status_update_json(apps, apks + archapks)
|
||||
|
||||
logging.info(_("Finished"))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue