mirror of
https://github.com/f-droid/fdroidserver.git
synced 2025-11-12 10:10:30 +03:00
define "string map" type for new Anti-Features explanations
closes #683
This commit is contained in:
Hans-Christoph Steiner
parent
6e62ea3614
commit
061ca38afd
27 changed files with 1188 additions and 194 deletions
|
|
@ -546,6 +546,13 @@ def package_metadata(app, repodir):
|
|||
|
||||
|
||||
def convert_version(version, app, repodir):
|
||||
"""Convert the internal representation of Builds: into index-v2 versions.
|
||||
|
||||
The diff algorithm of index-v2 uses null/None to mean a field to
|
||||
be removed, so this function handles any Nones that are in the
|
||||
metadata file.
|
||||
|
||||
"""
|
||||
ver = {}
|
||||
if "added" in version:
|
||||
ver["added"] = convert_datetime(version["added"])
|
||||
|
|
@ -555,7 +562,7 @@ def convert_version(version, app, repodir):
|
|||
ver["file"] = {
|
||||
"name": "/{}".format(version["apkName"]),
|
||||
version["hashType"]: version["hash"],
|
||||
"size": version["size"]
|
||||
"size": version["size"],
|
||||
}
|
||||
|
||||
ipfsCIDv1 = version.get("ipfsCIDv1")
|
||||
|
|
@ -619,24 +626,14 @@ def convert_version(version, app, repodir):
|
|||
else:
|
||||
manifest[en].append({"name": perm[0]})
|
||||
|
||||
antiFeatures = dict()
|
||||
if "AntiFeatures" in app and app["AntiFeatures"]:
|
||||
for antif in app["AntiFeatures"]:
|
||||
# TODO: get reasons from fdroiddata
|
||||
# ver["antiFeatures"][antif] = {"en-US": "reason"}
|
||||
antiFeatures[antif] = dict()
|
||||
|
||||
if "antiFeatures" in version and version["antiFeatures"]:
|
||||
for antif in version["antiFeatures"]:
|
||||
# TODO: get reasons from fdroiddata
|
||||
# ver["antiFeatures"][antif] = {"en-US": "reason"}
|
||||
antiFeatures[antif] = dict()
|
||||
|
||||
if app.get("NoSourceSince"):
|
||||
antiFeatures["NoSourceSince"] = dict()
|
||||
|
||||
# index-v2 has only per-version antifeatures, not per package.
|
||||
antiFeatures = app.get('AntiFeatures', {})
|
||||
for name, descdict in version.get('antiFeatures', dict()).items():
|
||||
antiFeatures[name] = descdict
|
||||
if antiFeatures:
|
||||
ver["antiFeatures"] = dict(sorted(antiFeatures.items()))
|
||||
ver['antiFeatures'] = {
|
||||
k: dict(sorted(antiFeatures[k].items())) for k in sorted(antiFeatures)
|
||||
}
|
||||
|
||||
if "versionCode" in version:
|
||||
if version["versionCode"] > app["CurrentVersionCode"]:
|
||||
|
|
@ -881,9 +878,8 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_
|
|||
for ikey, iname in sorted(lvalue.items()):
|
||||
lordered[lkey][ikey] = iname
|
||||
app_dict['localized'] = lordered
|
||||
antiFeatures = app_dict.get('antiFeatures', [])
|
||||
if apps[app_dict["packageName"]].get("NoSourceSince"):
|
||||
antiFeatures.append("NoSourceSince")
|
||||
# v1 uses a list of keys for Anti-Features
|
||||
antiFeatures = app_dict.get('antiFeatures', dict()).keys()
|
||||
if antiFeatures:
|
||||
app_dict['antiFeatures'] = sorted(set(antiFeatures))
|
||||
|
||||
|
|
@ -915,6 +911,9 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_
|
|||
continue
|
||||
if k in ('icon', 'icons', 'icons_src', 'ipfsCIDv1', 'name'):
|
||||
continue
|
||||
if k == 'antiFeatures':
|
||||
d[k] = sorted(v.keys())
|
||||
continue
|
||||
d[k] = v
|
||||
|
||||
json_name = 'index-v1.json'
|
||||
|
|
@ -1160,8 +1159,6 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing
|
|||
antiFeatures = list(app.AntiFeatures)
|
||||
if 'antiFeatures' in apklist[0]:
|
||||
antiFeatures.extend(apklist[0]['antiFeatures'])
|
||||
if app.get("NoSourceSince"):
|
||||
antiFeatures.append("NoSourceSince")
|
||||
if antiFeatures:
|
||||
afout = sorted(set(antiFeatures))
|
||||
addElementNonEmpty('antifeatures', ','.join(afout), doc, apel)
|
||||
|
|
|
|||
|
|
@ -624,6 +624,17 @@ def check_app_field_types(app):
|
|||
fieldtype=v.__class__.__name__,
|
||||
)
|
||||
)
|
||||
elif t == metadata.TYPE_STRINGMAP and not isinstance(v, dict):
|
||||
yield (
|
||||
_(
|
||||
"{appid}: {field} must be a '{type}', but it is a '{fieldtype}'!"
|
||||
).format(
|
||||
appid=app.id,
|
||||
field=field,
|
||||
type='dict',
|
||||
fieldtype=v.__class__.__name__,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def check_antiFeatures(app):
|
||||
|
|
|
|||
|
|
@ -114,7 +114,7 @@ class App(dict):
|
|||
super().__init__()
|
||||
|
||||
self.Disabled = None
|
||||
self.AntiFeatures = []
|
||||
self.AntiFeatures = dict()
|
||||
self.Provides = None
|
||||
self.Categories = []
|
||||
self.License = 'Unknown'
|
||||
|
|
@ -182,12 +182,13 @@ TYPE_SCRIPT = 5
|
|||
TYPE_MULTILINE = 6
|
||||
TYPE_BUILD = 7
|
||||
TYPE_INT = 8
|
||||
TYPE_STRINGMAP = 9
|
||||
|
||||
fieldtypes = {
|
||||
'Description': TYPE_MULTILINE,
|
||||
'MaintainerNotes': TYPE_MULTILINE,
|
||||
'Categories': TYPE_LIST,
|
||||
'AntiFeatures': TYPE_LIST,
|
||||
'AntiFeatures': TYPE_STRINGMAP,
|
||||
'AllowedAPKSigningKeys': TYPE_LIST,
|
||||
'Builds': TYPE_BUILD,
|
||||
'VercodeOperation': TYPE_LIST,
|
||||
|
|
@ -277,7 +278,7 @@ class Build(dict):
|
|||
self.antcommands = []
|
||||
self.postbuild = ''
|
||||
self.novcheck = False
|
||||
self.antifeatures = []
|
||||
self.antifeatures = dict()
|
||||
if copydict:
|
||||
super().__init__(copydict)
|
||||
return
|
||||
|
|
@ -358,7 +359,7 @@ flagtypes = {
|
|||
'forceversion': TYPE_BOOL,
|
||||
'forcevercode': TYPE_BOOL,
|
||||
'novcheck': TYPE_BOOL,
|
||||
'antifeatures': TYPE_LIST,
|
||||
'antifeatures': TYPE_STRINGMAP,
|
||||
'timeout': TYPE_INT,
|
||||
}
|
||||
|
||||
|
|
@ -649,8 +650,6 @@ def parse_metadata(metadatapath):
|
|||
metadatapath = Path(metadatapath)
|
||||
app = App()
|
||||
app.metadatapath = metadatapath.as_posix()
|
||||
if metadatapath.stem != '.fdroid':
|
||||
app.id = metadatapath.stem
|
||||
if metadatapath.suffix == '.yml':
|
||||
with metadatapath.open('r', encoding='utf-8') as mf:
|
||||
app.update(parse_yaml_metadata(mf))
|
||||
|
|
@ -659,6 +658,10 @@ def parse_metadata(metadatapath):
|
|||
_('Unknown metadata format: {path} (use: *.yml)').format(path=metadatapath)
|
||||
)
|
||||
|
||||
if metadatapath.stem != '.fdroid':
|
||||
app.id = metadatapath.stem
|
||||
parse_localized_antifeatures(app)
|
||||
|
||||
if metadatapath.name != '.fdroid.yml' and app.Repo:
|
||||
build_dir = common.get_build_dir(app)
|
||||
metadata_in_repo = build_dir / '.fdroid.yml'
|
||||
|
|
@ -770,6 +773,116 @@ def parse_yaml_metadata(mf):
|
|||
return yamldata
|
||||
|
||||
|
||||
def parse_localized_antifeatures(app):
|
||||
"""Read in localized Anti-Features files from the filesystem.
|
||||
|
||||
To support easy integration with Weblate and other translation
|
||||
systems, there is a special type of metadata that can be
|
||||
maintained in a Fastlane-style directory layout, where each field
|
||||
is represented by a text file on directories that specified which
|
||||
app it belongs to, which locale, etc. This function reads those
|
||||
in and puts them into the internal dict, to be merged with any
|
||||
related data that came from the metadata.yml file.
|
||||
|
||||
This needs to be run after parse_yaml_metadata() since that
|
||||
normalizes the data structure. Also, these values are lower
|
||||
priority than what comes from the metadata file. So this should
|
||||
not overwrite anything parse_yaml_metadata() puts into the App
|
||||
instance.
|
||||
|
||||
metadata/<Application ID>/<locale>/antifeatures/<Version Code>_<Anti-Feature>.txt
|
||||
metadata/<Application ID>/<locale>/antifeatures/<Anti-Feature>.txt
|
||||
|
||||
└── metadata/
|
||||
└── <Application ID>/
|
||||
├── en-US/
|
||||
│ └── antifeatures/
|
||||
│ ├── 123_Ads.txt -> "includes ad lib"
|
||||
│ ├── 123_Tracking.txt -> "standard suspects"
|
||||
│ └── NoSourceSince.txt -> "it vanished"
|
||||
│
|
||||
└── zh-CN/
|
||||
└── antifeatures/
|
||||
└── 123_Ads.txt -> "包括广告图书馆"
|
||||
|
||||
Gets parsed into the metadata data structure:
|
||||
|
||||
AntiFeatures:
|
||||
NoSourceSince:
|
||||
en-US: it vanished
|
||||
Builds:
|
||||
- versionCode: 123
|
||||
antifeatures:
|
||||
Ads:
|
||||
en-US: includes ad lib
|
||||
zh-CN: 包括广告图书馆
|
||||
Tracking:
|
||||
en-US: standard suspects
|
||||
|
||||
"""
|
||||
app_dir = Path('metadata', app['id'])
|
||||
if not app_dir.is_dir():
|
||||
return
|
||||
af_dup_msg = _('Duplicate Anti-Feature declaration at {path} was ignored!')
|
||||
|
||||
if app.get('AntiFeatures'):
|
||||
app_has_AntiFeatures = True
|
||||
else:
|
||||
app_has_AntiFeatures = False
|
||||
|
||||
has_versionCode = re.compile(r'^-?[0-9]+_.*')
|
||||
has_antifeatures_from_app = set()
|
||||
for build in app.get('Builds', []):
|
||||
antifeatures = build.get('antifeatures')
|
||||
if antifeatures:
|
||||
has_antifeatures_from_app.add(build['versionCode'])
|
||||
|
||||
for f in sorted(app_dir.glob('*/antifeatures/*.txt')):
|
||||
path = f.as_posix()
|
||||
left = path.index('/', 9) # 9 is length of "metadata/"
|
||||
right = path.index('/', left + 1)
|
||||
locale = path[left + 1 : right]
|
||||
description = f.read_text()
|
||||
if has_versionCode.match(f.stem):
|
||||
i = f.stem.index('_')
|
||||
versionCode = int(f.stem[:i])
|
||||
antifeature = f.stem[i + 1 :]
|
||||
if versionCode in has_antifeatures_from_app:
|
||||
logging.error(af_dup_msg.format(path=f))
|
||||
continue
|
||||
if 'Builds' not in app:
|
||||
app['Builds'] = []
|
||||
found = False
|
||||
for build in app['Builds']:
|
||||
# loop though builds again, there might be duplicate versionCodes
|
||||
if versionCode == build['versionCode']:
|
||||
found = True
|
||||
if 'antifeatures' not in build:
|
||||
build['antifeatures'] = dict()
|
||||
if antifeature not in build['antifeatures']:
|
||||
build['antifeatures'][antifeature] = dict()
|
||||
build['antifeatures'][antifeature][locale] = description
|
||||
if not found:
|
||||
app['Builds'].append(
|
||||
{
|
||||
'versionCode': versionCode,
|
||||
'antifeatures': {
|
||||
antifeature: {locale: description},
|
||||
},
|
||||
}
|
||||
)
|
||||
elif app_has_AntiFeatures:
|
||||
logging.error(af_dup_msg.format(path=f))
|
||||
continue
|
||||
else:
|
||||
if 'AntiFeatures' not in app:
|
||||
app['AntiFeatures'] = dict()
|
||||
if f.stem not in app['AntiFeatures']:
|
||||
app['AntiFeatures'][f.stem] = dict()
|
||||
with f.open() as fp:
|
||||
app['AntiFeatures'][f.stem][locale] = fp.read()
|
||||
|
||||
|
||||
def _normalize_type_string(v):
|
||||
"""Normalize any data to TYPE_STRING.
|
||||
|
||||
|
|
@ -787,6 +900,61 @@ def _normalize_type_string(v):
|
|||
return str(v)
|
||||
|
||||
|
||||
def _normalize_type_stringmap(k, v):
|
||||
"""Normalize any data to TYPE_STRINGMAP.
|
||||
|
||||
The internal representation of this format is a dict of dicts,
|
||||
where the outer dict's keys are things like tag names of
|
||||
Anti-Features, the inner dict's keys are locales, and the ultimate
|
||||
values are human readable text.
|
||||
|
||||
Metadata entries like AntiFeatures: can be written in many
|
||||
forms, including a simple one-entry string, a list of strings,
|
||||
a dict with keys and descriptions as values, or a dict with
|
||||
localization.
|
||||
|
||||
Returns
|
||||
-------
|
||||
A dictionary with string keys, where each value is either a string
|
||||
message or a dict with locale keys and string message values.
|
||||
|
||||
"""
|
||||
if v is None:
|
||||
return dict()
|
||||
if isinstance(v, str) or isinstance(v, int) or isinstance(v, float):
|
||||
return {_normalize_type_string(v): dict()}
|
||||
if isinstance(v, list) or isinstance(v, tuple) or isinstance(v, set):
|
||||
retdict = dict()
|
||||
for i in v:
|
||||
if isinstance(i, dict):
|
||||
# transitional format
|
||||
if len(i) != 1:
|
||||
_warn_or_exception(
|
||||
_(
|
||||
"'{value}' is not a valid {field}, should be {pattern}"
|
||||
).format(field=k, value=v, pattern='key: value')
|
||||
)
|
||||
afname = _normalize_type_string(next(iter(i)))
|
||||
desc = _normalize_type_string(next(iter(i.values())))
|
||||
retdict[afname] = {common.DEFAULT_LOCALE: desc}
|
||||
else:
|
||||
retdict[_normalize_type_string(i)] = {}
|
||||
return retdict
|
||||
|
||||
retdict = dict()
|
||||
for af, afdict in v.items():
|
||||
key = _normalize_type_string(af)
|
||||
if afdict:
|
||||
if isinstance(afdict, dict):
|
||||
retdict[key] = afdict
|
||||
else:
|
||||
retdict[key] = {common.DEFAULT_LOCALE: _normalize_type_string(afdict)}
|
||||
else:
|
||||
retdict[key] = dict()
|
||||
|
||||
return retdict
|
||||
|
||||
|
||||
def post_parse_yaml_metadata(yamldata):
|
||||
"""Convert human-readable metadata data structures into consistent data structures.
|
||||
|
||||
|
|
@ -808,6 +976,9 @@ def post_parse_yaml_metadata(yamldata):
|
|||
elif _fieldtype == TYPE_STRING:
|
||||
if v or v == 0:
|
||||
yamldata[k] = _normalize_type_string(v)
|
||||
elif _fieldtype == TYPE_STRINGMAP:
|
||||
if v or v == 0: # TODO probably want just `if v:`
|
||||
yamldata[k] = _normalize_type_stringmap(k, v)
|
||||
else:
|
||||
if type(v) in (float, int):
|
||||
yamldata[k] = str(v)
|
||||
|
|
@ -843,12 +1014,24 @@ def post_parse_yaml_metadata(yamldata):
|
|||
build_flag=k, value=v
|
||||
)
|
||||
)
|
||||
elif _flagtype == TYPE_STRINGMAP:
|
||||
if v or v == 0:
|
||||
build[k] = _normalize_type_stringmap(k, v)
|
||||
|
||||
builds.append(build)
|
||||
|
||||
if builds:
|
||||
yamldata['Builds'] = sorted(builds, key=lambda build: build['versionCode'])
|
||||
|
||||
no_source_since = yamldata.get("NoSourceSince")
|
||||
# do not overwrite the description if it is there
|
||||
if no_source_since and not yamldata.get('AntiFeatures', {}).get('NoSourceSince'):
|
||||
if 'AntiFeatures' not in yamldata:
|
||||
yamldata['AntiFeatures'] = dict()
|
||||
yamldata['AntiFeatures']['NoSourceSince'] = {
|
||||
common.DEFAULT_LOCALE: no_source_since
|
||||
}
|
||||
|
||||
|
||||
def write_yaml(mf, app):
|
||||
"""Write metadata in yaml format.
|
||||
|
|
|
|||
|
|
@ -52,8 +52,9 @@ def remove_blank_flags_from_builds(builds):
|
|||
for build in builds:
|
||||
new = dict()
|
||||
for k in metadata.build_flags:
|
||||
v = build[k]
|
||||
if v is None or v is False or v == [] or v == '':
|
||||
v = build.get(k)
|
||||
# 0 is valid value, it should not be stripped
|
||||
if v is None or v is False or v == '' or v == dict() or v == list():
|
||||
continue
|
||||
new[k] = v
|
||||
newbuilds.append(new)
|
||||
|
|
|
|||
|
|
@ -294,7 +294,7 @@ class ExodusSignatureDataController(SignatureDataController):
|
|||
"warn_code_signatures": [tracker["code_signature"]],
|
||||
# exodus also provides network signatures, unused atm.
|
||||
# "network_signatures": [tracker["network_signature"]],
|
||||
"AntiFeatures": ["Tracking"],
|
||||
"AntiFeatures": ["Tracking"], # TODO
|
||||
"license": "NonFree" # We assume all trackers in exodus
|
||||
# are non-free, although free
|
||||
# trackers like piwik, acra,
|
||||
|
|
|
|||
|
|
@ -158,7 +158,7 @@ def status_update_json(apps, apks):
|
|||
|
||||
for appid in apps:
|
||||
app = apps[appid]
|
||||
for af in app.get('AntiFeatures', []):
|
||||
for af in app.get('AntiFeatures', dict()):
|
||||
antiFeatures = output['antiFeatures'] # JSON camelCase
|
||||
if af not in antiFeatures:
|
||||
antiFeatures[af] = dict()
|
||||
|
|
@ -351,7 +351,8 @@ def get_cache():
|
|||
if not isinstance(v, dict):
|
||||
continue
|
||||
if 'antiFeatures' in v:
|
||||
v['antiFeatures'] = set(v['antiFeatures'])
|
||||
if not isinstance(v['antiFeatures'], dict):
|
||||
v['antiFeatures'] = {k: {} for k in sorted(v['antiFeatures'])}
|
||||
if 'added' in v:
|
||||
v['added'] = datetime.fromtimestamp(v['added'])
|
||||
|
||||
|
|
@ -400,7 +401,7 @@ def has_known_vulnerability(filename):
|
|||
Janus is similar to Master Key but is perhaps easier to scan for.
|
||||
https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
|
||||
"""
|
||||
found_vuln = False
|
||||
found_vuln = ''
|
||||
|
||||
# statically load this pattern
|
||||
if not hasattr(has_known_vulnerability, "pattern"):
|
||||
|
|
@ -431,15 +432,23 @@ def has_known_vulnerability(filename):
|
|||
logging.debug(_('"{path}" contains recent {name} ({version})')
|
||||
.format(path=filename, name=name, version=version))
|
||||
else:
|
||||
logging.warning(_('"{path}" contains outdated {name} ({version})')
|
||||
.format(path=filename, name=name, version=version))
|
||||
found_vuln = True
|
||||
msg = '"{path}" contains outdated {name} ({version})'
|
||||
logging.warning(
|
||||
_(msg).format(path=filename, name=name, version=version)
|
||||
)
|
||||
found_vuln += msg.format(
|
||||
path=filename, name=name, version=version
|
||||
)
|
||||
found_vuln += '\n'
|
||||
break
|
||||
elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
|
||||
if name in files_in_apk:
|
||||
logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
|
||||
.format(apkfilename=filename, name=name))
|
||||
found_vuln = True
|
||||
msg = '{apkfilename} has multiple {name} files, looks like Master Key exploit!'
|
||||
logging.warning(
|
||||
_(msg).format(apkfilename=filename, name=name)
|
||||
)
|
||||
found_vuln += msg.format(apkfilename=filename, name=name)
|
||||
found_vuln += '\n'
|
||||
files_in_apk.add(name)
|
||||
return found_vuln
|
||||
|
||||
|
|
@ -545,7 +554,7 @@ def translate_per_build_anti_features(apps, apks):
|
|||
if d:
|
||||
afl = d.get(apk['versionCode'])
|
||||
if afl:
|
||||
apk['antiFeatures'].update(afl)
|
||||
apk['antiFeatures'].update(afl) # TODO
|
||||
|
||||
|
||||
def _get_localized_dict(app, locale):
|
||||
|
|
@ -1228,7 +1237,7 @@ def scan_apk(apk_file, require_signature=True):
|
|||
'features': [],
|
||||
'icons_src': {},
|
||||
'icons': {},
|
||||
'antiFeatures': set(),
|
||||
'antiFeatures': {},
|
||||
}
|
||||
ipfsCIDv1 = common.calculate_IPFS_cid(apk_file)
|
||||
if ipfsCIDv1:
|
||||
|
|
@ -1263,8 +1272,9 @@ def scan_apk(apk_file, require_signature=True):
|
|||
apk['minSdkVersion'] = 3 # aapt defaults to 3 as the min
|
||||
|
||||
# Check for known vulnerabilities
|
||||
if has_known_vulnerability(apk_file):
|
||||
apk['antiFeatures'].add('KnownVuln')
|
||||
hkv = has_known_vulnerability(apk_file)
|
||||
if hkv:
|
||||
apk['antiFeatures']['KnownVuln'] = {DEFAULT_LOCALE: hkv}
|
||||
|
||||
return apk
|
||||
|
||||
|
|
@ -1545,7 +1555,7 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal
|
|||
if repodir == 'archive' or allow_disabled_algorithms:
|
||||
try:
|
||||
common.verify_deprecated_jar_signature(apkfile)
|
||||
apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
|
||||
apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm']) # TODO
|
||||
except VerificationException:
|
||||
skipapk = True
|
||||
else:
|
||||
|
|
@ -1885,7 +1895,7 @@ def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversi
|
|||
for apk in all_app_apks:
|
||||
if len(keep) == keepversions:
|
||||
break
|
||||
if 'antiFeatures' not in apk:
|
||||
if 'antiFeatures' not in apk: # TODO
|
||||
keep.append(apk)
|
||||
elif 'DisabledAlgorithm' not in apk['antiFeatures'] or disabled_algorithms_allowed():
|
||||
keep.append(apk)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue