From 0735bfa7e530f6a1b5773b229e0dacc1bb86269d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 9 May 2023 19:55:26 +0200 Subject: [PATCH 01/16] remove obsolete test case --- tests/run-tests | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/run-tests b/tests/run-tests index 5b114cdf..bfdb2a8c 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -213,16 +213,6 @@ if use_apksigner; then fi -#------------------------------------------------------------------------------# -echo_header "TODO remove once buildserver image is upgraded to bullseye with apksigner" - -if java -version 2>&1 | grep -F 1.8.0; then - echo "Skipping the rest because they require apksigner 30.0.0+ which does not run on Java8" - echo SUCCESS - exit -fi - - #------------------------------------------------------------------------------# echo_header "test UTF-8 metadata" From f7830a41f1e07cd3adaea7841292773e43a365f0 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 9 May 2023 20:09:28 +0200 Subject: [PATCH 02/16] deploy: ensure mirrors and binary transparency always create 'master' If there was a global default on a machine that was something other than 'master', these things would crash with: Traceback (most recent call last): File "/home/hans/code/fdroid/server/fdroid", line 22, in fdroidserver.__main__.main() File "/home/hans/code/fdroid/server/fdroidserver/__main__.py", line 230, in main raise e File "/home/hans/code/fdroid/server/fdroidserver/__main__.py", line 211, in main mod.main() File "/home/hans/code/fdroid/server/fdroidserver/deploy.py", line 833, in main push_binary_transparency(BINARY_TRANSPARENCY_DIR, File "/home/hans/code/fdroid/server/fdroidserver/deploy.py", line 705, in push_binary_transparency local.pull('master') File "/usr/lib/python3/dist-packages/git/remote.py", line 1045, in pull res = self._get_fetch_info_from_stderr(proc, progress, kill_after_timeout=kill_after_timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3/dist-packages/git/remote.py", line 848, in _get_fetch_info_from_stderr proc.wait(stderr=stderr_text) File "/usr/lib/python3/dist-packages/git/cmd.py", line 604, in wait raise GitCommandError(remove_password_if_present(self.args), status, errstr) git.exc.GitCommandError: Cmd('git') failed due to: exit code(1) cmdline: git pull -v -- local master stderr: 'fatal: couldn't find remote ref master' --- .gitlab-ci.yml | 4 ++++ fdroidserver/btlog.py | 2 +- fdroidserver/deploy.py | 14 +++++++++----- tests/run-tests | 6 +++--- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b9317954..7b790b2a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -125,6 +125,10 @@ ubuntu_lts_ppa: - apt-get update - apt-get dist-upgrade - apt-get install --install-recommends dexdump fdroidserver git jq python3-setuptools sdkmanager + + # Test things work with a default branch other than 'master' + - git config --global init.defaultBranch thisisnotmasterormain + - cd tests - ./run-tests diff --git a/fdroidserver/btlog.py b/fdroidserver/btlog.py index 870ff93b..ea79bc8a 100755 --- a/fdroidserver/btlog.py +++ b/fdroidserver/btlog.py @@ -65,7 +65,7 @@ def make_binary_transparency_log( else: if not os.path.exists(btrepo): os.mkdir(btrepo) - gitrepo = git.Repo.init(btrepo) + gitrepo = git.Repo.init(btrepo, initial_branch=deploy.GIT_BRANCH) if not url: url = common.config['repo_url'].rstrip('/') diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 0415f7f4..54496f78 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -39,6 +39,8 @@ config = None options = None start_timestamp = time.gmtime() +GIT_BRANCH = 'master' + BINARY_TRANSPARENCY_DIR = 'binary_transparency' AUTO_S3CFG = '.fdroid-deploy-s3cfg' @@ -407,7 +409,7 @@ def update_servergitmirrors(servergitmirrors, repo_section): elif 'identity_file' in config: ssh_cmd += ' -oIdentitiesOnly=yes -i "%s"' % config['identity_file'] - repo = git.Repo.init(git_mirror_path) + repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH) enabled_remotes = [] for remote_url in servergitmirrors: @@ -480,7 +482,9 @@ def update_servergitmirrors(servergitmirrors, repo_section): logging.debug(_('Pushing to {url}').format(url=remote.url)) with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): - pushinfos = remote.push('master', force=True, set_upstream=True, progress=progress) + pushinfos = remote.push( + GIT_BRANCH, force=True, set_upstream=True, progress=progress + ) for pushinfo in pushinfos: if pushinfo.flags & (git.remote.PushInfo.ERROR | git.remote.PushInfo.REJECTED @@ -691,7 +695,7 @@ def push_binary_transparency(git_repo_path, git_remote): remote_path = os.path.abspath(git_repo_path) if not os.path.isdir(os.path.join(git_remote, '.git')): os.makedirs(git_remote, exist_ok=True) - thumbdriverepo = git.Repo.init(git_remote) + thumbdriverepo = git.Repo.init(git_remote, initial_branch=GIT_BRANCH) local = thumbdriverepo.create_remote('local', remote_path) else: thumbdriverepo = git.Repo(git_remote) @@ -702,7 +706,7 @@ def push_binary_transparency(git_repo_path, git_remote): local.set_url(remote_path) else: local = thumbdriverepo.create_remote('local', remote_path) - local.pull('master') + local.pull(GIT_BRANCH) else: # from online machine to remote on a server on the internet gitrepo = git.Repo(git_repo_path) @@ -713,7 +717,7 @@ def push_binary_transparency(git_repo_path, git_remote): origin.set_url(git_remote) else: origin = gitrepo.create_remote('origin', git_remote) - origin.push('master') + origin.push(GIT_BRANCH) def main(): diff --git a/tests/run-tests b/tests/run-tests index bfdb2a8c..e076359a 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -1129,7 +1129,7 @@ echo_header "setup a new repo from scratch using ANDROID_HOME with git mirror" # fake git remote server for repo mirror SERVER_GIT_MIRROR=`create_test_dir` cd $SERVER_GIT_MIRROR -$git init +$git init --initial-branch=master $git config receive.denyCurrentBranch updateInstead REPOROOT=`create_test_dir` @@ -1191,7 +1191,7 @@ SERVERWEBROOT=`create_test_dir`/fdroid cd $OFFLINE_ROOT mkdir binary_transparency cd binary_transparency -$git init +$git init --initial-branch=master # fake git remote server for binary transparency log BINARY_TRANSPARENCY_REMOTE=`create_test_dir` @@ -1199,7 +1199,7 @@ BINARY_TRANSPARENCY_REMOTE=`create_test_dir` # fake git remote server for repo mirror SERVER_GIT_MIRROR=`create_test_dir` cd $SERVER_GIT_MIRROR -$git init +$git init --initial-branch=master $git config receive.denyCurrentBranch updateInstead cd $OFFLINE_ROOT From 3efe797bf8f84ecbe37268ab7428dad131e995d1 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 5 May 2023 10:15:18 +0200 Subject: [PATCH 03/16] gitlab-ci: CI_BUILD_* vars were renamed to other things https://web.archive.org/web/20190110134948/https://docs.gitlab.com/ee/ci/variables/#gitlab-90-renaming Looks like GitLab v16 is finally removing the old names. sed -i \ -e s,CI_BUILD_TOKEN,CI_JOB_TOKEN,g \ -e s,CI_BUILD_REF_SLUG,CI_COMMIT_REF_SLUG,g \ -e s,CI_BUILD_REF_NAME,CI_COMMIT_REF_NAME,g \ -e s,CI_BUILD_REPO,CI_REPOSITORY_URL,g \ .gitlab-ci.yml --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7b790b2a..cc40911d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -588,12 +588,12 @@ docker: RELEASE_IMAGE: $CI_REGISTRY_IMAGE:buildserver script: # git ref names can contain many chars that are not allowed in docker tags - - export TEST_IMAGE=$CI_REGISTRY_IMAGE:$(printf $CI_BUILD_REF_NAME | sed 's,[^a-zA-Z0-9_.-],_,g') + - export TEST_IMAGE=$CI_REGISTRY_IMAGE:$(printf $CI_COMMIT_REF_NAME | sed 's,[^a-zA-Z0-9_.-],_,g') - cd buildserver - docker build -t $TEST_IMAGE --build-arg GIT_REV_PARSE_HEAD=$(git rev-parse HEAD) . - docker tag $TEST_IMAGE $RELEASE_IMAGE - docker tag $TEST_IMAGE ${RELEASE_IMAGE}-bullseye - - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin registry.gitlab.com + - echo $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin registry.gitlab.com # This avoids filling up gitlab.com free tier accounts with unused docker images. - if test -z "$FDROID_PUSH_DOCKER_IMAGE"; then echo "Skipping docker push to save quota on your gitlab namespace."; From 8ccc89ad4eed06a8ddd61ccba85590f9727f590d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 5 May 2023 10:47:27 +0200 Subject: [PATCH 04/16] index: fix requestsdict check order of operations If requestsdict is None, the old logic would still check requestsdict["uninstall"]) and crash there. --- fdroidserver/index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/index.py b/fdroidserver/index.py index d93ec586..162472ed 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -735,7 +735,7 @@ def make_v2(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_ output = collections.OrderedDict() output["repo"] = v2_repo(repodict, repodir, archive) - if requestsdict and requestsdict["install"] or requestsdict["uninstall"]: + if requestsdict and (requestsdict["install"] or requestsdict["uninstall"]): output["repo"]["requests"] = requestsdict # establish sort order of the index From f9864dc3a2b754f069b2178756e065a654ea6b12 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 4 May 2023 14:51:04 +0200 Subject: [PATCH 05/16] rewritemeta: split into remove_blank_flags_from_builds() This takes this key bit of functionality, splits it out as its own function, and adds some unit tests. --- fdroidserver/rewritemeta.py | 29 ++++++--- tests/rewritemeta.TestCase | 122 +++++++++++++++++++++++++++++++++++- 2 files changed, 138 insertions(+), 13 deletions(-) diff --git a/fdroidserver/rewritemeta.py b/fdroidserver/rewritemeta.py index 0e7ad021..7413b029 100644 --- a/fdroidserver/rewritemeta.py +++ b/fdroidserver/rewritemeta.py @@ -44,6 +44,22 @@ def proper_format(app): return content == cur_content +def remove_blank_flags_from_builds(builds): + """Remove unset entries from Builds so they are not written out.""" + if not builds: + return list() + newbuilds = list() + for build in builds: + new = dict() + for k in metadata.build_flags: + v = build[k] + if v is None or v is False or v == [] or v == '': + continue + new[k] = v + newbuilds.append(new) + return newbuilds + + def main(): global config, options @@ -82,16 +98,9 @@ def main(): print(path) continue - newbuilds = [] - for build in app.get('Builds', []): - new = metadata.Build() - for k in metadata.build_flags: - v = build[k] - if v is None or v is False or v == [] or v == '': - continue - new[k] = v - newbuilds.append(new) - app['Builds'] = newbuilds + builds = remove_blank_flags_from_builds(app.get('Builds')) + if builds: + app['Builds'] = builds # rewrite to temporary file before overwriting existing # file in case there's a bug in write_metadata diff --git a/tests/rewritemeta.TestCase b/tests/rewritemeta.TestCase index 283e9c44..0bcc704e 100755 --- a/tests/rewritemeta.TestCase +++ b/tests/rewritemeta.TestCase @@ -9,15 +9,14 @@ import tempfile import textwrap from pathlib import Path -from testcommon import TmpCwd +from testcommon import TmpCwd, mkdtemp localmodule = Path(__file__).resolve().parent.parent print('localmodule: ' + str(localmodule)) if localmodule not in sys.path: sys.path.insert(0, str(localmodule)) -from fdroidserver import common -from fdroidserver import rewritemeta +from fdroidserver import common, metadata, rewritemeta class RewriteMetaTest(unittest.TestCase): @@ -27,6 +26,123 @@ class RewriteMetaTest(unittest.TestCase): logging.basicConfig(level=logging.DEBUG) self.basedir = localmodule / 'tests' os.chdir(self.basedir) + metadata.warnings_action = 'error' + self._td = mkdtemp() + self.testdir = self._td.name + + def tearDown(self): + self._td.cleanup() + + def test_remove_blank_flags_from_builds_com_politedroid_3(self): + """Unset fields in Builds: entries should be removed.""" + appid = 'com.politedroid' + app = metadata.read_metadata({appid: -1})[appid] + builds = rewritemeta.remove_blank_flags_from_builds(app.get('Builds')) + self.assertEqual( + builds[0], + { + 'versionName': '1.2', + 'versionCode': 3, + 'commit': '6a548e4b19', + 'target': 'android-10', + 'antifeatures': [ + 'KnownVuln', + 'UpstreamNonFree', + 'NonFreeAssets', + ], + }, + ) + + def test_remove_blank_flags_from_builds_com_politedroid_4(self): + """Unset fields in Builds: entries should be removed.""" + appid = 'com.politedroid' + app = metadata.read_metadata({appid: -1})[appid] + builds = rewritemeta.remove_blank_flags_from_builds(app.get('Builds')) + self.assertEqual( + builds[1], + { + 'versionName': '1.3', + 'versionCode': 4, + 'commit': 'ad865b57bf3ac59580f38485608a9b1dda4fa7dc', + 'target': 'android-15', + }, + ) + + def test_remove_blank_flags_from_builds_no_builds(self): + """Unset fields in Builds: entries should be removed.""" + self.assertEqual( + rewritemeta.remove_blank_flags_from_builds(None), + list(), + ) + self.assertEqual( + rewritemeta.remove_blank_flags_from_builds(dict()), + list(), + ) + + def test_rewrite_no_builds(self): + os.chdir(self.testdir) + Path('metadata').mkdir() + with Path('metadata/a.yml').open('w') as f: + f.write('AutoName: a') + rewritemeta.main() + self.assertEqual( + Path('metadata/a.yml').read_text(encoding='utf-8'), + textwrap.dedent( + '''\ + License: Unknown + + AutoName: a + + AutoUpdateMode: None + UpdateCheckMode: None + ''' + ), + ) + + def test_rewrite_empty_build_field(self): + os.chdir(self.testdir) + Path('metadata').mkdir() + with Path('metadata/a.yml').open('w') as fp: + fp.write( + textwrap.dedent( + """ + License: Apache-2.0 + Builds: + - versionCode: 4 + versionName: a + rm: + """ + ) + ) + rewritemeta.main() + self.assertEqual( + Path('metadata/a.yml').read_text(encoding='utf-8'), + textwrap.dedent( + '''\ + License: Apache-2.0 + + Builds: + - versionName: a + versionCode: 4 + + AutoUpdateMode: None + UpdateCheckMode: None + ''' + ), + ) + + def test_remove_blank_flags_from_builds_app_with_special_build_params(self): + appid = 'app.with.special.build.params' + app = metadata.read_metadata({appid: -1})[appid] + builds = rewritemeta.remove_blank_flags_from_builds(app.get('Builds')) + self.assertEqual( + builds[-1], + { + 'versionName': '2.1.2', + 'versionCode': 51, + 'disable': 'Labelled as pre-release, so skipped', + }, + ) def test_rewrite_scenario_trivial(self): sys.argv = ['rewritemeta', 'a', 'b'] From 49362b5fd1eb2c91a6e12a744ff3fa2dbd1c55fc Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 24 Apr 2023 17:34:47 +0200 Subject: [PATCH 06/16] move load_locale() and file_entry() to be accessible by all modules * load_locale -> common.load_localized_config() since common handles config * file_entry -> metadata.file_entry() since metadata handles data format --- fdroidserver/common.py | 36 ++++++++++++++++++++++++++ fdroidserver/index.py | 58 +++++++++++------------------------------- fdroidserver/update.py | 6 ++--- 3 files changed, 54 insertions(+), 46 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 550e7b5a..d704212e 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -487,6 +487,42 @@ def read_config(opts=None): return config +def file_entry(filename, hash_value=None): + meta = {} + meta["name"] = "/" + filename.split("/", 1)[1] + meta["sha256"] = hash_value or common.sha256sum(filename) + meta["size"] = os.stat(filename).st_size + return meta + + +def load_localized_config(name, repodir): + lst = {} + for f in Path().glob("config/**/{name}.yml".format(name=name)): + locale = f.parts[1] + if len(f.parts) == 2: + locale = "en-US" + with open(f, encoding="utf-8") as fp: + elem = yaml.safe_load(fp) + for akey, avalue in elem.items(): + if akey not in lst: + lst[akey] = {} + for key, value in avalue.items(): + if key not in lst[akey]: + lst[akey][key] = {} + if key == "icon": + shutil.copy( + os.path.join("config", value), + os.path.join(repodir, "icons") + ) + lst[akey][key][locale] = file_entry( + os.path.join(repodir, "icons", value) + ) + else: + lst[akey][key][locale] = value + + return lst + + def parse_human_readable_size(size): units = { 'b': 1, diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 162472ed..fb080550 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -29,7 +29,6 @@ import re import shutil import tempfile import urllib.parse -import yaml import zipfile import calendar import qrcode @@ -472,37 +471,6 @@ def dict_diff(source, target): return result -def file_entry(filename, hash_value=None): - meta = {} - meta["name"] = "/" + filename.split("/", 1)[1] - meta["sha256"] = hash_value or common.sha256sum(filename) - meta["size"] = os.stat(filename).st_size - return meta - - -def load_locale(name, repodir): - lst = {} - for yml in Path().glob("config/**/{name}.yml".format(name=name)): - locale = yml.parts[1] - if len(yml.parts) == 2: - locale = "en-US" - with open(yml, encoding="utf-8") as fp: - elem = yaml.safe_load(fp) - for akey, avalue in elem.items(): - if akey not in lst: - lst[akey] = {} - for key, value in avalue.items(): - if key not in lst[akey]: - lst[akey][key] = {} - if key == "icon": - shutil.copy(os.path.join("config", value), os.path.join(repodir, "icons")) - lst[akey][key][locale] = file_entry(os.path.join(repodir, "icons", value)) - else: - lst[akey][key][locale] = value - - return lst - - def convert_datetime(obj): if isinstance(obj, datetime): # Java prefers milliseconds @@ -568,7 +536,9 @@ def package_metadata(app, repodir): # TODO handle different resolutions if app.get("icon"): - meta["icon"] = {"en-US": file_entry(os.path.join(repodir, "icons", app["icon"]))} + meta["icon"] = { + "en-US": common.file_entry(os.path.join(repodir, "icons", app["icon"])) + } if "iconv2" in app: meta["icon"] = app["iconv2"] @@ -594,16 +564,16 @@ def convert_version(version, app, repodir): ver["file"]["ipfsCIDv1"] = ipfsCIDv1 if "srcname" in version: - ver["src"] = file_entry(os.path.join(repodir, version["srcname"])) + ver["src"] = common.file_entry(os.path.join(repodir, version["srcname"])) if "obbMainFile" in version: - ver["obbMainFile"] = file_entry( + ver["obbMainFile"] = common.file_entry( os.path.join(repodir, version["obbMainFile"]), version["obbMainFileSha256"], ) if "obbPatchFile" in version: - ver["obbPatchFile"] = file_entry( + ver["obbPatchFile"] = common.file_entry( os.path.join(repodir, version["obbPatchFile"]), version["obbPatchFileSha256"], ) @@ -686,9 +656,11 @@ def v2_repo(repodict, repodir, archive): repo["name"] = {"en-US": repodict["name"]} repo["description"] = {"en-US": repodict["description"]} - repo["icon"] = {"en-US": file_entry("{}/icons/{}".format(repodir, repodict["icon"]))} + repo["icon"] = { + "en-US": common.file_entry("{}/icons/{}".format(repodir, repodict["icon"])) + } - config = load_locale("config", repodir) + config = common.load_localized_config("config", repodir) if config: repo["name"] = config["archive" if archive else "repo"]["name"] repo["description"] = config["archive" if archive else "repo"]["description"] @@ -702,15 +674,15 @@ def v2_repo(repodict, repodir, archive): repo["timestamp"] = repodict["timestamp"] - antiFeatures = load_locale("antiFeatures", repodir) + antiFeatures = common.load_localized_config("antiFeatures", repodir) if antiFeatures: repo["antiFeatures"] = antiFeatures - categories = load_locale("categories", repodir) + categories = common.load_localized_config("categories", repodir) if categories: repo["categories"] = categories - channels = load_locale("channels", repodir) + channels = common.load_localized_config("channels", repodir) if channels: repo["releaseChannels"] = channels @@ -792,7 +764,7 @@ def make_v2(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_ else: json.dump(output, fp, default=_index_encoder_default, ensure_ascii=False) - entry["index"] = file_entry(index_file) + entry["index"] = common.file_entry(index_file) entry["index"]["numPackages"] = len(output.get("packages", [])) indexes = sorted(Path().glob("tmp/{}*.json".format(repodir)), key=lambda x: x.name) @@ -819,7 +791,7 @@ def make_v2(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_ else: json.dump(diff, fp, default=_index_encoder_default, ensure_ascii=False) - entry["diffs"][old["repo"]["timestamp"]] = file_entry(diff_file) + entry["diffs"][old["repo"]["timestamp"]] = common.file_entry(diff_file) entry["diffs"][old["repo"]["timestamp"]]["numPackages"] = len(diff.get("packages", [])) json_name = "entry.json" diff --git a/fdroidserver/update.py b/fdroidserver/update.py index b9490a53..aec69237 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1037,7 +1037,7 @@ def insert_localized_app_metadata(apps): base = "iconv2" if base not in apps[packageName] or not isinstance(apps[packageName][base], collections.OrderedDict): apps[packageName][base] = collections.OrderedDict() - apps[packageName][base][locale] = index.file_entry(dst) + apps[packageName][base][locale] = common.file_entry(dst) for d in dirs: if d in SCREENSHOT_DIRS: if locale == 'images': @@ -1090,7 +1090,7 @@ def insert_localized_app_metadata(apps): base = "iconv2" if base not in apps[packageName] or not isinstance(apps[packageName][base], collections.OrderedDict): apps[packageName][base] = collections.OrderedDict() - apps[packageName][base][locale] = index.file_entry(index_file) + apps[packageName][base][locale] = common.file_entry(index_file) elif screenshotdir in SCREENSHOT_DIRS: # there can any number of these per locale logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f)) @@ -1105,7 +1105,7 @@ def insert_localized_app_metadata(apps): apps[packageName]["screenshots"][newKey] = collections.OrderedDict() if locale not in apps[packageName]["screenshots"][newKey]: apps[packageName]["screenshots"][newKey][locale] = [] - apps[packageName]["screenshots"][newKey][locale].append(index.file_entry(f)) + apps[packageName]["screenshots"][newKey][locale].append(common.file_entry(f)) else: logging.warning(_('Unsupported graphics file found: {path}').format(path=f)) From 74a23284e1b94d0a587fd6b61a43a380957f93a8 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 4 May 2023 12:34:32 +0200 Subject: [PATCH 07/16] common: load_localized_config() should make repo/ if not present For 1,000,000 checks, this adds: * ~4 seconds of runtime on a server with very slow disks. * ~0.7 seconds of runtime on my laptop with a fast SSD. --- fdroidserver/common.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index d704212e..d6c188be 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -510,12 +510,12 @@ def load_localized_config(name, repodir): if key not in lst[akey]: lst[akey][key] = {} if key == "icon": - shutil.copy( - os.path.join("config", value), - os.path.join(repodir, "icons") - ) + icons_dir = os.path.join(repodir, 'icons') + if not os.path.exists(icons_dir): + os.mkdir(icons_dir) + shutil.copy(os.path.join("config", value), icons_dir) lst[akey][key][locale] = file_entry( - os.path.join(repodir, "icons", value) + os.path.join(icons_dir, value) ) else: lst[akey][key][locale] = value From b04c7ff539e81ae7f2be075b501ea1bc5cb8fa9a Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 10 May 2023 13:16:53 +0200 Subject: [PATCH 08/16] load_localized_config() returns a dict in a stable order I renamed the variables while I was at it, to make it clearer. --- fdroidserver/common.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index d6c188be..da802ad9 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -496,31 +496,41 @@ def file_entry(filename, hash_value=None): def load_localized_config(name, repodir): - lst = {} + """Load localized config files and put them into internal dict format. + + This will maintain the order as came from the data files, e.g + YAML. The locale comes from unsorted paths on the filesystem, so + that is separately sorted. + + """ + ret = dict() for f in Path().glob("config/**/{name}.yml".format(name=name)): locale = f.parts[1] if len(f.parts) == 2: locale = "en-US" with open(f, encoding="utf-8") as fp: elem = yaml.safe_load(fp) - for akey, avalue in elem.items(): - if akey not in lst: - lst[akey] = {} - for key, value in avalue.items(): - if key not in lst[akey]: - lst[akey][key] = {} + for afname, field_dict in elem.items(): + if afname not in ret: + ret[afname] = dict() + for key, value in field_dict.items(): + if key not in ret[afname]: + ret[afname][key] = dict() if key == "icon": icons_dir = os.path.join(repodir, 'icons') if not os.path.exists(icons_dir): os.mkdir(icons_dir) shutil.copy(os.path.join("config", value), icons_dir) - lst[akey][key][locale] = file_entry( + ret[afname][key][locale] = file_entry( os.path.join(icons_dir, value) ) else: - lst[akey][key][locale] = value + ret[afname][key][locale] = value - return lst + for elem in ret.values(): + for afname in elem: + elem[afname] = {locale: v for locale, v in sorted(elem[afname].items())} + return ret def parse_human_readable_size(size): From d6dba05ec35da918ae3864774f3ba50ad73fe5b9 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 24 Apr 2023 19:53:06 +0200 Subject: [PATCH 09/16] test load_localized_locale() and translated Anti-Features --- MANIFEST.in | 16 + tests/common.TestCase | 32 ++ tests/config/antiFeatures.yml | 45 ++ tests/config/de/antiFeatures.yml | 44 ++ tests/config/fa/antiFeatures.yml | 43 ++ tests/config/ic_antifeature_ads.xml | 15 + .../ic_antifeature_disabledalgorithm.xml | 21 + tests/config/ic_antifeature_knownvuln.xml | 15 + tests/config/ic_antifeature_nonfreeadd.xml | 19 + tests/config/ic_antifeature_nonfreeassets.xml | 19 + tests/config/ic_antifeature_nonfreedep.xml | 22 + tests/config/ic_antifeature_nonfreenet.xml | 37 ++ tests/config/ic_antifeature_nosourcesince.xml | 16 + tests/config/ic_antifeature_nsfw.xml | 4 + tests/config/ic_antifeature_tracking.xml | 22 + .../config/ic_antifeature_upstreamnonfree.xml | 19 + tests/config/ro/antiFeatures.yml | 44 ++ tests/config/zh-rCN/antiFeatures.yml | 43 ++ tests/repo/entry.json | 4 +- tests/repo/index-v2.json | 471 ++++++++++++++++++ tests/run-tests | 7 +- 21 files changed, 955 insertions(+), 3 deletions(-) create mode 100644 tests/config/antiFeatures.yml create mode 100644 tests/config/de/antiFeatures.yml create mode 100644 tests/config/fa/antiFeatures.yml create mode 100644 tests/config/ic_antifeature_ads.xml create mode 100644 tests/config/ic_antifeature_disabledalgorithm.xml create mode 100644 tests/config/ic_antifeature_knownvuln.xml create mode 100644 tests/config/ic_antifeature_nonfreeadd.xml create mode 100644 tests/config/ic_antifeature_nonfreeassets.xml create mode 100644 tests/config/ic_antifeature_nonfreedep.xml create mode 100644 tests/config/ic_antifeature_nonfreenet.xml create mode 100644 tests/config/ic_antifeature_nosourcesince.xml create mode 100644 tests/config/ic_antifeature_nsfw.xml create mode 100644 tests/config/ic_antifeature_tracking.xml create mode 100644 tests/config/ic_antifeature_upstreamnonfree.xml create mode 100644 tests/config/ro/antiFeatures.yml create mode 100644 tests/config/zh-rCN/antiFeatures.yml diff --git a/MANIFEST.in b/MANIFEST.in index 6fb0a57e..e534b0da 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -545,6 +545,22 @@ include tests/check-fdroid-apk include tests/checkupdates.TestCase include tests/common.TestCase include tests/config.py +include tests/config/antiFeatures.yml +include tests/config/de/antiFeatures.yml +include tests/config/fa/antiFeatures.yml +include tests/config/ic_antifeature_ads.xml +include tests/config/ic_antifeature_disabledalgorithm.xml +include tests/config/ic_antifeature_knownvuln.xml +include tests/config/ic_antifeature_nonfreeadd.xml +include tests/config/ic_antifeature_nonfreeassets.xml +include tests/config/ic_antifeature_nonfreedep.xml +include tests/config/ic_antifeature_nonfreenet.xml +include tests/config/ic_antifeature_nosourcesince.xml +include tests/config/ic_antifeature_nsfw.xml +include tests/config/ic_antifeature_tracking.xml +include tests/config/ic_antifeature_upstreamnonfree.xml +include tests/config/ro/antiFeatures.yml +include tests/config/zh-rCN/antiFeatures.yml include tests/corrupt-featureGraphic.png include tests/deploy.TestCase include tests/dummy-keystore.jks diff --git a/tests/common.TestCase b/tests/common.TestCase index 758b4881..8820004d 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -2667,6 +2667,38 @@ class CommonTest(unittest.TestCase): config['smartcardoptions'], ) + def test_load_localized_config(self): + """It should load""" + antiFeatures = fdroidserver.common.load_localized_config('antiFeatures', 'repo') + self.assertEqual( + [ + 'Ads', + 'DisabledAlgorithm', + 'KnownVuln', + 'NSFW', + 'NoSourceSince', + 'NonFreeAdd', + 'NonFreeAssets', + 'NonFreeDep', + 'NonFreeNet', + 'Tracking', + 'UpstreamNonFree', + ], + list(antiFeatures.keys()), + ) + self.assertEqual( + ['de', 'en-US', 'fa', 'ro', 'zh-rCN'], + list(antiFeatures['Ads']['description'].keys()), + ) + self.assertEqual( + ['en-US'], + list(antiFeatures['NoSourceSince']['description'].keys()), + ) + # it should have copied the icon files into place + for v in antiFeatures.values(): + p = Path(os.path.dirname(__file__) + '/repo' + v['icon']['en-US']['name']) + self.assertTrue(p.exists()) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) diff --git a/tests/config/antiFeatures.yml b/tests/config/antiFeatures.yml new file mode 100644 index 00000000..e4c1a107 --- /dev/null +++ b/tests/config/antiFeatures.yml @@ -0,0 +1,45 @@ +Ads: + description: This app contains advertising + icon: ic_antifeature_ads.xml + name: Ads +DisabledAlgorithm: + description: This app has a weak security signature + icon: ic_antifeature_disabledalgorithm.xml + name: Signed Using An Unsafe Algorithm +KnownVuln: + description: This app contains a known security vulnerability + icon: ic_antifeature_knownvuln.xml + name: Known Vulnerability +NSFW: + description: This app contains content that should not be publicized or visible + everywhere + icon: ic_antifeature_nsfw.xml + name: NSFW +NoSourceSince: + description: The source code is no longer available, no updates possible. + icon: ic_antifeature_nosourcesince.xml + name: Newer Source Not Available +NonFreeAdd: + description: This app promotes non-free add-ons + icon: ic_antifeature_nonfreeadd.xml + name: Non-Free Addons +NonFreeAssets: + description: This app contains non-free assets + icon: ic_antifeature_nonfreeassets.xml + name: Non-Free Assets +NonFreeDep: + description: This app depends on other non-free apps + icon: ic_antifeature_nonfreedep.xml + name: Non-Free Dependencies +NonFreeNet: + description: This app promotes or depends entirely on a non-free network service + icon: ic_antifeature_nonfreenet.xml + name: Non-Free Network Services +Tracking: + description: This app tracks and reports your activity + icon: ic_antifeature_tracking.xml + name: Tracking +UpstreamNonFree: + description: The upstream source code is not entirely Free + icon: ic_antifeature_upstreamnonfree.xml + name: Upstream Non-Free diff --git a/tests/config/de/antiFeatures.yml b/tests/config/de/antiFeatures.yml new file mode 100644 index 00000000..3053e41a --- /dev/null +++ b/tests/config/de/antiFeatures.yml @@ -0,0 +1,44 @@ +Ads: + description: Diese App enthält Werbung + icon: ic_antifeature_ads.xml + name: Werbung +DisabledAlgorithm: + description: Diese App hat eine schwache Sicherheitssignatur + icon: ic_antifeature_disabledalgorithm.xml + name: Mit einem unsicheren Algorithmus signiert +KnownVuln: + description: Diese App enthält eine bekannte Sicherheitslücke + icon: ic_antifeature_knownvuln.xml + name: Bekannte Sicherheitslücke +NSFW: + description: Diese App enthält Inhalte, die nicht überall veröffentlicht oder sichtbar + sein sollten + icon: ic_antifeature_nsfw.xml + name: NSFW +NoSourceSince: + icon: ic_antifeature_nosourcesince.xml + name: Der Quellcode ist nicht mehr erhältlich, keine Aktualisierungen möglich. +NonFreeAdd: + description: Diese App bewirbt nicht-quelloffene Erweiterungen + icon: ic_antifeature_nonfreeadd.xml + name: Nicht-quelloffene Erweiterungen +NonFreeAssets: + description: Diese App enthält nicht-quelloffene Bestandteile + icon: ic_antifeature_nonfreeassets.xml + name: Nicht-quelloffene Bestandteile +NonFreeDep: + description: Diese App ist abhängig von anderen nicht-quelloffenen Apps + icon: ic_antifeature_nonfreedep.xml + name: Nicht-quelloffene Abhängigkeiten +NonFreeNet: + description: Diese App bewirbt nicht-quelloffene Netzwerkdienste + icon: ic_antifeature_nonfreenet.xml + name: Nicht-quelloffene Netzwerkdienste +Tracking: + description: Diese App verfolgt und versendet Ihre Aktivitäten + icon: ic_antifeature_tracking.xml + name: Tracking +UpstreamNonFree: + description: Der Originalcode ist nicht völlig quelloffen + icon: ic_antifeature_upstreamnonfree.xml + name: Originalcode nicht-quelloffen diff --git a/tests/config/fa/antiFeatures.yml b/tests/config/fa/antiFeatures.yml new file mode 100644 index 00000000..554dcee9 --- /dev/null +++ b/tests/config/fa/antiFeatures.yml @@ -0,0 +1,43 @@ +Ads: + description: این کاره دارای تبلیغات است + icon: ic_antifeature_ads.xml + name: تبلیغات +DisabledAlgorithm: + description: این کاره، امضای امنیتی ضعیفی دارد + icon: ic_antifeature_disabledalgorithm.xml + name: امضا شده با الگوریتمی ناامن +KnownVuln: + description: این کاره، آسیب‌پذیری امنیتی شناخته‌شده‌ای دارد + icon: ic_antifeature_knownvuln.xml + name: آسیب‌پذیری شناخته +NSFW: + description: این کاره محتوایی دارد که نباید عمومی شده یا همه‌حا نمایان باشد + icon: ic_antifeature_nsfw.xml + name: NSFW +NoSourceSince: + icon: ic_antifeature_nosourcesince.xml + name: کد مبدأ دیگر در دسترس نیست. به‌روز رسانی ناممکن است. +NonFreeAdd: + description: این کاره، افزونه‌های ناآزاد را تبلیغ می‌کند + icon: ic_antifeature_nonfreeadd.xml + name: افزونه‌های ناآزاد +NonFreeAssets: + description: این کاره دارای بخش‌های ناآزاد است + icon: ic_antifeature_nonfreeassets.xml + name: بخش‌های ناآزاد +NonFreeDep: + description: این کاره به دیگر کاره‌های ناآزاد وابسته است + icon: ic_antifeature_nonfreedep.xml + name: وابستگی‌های ناآزاد +NonFreeNet: + description: این کاره، خدمات شبکه‌های ناآزاد را ترویج می‌کند + icon: ic_antifeature_nonfreenet.xml + name: خدمات شبکه‌ای ناآزاد +Tracking: + description: این کاره، فعّالیتتان را ردیابی و گزارش می‌کند + icon: ic_antifeature_tracking.xml + name: ردیابی +UpstreamNonFree: + description: کد مبدأ بالادستی کاملاً آزاد نیست + icon: ic_antifeature_upstreamnonfree.xml + name: بالادست ناآزاد diff --git a/tests/config/ic_antifeature_ads.xml b/tests/config/ic_antifeature_ads.xml new file mode 100644 index 00000000..99eb6f5e --- /dev/null +++ b/tests/config/ic_antifeature_ads.xml @@ -0,0 +1,15 @@ + + + + diff --git a/tests/config/ic_antifeature_disabledalgorithm.xml b/tests/config/ic_antifeature_disabledalgorithm.xml new file mode 100644 index 00000000..0b231666 --- /dev/null +++ b/tests/config/ic_antifeature_disabledalgorithm.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/tests/config/ic_antifeature_knownvuln.xml b/tests/config/ic_antifeature_knownvuln.xml new file mode 100644 index 00000000..cfa81345 --- /dev/null +++ b/tests/config/ic_antifeature_knownvuln.xml @@ -0,0 +1,15 @@ + + + + diff --git a/tests/config/ic_antifeature_nonfreeadd.xml b/tests/config/ic_antifeature_nonfreeadd.xml new file mode 100644 index 00000000..adca82ca --- /dev/null +++ b/tests/config/ic_antifeature_nonfreeadd.xml @@ -0,0 +1,19 @@ + + + + + + diff --git a/tests/config/ic_antifeature_nonfreeassets.xml b/tests/config/ic_antifeature_nonfreeassets.xml new file mode 100644 index 00000000..20619b4e --- /dev/null +++ b/tests/config/ic_antifeature_nonfreeassets.xml @@ -0,0 +1,19 @@ + + + + + + diff --git a/tests/config/ic_antifeature_nonfreedep.xml b/tests/config/ic_antifeature_nonfreedep.xml new file mode 100644 index 00000000..2b37b8d8 --- /dev/null +++ b/tests/config/ic_antifeature_nonfreedep.xml @@ -0,0 +1,22 @@ + + + + + + + diff --git a/tests/config/ic_antifeature_nonfreenet.xml b/tests/config/ic_antifeature_nonfreenet.xml new file mode 100644 index 00000000..4d726f09 --- /dev/null +++ b/tests/config/ic_antifeature_nonfreenet.xml @@ -0,0 +1,37 @@ + + + + + + + + + + + + diff --git a/tests/config/ic_antifeature_nosourcesince.xml b/tests/config/ic_antifeature_nosourcesince.xml new file mode 100644 index 00000000..b524fc42 --- /dev/null +++ b/tests/config/ic_antifeature_nosourcesince.xml @@ -0,0 +1,16 @@ + + + + + diff --git a/tests/config/ic_antifeature_nsfw.xml b/tests/config/ic_antifeature_nsfw.xml new file mode 100644 index 00000000..a1bf3b84 --- /dev/null +++ b/tests/config/ic_antifeature_nsfw.xml @@ -0,0 +1,4 @@ + + + diff --git a/tests/config/ic_antifeature_tracking.xml b/tests/config/ic_antifeature_tracking.xml new file mode 100644 index 00000000..984fe20c --- /dev/null +++ b/tests/config/ic_antifeature_tracking.xml @@ -0,0 +1,22 @@ + + + + + + + diff --git a/tests/config/ic_antifeature_upstreamnonfree.xml b/tests/config/ic_antifeature_upstreamnonfree.xml new file mode 100644 index 00000000..f3598e67 --- /dev/null +++ b/tests/config/ic_antifeature_upstreamnonfree.xml @@ -0,0 +1,19 @@ + + + + + + diff --git a/tests/config/ro/antiFeatures.yml b/tests/config/ro/antiFeatures.yml new file mode 100644 index 00000000..97d61172 --- /dev/null +++ b/tests/config/ro/antiFeatures.yml @@ -0,0 +1,44 @@ +Ads: + description: Aplicația conține reclamă + icon: ic_antifeature_ads.xml + name: Reclame +DisabledAlgorithm: + description: Aplicația are o semnătură slab securizată + icon: ic_antifeature_disabledalgorithm.xml + name: Algoritm nesigur semnătură +KnownVuln: + description: Aplicația conține o vulnerabilitate de securitate cunoscută + icon: ic_antifeature_knownvuln.xml + name: Vulnerabilitate cunoscută +NSFW: + description: Această aplicație conține conținut care nu ar trebui să fie făcut public + sau vizibil peste tot + icon: ic_antifeature_nsfw.xml + name: NSFW +NoSourceSince: + icon: ic_antifeature_nosourcesince.xml + name: Codul sursă nu mai este disponibil, nu mai există posibilitatea de a actualiza. +NonFreeAdd: + description: Aplicația promovează anexe ce nu sunt software liber + icon: ic_antifeature_nonfreeadd.xml + name: Anexe ne-libere +NonFreeAssets: + description: Aceasta aplicație conține resurse ce nu sunt la disponibile la liber + icon: ic_antifeature_nonfreeassets.xml + name: Resurse ne-libere +NonFreeDep: + description: Aplicația depinde de alte aplicații ce nu sunt software liber + icon: ic_antifeature_nonfreedep.xml + name: Dependențe ne-libere +NonFreeNet: + description: Aplicația promovează servicii de rețea ce nu sunt accesibile la liber + icon: ic_antifeature_nonfreenet.xml + name: Servicii de rețea ne-libere +Tracking: + description: Aplicația îți înregistrează și raportează activitatea undeva + icon: ic_antifeature_tracking.xml + name: Urmărire +UpstreamNonFree: + description: Codul sursa originar nu este în totalitatea lui software liber + icon: ic_antifeature_upstreamnonfree.xml + name: Surse ne-libere diff --git a/tests/config/zh-rCN/antiFeatures.yml b/tests/config/zh-rCN/antiFeatures.yml new file mode 100644 index 00000000..a1b287b9 --- /dev/null +++ b/tests/config/zh-rCN/antiFeatures.yml @@ -0,0 +1,43 @@ +Ads: + description: 此应用包含广告 + icon: ic_antifeature_ads.xml + name: 广告 +DisabledAlgorithm: + description: 此应用的安全签名较弱 + icon: ic_antifeature_disabledalgorithm.xml + name: 使用不安全算法签名 +KnownVuln: + description: 此应用包含已知的安全漏洞 + icon: ic_antifeature_knownvuln.xml + name: 含有已知漏洞 +NSFW: + description: 此应用包含不应宣扬或随处可见的内容 + icon: ic_antifeature_nsfw.xml + name: NSFW +NoSourceSince: + icon: ic_antifeature_nosourcesince.xml + name: 源代码不再可用，无法更新。 +NonFreeAdd: + description: 此应用推广非自由的附加组件 + icon: ic_antifeature_nonfreeadd.xml + name: 非自由附加组件 +NonFreeAssets: + description: 此应用包含非自由资源 + icon: ic_antifeature_nonfreeassets.xml + name: 非自由资产 +NonFreeDep: + description: 此应用依赖于其它非自由应用 + icon: ic_antifeature_nonfreedep.xml + name: 非自由依赖项 +NonFreeNet: + description: 此应用推广非自由的网络服务 + icon: ic_antifeature_nonfreenet.xml + name: 非自由网络服务 +Tracking: + description: 此应用会记录并报告你的活动 + icon: ic_antifeature_tracking.xml + name: 跟踪用户 +UpstreamNonFree: + description: 上游源代码不是完全自由的 + icon: ic_antifeature_upstreamnonfree.xml + name: 上游代码非自由 diff --git a/tests/repo/entry.json b/tests/repo/entry.json index 2fb77d2a..f13e622b 100644 --- a/tests/repo/entry.json +++ b/tests/repo/entry.json @@ -3,8 +3,8 @@ "version": 20002, "index": { "name": "/index-v2.json", - "sha256": "07fa4500736ae77fcc6434e4d70ab315b8e018aef52c2afca9f2834ddc73747d", - "size": 32946, + "sha256": "a3c7e88a522a7228937e5c3d760fc239e3578e292035d88478d32fec9ff5eb54", + "size": 52314, "numPackages": 10 }, "diffs": {} diff --git a/tests/repo/index-v2.json b/tests/repo/index-v2.json index 7addd167..b59f17bf 100644 --- a/tests/repo/index-v2.json +++ b/tests/repo/index-v2.json @@ -27,6 +27,477 @@ } ], "timestamp": 1676634233000, + "antiFeatures": { + "Ads": { + "description": { + "de": "Diese App enthält Werbung", + "en-US": "This app contains advertising", + "fa": "این کاره دارای تبلیغات است", + "ro": "Aplicația conține reclamă", + "zh-rCN": "此应用包含广告" + }, + "icon": { + "de": { + "name": "/icons/ic_antifeature_ads.xml", + "sha256": "b333528573134c5de73484862a1b567a0bdfd6878d183f8500287abadc0ba60e", + "size": 1564 + }, + "en-US": { + "name": "/icons/ic_antifeature_ads.xml", + "sha256": "b333528573134c5de73484862a1b567a0bdfd6878d183f8500287abadc0ba60e", + "size": 1564 + }, + "fa": { + "name": "/icons/ic_antifeature_ads.xml", + "sha256": "b333528573134c5de73484862a1b567a0bdfd6878d183f8500287abadc0ba60e", + "size": 1564 + }, + "ro": { + "name": "/icons/ic_antifeature_ads.xml", + "sha256": "b333528573134c5de73484862a1b567a0bdfd6878d183f8500287abadc0ba60e", + "size": 1564 + }, + "zh-rCN": { + "name": "/icons/ic_antifeature_ads.xml", + "sha256": "b333528573134c5de73484862a1b567a0bdfd6878d183f8500287abadc0ba60e", + "size": 1564 + } + }, + "name": { + "de": "Werbung", + "en-US": "Ads", + "fa": "تبلیغات", + "ro": "Reclame", + "zh-rCN": "广告" + } + }, + "DisabledAlgorithm": { + "description": { + "de": "Diese App hat eine schwache Sicherheitssignatur", + "en-US": "This app has a weak security signature", + "fa": "این کاره، امضای امنیتی ضعیفی دارد", + "ro": "Aplicația are o semnătură slab securizată", + "zh-rCN": "此应用的安全签名较弱" + }, + "icon": { + "de": { + "name": "/icons/ic_antifeature_disabledalgorithm.xml", + "sha256": "94dea590c7c0aa37d351ab62a69fc7eefbc2cdbb84b79df3934c2e9332e1dcfb", + "size": 2313 + }, + "en-US": { + "name": "/icons/ic_antifeature_disabledalgorithm.xml", + "sha256": "94dea590c7c0aa37d351ab62a69fc7eefbc2cdbb84b79df3934c2e9332e1dcfb", + "size": 2313 + }, + "fa": { + "name": "/icons/ic_antifeature_disabledalgorithm.xml", + "sha256": "94dea590c7c0aa37d351ab62a69fc7eefbc2cdbb84b79df3934c2e9332e1dcfb", + "size": 2313 + }, + "ro": { + "name": "/icons/ic_antifeature_disabledalgorithm.xml", + "sha256": "94dea590c7c0aa37d351ab62a69fc7eefbc2cdbb84b79df3934c2e9332e1dcfb", + "size": 2313 + }, + "zh-rCN": { + "name": "/icons/ic_antifeature_disabledalgorithm.xml", + "sha256": "94dea590c7c0aa37d351ab62a69fc7eefbc2cdbb84b79df3934c2e9332e1dcfb", + "size": 2313 + } + }, + "name": { + "de": "Mit einem unsicheren Algorithmus signiert", + "en-US": "Signed Using An Unsafe Algorithm", + "fa": "امضا شده با الگوریتمی ناامن", + "ro": "Algoritm nesigur semnătură", + "zh-rCN": "使用不安全算法签名" + } + }, + "KnownVuln": { + "description": { + "de": "Diese App enthält eine bekannte Sicherheitslücke", + "en-US": "This app contains a known security vulnerability", + "fa": "این کاره، آسیب‌پذیری امنیتی شناخته‌شده‌ای دارد", + "ro": "Aplicația conține o vulnerabilitate de securitate cunoscută", + "zh-rCN": "此应用包含已知的安全漏洞" + }, + "icon": { + "de": { + "name": "/icons/ic_antifeature_knownvuln.xml", + "sha256": "743ddcad0120896b03bf62bca9b3b9902878ac9366959a0b77b2c50beeb37f9d", + "size": 1415 + }, + "en-US": { + "name": "/icons/ic_antifeature_knownvuln.xml", + "sha256": "743ddcad0120896b03bf62bca9b3b9902878ac9366959a0b77b2c50beeb37f9d", + "size": 1415 + }, + "fa": { + "name": "/icons/ic_antifeature_knownvuln.xml", + "sha256": "743ddcad0120896b03bf62bca9b3b9902878ac9366959a0b77b2c50beeb37f9d", + "size": 1415 + }, + "ro": { + "name": "/icons/ic_antifeature_knownvuln.xml", + "sha256": "743ddcad0120896b03bf62bca9b3b9902878ac9366959a0b77b2c50beeb37f9d", + "size": 1415 + }, + "zh-rCN": { + "name": "/icons/ic_antifeature_knownvuln.xml", + "sha256": "743ddcad0120896b03bf62bca9b3b9902878ac9366959a0b77b2c50beeb37f9d", + "size": 1415 + } + }, + "name": { + "de": "Bekannte Sicherheitslücke", + "en-US": "Known Vulnerability", + "fa": "آسیب‌پذیری شناخته", + "ro": "Vulnerabilitate cunoscută", + "zh-rCN": "含有已知漏洞" + } + }, + "NSFW": { + "description": { + "de": "Diese App enthält Inhalte, die nicht überall veröffentlicht oder sichtbar sein sollten", + "en-US": "This app contains content that should not be publicized or visible everywhere", + "fa": "این کاره محتوایی دارد که نباید عمومی شده یا همه‌حا نمایان باشد", + "ro": "Această aplicație conține conținut care nu ar trebui să fie făcut public sau vizibil peste tot", + "zh-rCN": "此应用包含不应宣扬或随处可见的内容" + }, + "icon": { + "de": { + "name": "/icons/ic_antifeature_nsfw.xml", + "sha256": "acab2a7a846700529cd7f2b7a7980f7d04a291f22db8434f3e966f7350ed1465", + "size": 871 + }, + "en-US": { + "name": "/icons/ic_antifeature_nsfw.xml", + "sha256": "acab2a7a846700529cd7f2b7a7980f7d04a291f22db8434f3e966f7350ed1465", + "size": 871 + }, + "fa": { + "name": "/icons/ic_antifeature_nsfw.xml", + "sha256": "acab2a7a846700529cd7f2b7a7980f7d04a291f22db8434f3e966f7350ed1465", + "size": 871 + }, + "ro": { + "name": "/icons/ic_antifeature_nsfw.xml", + "sha256": "acab2a7a846700529cd7f2b7a7980f7d04a291f22db8434f3e966f7350ed1465", + "size": 871 + }, + "zh-rCN": { + "name": "/icons/ic_antifeature_nsfw.xml", + "sha256": "acab2a7a846700529cd7f2b7a7980f7d04a291f22db8434f3e966f7350ed1465", + "size": 871 + } + }, + "name": { + "de": "NSFW", + "en-US": "NSFW", + "fa": "NSFW", + "ro": "NSFW", + "zh-rCN": "NSFW" + } + }, + "NoSourceSince": { + "description": { + "en-US": "The source code is no longer available, no updates possible." + }, + "icon": { + "de": { + "name": "/icons/ic_antifeature_nosourcesince.xml", + "sha256": "69c880b075967fe9598c777e18d600e1c1612bf061111911421fe8f6b9d88d4f", + "size": 1102 + }, + "en-US": { + "name": "/icons/ic_antifeature_nosourcesince.xml", + "sha256": "69c880b075967fe9598c777e18d600e1c1612bf061111911421fe8f6b9d88d4f", + "size": 1102 + }, + "fa": { + "name": "/icons/ic_antifeature_nosourcesince.xml", + "sha256": "69c880b075967fe9598c777e18d600e1c1612bf061111911421fe8f6b9d88d4f", + "size": 1102 + }, + "ro": { + "name": "/icons/ic_antifeature_nosourcesince.xml", + "sha256": "69c880b075967fe9598c777e18d600e1c1612bf061111911421fe8f6b9d88d4f", + "size": 1102 + }, + "zh-rCN": { + "name": "/icons/ic_antifeature_nosourcesince.xml", + "sha256": "69c880b075967fe9598c777e18d600e1c1612bf061111911421fe8f6b9d88d4f", + "size": 1102 + } + }, + "name": { + "de": "Der Quellcode ist nicht mehr erhältlich, keine Aktualisierungen möglich.", + "en-US": "Newer Source Not Available", + "fa": "کد مبدأ دیگر در دسترس نیست. به‌روز رسانی ناممکن است.", + "ro": "Codul sursă nu mai este disponibil, nu mai există posibilitatea de a actualiza.", + "zh-rCN": "源代码不再可用，无法更新。" + } + }, + "NonFreeAdd": { + "description": { + "de": "Diese App bewirbt nicht-quelloffene Erweiterungen", + "en-US": "This app promotes non-free add-ons", + "fa": "این کاره، افزونه‌های ناآزاد را تبلیغ می‌کند", + "ro": "Aplicația promovează anexe ce nu sunt software liber", + "zh-rCN": "此应用推广非自由的附加组件" + }, + "icon": { + "de": { + "name": "/icons/ic_antifeature_nonfreeadd.xml", + "sha256": "a1d1f2070bdaabf80ca5a55bccef98c82031ea2f31cc040be5ec009f44ddeef2", + "size": 1846 + }, + "en-US": { + "name": "/icons/ic_antifeature_nonfreeadd.xml", + "sha256": "a1d1f2070bdaabf80ca5a55bccef98c82031ea2f31cc040be5ec009f44ddeef2", + "size": 1846 + }, + "fa": { + "name": "/icons/ic_antifeature_nonfreeadd.xml", + "sha256": "a1d1f2070bdaabf80ca5a55bccef98c82031ea2f31cc040be5ec009f44ddeef2", + "size": 1846 + }, + "ro": { + "name": "/icons/ic_antifeature_nonfreeadd.xml", + "sha256": "a1d1f2070bdaabf80ca5a55bccef98c82031ea2f31cc040be5ec009f44ddeef2", + "size": 1846 + }, + "zh-rCN": { + "name": "/icons/ic_antifeature_nonfreeadd.xml", + "sha256": "a1d1f2070bdaabf80ca5a55bccef98c82031ea2f31cc040be5ec009f44ddeef2", + "size": 1846 + } + }, + "name": { + "de": "Nicht-quelloffene Erweiterungen", + "en-US": "Non-Free Addons", + "fa": "افزونه‌های ناآزاد", + "ro": "Anexe ne-libere", + "zh-rCN": "非自由附加组件" + } + }, + "NonFreeAssets": { + "description": { + "de": "Diese App enthält nicht-quelloffene Bestandteile", + "en-US": "This app contains non-free assets", + "fa": "این کاره دارای بخش‌های ناآزاد است", + "ro": "Aceasta aplicație conține resurse ce nu sunt la disponibile la liber", + "zh-rCN": "此应用包含非自由资源" + }, + "icon": { + "de": { + "name": "/icons/ic_antifeature_nonfreeassets.xml", + "sha256": "b39fe384386fc67fb30fa2f91402594110e2e42c961d76adc93141b8bd774008", + "size": 1784 + }, + "en-US": { + "name": "/icons/ic_antifeature_nonfreeassets.xml", + "sha256": "b39fe384386fc67fb30fa2f91402594110e2e42c961d76adc93141b8bd774008", + "size": 1784 + }, + "fa": { + "name": "/icons/ic_antifeature_nonfreeassets.xml", + "sha256": "b39fe384386fc67fb30fa2f91402594110e2e42c961d76adc93141b8bd774008", + "size": 1784 + }, + "ro": { + "name": "/icons/ic_antifeature_nonfreeassets.xml", + "sha256": "b39fe384386fc67fb30fa2f91402594110e2e42c961d76adc93141b8bd774008", + "size": 1784 + }, + "zh-rCN": { + "name": "/icons/ic_antifeature_nonfreeassets.xml", + "sha256": "b39fe384386fc67fb30fa2f91402594110e2e42c961d76adc93141b8bd774008", + "size": 1784 + } + }, + "name": { + "de": "Nicht-quelloffene Bestandteile", + "en-US": "Non-Free Assets", + "fa": "بخش‌های ناآزاد", + "ro": "Resurse ne-libere", + "zh-rCN": "非自由资产" + } + }, + "NonFreeDep": { + "description": { + "de": "Diese App ist abhängig von anderen nicht-quelloffenen Apps", + "en-US": "This app depends on other non-free apps", + "fa": "این کاره به دیگر کاره‌های ناآزاد وابسته است", + "ro": "Aplicația depinde de alte aplicații ce nu sunt software liber", + "zh-rCN": "此应用依赖于其它非自由应用" + }, + "icon": { + "de": { + "name": "/icons/ic_antifeature_nonfreedep.xml", + "sha256": "c1b4052a8f58125b2120d9ca07adb725d47bfa7cfcea80c4d6bbbc432b5cb83a", + "size": 1396 + }, + "en-US": { + "name": "/icons/ic_antifeature_nonfreedep.xml", + "sha256": "c1b4052a8f58125b2120d9ca07adb725d47bfa7cfcea80c4d6bbbc432b5cb83a", + "size": 1396 + }, + "fa": { + "name": "/icons/ic_antifeature_nonfreedep.xml", + "sha256": "c1b4052a8f58125b2120d9ca07adb725d47bfa7cfcea80c4d6bbbc432b5cb83a", + "size": 1396 + }, + "ro": { + "name": "/icons/ic_antifeature_nonfreedep.xml", + "sha256": "c1b4052a8f58125b2120d9ca07adb725d47bfa7cfcea80c4d6bbbc432b5cb83a", + "size": 1396 + }, + "zh-rCN": { + "name": "/icons/ic_antifeature_nonfreedep.xml", + "sha256": "c1b4052a8f58125b2120d9ca07adb725d47bfa7cfcea80c4d6bbbc432b5cb83a", + "size": 1396 + } + }, + "name": { + "de": "Nicht-quelloffene Abhängigkeiten", + "en-US": "Non-Free Dependencies", + "fa": "وابستگی‌های ناآزاد", + "ro": "Dependențe ne-libere", + "zh-rCN": "非自由依赖项" + } + }, + "NonFreeNet": { + "description": { + "de": "Diese App bewirbt nicht-quelloffene Netzwerkdienste", + "en-US": "This app promotes or depends entirely on a non-free network service", + "fa": "این کاره، خدمات شبکه‌های ناآزاد را ترویج می‌کند", + "ro": "Aplicația promovează servicii de rețea ce nu sunt accesibile la liber", + "zh-rCN": "此应用推广非自由的网络服务" + }, + "icon": { + "de": { + "name": "/icons/ic_antifeature_nonfreenet.xml", + "sha256": "7fff45c847ed2ecc94e85ba2341685c8f113fa5fdf7267a25637dc38ee0275f6", + "size": 3038 + }, + "en-US": { + "name": "/icons/ic_antifeature_nonfreenet.xml", + "sha256": "7fff45c847ed2ecc94e85ba2341685c8f113fa5fdf7267a25637dc38ee0275f6", + "size": 3038 + }, + "fa": { + "name": "/icons/ic_antifeature_nonfreenet.xml", + "sha256": "7fff45c847ed2ecc94e85ba2341685c8f113fa5fdf7267a25637dc38ee0275f6", + "size": 3038 + }, + "ro": { + "name": "/icons/ic_antifeature_nonfreenet.xml", + "sha256": "7fff45c847ed2ecc94e85ba2341685c8f113fa5fdf7267a25637dc38ee0275f6", + "size": 3038 + }, + "zh-rCN": { + "name": "/icons/ic_antifeature_nonfreenet.xml", + "sha256": "7fff45c847ed2ecc94e85ba2341685c8f113fa5fdf7267a25637dc38ee0275f6", + "size": 3038 + } + }, + "name": { + "de": "Nicht-quelloffene Netzwerkdienste", + "en-US": "Non-Free Network Services", + "fa": "خدمات شبکه‌ای ناآزاد", + "ro": "Servicii de rețea ne-libere", + "zh-rCN": "非自由网络服务" + } + }, + "Tracking": { + "description": { + "de": "Diese App verfolgt und versendet Ihre Aktivitäten", + "en-US": "This app tracks and reports your activity", + "fa": "این کاره، فعّالیتتان را ردیابی و گزارش می‌کند", + "ro": "Aplicația îți înregistrează și raportează activitatea undeva", + "zh-rCN": "此应用会记录并报告你的活动" + }, + "icon": { + "de": { + "name": "/icons/ic_antifeature_tracking.xml", + "sha256": "4779337b5b0a12c4b4a8a83d0d8a994a2477460db702784df4c8d3e3730be961", + "size": 2493 + }, + "en-US": { + "name": "/icons/ic_antifeature_tracking.xml", + "sha256": "4779337b5b0a12c4b4a8a83d0d8a994a2477460db702784df4c8d3e3730be961", + "size": 2493 + }, + "fa": { + "name": "/icons/ic_antifeature_tracking.xml", + "sha256": "4779337b5b0a12c4b4a8a83d0d8a994a2477460db702784df4c8d3e3730be961", + "size": 2493 + }, + "ro": { + "name": "/icons/ic_antifeature_tracking.xml", + "sha256": "4779337b5b0a12c4b4a8a83d0d8a994a2477460db702784df4c8d3e3730be961", + "size": 2493 + }, + "zh-rCN": { + "name": "/icons/ic_antifeature_tracking.xml", + "sha256": "4779337b5b0a12c4b4a8a83d0d8a994a2477460db702784df4c8d3e3730be961", + "size": 2493 + } + }, + "name": { + "de": "Tracking", + "en-US": "Tracking", + "fa": "ردیابی", + "ro": "Urmărire", + "zh-rCN": "跟踪用户" + } + }, + "UpstreamNonFree": { + "description": { + "de": "Der Originalcode ist nicht völlig quelloffen", + "en-US": "The upstream source code is not entirely Free", + "fa": "کد مبدأ بالادستی کاملاً آزاد نیست", + "ro": "Codul sursa originar nu este în totalitatea lui software liber", + "zh-rCN": "上游源代码不是完全自由的" + }, + "icon": { + "de": { + "name": "/icons/ic_antifeature_upstreamnonfree.xml", + "sha256": "06a9af843ff56ecd7a270f98c0b19b3154edf3ffa854e6d50a84ef00d0ce1a86", + "size": 1442 + }, + "en-US": { + "name": "/icons/ic_antifeature_upstreamnonfree.xml", + "sha256": "06a9af843ff56ecd7a270f98c0b19b3154edf3ffa854e6d50a84ef00d0ce1a86", + "size": 1442 + }, + "fa": { + "name": "/icons/ic_antifeature_upstreamnonfree.xml", + "sha256": "06a9af843ff56ecd7a270f98c0b19b3154edf3ffa854e6d50a84ef00d0ce1a86", + "size": 1442 + }, + "ro": { + "name": "/icons/ic_antifeature_upstreamnonfree.xml", + "sha256": "06a9af843ff56ecd7a270f98c0b19b3154edf3ffa854e6d50a84ef00d0ce1a86", + "size": 1442 + }, + "zh-rCN": { + "name": "/icons/ic_antifeature_upstreamnonfree.xml", + "sha256": "06a9af843ff56ecd7a270f98c0b19b3154edf3ffa854e6d50a84ef00d0ce1a86", + "size": 1442 + } + }, + "name": { + "de": "Originalcode nicht-quelloffen", + "en-US": "Upstream Non-Free", + "fa": "بالادست ناآزاد", + "ro": "Surse ne-libere", + "zh-rCN": "上游代码非自由" + } + } + }, "requests": { "install": [ "org.adaway" diff --git a/tests/run-tests b/tests/run-tests index e076359a..49acb5aa 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -261,7 +261,12 @@ REPOROOT=`create_test_dir` GNUPGHOME=$REPOROOT/gnupghome cd $REPOROOT fdroid_init_with_prebuilt_keystore -cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $REPOROOT/ +cp -a \ + $WORKSPACE/tests/config \ + $WORKSPACE/tests/metadata \ + $WORKSPACE/tests/repo \ + $WORKSPACE/tests/stats \ + $REPOROOT/ cp -a $WORKSPACE/tests/gnupghome $GNUPGHOME chmod 0700 $GNUPGHOME echo "install_list: org.adaway" >> config.yml From d5a1439457607e66a3513e89b0734322a5c57c80 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 24 Apr 2023 20:10:17 +0200 Subject: [PATCH 10/16] lint: Anti-Features validator uses names from config --- fdroidserver/lint.py | 35 ++++++++++++++++++++++ fdroidserver/metadata.py | 4 --- tests/lint.TestCase | 63 ++++++++++++++++++++++++++++++---------- tests/metadata.TestCase | 20 ------------- 4 files changed, 83 insertions(+), 39 deletions(-) diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index fb94e258..1283409b 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -220,6 +220,19 @@ locale_pattern = re.compile(r"[a-z]{2,3}(-([A-Z][a-zA-Z]+|\d+|[a-z]+))*") versioncode_check_pattern = re.compile(r"(\\d|\[(0-9|\\d)_?(a-fA-F)?])[+]") +ANTIFEATURES_KEYS = None +ANTIFEATURES_PATTERN = None + + +def load_antiFeatures_config(): + """Lazy loading, since it might read a lot of files.""" + global ANTIFEATURES_KEYS, ANTIFEATURES_PATTERN + k = 'antiFeatures' # internal dict uses camelCase key name + if not ANTIFEATURES_KEYS or k not in common.config: + common.config[k] = common.load_localized_config(k, 'repo') + ANTIFEATURES_KEYS = sorted(common.config[k].keys()) + ANTIFEATURES_PATTERN = ','.join(ANTIFEATURES_KEYS) + def check_regexes(app): for f, checks in regex_checks.items(): @@ -613,6 +626,26 @@ def check_app_field_types(app): ) +def check_antiFeatures(app): + """Check the Anti-Features keys match those declared in the config.""" + pattern = ANTIFEATURES_PATTERN + msg = _("'{value}' is not a valid {field} in {appid}. Regex pattern: {pattern}") + + field = 'AntiFeatures' # App entries use capitalized CamelCase + for value in app.get(field, []): + if value not in ANTIFEATURES_KEYS: + yield msg.format(value=value, field=field, appid=app.id, pattern=pattern) + + field = 'antifeatures' # Build entries use all lowercase + for build in app.get('Builds', []): + build_antiFeatures = build.get(field, []) + for value in build_antiFeatures: + if value not in ANTIFEATURES_KEYS: + yield msg.format( + value=value, field=field, appid=app.id, pattern=pattern + ) + + def check_for_unsupported_metadata_files(basedir=""): """Check whether any non-metadata files are in metadata/.""" basedir = Path(basedir) @@ -745,6 +778,7 @@ def main(): metadata.warnings_action = options.W config = common.read_config(options) + load_antiFeatures_config() # Get all apps... allapps = metadata.read_metadata(options.appid) @@ -801,6 +835,7 @@ def main(): app_check_funcs = [ check_app_field_types, + check_antiFeatures, check_regexes, check_update_check_data_url, check_update_check_data_int, diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 896c224e..5341a3a7 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -448,10 +448,6 @@ valuetypes = { r'^[0-9]+ versions$', ["ArchivePolicy"]), - FieldValidator("Anti-Feature", - r'^(Ads|Tracking|NonFreeNet|NonFreeDep|NonFreeAdd|UpstreamNonFree|NonFreeAssets|KnownVuln|ApplicationDebuggable|NoSourceSince|NSFW)$', - ["AntiFeatures"]), - FieldValidator("Auto Update Mode", r"^(Version.*|None)$", ["AutoUpdateMode"]), diff --git a/tests/lint.TestCase b/tests/lint.TestCase index 6fbe76cf..544b18c8 100755 --- a/tests/lint.TestCase +++ b/tests/lint.TestCase @@ -132,23 +132,9 @@ class LintTest(unittest.TestCase): app.Description = 'These are some back' fields = { - 'AntiFeatures': { - 'good': [ - [ - 'KnownVuln', - ], - ['NonFreeNet', 'KnownVuln'], - ], - 'bad': [ - 'KnownVuln', - 'NonFreeNet,KnownVuln', - ], - }, 'Categories': { 'good': [ - [ - 'Sports & Health', - ], + ['Sports & Health'], ['Multimedia', 'Graphics'], ], 'bad': [ @@ -328,6 +314,53 @@ class LintTest(unittest.TestCase): self.assertFalse(anywarns) +class LintAntiFeaturesTest(unittest.TestCase): + def setUp(self): + self.basedir = localmodule / 'tests' + os.chdir(self.basedir) + fdroidserver.common.config = dict() + fdroidserver.lint.load_antiFeatures_config() + + def test_check_antiFeatures_empty(self): + app = fdroidserver.metadata.App() + self.assertEqual([], list(fdroidserver.lint.check_antiFeatures(app))) + + def test_check_antiFeatures_empty_AntiFeatures(self): + app = fdroidserver.metadata.App() + app['AntiFeatures'] = [] + self.assertEqual([], list(fdroidserver.lint.check_antiFeatures(app))) + + def test_check_antiFeatures(self): + app = fdroidserver.metadata.App() + app['AntiFeatures'] = ['Ads', 'UpstreamNonFree'] + self.assertEqual([], list(fdroidserver.lint.check_antiFeatures(app))) + + def test_check_antiFeatures_fails_one(self): + app = fdroidserver.metadata.App() + app['AntiFeatures'] = ['Ad'] + self.assertEqual(1, len(list(fdroidserver.lint.check_antiFeatures(app)))) + + def test_check_antiFeatures_fails_many(self): + app = fdroidserver.metadata.App() + app['AntiFeatures'] = ['Adss', 'Tracker', 'NoSourceSince', 'FAKE', 'NonFree'] + self.assertEqual(4, len(list(fdroidserver.lint.check_antiFeatures(app)))) + + def test_check_antiFeatures_build_empty(self): + app = fdroidserver.metadata.App() + app['Builds'] = [{'antifeatures': []}] + self.assertEqual([], list(fdroidserver.lint.check_antiFeatures(app))) + + def test_check_antiFeatures_build(self): + app = fdroidserver.metadata.App() + app['Builds'] = [{'antifeatures': ['Tracking']}] + self.assertEqual(0, len(list(fdroidserver.lint.check_antiFeatures(app)))) + + def test_check_antiFeatures_build_fail(self): + app = fdroidserver.metadata.App() + app['Builds'] = [{'antifeatures': ['Ads', 'Tracker']}] + self.assertEqual(1, len(list(fdroidserver.lint.check_antiFeatures(app)))) + + if __name__ == "__main__": parser = optparse.OptionParser() parser.add_option( diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index b4301fba..82d5c3e5 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -178,26 +178,6 @@ class MetadataTest(unittest.TestCase): 'fake.app.id', ) - def test_check_metadata_AntiFeatures(self): - fdroidserver.metadata.warnings_action = 'error' - - app = fdroidserver.metadata.App() - self.assertIsNone(metadata.check_metadata(app)) - - app['AntiFeatures'] = [] - self.assertIsNone(metadata.check_metadata(app)) - - app['AntiFeatures'] = ['Ads', 'UpstreamNonFree'] - self.assertIsNone(metadata.check_metadata(app)) - - app['AntiFeatures'] = ['Ad'] - with self.assertRaises(fdroidserver.exception.MetaDataException): - metadata.check_metadata(app) - - app['AntiFeatures'] = ['Adss'] - with self.assertRaises(fdroidserver.exception.MetaDataException): - metadata.check_metadata(app) - def test_valid_funding_yml_regex(self): """Check the regex can find all the cases""" with (self.basedir / 'funding-usernames.yaml').open() as fp: From c2bc52dd85219a2009f17ab1058e54ae2b8f830d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 21 Apr 2023 11:06:42 +0200 Subject: [PATCH 11/16] use constant for default locale --- fdroidserver/common.py | 7 +++++-- fdroidserver/index.py | 21 ++++++++++----------- fdroidserver/update.py | 11 ++++++----- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index da802ad9..f401b71f 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -77,6 +77,9 @@ from . import apksigcopier, common # The path to this fdroidserver distribution FDROID_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..')) +# There needs to be a default, and this is the most common for software. +DEFAULT_LOCALE = 'en-US' + # this is the build-tools version, aapt has a separate version that # has to be manually set in test_aapt_version() MINIMUM_AAPT_BUILD_TOOLS_VERSION = '26.0.0' @@ -507,7 +510,7 @@ def load_localized_config(name, repodir): for f in Path().glob("config/**/{name}.yml".format(name=name)): locale = f.parts[1] if len(f.parts) == 2: - locale = "en-US" + locale = DEFAULT_LOCALE with open(f, encoding="utf-8") as fp: elem = yaml.safe_load(fp) for afname, field_dict in elem.items(): @@ -3912,7 +3915,7 @@ def get_app_display_name(app): if app.get('Name'): return app['Name'] if app.get('localized'): - localized = app['localized'].get('en-US') + localized = app['localized'].get(DEFAULT_LOCALE) if not localized: for v in app['localized'].values(): localized = v diff --git a/fdroidserver/index.py b/fdroidserver/index.py index fb080550..812d5f3d 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -42,7 +42,7 @@ from . import common from . import metadata from . import net from . import signindex -from fdroidserver.common import FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints +from fdroidserver.common import DEFAULT_LOCALE, FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints from fdroidserver.exception import FDroidException, VerificationException @@ -518,14 +518,14 @@ def package_metadata(app, repodir): ): element_new = element[:1].lower() + element[1:] if element in app and app[element]: - meta[element_new] = {"en-US": convert_datetime(app[element])} + meta[element_new] = {DEFAULT_LOCALE: convert_datetime(app[element])} elif "localized" in app: localized = {k: v[element_new] for k, v in app["localized"].items() if element_new in v} if localized: meta[element_new] = localized if "name" not in meta and app["AutoName"]: - meta["name"] = {"en-US": app["AutoName"]} + meta["name"] = {DEFAULT_LOCALE: app["AutoName"]} # fdroidserver/metadata.py App default if meta["license"] == "Unknown": @@ -536,9 +536,8 @@ def package_metadata(app, repodir): # TODO handle different resolutions if app.get("icon"): - meta["icon"] = { - "en-US": common.file_entry(os.path.join(repodir, "icons", app["icon"])) - } + icon_path = os.path.join(repodir, "icons", app["icon"]) + meta["icon"] = {DEFAULT_LOCALE: common.file_entry(icon_path)} if "iconv2" in app: meta["icon"] = app["iconv2"] @@ -654,10 +653,10 @@ def convert_version(version, app, repodir): def v2_repo(repodict, repodir, archive): repo = {} - repo["name"] = {"en-US": repodict["name"]} - repo["description"] = {"en-US": repodict["description"]} + repo["name"] = {DEFAULT_LOCALE: repodict["name"]} + repo["description"] = {DEFAULT_LOCALE: repodict["description"]} repo["icon"] = { - "en-US": common.file_entry("{}/icons/{}".format(repodir, repodict["icon"])) + DEFAULT_LOCALE: common.file_entry("%s/icons/%s" % (repodir, repodict["icon"])) } config = common.load_localized_config("config", repodir) @@ -1022,7 +1021,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing lkey = key[:1].lower() + key[1:] localized = app.get('localized') if not value and localized: - for lang in ['en-US'] + [x for x in localized.keys()]: + for lang in [DEFAULT_LOCALE] + [x for x in localized.keys()]: if not lang.startswith('en'): continue if lang in localized: @@ -1266,7 +1265,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing namefield = common.config['current_version_name_source'] name = app.get(namefield) if not name and namefield == 'Name': - name = app.get('localized', {}).get('en-US', {}).get('name') + name = app.get('localized', {}).get(DEFAULT_LOCALE, {}).get('name') if not name: name = app.id sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', str(name).encode('utf-8')) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index aec69237..a2564ac9 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -51,6 +51,7 @@ from . import _ from . import common from . import index from . import metadata +from .common import DEFAULT_LOCALE from .exception import BuildException, FDroidException, VerificationException from PIL import Image, PngImagePlugin @@ -2034,7 +2035,7 @@ def insert_missing_app_names_from_apks(apps, apks): The name from the APK is set as the default name for the app if there is no other default set, e.g. app['Name'] or - app['localized']['en-US']['name']. The en-US locale is defined in + app['localized'][DEFAULT_LOCALE]['name']. The default is defined in the F-Droid ecosystem as the locale of last resort, as in the one that should always be present. en-US is used since it is the locale of the source strings. @@ -2050,7 +2051,7 @@ def insert_missing_app_names_from_apks(apps, apks): for appid, app in apps.items(): if app.get('Name') is not None: continue - if app.get('localized', {}).get('en-US', {}).get('name') is not None: + if app.get('localized', {}).get(DEFAULT_LOCALE, {}).get('name') is not None: continue bestver = UNSET_VERSION_CODE @@ -2063,9 +2064,9 @@ def insert_missing_app_names_from_apks(apps, apks): if bestver != UNSET_VERSION_CODE: if 'localized' not in app: app['localized'] = {} - if 'en-US' not in app['localized']: - app['localized']['en-US'] = {} - app['localized']['en-US']['name'] = bestapk.get('name') + if DEFAULT_LOCALE not in app['localized']: + app['localized'][DEFAULT_LOCALE] = {} + app['localized'][DEFAULT_LOCALE]['name'] = bestapk.get('name') def get_apps_with_packages(apps, apks): From bb999866302b21c29be415e00ffc1301ebd8b09a Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 3 May 2023 13:39:37 +0200 Subject: [PATCH 12/16] metadata: fix crash if .fdroid.yml but its not a git repo --- fdroidserver/metadata.py | 2 +- tests/metadata.TestCase | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 5341a3a7..787de9a0 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -672,7 +672,7 @@ def parse_metadata(metadatapath): # pylint: disable-next=no-member except git.exc.InvalidGitRepositoryError: logging.debug( - _('Including metadata from {path}').format(metadata_in_repo) + _('Including metadata from {path}').format(path=metadata_in_repo) ) app_in_repo = parse_metadata(metadata_in_repo) for k, v in app_in_repo.items(): diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index 82d5c3e5..40fdbd05 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -15,7 +15,7 @@ import textwrap from collections import OrderedDict from pathlib import Path -from testcommon import TmpCwd +from testcommon import TmpCwd, mkdtemp localmodule = Path(__file__).resolve().parent.parent print('localmodule: ' + str(localmodule)) @@ -40,10 +40,13 @@ class MetadataTest(unittest.TestCase): logging.basicConfig(level=logging.DEBUG) self.basedir = localmodule / 'tests' os.chdir(self.basedir) + self._td = mkdtemp() + self.testdir = self._td.name fdroidserver.metadata.warnings_action = 'error' def tearDown(self): # auto-generated dirs by functions, not tests, so they are not always cleaned up + self._td.cleanup() try: os.rmdir("srclibs") except OSError: @@ -228,6 +231,19 @@ class MetadataTest(unittest.TestCase): # yaml.register_class(metadata.Build) # yaml.dump(frommeta, fp) + def test_dot_fdroid_yml_works_without_git(self): + """Parsing should work if .fdroid.yml is present and it is not a git repo.""" + os.chdir(self.testdir) + yml = Path('metadata/test.yml') + yml.parent.mkdir() + with yml.open('w') as fp: + fp.write('Repo: https://example.com/not/git/or/anything') + fdroid_yml = Path('build/test/.fdroid.yml') + fdroid_yml.parent.mkdir(parents=True) + with fdroid_yml.open('w') as fp: + fp.write('OpenCollective: test') + metadata.parse_metadata(yml) # should not throw an exception + def test_rewrite_yaml_fakeotaupdate(self): with tempfile.TemporaryDirectory() as testdir: testdir = Path(testdir) From 86b643f87b6f172199b8478666adfe6d11687cae Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 2 May 2023 19:28:34 +0200 Subject: [PATCH 13/16] metadata: test to check that metadata/*.yml overrides .fdroid.yml This actually uncovered that .fdroid.yml isn't really working. But that is a problem for another day. --- .gitignore | 6 +++--- .../info.guardianproject.urzip/.fdroid.yml | 4 ++++ tests/metadata.TestCase | 20 +++++++++++++++---- 3 files changed, 23 insertions(+), 7 deletions(-) create mode 100644 tests/build/info.guardianproject.urzip/.fdroid.yml diff --git a/.gitignore b/.gitignore index c8785a33..04e92ad6 100644 --- a/.gitignore +++ b/.gitignore @@ -7,11 +7,11 @@ TAGS .ropeproject/ # files generated by build -build/ -dist/ +/build/ +/dist/ env/ ENV/ -fdroidserver.egg-info/ +/fdroidserver.egg-info/ pylint.parseable /.testfiles/ README.rst diff --git a/tests/build/info.guardianproject.urzip/.fdroid.yml b/tests/build/info.guardianproject.urzip/.fdroid.yml new file mode 100644 index 00000000..7f2b3c1c --- /dev/null +++ b/tests/build/info.guardianproject.urzip/.fdroid.yml @@ -0,0 +1,4 @@ + +Summary: This should be overridden by metadata/info.guardianproject.urzip.yml +Builds: + - versionCode: 50 diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index 40fdbd05..75df353a 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -14,6 +14,7 @@ import tempfile import textwrap from collections import OrderedDict from pathlib import Path +from unittest import mock from testcommon import TmpCwd, mkdtemp @@ -201,8 +202,10 @@ class MetadataTest(unittest.TestCase): m, 'this is a valid %s username: {%s}' % (k, entry) ) - def test_read_metadata(self): + @mock.patch('git.Repo') + def test_read_metadata(self, git_repo): """Read specified metadata files included in tests/, compare to stored output""" + self.maxDiff = None config = dict() @@ -231,6 +234,12 @@ class MetadataTest(unittest.TestCase): # yaml.register_class(metadata.Build) # yaml.dump(frommeta, fp) + @mock.patch('git.Repo') + def test_metadata_overrides_dot_fdroid_yml(self, git_Repo): + """Fields in metadata files should override anything in .fdroid.yml.""" + app = metadata.parse_metadata('metadata/info.guardianproject.urzip.yml') + self.assertEqual(app['Summary'], '一个实用工具，获取已安装在您的设备上的应用的有关信息') + def test_dot_fdroid_yml_works_without_git(self): """Parsing should work if .fdroid.yml is present and it is not a git repo.""" os.chdir(self.testdir) @@ -244,7 +253,8 @@ class MetadataTest(unittest.TestCase): fp.write('OpenCollective: test') metadata.parse_metadata(yml) # should not throw an exception - def test_rewrite_yaml_fakeotaupdate(self): + @mock.patch('git.Repo') + def test_rewrite_yaml_fakeotaupdate(self, git_Repo): with tempfile.TemporaryDirectory() as testdir: testdir = Path(testdir) fdroidserver.common.config = {'accepted_formats': ['yml']} @@ -266,7 +276,8 @@ class MetadataTest(unittest.TestCase): (Path('metadata-rewrite-yml') / file_name).read_text(encoding='utf-8'), ) - def test_rewrite_yaml_fdroidclient(self): + @mock.patch('git.Repo') + def test_rewrite_yaml_fdroidclient(self, git_Repo): with tempfile.TemporaryDirectory() as testdir: testdir = Path(testdir) fdroidserver.common.config = {'accepted_formats': ['yml']} @@ -287,7 +298,8 @@ class MetadataTest(unittest.TestCase): (Path('metadata-rewrite-yml') / file_name).read_text(encoding='utf-8'), ) - def test_rewrite_yaml_special_build_params(self): + @mock.patch('git.Repo') + def test_rewrite_yaml_special_build_params(self, git_Repo): with tempfile.TemporaryDirectory() as testdir: testdir = Path(testdir) From 8bc9a3da73ba018c95b67a211c6e979c0b4226d2 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 3 May 2023 13:01:42 +0200 Subject: [PATCH 14/16] test_parse_yaml_metadata_continue_on_warning checks logging calls --- tests/metadata.TestCase | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index 75df353a..b90abf9f 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -458,7 +458,6 @@ class MetadataTest(unittest.TestCase): with self.assertRaises(TypeError): metadata.parse_yaml_metadata(mf) - mf.name = 'mock_filename.yaml' self.assertEqual(fdroidserver.metadata.parse_yaml_metadata(mf), dict()) def test_parse_yaml_metadata_unknown_app_field(self): @@ -489,7 +488,9 @@ class MetadataTest(unittest.TestCase): with self.assertRaises(MetaDataException): fdroidserver.metadata.parse_yaml_metadata(mf) - def test_parse_yaml_metadata_continue_on_warning(self): + @mock.patch('logging.warning') + @mock.patch('logging.error') + def test_parse_yaml_metadata_continue_on_warning(self, _error, _warning): """When errors are disabled, parsing should provide something that can work. When errors are disabled, then it should try to give data that @@ -503,6 +504,8 @@ class MetadataTest(unittest.TestCase): fdroidserver.metadata.warnings_action = None mf = _get_mock_mf('[AntiFeatures: Tracking]') self.assertEqual(fdroidserver.metadata.parse_yaml_metadata(mf), dict()) + _warning.assert_called_once() + _error.assert_called_once() def test_parse_yaml_srclib_corrupt_file(self): with tempfile.TemporaryDirectory() as testdir: From 024d309262749039495507ba384b8ac850b4be29 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 3 May 2023 19:20:45 +0200 Subject: [PATCH 15/16] index: rename app var to app_dict, its not an App instance Throughout the code, variables named "app" are instances of the App class. In this case, this is related, but it is a dict not an App instance, since it is being prepared for including in the index-v1.json. --- fdroidserver/index.py | 20 ++++++++++---------- fdroidserver/update.py | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 812d5f3d..48b51ffe 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -843,10 +843,10 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_ appslist = [] output['apps'] = appslist - for packageName, appdict in apps.items(): + for packageName, app_dict in apps.items(): d = collections.OrderedDict() appslist.append(d) - for k, v in sorted(appdict.items()): + for k, v in sorted(app_dict.items()): if not v: continue if k in ('Builds', 'metadatapath', @@ -872,20 +872,20 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_ d[k] = v # establish sort order in lists, sets, and localized dicts - for app in output['apps']: - localized = app.get('localized') + for app_dict in output['apps']: + localized = app_dict.get('localized') if localized: lordered = collections.OrderedDict() for lkey, lvalue in sorted(localized.items()): lordered[lkey] = collections.OrderedDict() for ikey, iname in sorted(lvalue.items()): lordered[lkey][ikey] = iname - app['localized'] = lordered - antiFeatures = app.get('antiFeatures', []) - if apps[app["packageName"]].get("NoSourceSince"): + app_dict['localized'] = lordered + antiFeatures = app_dict.get('antiFeatures', []) + if apps[app_dict["packageName"]].get("NoSourceSince"): antiFeatures.append("NoSourceSince") if antiFeatures: - app['antiFeatures'] = sorted(set(antiFeatures)) + app_dict['antiFeatures'] = sorted(set(antiFeatures)) output_packages = collections.OrderedDict() output['packages'] = output_packages @@ -1067,8 +1067,8 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing root.appendChild(element) element.setAttribute('packageName', packageName) - for appid, appdict in apps.items(): - app = metadata.App(appdict) + for appid, app_dict in apps.items(): + app = metadata.App(app_dict) if app.get('Disabled') is not None: continue diff --git a/fdroidserver/update.py b/fdroidserver/update.py index a2564ac9..45d66d33 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -2343,10 +2343,10 @@ def main(): add_apks_to_per_app_repos(repodirs[0], apks) for appid, app in apps.items(): repodir = os.path.join(appid, 'fdroid', 'repo') - appdict = dict() - appdict[appid] = app + app_dict = dict() + app_dict[appid] = app if os.path.isdir(repodir): - index.make(appdict, apks, repodir, False) + index.make(app_dict, apks, repodir, False) else: logging.info(_('Skipping index generation for {appid}').format(appid=appid)) return From af5b067396ccf79da97d517e5e1aa967402973ee Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 9 May 2023 11:55:18 +0200 Subject: [PATCH 16/16] gitlab-ci: bump version to compare in metadata_v0 job The relevant change comes from !1332 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cc40911d..f26a093a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,7 +41,7 @@ metadata_v0: image: registry.gitlab.com/fdroid/fdroidserver:buildserver variables: GIT_DEPTH: 1000 - RELEASE_COMMIT_ID: 58cfce106b6d68dc8ebde7842cf01225f5adfd1b # 2.2b + RELEASE_COMMIT_ID: 0124b9dde99f9cab19c034cbc7d8cc6005a99b48 # 2.3a0 script: - git fetch https://gitlab.com/fdroid/fdroidserver.git $RELEASE_COMMIT_ID - cd tests