From 96fc49d7fc128aae6638216ff4c5d3d505eba7a4 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 7 Dec 2023 17:38:34 +0100 Subject: [PATCH] lint: check syntax of countryCode: fields for mirrors --- .gitlab-ci.yml | 6 ++- fdroidserver/lint.py | 23 +++++++++++ tests/get-country-region-data.py | 47 +++++++++++++++++++++ tests/lint.TestCase | 70 ++++++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+), 1 deletion(-) create mode 100755 tests/get-country-region-data.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ebdc30ce..b867e2b7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,11 +15,12 @@ variables: # * python3-babel for compiling localization files # * gnupg-agent for the full signing setup # * python3-clint for fancy progress bars for users +# * python3-pycountry for linting config/mirrors.yml buildserver run-tests: image: registry.gitlab.com/fdroid/fdroidserver:buildserver script: - apt-get update - - apt-get install gnupg-agent python3-babel python3-clint + - apt-get install gnupg-agent python3-babel python3-clint python3-pycountry - ./tests/run-tests # make sure that translations do not cause stacktraces - cd $CI_PROJECT_DIR/locale @@ -152,6 +153,9 @@ ubuntu_jammy_pip: - $pip install sdkmanager - sdkmanager 'build-tools;33.0.0' + # pycountry is only for linting config/mirrors.yml, so its not in setup.py + - $pip install pycountry + - $pip install dist/fdroidserver-*.tar.gz - tar xzf dist/fdroidserver-*.tar.gz - cd fdroidserver-* diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index 04b03b25..150258ad 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -17,6 +17,7 @@ # along with this program. If not, see . from argparse import ArgumentParser +import difflib import re import sys import platform @@ -752,6 +753,28 @@ def lint_config(arg): data = ruamel.yaml.YAML(typ='safe').load(fp) common.config_type_check(arg, data) + if path.name == 'mirrors.yml': + import pycountry + + valid_country_codes = [c.alpha_2 for c in pycountry.countries] + for mirror in data: + code = mirror.get('countryCode') + if code and code not in valid_country_codes: + passed = False + msg = _( + '{path}: "{code}" is not a valid ISO_3166-1 alpha-2 country code!' + ).format(path=str(path), code=code) + if code.upper() in valid_country_codes: + m = [code.upper()] + else: + m = difflib.get_close_matches( + code.upper(), valid_country_codes, 2, 0.5 + ) + if m: + msg += ' ' + msg += _('Did you mean {code}?').format(code=', '.join(sorted(m))) + print(msg) + return passed diff --git a/tests/get-country-region-data.py b/tests/get-country-region-data.py new file mode 100755 index 00000000..f0f52e4b --- /dev/null +++ b/tests/get-country-region-data.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# +# This generates a list of ISO_3166-1 alpha 2 country codes for use in lint. + +import collections +import os +import re +import requests +import requests_cache +import sys +import tempfile + + +def main(): + # we want all the data + url = 'https://api.worldbank.org/v2/country?format=json&per_page=500' + r = requests.get(url, timeout=30) + data = r.json() + if data[0]['pages'] != 1: + print( + 'ERROR: %d pages in data, this script only reads one page!' + % data[0]['pages'] + ) + sys.exit(1) + + iso2Codes = set() + ISO3166_1_alpha_2_codes = set() + names = dict() + regions = collections.defaultdict(set) + for country in data[1]: + iso2Code = country['iso2Code'] + iso2Codes.add(iso2Code) + if country['region']['value'] == 'Aggregates': + continue + if re.match(r'[A-Z][A-Z]', iso2Code): + ISO3166_1_alpha_2_codes.add(iso2Code) + names[iso2Code] = country['name'] + regions[country['region']['value']].add(country['name']) + for code in sorted(ISO3166_1_alpha_2_codes): + print(f" '{code}', # " + names[code]) + + +if __name__ == "__main__": + requests_cache.install_cache( + os.path.join(tempfile.gettempdir(), os.path.basename(__file__) + '.cache') + ) + main() diff --git a/tests/lint.TestCase b/tests/lint.TestCase index d69382f0..55c314b0 100755 --- a/tests/lint.TestCase +++ b/tests/lint.TestCase @@ -5,6 +5,7 @@ import logging import optparse import os +import ruamel.yaml import shutil import sys import tempfile @@ -368,6 +369,75 @@ class LintTest(unittest.TestCase): app = fdroidserver.metadata.App({'Categories': ['bar']}) self.assertEqual(0, len(list(fdroidserver.lint.check_categories(app)))) + def test_lint_config_basic_mirrors_yml(self): + os.chdir(self.testdir) + yaml = ruamel.yaml.YAML(typ='safe') + with Path('mirrors.yml').open('w') as fp: + yaml.dump([{'url': 'https://example.com/fdroid/repo'}], fp) + self.assertTrue(fdroidserver.lint.lint_config('mirrors.yml')) + + def test_lint_config_mirrors_yml_kenya_countryCode(self): + os.chdir(self.testdir) + yaml = ruamel.yaml.YAML(typ='safe') + with Path('mirrors.yml').open('w') as fp: + yaml.dump([{'url': 'https://foo.com/fdroid/repo', 'countryCode': 'KE'}], fp) + self.assertTrue(fdroidserver.lint.lint_config('mirrors.yml')) + + def test_lint_config_mirrors_yml_invalid_countryCode(self): + """WV is "indeterminately reserved" so it should never be used.""" + os.chdir(self.testdir) + yaml = ruamel.yaml.YAML(typ='safe') + with Path('mirrors.yml').open('w') as fp: + yaml.dump([{'url': 'https://foo.com/fdroid/repo', 'countryCode': 'WV'}], fp) + self.assertFalse(fdroidserver.lint.lint_config('mirrors.yml')) + + def test_lint_config_mirrors_yml_alpha3_countryCode(self): + """Only ISO 3166-1 alpha 2 are supported""" + os.chdir(self.testdir) + yaml = ruamel.yaml.YAML(typ='safe') + with Path('mirrors.yml').open('w') as fp: + yaml.dump([{'url': 'https://de.com/fdroid/repo', 'countryCode': 'DEU'}], fp) + self.assertFalse(fdroidserver.lint.lint_config('mirrors.yml')) + + def test_lint_config_mirrors_yml_one_invalid_countryCode(self): + """WV is "indeterminately reserved" so it should never be used.""" + os.chdir(self.testdir) + yaml = ruamel.yaml.YAML(typ='safe') + with Path('mirrors.yml').open('w') as fp: + yaml.dump( + [ + {'url': 'https://bar.com/fdroid/repo', 'countryCode': 'BA'}, + {'url': 'https://foo.com/fdroid/repo', 'countryCode': 'FO'}, + {'url': 'https://wv.com/fdroid/repo', 'countryCode': 'WV'}, + ], + fp, + ) + self.assertFalse(fdroidserver.lint.lint_config('mirrors.yml')) + + def test_lint_config_bad_mirrors_yml_dict(self): + os.chdir(self.testdir) + Path('mirrors.yml').write_text('baz: [foo, bar]\n') + with self.assertRaises(TypeError): + fdroidserver.lint.lint_config('mirrors.yml') + + def test_lint_config_bad_mirrors_yml_float(self): + os.chdir(self.testdir) + Path('mirrors.yml').write_text('1.0\n') + with self.assertRaises(TypeError): + fdroidserver.lint.lint_config('mirrors.yml') + + def test_lint_config_bad_mirrors_yml_int(self): + os.chdir(self.testdir) + Path('mirrors.yml').write_text('1\n') + with self.assertRaises(TypeError): + fdroidserver.lint.lint_config('mirrors.yml') + + def test_lint_config_bad_mirrors_yml_str(self): + os.chdir(self.testdir) + Path('mirrors.yml').write_text('foo\n') + with self.assertRaises(TypeError): + fdroidserver.lint.lint_config('mirrors.yml') + class LintAntiFeaturesTest(unittest.TestCase): def setUp(self):