lint: check syntax of countryCode: fields for mirrors

.gitlab-ci.yml

@@ -15,11 +15,12 @@ variables:
 # * python3-babel for compiling localization files
 # * gnupg-agent for the full signing setup
 # * python3-clint for fancy progress bars for users
+# * python3-pycountry for linting config/mirrors.yml
 buildserver run-tests:
   image: registry.gitlab.com/fdroid/fdroidserver:buildserver
   script:
     - apt-get update
-    - apt-get install gnupg-agent python3-babel python3-clint
+    - apt-get install gnupg-agent python3-babel python3-clint python3-pycountry
     - ./tests/run-tests
     # make sure that translations do not cause stacktraces
     - cd $CI_PROJECT_DIR/locale
@@ -152,6 +153,9 @@ ubuntu_jammy_pip:
     - $pip install sdkmanager
     - sdkmanager 'build-tools;33.0.0'
 
+    # pycountry is only for linting config/mirrors.yml, so its not in setup.py
+    - $pip install pycountry
+
     - $pip install dist/fdroidserver-*.tar.gz
     - tar xzf dist/fdroidserver-*.tar.gz
     - cd fdroidserver-*

fdroidserver/lint.py

@@ -17,6 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from argparse import ArgumentParser
+import difflib
 import re
 import sys
 import platform
@@ -752,6 +753,28 @@ def lint_config(arg):
         data = ruamel.yaml.YAML(typ='safe').load(fp)
     common.config_type_check(arg, data)
 
+    if path.name == 'mirrors.yml':
+        import pycountry
+
+        valid_country_codes = [c.alpha_2 for c in pycountry.countries]
+        for mirror in data:
+            code = mirror.get('countryCode')
+            if code and code not in valid_country_codes:
+                passed = False
+                msg = _(
+                    '{path}: "{code}" is not a valid ISO_3166-1 alpha-2 country code!'
+                ).format(path=str(path), code=code)
+                if code.upper() in valid_country_codes:
+                    m = [code.upper()]
+                else:
+                    m = difflib.get_close_matches(
+                        code.upper(), valid_country_codes, 2, 0.5
+                    )
+                if m:
+                    msg += ' '
+                    msg += _('Did you mean {code}?').format(code=', '.join(sorted(m)))
+                print(msg)
+
     return passed
 
 

tests/get-country-region-data.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+#
+# This generates a list of ISO_3166-1 alpha 2 country codes for use in lint.
+
+import collections
+import os
+import re
+import requests
+import requests_cache
+import sys
+import tempfile
+
+
+def main():
+    # we want all the data
+    url = 'https://api.worldbank.org/v2/country?format=json&per_page=500'
+    r = requests.get(url, timeout=30)
+    data = r.json()
+    if data[0]['pages'] != 1:
+        print(
+            'ERROR: %d pages in data, this script only reads one page!'
+            % data[0]['pages']
+        )
+        sys.exit(1)
+
+    iso2Codes = set()
+    ISO3166_1_alpha_2_codes = set()
+    names = dict()
+    regions = collections.defaultdict(set)
+    for country in data[1]:
+        iso2Code = country['iso2Code']
+        iso2Codes.add(iso2Code)
+        if country['region']['value'] == 'Aggregates':
+            continue
+        if re.match(r'[A-Z][A-Z]', iso2Code):
+            ISO3166_1_alpha_2_codes.add(iso2Code)
+            names[iso2Code] = country['name']
+            regions[country['region']['value']].add(country['name'])
+    for code in sorted(ISO3166_1_alpha_2_codes):
+        print(f"    '{code}',  # " + names[code])
+
+
+if __name__ == "__main__":
+    requests_cache.install_cache(
+        os.path.join(tempfile.gettempdir(), os.path.basename(__file__) + '.cache')
+    )
+    main()

tests/lint.TestCase

@@ -5,6 +5,7 @@
 import logging
 import optparse
 import os
+import ruamel.yaml
 import shutil
 import sys
 import tempfile
@@ -368,6 +369,75 @@ class LintTest(unittest.TestCase):
         app = fdroidserver.metadata.App({'Categories': ['bar']})
         self.assertEqual(0, len(list(fdroidserver.lint.check_categories(app))))
 
+    def test_lint_config_basic_mirrors_yml(self):
+        os.chdir(self.testdir)
+        yaml = ruamel.yaml.YAML(typ='safe')
+        with Path('mirrors.yml').open('w') as fp:
+            yaml.dump([{'url': 'https://example.com/fdroid/repo'}], fp)
+        self.assertTrue(fdroidserver.lint.lint_config('mirrors.yml'))
+
+    def test_lint_config_mirrors_yml_kenya_countryCode(self):
+        os.chdir(self.testdir)
+        yaml = ruamel.yaml.YAML(typ='safe')
+        with Path('mirrors.yml').open('w') as fp:
+            yaml.dump([{'url': 'https://foo.com/fdroid/repo', 'countryCode': 'KE'}], fp)
+        self.assertTrue(fdroidserver.lint.lint_config('mirrors.yml'))
+
+    def test_lint_config_mirrors_yml_invalid_countryCode(self):
+        """WV is "indeterminately reserved" so it should never be used."""
+        os.chdir(self.testdir)
+        yaml = ruamel.yaml.YAML(typ='safe')
+        with Path('mirrors.yml').open('w') as fp:
+            yaml.dump([{'url': 'https://foo.com/fdroid/repo', 'countryCode': 'WV'}], fp)
+        self.assertFalse(fdroidserver.lint.lint_config('mirrors.yml'))
+
+    def test_lint_config_mirrors_yml_alpha3_countryCode(self):
+        """Only ISO 3166-1 alpha 2 are supported"""
+        os.chdir(self.testdir)
+        yaml = ruamel.yaml.YAML(typ='safe')
+        with Path('mirrors.yml').open('w') as fp:
+            yaml.dump([{'url': 'https://de.com/fdroid/repo', 'countryCode': 'DEU'}], fp)
+        self.assertFalse(fdroidserver.lint.lint_config('mirrors.yml'))
+
+    def test_lint_config_mirrors_yml_one_invalid_countryCode(self):
+        """WV is "indeterminately reserved" so it should never be used."""
+        os.chdir(self.testdir)
+        yaml = ruamel.yaml.YAML(typ='safe')
+        with Path('mirrors.yml').open('w') as fp:
+            yaml.dump(
+                [
+                    {'url': 'https://bar.com/fdroid/repo', 'countryCode': 'BA'},
+                    {'url': 'https://foo.com/fdroid/repo', 'countryCode': 'FO'},
+                    {'url': 'https://wv.com/fdroid/repo', 'countryCode': 'WV'},
+                ],
+                fp,
+            )
+        self.assertFalse(fdroidserver.lint.lint_config('mirrors.yml'))
+
+    def test_lint_config_bad_mirrors_yml_dict(self):
+        os.chdir(self.testdir)
+        Path('mirrors.yml').write_text('baz: [foo, bar]\n')
+        with self.assertRaises(TypeError):
+            fdroidserver.lint.lint_config('mirrors.yml')
+
+    def test_lint_config_bad_mirrors_yml_float(self):
+        os.chdir(self.testdir)
+        Path('mirrors.yml').write_text('1.0\n')
+        with self.assertRaises(TypeError):
+            fdroidserver.lint.lint_config('mirrors.yml')
+
+    def test_lint_config_bad_mirrors_yml_int(self):
+        os.chdir(self.testdir)
+        Path('mirrors.yml').write_text('1\n')
+        with self.assertRaises(TypeError):
+            fdroidserver.lint.lint_config('mirrors.yml')
+
+    def test_lint_config_bad_mirrors_yml_str(self):
+        os.chdir(self.testdir)
+        Path('mirrors.yml').write_text('foo\n')
+        with self.assertRaises(TypeError):
+            fdroidserver.lint.lint_config('mirrors.yml')
+
 
 class LintAntiFeaturesTest(unittest.TestCase):
     def setUp(self):