standardize config on ruamel.yaml with a YAML 1.2 config

This is a key piece of the ongoing `PUBLISH` _config.yml_ migration. There was uneven implementation of which YAML parser to use, and that could lead to bugs where one parser might read a value one way, and a different parser will read the value a different way. I wanted to be sure that YAML 1.2 would always work.

This makes all code that handles config files use the same `ruamel.yaml` parsers.  This only touches other usages of YAML parsers when there is overlap.  This does not port all of _fdroidserver_ to `ruamel.yaml` and YAML 1.2.  The metadata files should already be YAML 1.2 anyway.

# Conflicts:
#	fdroidserver/lint.py

tests/metadata/dump/app.with.special.build.params.yaml

@@ -1,3 +1,5 @@
+%YAML 1.2
+---
 AllowedAPKSigningKeys: []
 AntiFeatures:
   UpstreamNonFree: {}

tests/metadata/dump/com.politedroid.yaml

@@ -1,3 +1,5 @@
+%YAML 1.2
+---
 AllowedAPKSigningKeys: []
 AntiFeatures:
   NoSourceSince:

tests/metadata/dump/org.adaway.yaml

@@ -1,3 +1,5 @@
+%YAML 1.2
+---
 AllowedAPKSigningKeys: []
 AntiFeatures: {}
 ArchivePolicy: null

tests/metadata/dump/org.smssecure.smssecure.yaml

@@ -1,3 +1,5 @@
+%YAML 1.2
+---
 AllowedAPKSigningKeys: []
 AntiFeatures: {}
 ArchivePolicy: null

tests/metadata/dump/org.videolan.vlc.yaml

@@ -1,3 +1,5 @@
+%YAML 1.2
+---
 AllowedAPKSigningKeys: []
 AntiFeatures: {}
 ArchivePolicy: 9

tests/test_common.py

@@ -17,7 +17,6 @@ import tempfile
 import time
 import unittest
 import textwrap
-import yaml
 import gzip
 from argparse import ArgumentParser
 from datetime import datetime, timezone
@@ -32,6 +31,7 @@ import fdroidserver.common
 import fdroidserver.metadata
 from .shared_test_code import TmpCwd, mkdtemp
 from fdroidserver.common import ANTIFEATURES_CONFIG_NAME, CATEGORIES_CONFIG_NAME
+from fdroidserver._yaml import yaml, yaml_dumper
 from fdroidserver.exception import FDroidException, VCSException,\
     MetaDataException, VerificationException
 from fdroidserver.looseversion import LooseVersion
@@ -77,6 +77,26 @@ class CommonTest(unittest.TestCase):
         if os.path.exists(self.tmpdir):
             shutil.rmtree(self.tmpdir)
 
+    def test_yaml_1_2(self):
+        """Return a ruamel.yaml instance that supports YAML 1.2
+
+        There should be no "Norway Problem", and other things like this:
+        https://yaml.org/spec/1.2.2/ext/changes/
+
+        YAML 1.2 says "underlines _ cannot be used within numerical
+        values", but ruamel.yaml seems to ignore that. 1_0 should be a
+        string, but it is read as a 10.
+
+        """
+        os.chdir(self.testdir)
+        yaml12file = Path('YAML 1.2.yml')
+        yaml12file.write_text('[true, no, 0b010, 010, 0o10, "\\/"]', encoding='utf-8')
+        with yaml12file.open() as fp:
+            self.assertEqual(
+                [True, 'no', 2, 10, 8, '/'],
+                yaml.load(fp),
+            )
+
     def test_parse_human_readable_size(self):
         for k, v in (
             (9827, 9827),
@@ -417,7 +437,7 @@ class CommonTest(unittest.TestCase):
         metadata['RepoType'] = 'git'
         metadata['Repo'] = git_url
         with open(os.path.join('metadata', packageName + '.yml'), 'w') as fp:
-            yaml.dump(metadata, fp)
+            yaml_dumper.dump(metadata, fp)
 
         gitrepo = os.path.join(self.tmpdir, 'build', packageName)
         vcs0 = fdroidserver.common.getvcs('git', git_url, gitrepo)
@@ -1913,7 +1933,7 @@ class CommonTest(unittest.TestCase):
         os.chdir(self.tmpdir)
         teststr = '/πÇÇ现代通用字-български-عربي1/ö/yml'
         with open(fdroidserver.common.CONFIG_FILE, 'w', encoding='utf-8') as fp:
-            yaml.dump({'apksigner': teststr}, fp)
+            yaml_dumper.dump({'apksigner': teststr}, fp)
         self.assertTrue(os.path.exists(fdroidserver.common.CONFIG_FILE))
         config = fdroidserver.common.read_config()
         self.assertEqual(teststr, config.get('apksigner'))
@@ -1937,7 +1957,7 @@ class CommonTest(unittest.TestCase):
     def test_with_config_yml_is_not_mixed_type(self):
         os.chdir(self.tmpdir)
         Path(fdroidserver.common.CONFIG_FILE).write_text('k: v\napksigner = /bin/apk')
-        with self.assertRaises(yaml.scanner.ScannerError):
+        with self.assertRaises(ruamel.yaml.scanner.ScannerError):
             fdroidserver.common.read_config()
 
     def test_config_perm_warning(self):
@@ -2613,7 +2633,7 @@ class CommonTest(unittest.TestCase):
                 ' -providerClass sun.security.pkcs11.SunPKCS11'
                 ' -providerArg opensc-fdroid.cfg'
             }
-            yaml.dump(d, fp)
+            yaml_dumper.dump(d, fp)
         config = fdroidserver.common.read_config()
         fdroidserver.common.config = config
         self.assertTrue(isinstance(d['smartcardoptions'], str))
@@ -2829,21 +2849,21 @@ class CommonTest(unittest.TestCase):
 
     def test_parse_mirrors_config_str(self):
         s = 'foo@example.com:/var/www'
-        mirrors = ruamel.yaml.YAML(typ='safe').load("""'%s'""" % s)
+        mirrors = yaml.load("""'%s'""" % s)
         self.assertEqual(
             [{'url': s}], fdroidserver.common.parse_mirrors_config(mirrors)
         )
 
     def test_parse_mirrors_config_list(self):
         s = 'foo@example.com:/var/www'
-        mirrors = ruamel.yaml.YAML(typ='safe').load("""- '%s'""" % s)
+        mirrors = yaml.load("""- '%s'""" % s)
         self.assertEqual(
             [{'url': s}], fdroidserver.common.parse_mirrors_config(mirrors)
         )
 
     def test_parse_mirrors_config_dict(self):
         s = 'foo@example.com:/var/www'
-        mirrors = ruamel.yaml.YAML(typ='safe').load("""- url: '%s'""" % s)
+        mirrors = yaml.load("""- url: '%s'""" % s)
         self.assertEqual(
             [{'url': s}], fdroidserver.common.parse_mirrors_config(mirrors)
         )

tests/test_integration.py

@@ -11,13 +11,12 @@ from datetime import datetime, timezone
 from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
 
-from ruamel.yaml import YAML
-
 try:
     from androguard.core.bytecodes.apk import get_apkid  # androguard <4
 except ModuleNotFoundError:
     from androguard.core.apk import get_apkid
 
+from fdroidserver._yaml import yaml, yaml_dumper
 from .shared_test_code import mkdir_testfiles
 
 # TODO: port generic tests that use index.xml to index-v2 (test that
@@ -81,7 +80,6 @@ class IntegrationTest(unittest.TestCase):
     @staticmethod
     def update_yaml(path, items, replace=False):
         """Update a .yml file, e.g. config.yml, with the given items."""
-        yaml = YAML()
         doc = {}
         if not replace:
             try:
@@ -91,7 +89,7 @@ class IntegrationTest(unittest.TestCase):
                 pass
         doc.update(items)
         with open(path, "w") as f:
-            yaml.dump(doc, f)
+            yaml_dumper.dump(doc, f)
 
     @staticmethod
     def remove_lines(path, unwanted_strings):

tests/test_lint.py

@@ -7,13 +7,12 @@ import tempfile
 import unittest
 from pathlib import Path
 
-import ruamel.yaml
-
 from .shared_test_code import mkdtemp
 
 import fdroidserver.common
 import fdroidserver.lint
 import fdroidserver.metadata
+from fdroidserver._yaml import yaml_dumper
 
 basedir = Path(__file__).parent
 
@@ -365,40 +364,41 @@ class LintTest(unittest.TestCase):
 
     def test_lint_config_basic_mirrors_yml(self):
         os.chdir(self.testdir)
-        yaml = ruamel.yaml.YAML(typ='safe')
         with Path('mirrors.yml').open('w') as fp:
-            yaml.dump([{'url': 'https://example.com/fdroid/repo'}], fp)
+            yaml_dumper.dump([{'url': 'https://example.com/fdroid/repo'}], fp)
         self.assertTrue(fdroidserver.lint.lint_config('mirrors.yml'))
 
     def test_lint_config_mirrors_yml_kenya_countryCode(self):
         os.chdir(self.testdir)
-        yaml = ruamel.yaml.YAML(typ='safe')
         with Path('mirrors.yml').open('w') as fp:
-            yaml.dump([{'url': 'https://foo.com/fdroid/repo', 'countryCode': 'KE'}], fp)
+            yaml_dumper.dump(
+                [{'url': 'https://foo.com/fdroid/repo', 'countryCode': 'KE'}], fp
+            )
         self.assertTrue(fdroidserver.lint.lint_config('mirrors.yml'))
 
     def test_lint_config_mirrors_yml_invalid_countryCode(self):
         """WV is "indeterminately reserved" so it should never be used."""
         os.chdir(self.testdir)
-        yaml = ruamel.yaml.YAML(typ='safe')
         with Path('mirrors.yml').open('w') as fp:
-            yaml.dump([{'url': 'https://foo.com/fdroid/repo', 'countryCode': 'WV'}], fp)
+            yaml_dumper.dump(
+                [{'url': 'https://foo.com/fdroid/repo', 'countryCode': 'WV'}], fp
+            )
         self.assertFalse(fdroidserver.lint.lint_config('mirrors.yml'))
 
     def test_lint_config_mirrors_yml_alpha3_countryCode(self):
         """Only ISO 3166-1 alpha 2 are supported"""
         os.chdir(self.testdir)
-        yaml = ruamel.yaml.YAML(typ='safe')
         with Path('mirrors.yml').open('w') as fp:
-            yaml.dump([{'url': 'https://de.com/fdroid/repo', 'countryCode': 'DEU'}], fp)
+            yaml_dumper.dump(
+                [{'url': 'https://de.com/fdroid/repo', 'countryCode': 'DEU'}], fp
+            )
         self.assertFalse(fdroidserver.lint.lint_config('mirrors.yml'))
 
     def test_lint_config_mirrors_yml_one_invalid_countryCode(self):
         """WV is "indeterminately reserved" so it should never be used."""
         os.chdir(self.testdir)
-        yaml = ruamel.yaml.YAML(typ='safe')
         with Path('mirrors.yml').open('w') as fp:
-            yaml.dump(
+            yaml_dumper.dump(
                 [
                     {'url': 'https://bar.com/fdroid/repo', 'countryCode': 'BA'},
                     {'url': 'https://foo.com/fdroid/repo', 'countryCode': 'FO'},

tests/test_metadata.py

@@ -17,6 +17,7 @@ import fdroidserver
 from fdroidserver import metadata
 from fdroidserver.exception import MetaDataException
 from fdroidserver.common import DEFAULT_LOCALE
+from fdroidserver._yaml import yaml
 from .shared_test_code import TmpCwd, mkdtemp
 
 
@@ -178,7 +179,6 @@ class MetadataTest(unittest.TestCase):
     def test_valid_funding_yml_regex(self):
         """Check the regex can find all the cases"""
         with (basedir / 'funding-usernames.yaml').open() as fp:
-            yaml = ruamel.yaml.YAML(typ='safe')
             data = yaml.load(fp)
 
         for k, entries in data.items():
@@ -207,7 +207,6 @@ class MetadataTest(unittest.TestCase):
         fdroidserver.common.config = config
         fdroidserver.metadata.warnings_action = None
 
-        yaml = ruamel.yaml.YAML(typ='safe')
         apps = fdroidserver.metadata.read_metadata()
         for appid in (
             'app.with.special.build.params',
@@ -337,7 +336,6 @@ class MetadataTest(unittest.TestCase):
 
     def test_normalize_type_string_sha256(self):
         """SHA-256 values are TYPE_STRING, which YAML can parse as decimal ints."""
-        yaml = ruamel.yaml.YAML(typ='safe')
         for v in range(1, 1000):
             s = '%064d' % (v * (10**51))
             self.assertEqual(s, metadata._normalize_type_string(yaml.load(s)))
@@ -378,7 +376,6 @@ class MetadataTest(unittest.TestCase):
     def test_normalize_type_list(self):
         """TYPE_LIST is always a list of strings, no matter what YAML thinks."""
         k = 'placeholder'
-        yaml = ruamel.yaml.YAML(typ='safe')
         self.assertEqual(['1.0'], metadata._normalize_type_list(k, 1.0))
         self.assertEqual(['1234567890'], metadata._normalize_type_list(k, 1234567890))
         self.assertEqual(['false'], metadata._normalize_type_list(k, False))
@@ -441,7 +438,6 @@ class MetadataTest(unittest.TestCase):
     def test_post_parse_yaml_metadata_0padding_sha256(self):
         """SHA-256 values are strings, but YAML 1.2 will read some as decimal ints."""
         v = '0027293472934293872934729834729834729834729834792837487293847926'
-        yaml = ruamel.yaml.YAML(typ='safe')
         yamldata = yaml.load('AllowedAPKSigningKeys: ' + v)
         metadata.post_parse_yaml_metadata(yamldata)
         self.assertEqual(yamldata['AllowedAPKSigningKeys'], [v])
@@ -2287,7 +2283,6 @@ class PostMetadataParseTest(unittest.TestCase):
         maximum of two leading zeros, but this will handle more.
 
         """
-        yaml = ruamel.yaml.YAML(typ='safe', pure=True)
         str_sha256 = '0000000000000498456908409534729834729834729834792837487293847926'
         sha256 = yaml.load('a: ' + str_sha256)['a']
         self.assertEqual(*self._post_metadata_parse_app_int(sha256, int(str_sha256)))

tests/test_publish.py

@@ -13,7 +13,6 @@
 import json
 import os
 import pathlib
-import ruamel.yaml
 import shutil
 import sys
 import unittest
@@ -24,6 +23,7 @@ from fdroidserver import publish
 from fdroidserver import common
 from fdroidserver import metadata
 from fdroidserver import signatures
+from fdroidserver._yaml import yaml
 from fdroidserver.exception import FDroidException
 from .shared_test_code import mkdtemp, VerboseFalseOptions
 
@@ -116,7 +116,6 @@ class PublishTest(unittest.TestCase):
         }
         self.assertEqual(expected, common.load_stats_fdroid_signing_key_fingerprints())
 
-        yaml = ruamel.yaml.YAML(typ='safe')
         with open(common.CONFIG_FILE) as fp:
             config = yaml.load(fp)
         self.assertEqual(