purge config.py handling, it is no longer supported

.gitignore

@@ -27,7 +27,6 @@ tmp/
 /tests/repo/status
 
 # files used in manual testing
-/config.py
 /config.yml
 /tmp/
 /logs/

.gitlab-ci.yml

@@ -555,11 +555,11 @@ servergitmirrors:
     - ./tests/key-tricks.py
     - ssh-keyscan gitlab.com >> /root/.ssh/known_hosts
     - test -d /tmp/fdroid/repo || mkdir -p /tmp/fdroid/repo
-    - cp tests/config.py tests/keystore.jks /tmp/fdroid/
+    - cp tests/config.yml tests/keystore.jks /tmp/fdroid/
     - cp tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/
     - cd /tmp/fdroid
     - touch fdroid-icon.png
-    - printf "\nservergitmirrors = 'git@gitlab.com:fdroid/ci-test-servergitmirrors-repo.git'\n" >> config.py
+    - printf "\nservergitmirrors\x3a 'git@gitlab.com:fdroid/ci-test-servergitmirrors-repo.git'\n" >> config.yml
     - $PYTHONPATH/fdroid update --verbose --create-metadata
     - $PYTHONPATH/fdroid deploy --verbose
     - export DLURL=`grep -Eo 'https://gitlab.com/fdroid/ci-test-servergitmirrors-repo[^"]+' repo/index-v1.json`

MANIFEST.in

@@ -548,7 +548,7 @@ include tests/build-tools/28.0.3/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/generate.sh
 include tests/check-fdroid-apk
 include tests/com.fake.IpaApp_1000000000001.ipa
-include tests/config.py
+include tests/config.yml
 include tests/config/antiFeatures.yml
 include tests/config/categories.yml
 include tests/config/de/antiFeatures.yml

fdroidserver/common.py

@@ -531,8 +531,7 @@ def read_config():
     not required, just use defaults.
 
     config.yml is the preferred form because no code is executed when
-    reading it.  config.py is deprecated and supported for backwards
-    compatibility.
+    reading it.  config.py is deprecated and no longer supported.
 
     config.yml requires ASCII or UTF-8 encoding because this code does
     not auto-detect the file's encoding.  That is left up to the YAML
@@ -550,10 +549,6 @@ def read_config():
     config_file = 'config.yml'
     old_config_file = 'config.py'
 
-    if os.path.exists(config_file) and os.path.exists(old_config_file):
-        logging.error(_("""Conflicting config files! Using {newfile}, ignoring {oldfile}!""")
-                      .format(oldfile=old_config_file, newfile=config_file))
-
     if os.path.exists(config_file):
         logging.debug(_("Reading '{config_file}'").format(config_file=config_file))
         with open(config_file, encoding='utf-8') as fp:
@@ -561,19 +556,13 @@ def read_config():
         if not config:
             config = {}
         config_type_check(config_file, config)
-    elif os.path.exists(old_config_file):
-        logging.warning(_("""{oldfile} is deprecated, use {newfile}""")
-                        .format(oldfile=old_config_file, newfile=config_file))
-        with io.open(old_config_file, "rb") as fp:
-            code = compile(fp.read(), old_config_file, 'exec')
-            exec(code, None, config)  # nosec TODO automatically migrate
 
-        for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
-            if k in config:
-                if not type(config[k]) in (str, list, tuple):
+    if os.path.exists(old_config_file):
         logging.warning(
-                        _("'{field}' will be in random order! Use () or [] brackets if order is important!")
-                        .format(field=k))
+            _("""Ignoring deprecated {oldfile}, use {newfile}!""").format(
+                oldfile=old_config_file, newfile=config_file
+            )
+        )
 
     # smartcardoptions must be a list since its command line args for Popen
     smartcardoptions = config.get('smartcardoptions')
@@ -4203,7 +4192,7 @@ def load_stats_fdroid_signing_key_fingerprints():
 
 
 def write_to_config(thisconfig, key, value=None, config_file=None):
-    """Write a key/value to the local config.yml or config.py.
+    """Write a key/value to the local config.yml.
 
     NOTE: only supports writing string variables.
 
@@ -4222,8 +4211,6 @@ def write_to_config(thisconfig, key, value=None, config_file=None):
         value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
     if config_file:
         cfg = config_file
-    elif os.path.exists('config.py') and not os.path.exists('config.yml'):
-        cfg = 'config.py'
     else:
         cfg = 'config.yml'
 
@@ -4239,19 +4226,8 @@ def write_to_config(thisconfig, key, value=None, config_file=None):
         if not lines[-1].endswith('\n'):
             lines[-1] += '\n'
 
-    # regex for finding and replacing python string variable
-    # definitions/initializations
-    if cfg.endswith('.py'):
-        pattern = re.compile(r'^[\s#]*' + key + r'\s*=\s*"[^"]*"')
-        repl = key + ' = "' + value + '"'
-        pattern2 = re.compile(r'^[\s#]*' + key + r"\s*=\s*'[^']*'")
-        repl2 = key + " = '" + value + "'"
-    else:
-        # assume .yml as default
-        pattern = re.compile(r'^[\s#]*' + key + r':.*')
-        repl = yaml.dump({key: value}, default_flow_style=False)
-        pattern2 = pattern
-        repl2 = repl
+    pattern = re.compile(r'^[\s#]*' + key + r':.*\n')
+    repl = yaml.dump({key: value})
 
     # If we replaced this line once, we make sure won't be a
     # second instance of this line for this key in the document.
@@ -4259,18 +4235,15 @@ def write_to_config(thisconfig, key, value=None, config_file=None):
     # edit config file
     with open(cfg, 'w') as f:
         for line in lines:
-            if pattern.match(line) or pattern2.match(line):
+            if pattern.match(line):
                 if not didRepl:
                     line = pattern.sub(repl, line)
-                    line = pattern2.sub(repl2, line)
                     f.write(line)
                     didRepl = True
             else:
                 f.write(line)
         if not didRepl:
-            f.write('\n')
             f.write(repl)
-            f.write('\n')
 
 
 def parse_xml(path):

fdroidserver/init.py

@@ -138,7 +138,7 @@ def main():
             _("Android SDK not found at {path}!").format(path=test_config['sdk_path'])
         )
 
-    if not os.path.exists('config.yml') and not os.path.exists('config.py'):
+    if not os.path.exists('config.yml'):
         # 'metadata' and 'tmp' are created in fdroid
         if not os.path.exists('repo'):
             os.mkdir('repo')

tests/config.py

@@ -1,31 +0,0 @@
-
-# TODO convert to config.yml!
-
-repo_url = "https://MyFirstFDroidRepo.org/fdroid/repo"
-repo_name = "My First F-Droid Repo Demo"
-repo_description = """This is a repository of apps to be used with F-Droid. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitlab.com/fdroid."""
-
-archive_older = 3
-archive_url = "https://f-droid.org/archive"
-archive_name = "My First F-Droid Archive Demo"
-archive_description = """
-The repository of older versions of applications from the main demo repository.
-"""
-
-make_current_version_link = False
-
-repo_keyalias = "sova"
-keystore = "keystore.jks"
-keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="
-keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="
-keydname = "CN=sova, OU=F-Droid"
-
-mirrors = (
-    'http://foobarfoobarfoobar.onion/fdroid',
-    'https://foo.bar/fdroid',
-)
-
-install_list = 'org.adaway'
-uninstall_list = ('com.android.vending', 'com.facebook.orca', )
-
-repo_key_sha256 = "f49af3f11efddf20dffd70f5e3117b9976674167adca280e6b1932a0601b26f6"

tests/config.yml

@@ -0,0 +1,27 @@
+---
+
+repo_url: https://MyFirstFDroidRepo.org/fdroid/repo
+repo_name: My First F-Droid Repo Demo
+repo_description: This is a repository of apps to be used with F-Droid. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitlab.com/fdroid.
+
+archive_older: 3
+archive_url: https://f-droid.org/archive
+archive_name: My First F-Droid Archive Demo
+archive_description: The repository of older versions of applications from the main demo repository.
+
+make_current_version_link: false
+
+repo_keyalias: sova
+keystore: keystore.jks
+keystorepass: "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="
+keypass: "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="
+keydname: "CN=sova, OU=F-Droid"
+
+mirrors:
+  - http://foobarfoobarfoobar.onion/fdroid
+  - https://foo.bar/fdroid
+
+install_list: org.adaway
+uninstall_list: ['com.android.vending', 'com.facebook.orca']
+
+repo_key_sha256: f49af3f11efddf20dffd70f5e3117b9976674167adca280e6b1932a0601b26f6

tests/test_common.py

@@ -58,6 +58,9 @@ class CommonTest(unittest.TestCase):
             os.makedirs(self.tmpdir)
         os.chdir(basedir)
 
+        self.verbose = '-v' in sys.argv or '--verbose' in sys.argv
+        fdroidserver.common.set_console_logging(self.verbose)
+
         # these are declared as None at the top of the module file
         fdroidserver.common.config = None
         fdroidserver.common.options = None
@@ -642,65 +645,65 @@ class CommonTest(unittest.TestCase):
         )
 
     def test_write_to_config(self):
-        with tempfile.TemporaryDirectory() as tmpPath:
-            cfgPath = os.path.join(tmpPath, 'config.py')
-            with open(cfgPath, 'w') as f:
-                f.write(
+        """Test that config items can be added without messing up config.yml.
+
+        The '_orig' key are where the original string values of paths
+        are stored.  Paths have tilde expansion and env vars replaced
+        in fill_config_defaults().
+
+        """
+        os.chdir(self.testdir)
+        config_yml = 'config.yml'
+        Path(config_yml).write_text(
             textwrap.dedent(
                 """\
                 # abc
-                    # test = 'example value'
-                    default_me= '%%%'
+                # test: 'example value'
+                a_path: ~/android-sdk
 
                 # comment
-                    do_not_touch = "good value"
-                    default_me="!!!"
+                do_not_touch: good value
+                a_path: "!!!"
 
-                    key="123"    # inline"""
+                key: "123"    # inline"""
             )
         )
 
-            cfg = {'key': '111', 'default_me_orig': 'orig'}
-            fdroidserver.common.write_to_config(cfg, 'key', config_file=cfgPath)
-            fdroidserver.common.write_to_config(cfg, 'default_me', config_file=cfgPath)
-            fdroidserver.common.write_to_config(cfg, 'test', value='test value', config_file=cfgPath)
-            fdroidserver.common.write_to_config(cfg, 'new_key', value='new', config_file=cfgPath)
+        config = {'key': 111, 'a_path_orig': '~/android-sdk'}
+        fdroidserver.common.write_to_config(config, 'key', config_file=config_yml)
+        fdroidserver.common.write_to_config(
+            config, 'a_path', config_file=config_yml
+        )
+        fdroidserver.common.write_to_config(
+            config, 'test', value='test value', config_file=config_yml
+        )
+        fdroidserver.common.write_to_config(
+            config, 'new_key', value='new', config_file=config_yml
+        )
 
-            with open(cfgPath, 'r') as f:
+        with open(config_yml) as fp:
             self.assertEqual(
-                    f.read(),
+                fp.read(),
                 textwrap.dedent(
                     """\
                     # abc
-                    test = 'test value'
-                    default_me = 'orig'
+                    test: test value
+                    a_path: ~/android-sdk
 
                     # comment
-                    do_not_touch = "good value"
+                    do_not_touch: good value
 
-                    key = "111"    # inline
-
-                    new_key = "new"
+                    key: 111
+                    new_key: new
                     """
                 ),
             )
 
     def test_write_to_config_when_empty(self):
-        with tempfile.TemporaryDirectory() as tmpPath:
-            cfgPath = os.path.join(tmpPath, 'config.py')
-            with open(cfgPath, 'w') as f:
-                pass
-            fdroidserver.common.write_to_config({}, 'key', 'val', cfgPath)
-            with open(cfgPath, 'r') as f:
-                self.assertEqual(
-                    f.read(),
-                    textwrap.dedent(
-                        """\
-
-                key = "val"
-                """
-                    ),
-                )
+        config_yml = Path(self.testdir) / 'config.yml'
+        config_yml.write_text('')
+        fdroidserver.common.write_to_config({}, 'key', 'val', config_yml)
+        self.assertEqual(config_yml.read_text(), 'key: val\n')
 
     def test_apk_name_regex(self):
         good = [
@@ -1883,7 +1886,6 @@ class CommonTest(unittest.TestCase):
         """It should set defaults if no config file is found"""
         os.chdir(self.tmpdir)
         self.assertFalse(os.path.exists('config.yml'))
-        self.assertFalse(os.path.exists('config.py'))
         config = fdroidserver.common.read_config()
         self.assertIsNotNone(config.get('char_limits'))
 
@@ -1892,7 +1894,6 @@ class CommonTest(unittest.TestCase):
         os.chdir(self.tmpdir)
         open('config.yml', 'w').close()
         self.assertTrue(os.path.exists('config.yml'))
-        self.assertFalse(os.path.exists('config.py'))
         config = fdroidserver.common.read_config()
         self.assertIsNotNone(config.get('char_limits'))
 
@@ -1902,7 +1903,6 @@ class CommonTest(unittest.TestCase):
         with open('config.yml', 'w') as fp:
             fp.write('apksigner: yml')
         self.assertTrue(os.path.exists('config.yml'))
-        self.assertFalse(os.path.exists('config.py'))
         config = fdroidserver.common.read_config()
         self.assertEqual('yml', config.get('apksigner'))
 
@@ -1913,7 +1913,6 @@ class CommonTest(unittest.TestCase):
         with open('config.yml', 'w', encoding='utf-8') as fp:
             fp.write('apksigner: ' + teststr)
         self.assertTrue(os.path.exists('config.yml'))
-        self.assertFalse(os.path.exists('config.py'))
         config = fdroidserver.common.read_config()
         self.assertEqual(teststr, config.get('apksigner'))
 
@@ -1924,7 +1923,6 @@ class CommonTest(unittest.TestCase):
         with open('config.yml', 'w') as fp:
             yaml.dump({'apksigner': teststr}, fp)
         self.assertTrue(os.path.exists('config.yml'))
-        self.assertFalse(os.path.exists('config.py'))
         config = fdroidserver.common.read_config()
         self.assertEqual(teststr, config.get('apksigner'))
 
@@ -1936,7 +1934,6 @@ class CommonTest(unittest.TestCase):
             with open('config.yml', 'w') as fp:
                 fp.write("""keypass: {'env': 'SECRET'}""")
             self.assertTrue(os.path.exists('config.yml'))
-            self.assertFalse(os.path.exists('config.py'))
             config = fdroidserver.common.read_config()
             self.assertEqual(os.getenv('SECRET', 'fail'), config.get('keypass'))
 
@@ -1952,16 +1949,6 @@ class CommonTest(unittest.TestCase):
         with self.assertRaises(yaml.scanner.ScannerError):
             fdroidserver.common.read_config()
 
-    def test_with_config_py(self):
-        """Make sure it is still possible to use config.py alone."""
-        os.chdir(self.tmpdir)
-        with open('config.py', 'w') as fp:
-            fp.write('apksigner = "py"')
-        self.assertFalse(os.path.exists('config.yml'))
-        self.assertTrue(os.path.exists('config.py'))
-        config = fdroidserver.common.read_config()
-        self.assertEqual("py", config.get('apksigner'))
-
     def test_config_perm_warning(self):
         """Exercise the code path that issues a warning about unsafe permissions."""
         os.chdir(self.tmpdir)
@@ -1973,24 +1960,6 @@ class CommonTest(unittest.TestCase):
         os.remove(fp.name)
         fdroidserver.common.config = None
 
-        with open('config.py', 'w') as fp:
-            fp.write('keystore = "foo.jks"')
-        self.assertTrue(os.path.exists(fp.name))
-        os.chmod(fp.name, 0o666)  # nosec B103
-        fdroidserver.common.read_config()
-
-    def test_with_both_config_yml_py(self):
-        """If config.yml and config.py are present, config.py should be ignored."""
-        os.chdir(self.tmpdir)
-        with open('config.yml', 'w') as fp:
-            fp.write('apksigner: yml')
-        with open('config.py', 'w') as fp:
-            fp.write('apksigner = "py"')
-        self.assertTrue(os.path.exists('config.yml'))
-        self.assertTrue(os.path.exists('config.py'))
-        config = fdroidserver.common.read_config()
-        self.assertEqual('yml', config.get('apksigner'))
-
     def test_config_repo_url(self):
         """repo_url ends in /repo, archive_url ends in /archive."""
         os.chdir(self.tmpdir)
@@ -2037,28 +2006,12 @@ class CommonTest(unittest.TestCase):
         os.chdir(self.tmpdir)
         with open('config.yml', 'w') as fp:
             fp.write('apksigner: yml')
+        os.chmod('config.yml', 0o0600)
         self.assertTrue(os.path.exists(fp.name))
-        self.assertFalse(os.path.exists('config.py'))
         config = fdroidserver.common.read_config()
         self.assertFalse('keypass' in config)
         self.assertEqual('yml', config.get('apksigner'))
         fdroidserver.common.write_to_config(config, 'keypass', 'mysecretpassword')
-        with open(fp.name) as fp:
-            print(fp.read())
-        fdroidserver.common.config = None
-        config = fdroidserver.common.read_config()
-        self.assertEqual('mysecretpassword', config['keypass'])
-
-    def test_write_to_config_py(self):
-        os.chdir(self.tmpdir)
-        with open('config.py', 'w') as fp:
-            fp.write('apksigner = "py"')
-        self.assertTrue(os.path.exists(fp.name))
-        self.assertFalse(os.path.exists('config.yml'))
-        config = fdroidserver.common.read_config()
-        self.assertFalse('keypass' in config)
-        self.assertEqual('py', config.get('apksigner'))
-        fdroidserver.common.write_to_config(config, 'keypass', 'mysecretpassword')
         fdroidserver.common.config = None
         config = fdroidserver.common.read_config()
         self.assertEqual('mysecretpassword', config['keypass'])
@@ -2068,7 +2021,6 @@ class CommonTest(unittest.TestCase):
         with open('config.yml', 'w') as fp:
             fp.write('java_paths:\n  8: /usr/lib/jvm/java-8-openjdk\n')
         self.assertTrue(os.path.exists(fp.name))
-        self.assertFalse(os.path.exists('config.py'))
         config = fdroidserver.common.read_config()
         self.assertEqual('/usr/lib/jvm/java-8-openjdk', config.get('java_paths', {}).get('8'))
 
@@ -2143,10 +2095,11 @@ class CommonTest(unittest.TestCase):
 
     def test_loading_config_buildserver_yml(self):
         """Smoke check to make sure this file is properly parsed"""
-        os.chdir(self.tmpdir)
-        shutil.copy(os.path.join(basedir, '..', 'buildserver', 'config.buildserver.yml'),
-                    'config.yml')
-        self.assertFalse(os.path.exists('config.py'))
+        os.chdir(self.testdir)
+        shutil.copy(
+            os.path.join(basedir, '..', 'buildserver', 'config.buildserver.yml'),
+            'config.yml',
+        )
         fdroidserver.common.read_config()
 
     def test_setup_status_output(self):
@@ -2742,35 +2695,6 @@ class CommonTest(unittest.TestCase):
             config['smartcardoptions'],
         )
 
-    def test_get_smartcardoptions_config_py(self):
-        os.chdir(self.tmpdir)
-        with open('config.py', 'w') as fp:
-            fp.write(
-                textwrap.dedent(
-                    """
-                    smartcardoptions = '''
-                    \t-storetype\tPKCS11
-                    \t-providerClass\tsun.security.pkcs11.SunPKCS11
-                    \t-providerArg\t/etc/pkcs11_java.cfg
-
-                     '''
-                    """
-                )
-            )
-        config = fdroidserver.common.read_config()
-        fdroidserver.common.config = config
-        self.assertEqual(
-            [
-                '-storetype',
-                'PKCS11',
-                '-providerClass',
-                'sun.security.pkcs11.SunPKCS11',
-                '-providerArg',
-                '/etc/pkcs11_java.cfg',
-            ],
-            config['smartcardoptions'],
-        )
-
     def test_load_localized_config(self):
         """It should load"""
         antiFeatures = fdroidserver.common.load_localized_config(

tests/test_index.py

@@ -38,8 +38,8 @@ class IndexTest(unittest.TestCase):
         cls.index_v1_jar = basedir / 'repo' / 'index-v1.jar'
 
     def setUp(self):
-        (basedir / 'config.py').chmod(0o600)
-        os.chdir(basedir)  # so read_config() can find config.py
+        (basedir / 'config.yml').chmod(0o600)
+        os.chdir(basedir)  # so read_config() can find config.yml
 
         common.config = None
         common.options = Options
@@ -380,8 +380,7 @@ class IndexTest(unittest.TestCase):
             with zipfile.ZipFile(jarfile, 'w', zipfile.ZIP_DEFLATED) as jar:
                 jar.writestr('publishsigkeys.json', json.dumps(sigkeyfps))
             publish.sign_sig_key_fingerprint_list(jarfile)
-            with open('config.py', 'w'):
-                pass
+            Path('config.yml').write_text('')
 
             index.v1_sort_packages(
                 i, common.load_stats_fdroid_signing_key_fingerprints()
@@ -701,7 +700,7 @@ class IndexTest(unittest.TestCase):
         )
 
     def test_add_mirrors_to_repodict(self):
-        """Test based on the contents of tests/config.py"""
+        """Test based on the contents of tests/config.yml"""
         repodict = {'address': common.config['repo_url']}
         index.add_mirrors_to_repodict('repo', repodict)
         self.assertEqual(

tests/test_nightly.py

@@ -250,7 +250,6 @@ class NightlyTest(unittest.TestCase):
                         raise
 
         self.assertEqual(called, [['ssh', '-Tvi'], ['fdroid', 'deploy']])
-        self.assertFalse(os.path.exists('config.py'))
         git_url = 'git@github.com:f-droid/test-nightly'
         mirror_url = index.get_mirror_service_urls({"url": git_url})[0]
         expected = {
@@ -324,7 +323,6 @@ class NightlyTest(unittest.TestCase):
                         raise
 
         self.assertEqual(called, [['ssh', '-Tvi'], ['fdroid', 'deploy']])
-        self.assertFalse(os.path.exists('config.py'))
         expected = {
             'archive_description': 'Old nightly builds that have been archived.',
             'archive_name': 'fdroid/test-nightly archive',

tests/test_publish.py

@@ -13,10 +13,12 @@
 import json
 import os
 import pathlib
+import ruamel.yaml
 import shutil
 import sys
 import unittest
 import tempfile
+from pathlib import Path
 from unittest import mock
 
 from fdroidserver import publish
@@ -94,8 +96,7 @@ class PublishTest(unittest.TestCase):
         ]
 
         os.chdir(self.testdir)
-        with open('config.py', 'w') as f:
-            pass
+        Path('config.yml').write_text('')
 
         publish.store_stats_fdroid_signing_key_fingerprints(appids, indent=2)
 
@@ -116,10 +117,12 @@ class PublishTest(unittest.TestCase):
         }
         self.assertEqual(expected, common.load_stats_fdroid_signing_key_fingerprints())
 
-        with open('config.py', 'r') as f:
+        yaml = ruamel.yaml.YAML(typ='safe')
+        with open('config.yml') as fp:
+            config = yaml.load(fp)
         self.assertEqual(
-                '\nrepo_key_sha256 = "c58460800c7b250a619c30c13b07b7359a43e5af71a4352d86c58ae18c9f6d41"\n',
-                f.read(),
+            'c58460800c7b250a619c30c13b07b7359a43e5af71a4352d86c58ae18c9f6d41',
+            config['repo_key_sha256'],
         )
 
     def test_store_and_load_fdroid_signing_key_fingerprints_with_missmatch(self):