update: test metadata creation using internal and external templates

There is a hardcoded template in update.py, and there is also the
possibility for the user to create a template.yml.  This tests both of them
and cleans up the related code a bit.

#352 !310

fdroidserver/update.py

@@ -1638,6 +1638,53 @@ def add_apks_to_per_app_repos(repodir, apks):
             shutil.copy(apkascpath, apk['per_app_repo'])
 
 
+def create_metadata_from_template(apk):
+    '''create a new metadata file using internal or external template
+
+    Generate warnings for apk's with no metadata (or create skeleton
+    metadata files, if requested on the command line).  Though the
+    template file is YAML, this uses neither pyyaml nor ruamel.yaml
+    since those impose things on the metadata file made from the
+    template: field sort order, empty field value, formatting, etc.
+    '''
+
+    import yaml
+    if os.path.exists('template.yml'):
+        with open('template.yml') as f:
+            metatxt = f.read()
+        if 'name' in apk and apk['name'] != '':
+            metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
+                             r'\1 ' + apk['name'],
+                             metatxt,
+                             flags=re.IGNORECASE | re.MULTILINE)
+        else:
+            logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
+            metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
+                             r'\1 ' + apk['packageName'],
+                             metatxt,
+                             flags=re.IGNORECASE | re.MULTILINE)
+        with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
+            f.write(metatxt)
+    else:
+        app = dict()
+        app['Categories'] = [os.path.basename(os.getcwd())]
+        # include some blanks as part of the template
+        app['AuthorName'] = ''
+        app['Summary'] = ''
+        app['WebSite'] = ''
+        app['IssueTracker'] = ''
+        app['SourceCode'] = ''
+        app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
+        if 'name' in apk and apk['name'] != '':
+            app['Name'] = apk['name']
+        else:
+            logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
+            app['Name'] = apk['packageName']
+        with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
+            yaml.dump(app, f, default_flow_style=False)
+    logging.info("Generated skeleton metadata for " + apk['packageName'])
+
+
 config = None
 options = None
 
@@ -1757,47 +1804,11 @@ def main():
                                            options.use_date_from_apk)
     cachechanged = cachechanged or fcachechanged
     apks += files
-    # Generate warnings for apk's with no metadata (or create skeleton
-    # metadata files, if requested on the command line)
-    newmetadata = False
     for apk in apks:
         if apk['packageName'] not in apps:
             if options.create_metadata:
-                import yaml
+                create_metadata_from_template(apk)
-                with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
+                apps = metadata.read_metadata()
-                    # this should use metadata.App() and
-                    # metadata.write_yaml(), but since ruamel.yaml
-                    # 0.13 is not widely distributed yet, and it's
-                    # special tricks are not really needed here, this
-                    # uses the plain YAML lib
-                    if os.path.exists('template.yml'):
-                        with open('template.yml') as f:
-                            metatxt = f.read()
-                        if 'name' in apk and apk['name'] != '':
-                            metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$', r'\1 ' + apk['name'], metatxt, flags=re.IGNORECASE | re.MULTILINE)
-                        else:
-                            logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
-                            metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$', r'\1 ' + apk['packageName'], metatxt, flags=re.IGNORECASE | re.MULTILINE)
-                        with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
-                            f.write(metatxt)
-                    else:
-                        app = dict()
-                        app['Categories'] = [os.path.basename(os.getcwd())]
-                        # include some blanks as part of the template
-                        app['AuthorName'] = ''
-                        app['Summary'] = ''
-                        app['WebSite'] = ''
-                        app['IssueTracker'] = ''
-                        app['SourceCode'] = ''
-                        app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
-                        if 'name' in apk and apk['name'] != '':
-                            app['Name'] = apk['name']
-                        else:
-                            logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
-                            app['Name'] = apk['packageName']
-                        yaml.dump(app, f, default_flow_style=False)
-                    logging.info("Generated skeleton metadata for " + apk['packageName'])
-                    newmetadata = True
             else:
                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
                 if options.delete_unknown:
@@ -1810,10 +1821,6 @@ def main():
                 else:
                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
 
-    # update the metadata with the newly created ones included
-    if newmetadata:
-        apps = metadata.read_metadata()
-
     copy_triple_t_store_metadata(apps)
     insert_obbs(repodirs[0], apps, apks)
     insert_localized_app_metadata(apps)

tests/update.TestCase

@@ -491,6 +491,65 @@ class UpdateTest(unittest.TestCase):
                 foundtest = True
         self.assertTrue(foundtest)
 
+    def test_create_metadata_from_template(self):
+        tmpdir = os.path.join(localmodule, '.testfiles')
+        if not os.path.exists(tmpdir):
+            os.makedirs(tmpdir)
+        tmptestsdir = tempfile.mkdtemp(prefix='test_create_metadata_from_template-',
+                                       dir=tmpdir)
+        print('tmptestsdir', tmptestsdir)
+        os.chdir(tmptestsdir)
+        os.mkdir('repo')
+        shutil.copy(os.path.join(localmodule, 'tests', 'urzip.apk'), 'repo')
+
+        config = dict()
+        fdroidserver.common.fill_config_defaults(config)
+        config['ndk_paths'] = dict()
+        config['accepted_formats'] = ['json', 'txt', 'yml']
+        fdroidserver.common.config = config
+        fdroidserver.update.config = config
+
+        fdroidserver.update.options = type('', (), {})()
+        fdroidserver.update.options.clean = True
+        fdroidserver.update.options.delete_unknown = False
+        fdroidserver.update.options.rename_apks = False
+        fdroidserver.update.options.allow_disabled_algorithms = False
+
+        apps = fdroidserver.metadata.read_metadata(xref=True)
+        self.assertEqual(0, len(apps))
+        knownapks = fdroidserver.common.KnownApks()
+        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False)
+        self.assertEqual(1, len(apks))
+        apk = apks[0]
+
+        # test using internal template
+        fdroidserver.update.create_metadata_from_template(apk)
+        self.assertTrue(os.path.exists('metadata/info.guardianproject.urzip.yml'))
+        apps = fdroidserver.metadata.read_metadata(xref=True)
+        self.assertEqual(1, len(apps))
+        for app in apps.values():
+            self.assertEqual('urzip', app['Name'])
+            self.assertEqual(1, len(app['Categories']))
+            break
+
+        # test using external template.yml
+        os.remove('metadata/info.guardianproject.urzip.yml')
+        self.assertFalse(os.path.exists('metadata/info.guardianproject.urzip.yml'))
+        shutil.copy(os.path.join(localmodule, 'examples', 'template.yml'), tmptestsdir)
+        fdroidserver.update.create_metadata_from_template(apk)
+        self.assertTrue(os.path.exists('metadata/info.guardianproject.urzip.yml'))
+        apps = fdroidserver.metadata.read_metadata(xref=True)
+        self.assertEqual(1, len(apps))
+        for app in apps.values():
+            self.assertEqual('urzip', app['Name'])
+            self.assertEqual(1, len(app['Categories']))
+            self.assertEqual('Internet', app['Categories'][0])
+            break
+        with open('metadata/info.guardianproject.urzip.yml') as fp:
+            data = yaml.load(fp)
+        self.assertEqual('urzip', data['Name'])
+        self.assertEqual('urzip', data['Summary'])
+
 
 if __name__ == "__main__":
     parser = optparse.OptionParser()