metadata: remove git repo handling from metadata parsing

This is a vestige of implementing builds from a .fdroid.yml file directly
in the app's source repo.  It was never fully complete and seems to not be
used in any apps in fdroiddata.  This makes `fdroid build --all` runs much
faster since it does not need to do any git handling for apps that do not
have any new builds to run.

4e8e29794
948689281
a4e431080
e37be9b06
e775d330c

examples/fdroid_fetchsrclibs.py

@@ -21,12 +21,14 @@ def main():
     options = parser.parse_args()
     common.options = options
     pkgs = common.read_pkg_args(options.appid, True)
-    allapps = metadata.read_metadata(pkgs, check_vcs=True)
+    allapps = metadata.read_metadata(pkgs)
     apps = common.read_app_args(options.appid, allapps, True)
     srclib_dir = os.path.join('build', 'srclib')
     os.makedirs(srclib_dir, exist_ok=True)
     srclibpaths = []
     for appid, app in apps.items():
+        vcs, _ignored = common.setup_vcs(app)
+        vcs.gotorevision('HEAD', refresh=False)
         for build in app.get('Builds', []):
             for lib in build.srclibs:
                 srclibpaths.append(common.getsrclib(lib, srclib_dir, build=build))

fdroidserver/build.py

@@ -1012,7 +1012,7 @@ def main():
 
     # Read all app and srclib metadata
     pkgs = common.read_pkg_args(options.appid, True)
-    allapps = metadata.read_metadata(pkgs, options.refresh, sort_by_time=True, check_vcs=True)
+    allapps = metadata.read_metadata(pkgs, sort_by_time=True)
     apps = common.read_app_args(options.appid, allapps, True)
 
     for appid, app in list(apps.items()):

fdroidserver/metadata.py

@@ -18,6 +18,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import git
 import os
 import re
 import glob
@@ -541,7 +542,7 @@ def read_srclibs():
         srclibs[srclibname] = parse_yaml_srclib(metadatapath)
 
 
-def read_metadata(appids={}, refresh=True, sort_by_time=False, check_vcs=False):
+def read_metadata(appids={}, sort_by_time=False):
     """Return a list of App instances sorted newest first
 
     This reads all of the metadata files in a 'data' repository, then
@@ -597,7 +598,7 @@ def read_metadata(appids={}, refresh=True, sort_by_time=False, check_vcs=False):
         if appid in apps:
             _warn_or_exception(_("Found multiple metadata files for {appid}")
                                .format(appid=appid))
-        app = parse_metadata(metadatapath, check_vcs, refresh)
+        app = parse_metadata(metadatapath)
         check_metadata(app)
         apps[app.id] = app
 
@@ -730,15 +731,22 @@ def _decode_bool(s):
     _warn_or_exception(_("Invalid boolean '%s'") % s)
 
 
-def parse_metadata(metadatapath, check_vcs=False, refresh=True):
+def parse_metadata(metadatapath):
-    '''parse metadata file, optionally checking the git repo for metadata first'''
+    """parse metadata file, also checking the source repo for .fdroid.yml
+
+    If this is a metadata file from fdroiddata, it will first load the
+    source repo type and URL from fdroiddata, then read .fdroid.yml if
+    it exists, then include the rest of the metadata as specified in
+    fdroiddata, so that fdroiddata has precedence over the metadata in
+    the source code.
+
+    """
 
     app = App()
     app.metadatapath = metadatapath
-    name, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
+    metadata_file = os.path.basename(metadatapath)
-    if name == '.fdroid':
+    name, _ignored = fdroidserver.common.get_extension(metadata_file)
-        check_vcs = False
+    if name != '.fdroid':
-    else:
         app.id = name
 
     if metadatapath.endswith('.yml'):
@@ -748,16 +756,15 @@ def parse_metadata(metadatapath, check_vcs=False, refresh=True):
         _warn_or_exception(_('Unknown metadata format: {path} (use: *.yml)')
                            .format(path=metadatapath))
 
-    if check_vcs and app.Repo:
+    if metadata_file != '.fdroid.yml' and app.Repo:
         build_dir = fdroidserver.common.get_build_dir(app)
         metadata_in_repo = os.path.join(build_dir, '.fdroid.yml')
-        if not os.path.isfile(metadata_in_repo):
-            vcs, build_dir = fdroidserver.common.setup_vcs(app)
-            if isinstance(vcs, fdroidserver.common.vcs_git):
-                vcs.gotorevision('HEAD', refresh)  # HEAD since we can't know where else to go
         if os.path.isfile(metadata_in_repo):
-            logging.debug('Including metadata from ' + metadata_in_repo)
+            try:
-            # do not include fields already provided by main metadata file
+                commit_id = fdroidserver.common.get_head_commit_id(git.repo.Repo(build_dir))
+                logging.debug(_('Including metadata from %s@%s') % (metadata_in_repo, commit_id))
+            except git.exc.InvalidGitRepositoryError:
+                logging.debug(_('Including metadata from {path}').format(metadata_in_repo))
             app_in_repo = parse_metadata(metadata_in_repo)
             for k, v in app_in_repo.items():
                 if k not in app:
@@ -779,6 +786,17 @@ def parse_metadata(metadatapath, check_vcs=False, refresh=True):
 
 
 def parse_yaml_metadata(mf, app):
+    """Parse the .yml file and post-process it
+
+    Clean metadata .yml files can be used directly, but in order to
+    make a better user experience for people editing .yml files, there
+    is post processing.  .fdroid.yml is embedded in the app's source
+    repo, so it is "user-generated".  That means that it can have
+    weird things in it that need to be removed so they don't break the
+    overall process.
+
+    """
+
     try:
         yamldata = yaml.load(mf, Loader=SafeLoader)
     except yaml.YAMLError as e:
@@ -791,13 +809,15 @@ def parse_yaml_metadata(mf, app):
     deprecated_in_yaml = ['Provides']
 
     if yamldata:
-        for field in yamldata:
+        for field in tuple(yamldata.keys()):
-            if field not in yaml_app_fields:
+            if field not in yaml_app_fields + deprecated_in_yaml:
-                if field not in deprecated_in_yaml:
+                msg = (_("Unrecognised app field '{fieldname}' in '{path}'")
-                    _warn_or_exception(_("Unrecognised app field "
+                       .format(fieldname=field, path=mf.name))
-                                         "'{fieldname}' in '{path}'")
+                if os.path.basename(mf.name) == '.fdroid.yml':
-                                       .format(fieldname=field,
+                    logging.error(msg)
-                                               path=mf.name))
+                    del yamldata[field]
+                else:
+                    _warn_or_exception(msg)
 
         for deprecated_field in deprecated_in_yaml:
             if deprecated_field in yamldata:

tests/index.TestCase

@@ -289,7 +289,7 @@ class IndexTest(unittest.TestCase):
             'timestamp': datetime.datetime.now(),
             'version': 12,
         }
-        app = fdroidserver.metadata.parse_metadata(metadatafile, False, False)
+        app = fdroidserver.metadata.parse_metadata(metadatafile)
         app['icon'] = 'info.zwanenburg.caffeinetile.4.xml'
         app['CurrentVersionCode'] = '4'
         apps = {app.id: app}