Merge branch 'metadata-fixes-and-tests' into 'master'

metadata fixes and tests

Closes #261

See merge request !215

fdroidserver/common.py

@@ -1123,8 +1123,8 @@ def remove_debuggable_flags(root_dir):
                         os.path.join(root, 'AndroidManifest.xml'))
 
 
-vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
+vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
-vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
+vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
 
 
@@ -1160,12 +1160,11 @@ def parse_androidmanifests(paths, app):
             continue
 
         logging.debug("Parsing manifest at {0}".format(path))
-        gradle = has_extension(path, 'gradle')
         version = None
         vercode = None
         package = None
 
-        if gradle:
+        if has_extension(path, 'gradle'):
             with open(path, 'r') as f:
                 for line in f:
                     if gradle_comment.match(line):
@@ -1218,7 +1217,8 @@ def parse_androidmanifests(paths, app):
         if max_version is None and version is not None:
             max_version = version
 
-        if max_vercode is None or (vercode is not None and vercode > max_vercode):
+        if vercode is not None \
+           and (max_vercode is None or vercode > max_vercode):
             if not ignoresearch or not ignoresearch(version):
                 if version is not None:
                     max_version = version

fdroidserver/metadata.py

@@ -152,7 +152,7 @@ class App(dict):
         self.builds = []
         self.comments = {}
         self.added = None
-        self.lastupdated = None
+        self.lastUpdated = None
 
     def __getattr__(self, name):
         if name in self:
@@ -912,10 +912,10 @@ def parse_metadata(metadatapath, check_vcs=False):
         if os.path.isfile(metadata_in_repo):
             logging.debug('Including metadata from ' + metadata_in_repo)
             # do not include fields already provided by main metadata file
-            app_in_repo = parse_metadata(metadata_in_repo).field_dict()
+            app_in_repo = parse_metadata(metadata_in_repo)
             for k, v in app_in_repo.items():
-                if k not in app.field_dict():
+                if k not in app:
-                    app.set_field(k, v)
+                    app[k] = v
 
     post_metadata_parse(app)
 

fdroidserver/rewritemeta.py

@@ -45,13 +45,15 @@ def main():
 
     global config, options
 
+    supported = ['txt', 'yml']
+
     # Parse command line...
     parser = ArgumentParser(usage="%(prog)s [options] [APPID [APPID ...]]")
     common.setup_global_opts(parser)
     parser.add_argument("-l", "--list", action="store_true", default=False,
                         help="List files that would be reformatted")
     parser.add_argument("-t", "--to", default=None,
-                        help="Rewrite to a specific format")
+                        help="Rewrite to a specific format: " + ', '.join(supported))
     parser.add_argument("appid", nargs='*', help="app-id in the form APPID")
     metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
@@ -66,10 +68,8 @@ def main():
     if options.list and options.to is not None:
         parser.error("Cannot use --list and --to at the same time")
 
-    supported = ['txt', 'yml']
-
     if options.to is not None and options.to not in supported:
-        parser.error("Must give a valid format to --to")
+        parser.error("Unsupported metadata format, use: --to [" + ' '.join(supported) + "]")
 
     for appid, app in apps.items():
         base, ext = common.get_extension(app.metadatapath)

fdroidserver/update.py

@@ -115,7 +115,7 @@ def update_wiki(apps, sortedids, apks):
             appid,
             app.Name,
             time.strftime('%Y-%m-%d', app.added) if app.added else '',
-            time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
+            time.strftime('%Y-%m-%d', app.lastUpdated) if app.lastUpdated else '',
             app.SourceCode,
             app.IssueTracker,
             app.WebSite,
@@ -549,7 +549,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
         if file_extension == 'apk' or file_extension == 'obb':
             continue
         filename = os.path.join(repodir, name)
-        if not common.is_repo_file(name):
+        if not common.is_repo_file(filename):
             continue
         stat = os.stat(filename)
         if stat.st_size == 0:
@@ -1132,8 +1132,8 @@ def make_index(apps, sortedids, apks, repodir, archive):
         addElement('id', app.id, doc, apel)
         if app.added:
             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
-        if app.lastupdated:
+        if app.lastUpdated:
-            addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
+            addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastUpdated), doc, apel)
         addElement('name', app.Name, doc, apel)
         addElement('summary', app.Summary, doc, apel)
         if app.icon:
@@ -1612,12 +1612,12 @@ def main():
                 if 'added' in apk:
                     if not app.added or apk['added'] < app.added:
                         app.added = apk['added']
-                    if not app.lastupdated or apk['added'] > app.lastupdated:
+                    if not app.lastUpdated or apk['added'] > app.lastUpdated:
-                        app.lastupdated = apk['added']
+                        app.lastUpdated = apk['added']
 
         if not app.added:
             logging.debug("Don't know when " + appid + " was added")
-        if not app.lastupdated:
+        if not app.lastUpdated:
             logging.debug("Don't know when " + appid + " was last updated")
 
         if bestver == UNSET_VERSION_CODE:

tests/dump_internal_metadata_format.py

@@ -62,11 +62,14 @@ parser.add_option("-v", "--verbose", action="store_true", default=False,
                   help="Spew out even more information than normal")
 (fdroidserver.common.options, args) = parser.parse_args(['--verbose'])
 
-os.chdir('/home/hans/code/fdroid/fdroiddata')
+if not os.path.isdir('metadata'):
+    print("This script must be run in an F-Droid data folder with a 'metadata' subdir!")
+    sys.exit(1)
+
 # these need to be set to prevent code running on None, only
 # 'accepted_formats' is actually used in metadata.py
 config = dict()
-config['sdk_path'] = '/opt/android-sdk'
+config['sdk_path'] = os.getenv('ANDROID_HOME') or '/opt/android-sdk'
 config['ndk_paths'] = dict()
 config['accepted_formats'] = ['txt']
 fdroidserver.common.config = config
@@ -79,7 +82,6 @@ if not os.path.isdir(savedir):
 apps = fdroidserver.metadata.read_metadata(xref=True)
 for appid, app in apps.items():
     savepath = os.path.join(savedir, appid + '.yaml')
-    print('dumping', savepath)
     if hasattr(app, 'attr_to_field'):
         # for 0.7.0 and earlier, before https://gitlab.com/fdroid/fdroidserver/merge_requests/210
         app.__dict__['lastUpdated'] = app.__dict__['lastupdated']

tests/metadata/dump/org.adaway.yaml

@@ -1003,5 +1003,5 @@ comments:
   build:42:
   - '#RootCommands srclib needs changing on fdroidserver'
 id: org.adaway
-lastupdated: null
+lastUpdated: null
 metadatapath: metadata/org.adaway.json

tests/metadata/dump/org.smssecure.smssecure.yaml

@@ -335,5 +335,5 @@ builds:
   versionName: 0.6.0
 comments: {}
 id: org.smssecure.smssecure
-lastupdated: null
+lastUpdated: null
 metadatapath: metadata/org.smssecure.smssecure.txt

tests/metadata/dump/org.videolan.vlc.yaml

@@ -2243,5 +2243,5 @@ builds:
   versionName: 1.2.6
 comments: {}
 id: org.videolan.vlc
-lastupdated: null
+lastUpdated: null
 metadatapath: metadata/org.videolan.vlc.yml