From 8b85c860a6dddd7c05317455592f4a129c9c8a9c Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 28 Nov 2016 21:17:22 +0100 Subject: [PATCH 1/7] rename lastupdated to lastUpdated to match fdroid client This lets index-v1 be parsed directly into class instances because the field/instance var names match exactly. The original index v0 element must retain the 'lastupdated' name for backwards compatibility. --- fdroidserver/metadata.py | 2 +- fdroidserver/update.py | 12 ++++++------ tests/metadata/dump/org.adaway.yaml | 2 +- tests/metadata/dump/org.smssecure.smssecure.yaml | 2 +- tests/metadata/dump/org.videolan.vlc.yaml | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index a13e3945..31128c41 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -152,7 +152,7 @@ class App(dict): self.builds = [] self.comments = {} self.added = None - self.lastupdated = None + self.lastUpdated = None def __getattr__(self, name): if name in self: diff --git a/fdroidserver/update.py b/fdroidserver/update.py index ea536152..ba5fd706 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -115,7 +115,7 @@ def update_wiki(apps, sortedids, apks): appid, app.Name, time.strftime('%Y-%m-%d', app.added) if app.added else '', - time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '', + time.strftime('%Y-%m-%d', app.lastUpdated) if app.lastUpdated else '', app.SourceCode, app.IssueTracker, app.WebSite, @@ -1132,8 +1132,8 @@ def make_index(apps, sortedids, apks, repodir, archive): addElement('id', app.id, doc, apel) if app.added: addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel) - if app.lastupdated: - addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel) + if app.lastUpdated: + addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastUpdated), doc, apel) addElement('name', app.Name, doc, apel) addElement('summary', app.Summary, doc, apel) if app.icon: @@ -1612,12 +1612,12 @@ def main(): if 'added' in apk: if not app.added or apk['added'] < app.added: app.added = apk['added'] - if not app.lastupdated or apk['added'] > app.lastupdated: - app.lastupdated = apk['added'] + if not app.lastUpdated or apk['added'] > app.lastUpdated: + app.lastUpdated = apk['added'] if not app.added: logging.debug("Don't know when " + appid + " was added") - if not app.lastupdated: + if not app.lastUpdated: logging.debug("Don't know when " + appid + " was last updated") if bestver == UNSET_VERSION_CODE: diff --git a/tests/metadata/dump/org.adaway.yaml b/tests/metadata/dump/org.adaway.yaml index 2be4c114..7f6d1589 100644 --- a/tests/metadata/dump/org.adaway.yaml +++ b/tests/metadata/dump/org.adaway.yaml @@ -1003,5 +1003,5 @@ comments: build:42: - '#RootCommands srclib needs changing on fdroidserver' id: org.adaway -lastupdated: null +lastUpdated: null metadatapath: metadata/org.adaway.json diff --git a/tests/metadata/dump/org.smssecure.smssecure.yaml b/tests/metadata/dump/org.smssecure.smssecure.yaml index b8b4e692..0fdd3372 100644 --- a/tests/metadata/dump/org.smssecure.smssecure.yaml +++ b/tests/metadata/dump/org.smssecure.smssecure.yaml @@ -335,5 +335,5 @@ builds: versionName: 0.6.0 comments: {} id: org.smssecure.smssecure -lastupdated: null +lastUpdated: null metadatapath: metadata/org.smssecure.smssecure.txt diff --git a/tests/metadata/dump/org.videolan.vlc.yaml b/tests/metadata/dump/org.videolan.vlc.yaml index 4e0b5c9e..d905830c 100644 --- a/tests/metadata/dump/org.videolan.vlc.yaml +++ b/tests/metadata/dump/org.videolan.vlc.yaml @@ -2243,5 +2243,5 @@ builds: versionName: 1.2.6 comments: {} id: org.videolan.vlc -lastupdated: null +lastUpdated: null metadatapath: metadata/org.videolan.vlc.yml From d92ac1e8f6906ef40c08ade3753533e4bbc3a26f Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 28 Feb 2017 11:32:51 +0100 Subject: [PATCH 2/7] fix metadata dump test to run on CI servers --- tests/dump_internal_metadata_format.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/dump_internal_metadata_format.py b/tests/dump_internal_metadata_format.py index 4dc95890..6b657b06 100755 --- a/tests/dump_internal_metadata_format.py +++ b/tests/dump_internal_metadata_format.py @@ -62,11 +62,14 @@ parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") (fdroidserver.common.options, args) = parser.parse_args(['--verbose']) -os.chdir('/home/hans/code/fdroid/fdroiddata') +if not os.path.isdir('metadata'): + print("This script must be run in an F-Droid data folder with a 'metadata' subdir!") + sys.exit(1) + # these need to be set to prevent code running on None, only # 'accepted_formats' is actually used in metadata.py config = dict() -config['sdk_path'] = '/opt/android-sdk' +config['sdk_path'] = os.getenv('ANDROID_HOME') or '/opt/android-sdk' config['ndk_paths'] = dict() config['accepted_formats'] = ['txt'] fdroidserver.common.config = config @@ -79,7 +82,6 @@ if not os.path.isdir(savedir): apps = fdroidserver.metadata.read_metadata(xref=True) for appid, app in apps.items(): savepath = os.path.join(savedir, appid + '.yaml') - print('dumping', savepath) if hasattr(app, 'attr_to_field'): # for 0.7.0 and earlier, before https://gitlab.com/fdroid/fdroidserver/merge_requests/210 app.__dict__['lastUpdated'] = app.__dict__['lastupdated'] From 507ed4a1704b0cb1fd6110d4c961770e843a6065 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 27 Feb 2017 13:38:59 +0100 Subject: [PATCH 3/7] do not crash if no versionCode is found while parsing if vercode is None and max_vercode is None, then it gave None as an arg to the regexp pattern in ignoresearch(). closes #261 --- fdroidserver/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 5321b7b6..c13e2f94 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1160,12 +1160,11 @@ def parse_androidmanifests(paths, app): continue logging.debug("Parsing manifest at {0}".format(path)) - gradle = has_extension(path, 'gradle') version = None vercode = None package = None - if gradle: + if has_extension(path, 'gradle'): with open(path, 'r') as f: for line in f: if gradle_comment.match(line): @@ -1218,7 +1217,8 @@ def parse_androidmanifests(paths, app): if max_version is None and version is not None: max_version = version - if max_vercode is None or (vercode is not None and vercode > max_vercode): + if vercode is not None \ + and (max_vercode is None or vercode > max_vercode): if not ignoresearch or not ignoresearch(version): if version is not None: max_version = version From 564cfb108971f00bbbbed4a5567c74bb8d821ea5 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 27 Feb 2017 14:00:42 +0100 Subject: [PATCH 4/7] match versionCode/versionName when set as gradle variable For example, see org.kontalk: allprojects { ext { appVersionCode = 221 appVersionName = '4.0.1-preview' } } --- fdroidserver/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index c13e2f94..e01b7115 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1123,8 +1123,8 @@ def remove_debuggable_flags(root_dir): os.path.join(root, 'AndroidManifest.xml')) -vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search -vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search +vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search +vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search From f06e33690455defdc484c3b332acec4b51b227dc Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 7 Dec 2016 20:20:59 +0100 Subject: [PATCH 5/7] improve rewritemeta error msg about formats --- fdroidserver/rewritemeta.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fdroidserver/rewritemeta.py b/fdroidserver/rewritemeta.py index abd6b69a..f6e2910f 100644 --- a/fdroidserver/rewritemeta.py +++ b/fdroidserver/rewritemeta.py @@ -45,13 +45,15 @@ def main(): global config, options + supported = ['txt', 'yml'] + # Parse command line... parser = ArgumentParser(usage="%(prog)s [options] [APPID [APPID ...]]") common.setup_global_opts(parser) parser.add_argument("-l", "--list", action="store_true", default=False, help="List files that would be reformatted") parser.add_argument("-t", "--to", default=None, - help="Rewrite to a specific format") + help="Rewrite to a specific format: " + ', '.join(supported)) parser.add_argument("appid", nargs='*', help="app-id in the form APPID") metadata.add_metadata_arguments(parser) options = parser.parse_args() @@ -66,10 +68,8 @@ def main(): if options.list and options.to is not None: parser.error("Cannot use --list and --to at the same time") - supported = ['txt', 'yml'] - if options.to is not None and options.to not in supported: - parser.error("Must give a valid format to --to") + parser.error("Unsupported metadata format, use: --to [" + ' '.join(supported) + "]") for appid, app in apps.items(): base, ext = common.get_extension(app.metadatapath) From 5214889bbd69d16a62e26ad1a9f729870303588e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 8 Dec 2016 19:11:36 +0100 Subject: [PATCH 6/7] fix bug including media files 'name' is not the full path, so it'll always fail common.is_repo_file()'s os.path.isfile() check. Introduced in 56d51fcd6be992c7bbc38431db06817816c1e08e --- fdroidserver/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index ba5fd706..a81eda21 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -549,7 +549,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): if file_extension == 'apk' or file_extension == 'obb': continue filename = os.path.join(repodir, name) - if not common.is_repo_file(name): + if not common.is_repo_file(filename): continue stat = os.stat(filename) if stat.st_size == 0: From 4e8e29794482741721583ff428202235be7c7115 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 28 Feb 2017 10:37:09 +0100 Subject: [PATCH 7/7] metadata: port .fdroid.yml handling to App dict subclass This got overlooked in b7fc7f2228986d0210e221c9ec8ddcc2ad9b93bc, then caught in the ./makebuildserver tests, where it builds Checkey. --- fdroidserver/metadata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 31128c41..efd16fc9 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -912,10 +912,10 @@ def parse_metadata(metadatapath, check_vcs=False): if os.path.isfile(metadata_in_repo): logging.debug('Including metadata from ' + metadata_in_repo) # do not include fields already provided by main metadata file - app_in_repo = parse_metadata(metadata_in_repo).field_dict() + app_in_repo = parse_metadata(metadata_in_repo) for k, v in app_in_repo.items(): - if k not in app.field_dict(): - app.set_field(k, v) + if k not in app: + app[k] = v post_metadata_parse(app)