mirrors/fdroidserver
1
0
Fork 0
mirror of https://github.com/f-droid/fdroidserver.git synced 2025-09-13 22:42:29 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improved metadata handling

Browse source 
Two main points: Firstly, there is no longer a random mapping between
app['key'] and a corresponding field in the metadata file - the key and
field are now the same. Secondly, more information (including comments)
is retrieved from the metadata, to facilitate being able to re-write it
which is necessary for various support utilities.
This commit is contained in:
Ciaran Gultnieks 2012-01-10 18:57:07 +00:00
parent c9c824f57d
commit c469f0feed
5 changed files with 157 additions and 117 deletions
Download patch file Download diff file Expand all files Collapse all files

8
build.py
View file

@ -74,13 +74,13 @@ if not os.path.isdir(build_dir):
for app in apps: for app in apps:
if app['disabled']: if app['Disabled']:
print "Skipping %s: disabled" % app['id'] print "Skipping %s: disabled" % app['id']
elif not app['builds']: elif not app['builds']:
print "Skipping %s: no builds specified" % app['id'] print "Skipping %s: no builds specified" % app['id']
if (app['disabled'] is None and app['repo'] != '' if (app['Disabled'] is None and app['Repo'] != ''
and app['repotype'] != '' and (options.package is None or and app['Repo Type'] != '' and (options.package is None or
options.package == app['id']) and len(app['builds']) > 0): options.package == app['id']) and len(app['builds']) > 0):
print "Processing " + app['id'] print "Processing " + app['id']
@ -88,7 +88,7 @@ for app in apps:
build_dir = 'build/' + app['id'] build_dir = 'build/' + app['id']
# Set up vcs interface and make sure we have the latest code... # Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['repotype'], app['repo'], build_dir) vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
refreshed_source = False refreshed_source = False

2
checkmarket2.py
View file

@ -63,7 +63,7 @@ for app in apps:
print "...couldn't find version code" print "...couldn't find version code"
elif not version: elif not version:
print "...couldn't find version" print "...couldn't find version"
elif vercode == app['marketvercode'] and version == app['marketversion']: elif vercode == app['Market Version Code'] and version == app['Market Version']:
print "...up to date" print "...up to date"
else: else:
print '...updating to version:' + version + ' vercode:' + vercode print '...updating to version:' + version + ' vercode:' + vercode

200
common.py
View file

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# common.py - part of the FDroid server tools # common.py - part of the FDroid server tools
# Copyright (C) 2010-11, Ciaran Gultnieks, ciaran@ciarang.com # Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by # it under the terms of the GNU Affero General Public License as published by
@ -245,15 +245,53 @@ class vcs_bzr(vcs):
raise VCSException("Bzr update failed") raise VCSException("Bzr update failed")
# Get the type expected for a given metadata field.
def metafieldtype(name):
if name == 'Description':
return 'multiline'
if name == 'Requires Root':
return 'flag'
if name == 'Build Version':
return 'build'
if name == 'Use Built':
return 'obsolete'
return 'string'
# Parse metadata for a single application.
#
# 'metafile' - the filename to read. The package id for the application comes
# from this filename.
#
# Returns a dictionary containing all the details of the application. There are
# two major kinds of information in the dictionary. Keys beginning with capital
# letters correspond directory to identically named keys in the metadata file.
# Keys beginning with lower case letters are generated in one way or another,
# and are not found verbatim in the metadata.
#
# Known keys not originating from the metadata are:
#
# 'id' - the application's package ID
# 'builds' - a list of dictionaries containing build information
# for each defined build
# 'comments' - a list of comments from the metadata file. Each is
# a tuple of the form (field, comment) where field is
# the name of the field it preceded in the metadata
# file
# 'name' - the application's name, as displayed. This will
# always be None on return from this parser. The name
# is discovered from built APK files.
#
def parse_metadata(metafile, **kw): def parse_metadata(metafile, **kw):
def parse_buildline(value): def parse_buildline(lines):
value = "".join(lines)
parts = [p.replace("\\,", ",") parts = [p.replace("\\,", ",")
for p in re.split(r"(?<!\\),", value)] for p in re.split(r"(?<!\\),", value)]
if len(parts) < 3: if len(parts) < 3:
raise MetaDataException("Invalid build format: " + value + " in " + metafile.name) raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
thisbuild = {} thisbuild = {}
thisbuild['origlines'] = lines
thisbuild['version'] = parts[0] thisbuild['version'] = parts[0]
thisbuild['vercode'] = parts[1] thisbuild['vercode'] = parts[1]
thisbuild['commit'] = parts[2] thisbuild['commit'] = parts[2]
@ -268,29 +306,37 @@ def parse_metadata(metafile, **kw):
thisinfo['id'] = metafile.name[9:-4] thisinfo['id'] = metafile.name[9:-4]
if kw.get("verbose", False): if kw.get("verbose", False):
print "Reading metadata for " + thisinfo['id'] print "Reading metadata for " + thisinfo['id']
thisinfo['description'] = ''
thisinfo['name'] = None # Defaults for fields that come from metadata...
thisinfo['summary'] = '' thisinfo['Description'] = ''
thisinfo['license'] = 'Unknown' thisinfo['Summary'] = ''
thisinfo['web'] = '' thisinfo['License'] = 'Unknown'
thisinfo['source'] = '' thisinfo['Web Site'] = ''
thisinfo['tracker'] = '' thisinfo['Source Code'] = ''
thisinfo['donate'] = None thisinfo['Issue Tracker'] = ''
thisinfo['disabled'] = None thisinfo['Donate'] = None
thisinfo['antifeatures'] = None thisinfo['Disabled'] = None
thisinfo['marketversion'] = '' thisinfo['AntiFeatures'] = None
thisinfo['marketvercode'] = '0' thisinfo['Market Version'] = ''
thisinfo['repotype'] = '' thisinfo['Market Version Code'] = '0'
thisinfo['repo'] = '' thisinfo['Repo Type'] = ''
thisinfo['Repo'] = ''
thisinfo['Requires Root'] = False
# General defaults...
thisinfo['builds'] = [] thisinfo['builds'] = []
thisinfo['requiresroot'] = False thisinfo['name'] = None
thisinfo['comments'] = []
mode = 0 mode = 0
buildline = [] buildlines = []
curcomments = []
for line in metafile: for line in metafile:
line = line.rstrip('\r\n') line = line.rstrip('\r\n')
if line.startswith("#"): if line.startswith("#"):
continue curcomments.append(line)
if mode == 0: elif mode == 0:
if len(line) == 0: if len(line) == 0:
continue continue
index = line.find(':') index = line.find(':')
@ -298,83 +344,77 @@ def parse_metadata(metafile, **kw):
raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line) raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
field = line[:index] field = line[:index]
value = line[index+1:] value = line[index+1:]
if field == 'Description':
for comment in curcomments:
thisinfo['comments'].append((field,comment))
fieldtype = metafieldtype(field)
if fieldtype == 'multiline':
mode = 1 mode = 1
elif field == 'Name': if len(value) > 0:
thisinfo['name'] = value raise MetaDataException("Unexpected text on same line as " + field + " in " + metafile.name)
elif field == 'Summary': elif fieldtype == 'string':
thisinfo['summary'] = value thisinfo[field] = value
elif field == 'Source Code': elif fieldtype == 'flag':
thisinfo['source'] = value if value == 'Yes':
elif field == 'License': thisinfo[field] = True
thisinfo['license'] = value elif value == 'No':
elif field == 'Category': thisinfo[field] = False
thisinfo['category'] = value else:
elif field == 'Web Site': raise MetaDataException("Expected Yes or No for " + field + " in " + metafile.name)
thisinfo['web'] = value elif fieldtype == 'build':
elif field == 'Issue Tracker':
thisinfo['tracker'] = value
elif field == 'Donate':
thisinfo['donate'] = value
elif field == 'Disabled':
thisinfo['disabled'] = value
elif field == 'Use Built':
pass #Ignoring this - will be removed
elif field == 'AntiFeatures':
parts = value.split(",")
for part in parts:
if (part != "Ads" and
part != "Tracking" and
part != "NonFreeNet" and
part != "NonFreeDep" and
part != "NonFreeAdd"):
raise MetaDataException("Unrecognised antifeature '" + part + "' in " \
+ metafile.name)
thisinfo['antifeatures'] = value
elif field == 'Market Version':
thisinfo['marketversion'] = value
elif field == 'Market Version Code':
thisinfo['marketvercode'] = value
elif field == 'Repo Type':
thisinfo['repotype'] = value
elif field == 'Repo':
thisinfo['repo'] = value
elif field == 'Build Version':
if value.endswith("\\"): if value.endswith("\\"):
mode = 2 mode = 2
buildline = [value[:-1]] buildlines = [value[:-1]]
else: else:
thisinfo['builds'].append(parse_buildline(value)) thisinfo['builds'].append(parse_buildline([value]))
elif field == "Requires Root": elif fieldtype == 'obsolete':
if value == "Yes": pass # Just throw it away!
thisinfo['requiresroot'] = True
else: else:
raise MetaDataException("Unrecognised field " + field + " in " + metafile.name) raise MetaDataException("Unrecognised field type for " + field + " in " + metafile.name)
elif mode == 1: # multi-line description elif mode == 1: # Multiline field
if line == '.': if line == '.':
mode = 0 mode = 0
else: else:
if len(line) == 0: if len(line) == 0:
thisinfo['description'] += '\n\n' thisinfo[field] += '\n\n'
else: else:
if (not thisinfo['description'].endswith('\n') and if (not thisinfo[field].endswith('\n') and
len(thisinfo['description']) > 0): len(thisinfo[field]) > 0):
thisinfo['description'] += ' ' thisinfo[field] += ' '
thisinfo['description'] += line thisinfo[field] += line
elif mode == 2: # line continuation elif mode == 2: # Line continuation mode in Build Version
if line.endswith("\\"): if line.endswith("\\"):
buildline.append(line[:-1]) buildlines.append(line[:-1])
else: else:
buildline.append(line) buildlines.append(line)
thisinfo['builds'].append( thisinfo['builds'].append(
parse_buildline("".join(buildline))) parse_buildline(buildlines))
mode = 0 mode = 0
if mode == 1: if mode == 1:
raise MetaDataException("Description not terminated in " + metafile.name) raise MetaDataException(field + " not terminated in " + metafile.name)
if len(thisinfo['description']) == 0: elif mode == 2:
thisinfo['description'] = 'No description available' raise MetaDataException("Unterminated continuation in " + metafile.name)
if len(thisinfo['Description']) == 0:
thisinfo['Description'] = 'No description available'
# Ensure all AntiFeatures are recognised...
if thisinfo['AntiFeatures']:
parts = thisinfo['AntiFeatures'].split(",")
for part in parts:
if (part != "Ads" and
part != "Tracking" and
part != "NonFreeNet" and
part != "NonFreeDep" and
part != "NonFreeAdd"):
raise MetaDataException("Unrecognised antifeature '" + part + "' in " \
+ metafile.name)
return thisinfo return thisinfo
def read_metadata(verbose=False): def read_metadata(verbose=False):
apps = [] apps = []
for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))): for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):

4
scanner.py
View file

@ -55,7 +55,7 @@ for app in apps:
skip = False skip = False
if options.package and app['id'] != options.package: if options.package and app['id'] != options.package:
skip = True skip = True
elif app['disabled']: elif app['Disabled']:
print "Skipping %s: disabled" % app['id'] print "Skipping %s: disabled" % app['id']
skip = True skip = True
elif not app['builds']: elif not app['builds']:
@ -71,7 +71,7 @@ for app in apps:
build_dir = 'build/' + app['id'] build_dir = 'build/' + app['id']
# Set up vcs interface and make sure we have the latest code... # Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['repotype'], app['repo'], build_dir) vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
refreshed_source = False refreshed_source = False

60
update.py
View file

@ -2,7 +2,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# update.py - part of the FDroid server tools # update.py - part of the FDroid server tools
# Copyright (C) 2010-11, Ciaran Gultnieks, ciaran@ciarang.com # Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by # it under the terms of the GNU Affero General Public License as published by
@ -196,7 +196,7 @@ for app in apps:
if app['name'] is None: if app['name'] is None:
app['name'] = app['id'] app['name'] = app['id']
app['icon'] = '' app['icon'] = ''
if app['disabled'] is None: if app['Disabled'] is None:
print "WARNING: Application " + app['id'] + " has no packages" print "WARNING: Application " + app['id'] + " has no packages"
else: else:
if app['name'] is None: if app['name'] is None:
@ -283,14 +283,14 @@ apps_nopkg = 0
for app in apps: for app in apps:
if app['disabled'] is None: if app['Disabled'] is None:
# Get a list of the apks for this app... # Get a list of the apks for this app...
gotmarketver = False gotmarketver = False
apklist = [] apklist = []
for apk in apks: for apk in apks:
if apk['id'] == app['id']: if apk['id'] == app['id']:
if str(apk['versioncode']) == app['marketvercode']: if str(apk['versioncode']) == app['Market Version Code']:
gotmarketver = True gotmarketver = True
apklist.append(apk) apklist.append(apk)
@ -304,22 +304,22 @@ for app in apps:
addElement('id', app['id'], doc, apel) addElement('id', app['id'], doc, apel)
addElement('name', app['name'], doc, apel) addElement('name', app['name'], doc, apel)
addElement('summary', app['summary'], doc, apel) addElement('summary', app['Summary'], doc, apel)
addElement('icon', app['icon'], doc, apel) addElement('icon', app['icon'], doc, apel)
addElement('description', app['description'], doc, apel) addElement('description', app['Description'], doc, apel)
addElement('license', app['license'], doc, apel) addElement('license', app['License'], doc, apel)
if 'category' in app: if 'Category' in app:
addElement('category', app['category'], doc, apel) addElement('category', app['Category'], doc, apel)
addElement('web', app['web'], doc, apel) addElement('web', app['Web Site'], doc, apel)
addElement('source', app['source'], doc, apel) addElement('source', app['Source Code'], doc, apel)
addElement('tracker', app['tracker'], doc, apel) addElement('tracker', app['Issue Tracker'], doc, apel)
if app['donate'] != None: if app['Donate'] != None:
addElement('donate', app['donate'], doc, apel) addElement('donate', app['Donate'], doc, apel)
addElement('marketversion', app['marketversion'], doc, apel) addElement('marketversion', app['Market Version'], doc, apel)
addElement('marketvercode', app['marketvercode'], doc, apel) addElement('marketvercode', app['Market Version Code'], doc, apel)
if not (app['antifeatures'] is None): if not (app['AntiFeatures'] is None):
addElement('antifeatures', app['antifeatures'], doc, apel) addElement('antifeatures', app['AntiFeatures'], doc, apel)
if app['requiresroot']: if app['Requires Root']:
addElement('requirements', 'root', doc, apel) addElement('requirements', 'root', doc, apel)
# Sort the apk list into version order, just so the web site # Sort the apk list into version order, just so the web site
@ -370,38 +370,38 @@ for app in apps:
if options.buildreport: if options.buildreport:
if len(app['builds']) == 0: if len(app['builds']) == 0:
print ("WARNING: No builds defined for " + app['id'] + print ("WARNING: No builds defined for " + app['id'] +
" Source: " + app['source']) " Source: " + app['Source Code'])
warnings += 1 warnings += 1
else: else:
if app['marketvercode'] != '0': if app['Market Version Code'] != '0':
gotbuild = False gotbuild = False
for build in app['builds']: for build in app['builds']:
if build['vercode'] == app['marketvercode']: if build['vercode'] == app['Market Version Code']:
gotbuild = True gotbuild = True
if not gotbuild: if not gotbuild:
print ("WARNING: No build data for market version of " print ("WARNING: No build data for market version of "
+ app['id'] + " (" + app['marketversion'] + app['id'] + " (" + app['Market Version']
+ ") " + app['source']) + ") " + app['Source Code'])
warnings += 1 warnings += 1
# If we don't have the market version, check if there is a build # If we don't have the market version, check if there is a build
# with a commit ID starting with '!' - this means we can't build it # with a commit ID starting with '!' - this means we can't build it
# for some reason, and don't want hassling about it... # for some reason, and don't want hassling about it...
if not gotmarketver and app['marketvercode'] != '0': if not gotmarketver and app['Market Version Code'] != '0':
for build in app['builds']: for build in app['builds']:
if build['vercode'] == app['marketvercode']: if build['vercode'] == app['Market Version Code']:
gotmarketver = True gotmarketver = True
# Output a message of harassment if we don't have the market version: # Output a message of harassment if we don't have the market version:
if not gotmarketver and app['marketvercode'] != '0': if not gotmarketver and app['Market Version Code'] != '0':
addr = app['source'] addr = app['Source Code']
print "WARNING: Don't have market version (" + app['marketversion'] + ") of " + app['name'] print "WARNING: Don't have market version (" + app['Market Version'] + ") of " + app['name']
print " (" + app['id'] + ") " + addr print " (" + app['id'] + ") " + addr
warnings += 1 warnings += 1
if options.verbose: if options.verbose:
# A bit of extra debug info, basically for diagnosing # A bit of extra debug info, basically for diagnosing
# app developer mistakes: # app developer mistakes:
print " Market vercode:" + app['marketvercode'] print " Market vercode:" + app['Market Version Code']
print " Got:" print " Got:"
for apk in apks: for apk in apks:
if apk['id'] == app['id']: if apk['id'] == app['id']: