mirror of
https://github.com/f-droid/fdroidserver.git
synced 2025-10-05 17:01:06 +03:00
Merge branch 'yaml-is-king' into 'master'
remove all code for handling txt metadata See merge request fdroid/fdroidserver!772
This commit is contained in:
Hans-Christoph Steiner
commit
bd1f05e370
10 changed files with 45 additions and 688 deletions
|
|
@ -32,6 +32,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
|
|||
* build: deploying buildlogs with rsync
|
||||
([!651](https://gitlab.com/fdroid/fdroidserver/merge_requests/651))
|
||||
|
||||
### Removed
|
||||
* removed support for txt and json metadata
|
||||
([!772](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/772))
|
||||
|
||||
## [1.1.4] - 2019-08-15
|
||||
### Fixed
|
||||
* include bitcoin validation regex required by fdroiddata
|
||||
|
|
|
|||
|
|
@ -206,14 +206,11 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
|
|||
if os.path.isfile(os.path.join('srclibs', name + '.yml')):
|
||||
ftp.put(os.path.join('srclibs', name + '.yml'),
|
||||
name + '.yml')
|
||||
elif os.path.isfile(os.path.join('srclibs', name + '.txt')):
|
||||
ftp.put(os.path.join('srclibs', name + '.txt'),
|
||||
name + '.txt')
|
||||
else:
|
||||
raise BuildException("can not find metadata file for "
|
||||
"'{name}', please make sure it is "
|
||||
"present in your 'srclibs' folder."
|
||||
"(supported formats: txt, yml)"
|
||||
"(supported format: yml)"
|
||||
.format(name=name))
|
||||
# Copy the main app source code
|
||||
# (no need if it's a srclib)
|
||||
|
|
|
|||
|
|
@ -119,7 +119,6 @@ default_config = {
|
|||
'mvn3': "mvn",
|
||||
'gradle': os.path.join(FDROID_PATH, 'gradlew-fdroid'),
|
||||
'gradle_version_dir': os.path.join(os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), 'gradle'),
|
||||
'accepted_formats': ['txt', 'yml'],
|
||||
'sync_from_local_copy_dir': False,
|
||||
'allow_disabled_algorithms': False,
|
||||
'per_app_repos': False,
|
||||
|
|
@ -517,7 +516,7 @@ def get_local_metadata_files():
|
|||
'''get any metadata files local to an app's source repo
|
||||
|
||||
This tries to ignore anything that does not count as app metdata,
|
||||
including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
|
||||
including emacs cruft ending in ~
|
||||
|
||||
'''
|
||||
return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
|
||||
|
|
|
|||
|
|
@ -515,16 +515,12 @@ def check_for_unsupported_metadata_files(basedir=""):
|
|||
global config
|
||||
|
||||
return_value = False
|
||||
formats = config['accepted_formats']
|
||||
for f in glob.glob(basedir + 'metadata/*') + glob.glob(basedir + 'metadata/.*'):
|
||||
if os.path.isdir(f):
|
||||
exists = False
|
||||
for t in formats:
|
||||
exists = exists or os.path.exists(f + '.' + t)
|
||||
if not exists:
|
||||
if not os.path.exists(f + '.yml'):
|
||||
print(_('"%s/" has no matching metadata file!') % f)
|
||||
return_value = True
|
||||
elif os.path.splitext(f)[1][1:] in formats:
|
||||
elif os.path.splitext(f)[1][1:] == "yml":
|
||||
packageName = os.path.splitext(os.path.basename(f))[0]
|
||||
if not common.is_valid_package_name(packageName):
|
||||
print('"' + packageName + '" is an invalid package name!\n'
|
||||
|
|
@ -532,7 +528,7 @@ def check_for_unsupported_metadata_files(basedir=""):
|
|||
return_value = True
|
||||
else:
|
||||
print('"' + f.replace(basedir, '')
|
||||
+ '" is not a supported file format: (' + ','.join(formats) + ')')
|
||||
+ '" is not a supported file format (use: .yml)')
|
||||
return_value = True
|
||||
|
||||
return return_value
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@
|
|||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import glob
|
||||
|
|
@ -59,54 +58,6 @@ def warn_or_exception(value, cause=None):
|
|||
logging.warning(value)
|
||||
|
||||
|
||||
# To filter which ones should be written to the metadata files if
|
||||
# present
|
||||
app_fields = set([
|
||||
'Disabled',
|
||||
'AntiFeatures',
|
||||
'Provides', # deprecated, txt only
|
||||
'Categories',
|
||||
'License',
|
||||
'Author Name',
|
||||
'Author Email',
|
||||
'Author Web Site',
|
||||
'Web Site',
|
||||
'Source Code',
|
||||
'Issue Tracker',
|
||||
'Translation',
|
||||
'Changelog',
|
||||
'Donate',
|
||||
'FlattrID',
|
||||
'Liberapay',
|
||||
'LiberapayID',
|
||||
'OpenCollective',
|
||||
'Bitcoin',
|
||||
'Litecoin',
|
||||
'Name',
|
||||
'Auto Name',
|
||||
'Summary',
|
||||
'Description',
|
||||
'Requires Root',
|
||||
'Repo Type',
|
||||
'Repo',
|
||||
'Binaries',
|
||||
'Maintainer Notes',
|
||||
'Archive Policy',
|
||||
'Auto Update Mode',
|
||||
'Update Check Mode',
|
||||
'Update Check Ignore',
|
||||
'Vercode Operation',
|
||||
'Update Check Name',
|
||||
'Update Check Data',
|
||||
'Current Version',
|
||||
'Current Version Code',
|
||||
'No Source Since',
|
||||
'Build',
|
||||
|
||||
'comments', # For formats that don't do inline comments
|
||||
'builds', # For formats that do builds as a list
|
||||
])
|
||||
|
||||
yaml_app_field_order = [
|
||||
'Disabled',
|
||||
'AntiFeatures',
|
||||
|
|
@ -266,7 +217,9 @@ def fieldtype(name):
|
|||
|
||||
|
||||
# In the order in which they are laid out on files
|
||||
build_flags_order = [
|
||||
build_flags = [
|
||||
'versionName',
|
||||
'versionCode',
|
||||
'disable',
|
||||
'commit',
|
||||
'timeout',
|
||||
|
|
@ -301,10 +254,6 @@ build_flags_order = [
|
|||
'antifeatures',
|
||||
]
|
||||
|
||||
# old .txt format has version name/code inline in the 'Build:' line
|
||||
# but YAML and JSON have a explicit key for them
|
||||
build_flags = ['versionName', 'versionCode'] + build_flags_order
|
||||
|
||||
|
||||
class Build(dict):
|
||||
|
||||
|
|
@ -691,16 +640,6 @@ class DescriptionFormatter:
|
|||
self.html.close()
|
||||
|
||||
|
||||
# Parse multiple lines of description as written in a metadata file, returning
|
||||
# a single string in text format and wrapped to 80 columns.
|
||||
def description_txt(s):
|
||||
ps = DescriptionFormatter(None)
|
||||
for line in s.splitlines():
|
||||
ps.parseline(line)
|
||||
ps.end()
|
||||
return ps.text_txt
|
||||
|
||||
|
||||
# Parse multiple lines of description as written in a metadata file, returning
|
||||
# a single string in wiki format. Used for the Maintainer Notes field as well,
|
||||
# because it's the same format.
|
||||
|
|
@ -718,46 +657,6 @@ def description_html(s, linkres):
|
|||
return ps.text_html
|
||||
|
||||
|
||||
def parse_txt_srclib(metadatapath):
|
||||
|
||||
thisinfo = {}
|
||||
|
||||
# Defaults for fields that come from metadata
|
||||
thisinfo['RepoType'] = ''
|
||||
thisinfo['Repo'] = ''
|
||||
thisinfo['Subdir'] = None
|
||||
thisinfo['Prepare'] = None
|
||||
|
||||
if not os.path.exists(metadatapath):
|
||||
return thisinfo
|
||||
|
||||
metafile = open(metadatapath, "r")
|
||||
|
||||
n = 0
|
||||
for line in metafile:
|
||||
n += 1
|
||||
line = line.rstrip('\r\n')
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
|
||||
try:
|
||||
f, v = line.split(':', 1)
|
||||
except ValueError:
|
||||
warn_or_exception(_("Invalid metadata in %s:%d") % (line, n))
|
||||
|
||||
# collapse whitespaces in field names
|
||||
f = f.replace(' ', '')
|
||||
|
||||
if f == "Subdir":
|
||||
thisinfo[f] = v.split(',')
|
||||
else:
|
||||
thisinfo[f] = v
|
||||
|
||||
metafile.close()
|
||||
|
||||
return thisinfo
|
||||
|
||||
|
||||
def parse_yaml_srclib(metadatapath):
|
||||
|
||||
thisinfo = {'RepoType': '',
|
||||
|
|
@ -813,7 +712,7 @@ def read_srclibs():
|
|||
|
||||
The information read will be accessible as metadata.srclibs, which is a
|
||||
dictionary, keyed on srclib name, with the values each being a dictionary
|
||||
in the same format as that returned by the parse_txt_srclib function.
|
||||
in the same format as that returned by the parse_yaml_srclib function.
|
||||
|
||||
A MetaDataException is raised if there are any problems with the srclib
|
||||
metadata.
|
||||
|
|
@ -830,10 +729,6 @@ def read_srclibs():
|
|||
if not os.path.exists(srcdir):
|
||||
os.makedirs(srcdir)
|
||||
|
||||
for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.txt'))):
|
||||
srclibname = os.path.basename(metadatapath[:-4])
|
||||
srclibs[srclibname] = parse_txt_srclib(metadatapath)
|
||||
|
||||
for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.yml'))):
|
||||
srclibname = os.path.basename(metadatapath[:-4])
|
||||
srclibs[srclibname] = parse_yaml_srclib(metadatapath)
|
||||
|
|
@ -847,11 +742,6 @@ def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False):
|
|||
sorted based on creation time, newest first. Most of the time,
|
||||
the newer files are the most interesting.
|
||||
|
||||
If there are multiple metadata files for a single appid, then the first
|
||||
file that is parsed wins over all the others, and the rest throw an
|
||||
exception. So the original .txt format is parsed first, at least until
|
||||
newer formats stabilize.
|
||||
|
||||
check_vcs is the list of appids to check for .fdroid.yml in source
|
||||
|
||||
"""
|
||||
|
|
@ -866,11 +756,7 @@ def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False):
|
|||
if not os.path.exists(basedir):
|
||||
os.makedirs(basedir)
|
||||
|
||||
metadatafiles = (glob.glob(os.path.join('metadata', '*.txt'))
|
||||
+ glob.glob(os.path.join('metadata', '*.json'))
|
||||
+ glob.glob(os.path.join('metadata', '*.yml'))
|
||||
+ glob.glob('.fdroid.txt')
|
||||
+ glob.glob('.fdroid.json')
|
||||
metadatafiles = (glob.glob(os.path.join('metadata', '*.yml'))
|
||||
+ glob.glob('.fdroid.yml'))
|
||||
|
||||
if sort_by_time:
|
||||
|
|
@ -883,8 +769,6 @@ def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False):
|
|||
metadatafiles = sorted(metadatafiles)
|
||||
|
||||
for metadatapath in metadatafiles:
|
||||
if metadatapath == '.fdroid.txt':
|
||||
warn_or_exception(_('.fdroid.txt is not supported! Convert to .fdroid.yml or .fdroid.json.'))
|
||||
appid, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
|
||||
if appid != '.fdroid' and not fdroidserver.common.is_valid_package_name(appid):
|
||||
warn_or_exception(_("{appid} from {path} is not a valid Java Package Name!")
|
||||
|
|
@ -1044,10 +928,6 @@ def parse_metadata(metadatapath, check_vcs=False, refresh=True):
|
|||
'''parse metadata file, optionally checking the git repo for metadata first'''
|
||||
|
||||
_ignored, ext = fdroidserver.common.get_extension(metadatapath)
|
||||
accepted = fdroidserver.common.config['accepted_formats']
|
||||
if ext not in accepted:
|
||||
warn_or_exception(_('"{path}" is not an accepted format, convert to: {formats}')
|
||||
.format(path=metadatapath, formats=', '.join(accepted)))
|
||||
|
||||
app = App()
|
||||
app.metadatapath = metadatapath
|
||||
|
|
@ -1057,16 +937,12 @@ def parse_metadata(metadatapath, check_vcs=False, refresh=True):
|
|||
else:
|
||||
app.id = name
|
||||
|
||||
with open(metadatapath, 'r') as mf:
|
||||
if ext == 'txt':
|
||||
parse_txt_metadata(mf, app)
|
||||
elif ext == 'json':
|
||||
parse_json_metadata(mf, app)
|
||||
elif ext == 'yml':
|
||||
if ext == 'yml':
|
||||
with open(metadatapath, 'r') as mf:
|
||||
parse_yaml_metadata(mf, app)
|
||||
else:
|
||||
warn_or_exception(_('Unknown metadata format: {path}')
|
||||
.format(path=metadatapath))
|
||||
else:
|
||||
warn_or_exception(_('Unknown metadata format: {path} (use: .yml)')
|
||||
.format(path=metadatapath))
|
||||
|
||||
if check_vcs and app.Repo:
|
||||
build_dir = fdroidserver.common.get_build_dir(app)
|
||||
|
|
@ -1098,20 +974,6 @@ def parse_metadata(metadatapath, check_vcs=False, refresh=True):
|
|||
return app
|
||||
|
||||
|
||||
def parse_json_metadata(mf, app):
|
||||
|
||||
# fdroid metadata is only strings and booleans, no floats or ints.
|
||||
# TODO create schema using https://pypi.python.org/pypi/jsonschema
|
||||
jsoninfo = json.load(mf, parse_int=lambda s: s,
|
||||
parse_float=lambda s: s)
|
||||
app.update(jsoninfo)
|
||||
for f in ['Description', 'Maintainer Notes']:
|
||||
v = app.get(f)
|
||||
if v:
|
||||
app[f] = '\n'.join(v)
|
||||
return app
|
||||
|
||||
|
||||
def parse_yaml_metadata(mf, app):
|
||||
try:
|
||||
yamldata = yaml.load(mf, Loader=SafeLoader)
|
||||
|
|
@ -1254,7 +1116,7 @@ def write_yaml(mf, app):
|
|||
insert_newline = True
|
||||
else:
|
||||
if app.get(field) or field == 'Builds':
|
||||
# .txt calls it 'builds' internally, everywhere else its 'Builds'
|
||||
# .txt called it 'builds' internally, everywhere else its 'Builds'
|
||||
if field == 'Builds':
|
||||
if app.get('builds'):
|
||||
cm.update({field: _builds_to_yaml(app)})
|
||||
|
|
@ -1304,363 +1166,10 @@ build_line_sep = re.compile(r'(?<!\\),')
|
|||
build_cont = re.compile(r'^[ \t]')
|
||||
|
||||
|
||||
def parse_txt_metadata(mf, app):
|
||||
|
||||
linedesc = None
|
||||
|
||||
def add_buildflag(p, build):
|
||||
if not p.strip():
|
||||
warn_or_exception(_("Empty build flag at {linedesc}")
|
||||
.format(linedesc=linedesc))
|
||||
bv = p.split('=', 1)
|
||||
if len(bv) != 2:
|
||||
warn_or_exception(_("Invalid build flag at {line} in {linedesc}")
|
||||
.format(line=buildlines[0], linedesc=linedesc))
|
||||
|
||||
pk, pv = bv
|
||||
pk = pk.lstrip()
|
||||
if pk == 'update':
|
||||
pk = 'androidupdate' # avoid conflicting with Build(dict).update()
|
||||
t = flagtype(pk)
|
||||
if t == TYPE_LIST:
|
||||
pv = split_list_values(pv)
|
||||
build[pk] = pv
|
||||
elif t == TYPE_STRING or t == TYPE_SCRIPT:
|
||||
build[pk] = pv
|
||||
elif t == TYPE_BOOL:
|
||||
build[pk] = _decode_bool(pv)
|
||||
elif t == TYPE_INT:
|
||||
build[pk] = int(pv)
|
||||
|
||||
def parse_buildline(lines):
|
||||
v = "".join(lines)
|
||||
parts = [p.replace("\\,", ",") for p in re.split(build_line_sep, v)]
|
||||
if len(parts) < 3:
|
||||
warn_or_exception(_("Invalid build format: {value} in {name}")
|
||||
.format(value=v, name=mf.name))
|
||||
build = Build()
|
||||
build.versionName = parts[0]
|
||||
build.versionCode = parts[1]
|
||||
check_versionCode(build.versionCode)
|
||||
|
||||
if parts[2].startswith('!'):
|
||||
# For backwards compatibility, handle old-style disabling,
|
||||
# including attempting to extract the commit from the message
|
||||
build.disable = parts[2][1:]
|
||||
commit = 'unknown - see disabled'
|
||||
index = parts[2].rfind('at ')
|
||||
if index != -1:
|
||||
commit = parts[2][index + 3:]
|
||||
if commit.endswith(')'):
|
||||
commit = commit[:-1]
|
||||
build.commit = commit
|
||||
else:
|
||||
build.commit = parts[2]
|
||||
for p in parts[3:]:
|
||||
add_buildflag(p, build)
|
||||
|
||||
return build
|
||||
|
||||
def check_versionCode(versionCode):
|
||||
try:
|
||||
int(versionCode)
|
||||
except ValueError:
|
||||
warn_or_exception(_('Invalid versionCode: "{versionCode}" is not an integer!')
|
||||
.format(versionCode=versionCode))
|
||||
|
||||
def add_comments(key):
|
||||
if not curcomments:
|
||||
return
|
||||
app.comments[key] = list(curcomments)
|
||||
del curcomments[:]
|
||||
|
||||
mode = 0
|
||||
buildlines = []
|
||||
multiline_lines = []
|
||||
curcomments = []
|
||||
build = None
|
||||
vc_seen = set()
|
||||
|
||||
app.builds = []
|
||||
|
||||
c = 0
|
||||
for line in mf:
|
||||
c += 1
|
||||
linedesc = "%s:%d" % (mf.name, c)
|
||||
line = line.rstrip('\r\n')
|
||||
if mode == 3:
|
||||
if build_cont.match(line):
|
||||
if line.endswith('\\'):
|
||||
buildlines.append(line[:-1].lstrip())
|
||||
else:
|
||||
buildlines.append(line.lstrip())
|
||||
bl = ''.join(buildlines)
|
||||
add_buildflag(bl, build)
|
||||
del buildlines[:]
|
||||
else:
|
||||
if not build.commit and not build.disable:
|
||||
warn_or_exception(_("No commit specified for {versionName} in {linedesc}")
|
||||
.format(versionName=build.versionName, linedesc=linedesc))
|
||||
|
||||
app.builds.append(build)
|
||||
add_comments('build:' + build.versionCode)
|
||||
mode = 0
|
||||
|
||||
if mode == 0:
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("#"):
|
||||
curcomments.append(line[1:].strip())
|
||||
continue
|
||||
try:
|
||||
f, v = line.split(':', 1)
|
||||
except ValueError:
|
||||
warn_or_exception(_("Invalid metadata in: ") + linedesc)
|
||||
|
||||
if f not in app_fields:
|
||||
warn_or_exception(_('Unrecognised app field: ') + f)
|
||||
|
||||
# Translate obsolete fields...
|
||||
if f == 'Market Version':
|
||||
f = 'Current Version'
|
||||
if f == 'Market Version Code':
|
||||
f = 'Current Version Code'
|
||||
|
||||
f = f.replace(' ', '')
|
||||
|
||||
ftype = fieldtype(f)
|
||||
if ftype not in [TYPE_BUILD]:
|
||||
add_comments(f)
|
||||
if ftype == TYPE_MULTILINE:
|
||||
mode = 1
|
||||
if v:
|
||||
warn_or_exception(_("Unexpected text on same line as {field} in {linedesc}")
|
||||
.format(field=f, linedesc=linedesc))
|
||||
elif ftype == TYPE_STRING:
|
||||
app[f] = v
|
||||
elif ftype == TYPE_LIST:
|
||||
app[f] = split_list_values(v)
|
||||
elif ftype == TYPE_BUILD:
|
||||
vv = v.split(',')
|
||||
if len(vv) != 2:
|
||||
warn_or_exception(_('Build should have comma-separated '
|
||||
'versionName and versionCode, '
|
||||
'not "{value}", in {linedesc}')
|
||||
.format(value=v, linedesc=linedesc))
|
||||
build = Build()
|
||||
build.versionName = vv[0]
|
||||
build.versionCode = vv[1]
|
||||
check_versionCode(build.versionCode)
|
||||
|
||||
if build.versionCode in vc_seen:
|
||||
warn_or_exception(_('Duplicate build recipe found for versionCode {versionCode} in {linedesc}')
|
||||
.format(versionCode=build.versionCode, linedesc=linedesc))
|
||||
vc_seen.add(build.versionCode)
|
||||
del buildlines[:]
|
||||
mode = 3
|
||||
elif ftype == TYPE_OBSOLETE:
|
||||
warn_or_exception(_("'{field}' in {linedesc} is obsolete, see docs for current fields:")
|
||||
.format(field=f, linedesc=linedesc)
|
||||
+ '\nhttps://f-droid.org/docs/')
|
||||
else:
|
||||
warn_or_exception(_("Unrecognised field '{field}' in {linedesc}")
|
||||
.format(field=f, linedesc=linedesc))
|
||||
elif mode == 1: # Multiline field
|
||||
if line == '.':
|
||||
mode = 0
|
||||
app[f] = '\n'.join(multiline_lines)
|
||||
del multiline_lines[:]
|
||||
else:
|
||||
multiline_lines.append(line)
|
||||
elif mode == 2: # Line continuation mode in Build Version
|
||||
if line.endswith("\\"):
|
||||
buildlines.append(line[:-1])
|
||||
else:
|
||||
buildlines.append(line)
|
||||
build = parse_buildline(buildlines)
|
||||
app.builds.append(build)
|
||||
add_comments('build:' + app.builds[-1].versionCode)
|
||||
mode = 0
|
||||
add_comments(None)
|
||||
|
||||
# Mode at end of file should always be 0
|
||||
if mode == 1:
|
||||
warn_or_exception(_("{field} not terminated in {name}")
|
||||
.format(field=f, name=mf.name))
|
||||
if mode == 2:
|
||||
warn_or_exception(_("Unterminated continuation in {name}")
|
||||
.format(name=mf.name))
|
||||
if mode == 3:
|
||||
warn_or_exception(_("Unterminated build in {name}")
|
||||
.format(name=mf.name))
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def write_plaintext_metadata(mf, app, w_comment, w_field, w_build):
|
||||
|
||||
def field_to_attr(f):
|
||||
"""
|
||||
Translates human-readable field names to attribute names, e.g.
|
||||
'Auto Name' to 'AutoName'
|
||||
"""
|
||||
return f.replace(' ', '')
|
||||
|
||||
def attr_to_field(k):
|
||||
"""
|
||||
Translates attribute names to human-readable field names, e.g.
|
||||
'AutoName' to 'Auto Name'
|
||||
"""
|
||||
if k in app_fields:
|
||||
return k
|
||||
f = re.sub(r'([a-z])([A-Z])', r'\1 \2', k)
|
||||
return f
|
||||
|
||||
def w_comments(key):
|
||||
if key not in app.comments:
|
||||
return
|
||||
for line in app.comments[key]:
|
||||
w_comment(line)
|
||||
|
||||
def w_field_always(f, v=None):
|
||||
key = field_to_attr(f)
|
||||
if v is None:
|
||||
v = app.get(key)
|
||||
w_comments(key)
|
||||
w_field(f, v)
|
||||
|
||||
def w_field_nonempty(f, v=None):
|
||||
key = field_to_attr(f)
|
||||
if v is None:
|
||||
v = app.get(key)
|
||||
w_comments(key)
|
||||
if v:
|
||||
w_field(f, v)
|
||||
|
||||
w_field_nonempty('Disabled')
|
||||
w_field_nonempty('AntiFeatures')
|
||||
w_field_nonempty('Provides')
|
||||
w_field_always('Categories')
|
||||
w_field_always('License')
|
||||
w_field_nonempty('Author Name')
|
||||
w_field_nonempty('Author Email')
|
||||
w_field_nonempty('Author Web Site')
|
||||
w_field_always('Web Site')
|
||||
w_field_always('Source Code')
|
||||
w_field_always('Issue Tracker')
|
||||
w_field_nonempty('Translation')
|
||||
w_field_nonempty('Changelog')
|
||||
w_field_nonempty('Donate')
|
||||
w_field_nonempty('FlattrID')
|
||||
w_field_nonempty('LiberapayID')
|
||||
w_field_nonempty('OpenCollective')
|
||||
w_field_nonempty('Bitcoin')
|
||||
w_field_nonempty('Litecoin')
|
||||
mf.write('\n')
|
||||
w_field_nonempty('Name')
|
||||
w_field_nonempty('Auto Name')
|
||||
w_field_nonempty('Summary')
|
||||
w_field_nonempty('Description', description_txt(app.Description))
|
||||
mf.write('\n')
|
||||
if app.RequiresRoot:
|
||||
w_field_always('Requires Root', 'yes')
|
||||
mf.write('\n')
|
||||
if app.RepoType:
|
||||
w_field_always('Repo Type')
|
||||
w_field_always('Repo')
|
||||
if app.Binaries:
|
||||
w_field_always('Binaries')
|
||||
mf.write('\n')
|
||||
|
||||
for build in app.builds:
|
||||
|
||||
if build.versionName == "Ignore":
|
||||
continue
|
||||
|
||||
w_comments('build:%s' % build.versionCode)
|
||||
w_build(build)
|
||||
mf.write('\n')
|
||||
|
||||
if app.MaintainerNotes:
|
||||
w_field_always('Maintainer Notes', app.MaintainerNotes)
|
||||
mf.write('\n')
|
||||
|
||||
w_field_nonempty('Archive Policy')
|
||||
w_field_always('Auto Update Mode')
|
||||
w_field_always('Update Check Mode')
|
||||
w_field_nonempty('Update Check Ignore')
|
||||
w_field_nonempty('Vercode Operation')
|
||||
w_field_nonempty('Update Check Name')
|
||||
w_field_nonempty('Update Check Data')
|
||||
if app.CurrentVersion:
|
||||
w_field_always('Current Version')
|
||||
w_field_always('Current Version Code')
|
||||
if app.NoSourceSince:
|
||||
mf.write('\n')
|
||||
w_field_always('No Source Since')
|
||||
w_comments(None)
|
||||
|
||||
|
||||
# Write a metadata file in txt format.
|
||||
#
|
||||
# 'mf' - Writer interface (file, StringIO, ...)
|
||||
# 'app' - The app data
|
||||
def write_txt(mf, app):
|
||||
|
||||
def w_comment(line):
|
||||
mf.write("# %s\n" % line)
|
||||
|
||||
def w_field(f, v):
|
||||
t = fieldtype(f)
|
||||
if t == TYPE_LIST:
|
||||
v = ','.join(v)
|
||||
elif t == TYPE_MULTILINE:
|
||||
v = '\n' + v + '\n.'
|
||||
mf.write("%s:%s\n" % (f, v))
|
||||
|
||||
def w_build(build):
|
||||
mf.write("Build:%s,%s\n" % (build.versionName, build.versionCode))
|
||||
|
||||
for f in build_flags_order:
|
||||
v = build.get(f)
|
||||
if not v:
|
||||
continue
|
||||
|
||||
t = flagtype(f)
|
||||
if f == 'androidupdate':
|
||||
f = 'update' # avoid conflicting with Build(dict).update()
|
||||
mf.write(' %s=' % f)
|
||||
if t == TYPE_STRING or t == TYPE_INT:
|
||||
mf.write(v)
|
||||
elif t == TYPE_BOOL:
|
||||
mf.write('yes')
|
||||
elif t == TYPE_SCRIPT:
|
||||
first = True
|
||||
for s in v.split(' && '):
|
||||
if first:
|
||||
first = False
|
||||
else:
|
||||
mf.write(' && \\\n ')
|
||||
mf.write(s.strip())
|
||||
elif t == TYPE_LIST:
|
||||
mf.write(','.join(v))
|
||||
|
||||
mf.write('\n')
|
||||
|
||||
write_plaintext_metadata(mf, app, w_comment, w_field, w_build)
|
||||
|
||||
|
||||
def write_metadata(metadatapath, app):
|
||||
_ignored, ext = fdroidserver.common.get_extension(metadatapath)
|
||||
accepted = fdroidserver.common.config['accepted_formats']
|
||||
if ext not in accepted:
|
||||
warn_or_exception(_('Cannot write "{path}", not an accepted format, use: {formats}')
|
||||
.format(path=metadatapath, formats=', '.join(accepted)))
|
||||
|
||||
if ext == 'txt':
|
||||
with open(metadatapath, 'w') as mf:
|
||||
return write_txt(mf, app)
|
||||
elif ext == 'yml':
|
||||
if ext == 'yml':
|
||||
if importlib.util.find_spec('ruamel.yaml'):
|
||||
with open(metadatapath, 'w') as mf:
|
||||
return write_yaml(mf, app)
|
||||
|
|
|
|||
|
|
@ -248,7 +248,6 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
|
|||
config += "keypass = '%s'\n" % PASSWORD
|
||||
config += "keydname = '%s'\n" % DISTINGUISHED_NAME
|
||||
config += "make_current_version_link = False\n"
|
||||
config += "accepted_formats = ('txt', 'yml')\n"
|
||||
config += "update_stats = True\n"
|
||||
with open('config.py', 'w') as fp:
|
||||
fp.write(config)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
#
|
||||
# rewritemeta.py - part of the FDroid server tools
|
||||
# This cleans up the original .txt metadata file format.
|
||||
# This cleans up the original .yml metadata file format.
|
||||
# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
|
|
@ -21,6 +21,8 @@ from argparse import ArgumentParser
|
|||
import os
|
||||
import logging
|
||||
import io
|
||||
import tempfile
|
||||
import shutil
|
||||
|
||||
from . import _
|
||||
from . import common
|
||||
|
|
@ -30,9 +32,6 @@ config = None
|
|||
options = None
|
||||
|
||||
|
||||
SUPPORTED_FORMATS = ['txt', 'yml']
|
||||
|
||||
|
||||
def proper_format(app):
|
||||
s = io.StringIO()
|
||||
# TODO: currently reading entire file again, should reuse first
|
||||
|
|
@ -42,8 +41,6 @@ def proper_format(app):
|
|||
_ignored, extension = common.get_extension(app.metadatapath)
|
||||
if extension == 'yml':
|
||||
metadata.write_yaml(s, app)
|
||||
elif extension == 'txt':
|
||||
metadata.write_txt(s, app)
|
||||
content = s.getvalue()
|
||||
s.close()
|
||||
return content == cur_content
|
||||
|
|
@ -58,8 +55,6 @@ def main():
|
|||
common.setup_global_opts(parser)
|
||||
parser.add_argument("-l", "--list", action="store_true", default=False,
|
||||
help=_("List files that would be reformatted"))
|
||||
parser.add_argument("-t", "--to", default=None,
|
||||
help=_("Rewrite to a specific format: ") + ', '.join(SUPPORTED_FORMATS))
|
||||
parser.add_argument("appid", nargs='*', help=_("applicationId in the form APPID"))
|
||||
metadata.add_metadata_arguments(parser)
|
||||
options = parser.parse_args()
|
||||
|
|
@ -71,28 +66,15 @@ def main():
|
|||
allapps = metadata.read_metadata(xref=True)
|
||||
apps = common.read_app_args(options.appid, allapps, False)
|
||||
|
||||
if options.list and options.to is not None:
|
||||
parser.error(_("Cannot use --list and --to at the same time"))
|
||||
|
||||
if options.to is not None and options.to not in SUPPORTED_FORMATS:
|
||||
parser.error(_("Unsupported metadata format, use: --to [{supported}]")
|
||||
.format(supported=' '.join(SUPPORTED_FORMATS)))
|
||||
|
||||
for appid, app in apps.items():
|
||||
path = app.metadatapath
|
||||
base, ext = common.get_extension(path)
|
||||
if not options.to and ext not in SUPPORTED_FORMATS:
|
||||
if ext != "yml":
|
||||
logging.info(_("Ignoring {ext} file at '{path}'").format(ext=ext, path=path))
|
||||
continue
|
||||
elif options.to is not None:
|
||||
logging.info(_("Rewriting '{appid}' to '{path}'").format(appid=appid, path=options.to))
|
||||
else:
|
||||
logging.info(_("Rewriting '{appid}'").format(appid=appid))
|
||||
|
||||
to_ext = ext
|
||||
if options.to is not None:
|
||||
to_ext = options.to
|
||||
|
||||
if options.list:
|
||||
if not proper_format(app):
|
||||
print(path)
|
||||
|
|
@ -109,14 +91,12 @@ def main():
|
|||
newbuilds.append(new)
|
||||
app.builds = newbuilds
|
||||
|
||||
try:
|
||||
metadata.write_metadata(base + '.' + to_ext, app)
|
||||
# remove old format metadata if there was a format change
|
||||
# and rewriting to the new format worked
|
||||
if ext != to_ext:
|
||||
os.remove(path)
|
||||
finally:
|
||||
pass
|
||||
# rewrite to temporary file before overwriting existsing
|
||||
# file in case there's a bug in write_metadata
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmp_path = os.path.join(tmpdir, os.path.basename(path))
|
||||
metadata.write_metadata(tmp_path, app)
|
||||
shutil.move(tmp_path, path)
|
||||
|
||||
logging.debug(_("Finished"))
|
||||
|
||||
|
|
|
|||
|
|
@ -237,7 +237,7 @@ def update_wiki(apps, apks):
|
|||
wikidata += "=Maintainer Notes=\n"
|
||||
if app.MaintainerNotes:
|
||||
wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
|
||||
wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
|
||||
wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.yml current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.yml history]\n".format(appid)
|
||||
|
||||
# Get a list of all packages for this application...
|
||||
apklist = []
|
||||
|
|
|
|||
|
|
@ -637,88 +637,6 @@ class MetadataTest(unittest.TestCase):
|
|||
UpdateCheckMode: None
|
||||
"""))
|
||||
|
||||
def test_parse_txt_srclib(self):
|
||||
fdroidserver.metadata.warnings_action = 'error'
|
||||
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
||||
with open('JSoup.txt', 'w', encoding='utf-8') as f:
|
||||
f.write(textwrap.dedent('''\
|
||||
# Source details (the only mandatory fields)
|
||||
Repo Type:git
|
||||
Repo:https://github.com/jhy/jsoup.git
|
||||
|
||||
# Comma-separated list of subdirs to use. The first existing subdirectory
|
||||
# found between those given will be used. If none is found or provided, the
|
||||
# root of the repo directory will be used instead.
|
||||
Subdir:
|
||||
|
||||
# Any extra commands to prepare the source library
|
||||
Prepare:
|
||||
'''))
|
||||
srclib = fdroidserver.metadata.parse_txt_srclib('JSoup.txt')
|
||||
self.assertDictEqual({'Repo': 'https://github.com/jhy/jsoup.git',
|
||||
'RepoType': 'git',
|
||||
'Subdir': [''],
|
||||
'Prepare': ''},
|
||||
srclib)
|
||||
|
||||
def test_parse_txt_srclib_simple(self):
|
||||
fdroidserver.metadata.warnings_action = 'error'
|
||||
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
||||
with open('simple.txt', 'w', encoding='utf-8') as f:
|
||||
f.write(textwrap.dedent('''\
|
||||
# this should be simple
|
||||
Repo Type:git
|
||||
Repo:https://git.host/repo.git
|
||||
'''))
|
||||
srclib = fdroidserver.metadata.parse_txt_srclib('simple.txt')
|
||||
self.assertDictEqual({'Repo': 'https://git.host/repo.git',
|
||||
'RepoType': 'git',
|
||||
'Subdir': None,
|
||||
'Prepare': None},
|
||||
srclib)
|
||||
|
||||
def test_parse_txt_srclib_simple_blanks(self):
|
||||
fdroidserver.metadata.warnings_action = 'error'
|
||||
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
||||
with open('simple.txt', 'w', encoding='utf-8') as f:
|
||||
f.write(textwrap.dedent('''\
|
||||
# this should be simple
|
||||
|
||||
Repo Type:git
|
||||
|
||||
Repo:https://git.host/repo.git
|
||||
|
||||
Subdir:
|
||||
|
||||
Prepare:
|
||||
'''))
|
||||
srclib = fdroidserver.metadata.parse_txt_srclib('simple.txt')
|
||||
self.assertDictEqual({'Repo': 'https://git.host/repo.git',
|
||||
'RepoType': 'git',
|
||||
'Subdir': [''],
|
||||
'Prepare': ''},
|
||||
srclib)
|
||||
|
||||
def test_parse_txt_srclib_Changelog_cketti(self):
|
||||
fdroidserver.metadata.warnings_action = 'error'
|
||||
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
||||
with open('Changelog-cketti.txt', 'w', encoding='utf-8') as f:
|
||||
f.write(textwrap.dedent('''\
|
||||
Repo Type:git
|
||||
Repo:https://github.com/cketti/ckChangeLog
|
||||
|
||||
Subdir:library,ckChangeLog/src/main
|
||||
Prepare:[ -f project.properties ] || echo 'source.dir=java' > ant.properties && echo -e 'android.library=true\\ntarget=android-19' > project.properties
|
||||
'''))
|
||||
srclib = fdroidserver.metadata.parse_txt_srclib('Changelog-cketti.txt')
|
||||
self.assertDictEqual({'Repo': 'https://github.com/cketti/ckChangeLog',
|
||||
'RepoType': 'git',
|
||||
'Subdir': ['library', 'ckChangeLog/src/main'],
|
||||
'Prepare': "[ -f project.properties ] || echo 'source.dir=java' > "
|
||||
"ant.properties && echo -e "
|
||||
"'android.library=true\\ntarget=android-19' > project.properties"},
|
||||
srclib)
|
||||
|
||||
def test_parse_yaml_srclib_unknown_key(self):
|
||||
fdroidserver.metadata.warnings_action = 'error'
|
||||
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
||||
|
|
@ -885,11 +803,11 @@ class MetadataTest(unittest.TestCase):
|
|||
RepoType: git
|
||||
Repo: https://git.host/repo.git
|
||||
'''))
|
||||
with open('srclibs/simple-wb.txt', 'w', encoding='utf-8') as f:
|
||||
with open('srclibs/simple-wb.yml', 'w', encoding='utf-8') as f:
|
||||
f.write(textwrap.dedent('''\
|
||||
# this should be simple
|
||||
Repo Type:git
|
||||
Repo:https://git.host/repo.git
|
||||
RepoType: git
|
||||
Repo: https://git.host/repo.git
|
||||
|
||||
Subdir:
|
||||
Prepare:
|
||||
|
|
|
|||
|
|
@ -49,48 +49,13 @@ class RewriteMetaTest(unittest.TestCase):
|
|||
|
||||
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
||||
os.mkdir('metadata')
|
||||
with open('metadata/a.txt', 'w') as f:
|
||||
f.write('Auto Name:a')
|
||||
with open('metadata/a.yml', 'w') as f:
|
||||
f.write('AutoName: a')
|
||||
with open('metadata/b.yml', 'w') as f:
|
||||
f.write('AutoName: b')
|
||||
|
||||
rewritemeta.main()
|
||||
|
||||
with open('metadata/a.txt') as f:
|
||||
self.assertEqual(f.read(), textwrap.dedent('''\
|
||||
Categories:
|
||||
License:Unknown
|
||||
Web Site:
|
||||
Source Code:
|
||||
Issue Tracker:
|
||||
|
||||
Auto Name:a
|
||||
|
||||
Auto Update Mode:None
|
||||
Update Check Mode:None
|
||||
'''))
|
||||
|
||||
with open('metadata/b.yml') as f:
|
||||
self.assertEqual(f.read(), textwrap.dedent('''\
|
||||
License: Unknown
|
||||
|
||||
AutoName: b
|
||||
|
||||
AutoUpdateMode: None
|
||||
UpdateCheckMode: None
|
||||
'''))
|
||||
|
||||
def test_rewrite_scenario_txt_to_yml(self):
|
||||
|
||||
sys.argv = ['rewritemeta', '--to', 'yml', 'a']
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
||||
os.mkdir('metadata')
|
||||
with open('metadata/a.txt', 'w') as f:
|
||||
f.write('Auto Name:a')
|
||||
|
||||
rewritemeta.main()
|
||||
|
||||
with open('metadata/a.yml') as f:
|
||||
self.assertEqual(f.read(), textwrap.dedent('''\
|
||||
License: Unknown
|
||||
|
|
@ -101,25 +66,15 @@ class RewriteMetaTest(unittest.TestCase):
|
|||
UpdateCheckMode: None
|
||||
'''))
|
||||
|
||||
def test_rewrite_scenario_txt_to_yml_no_ruamel(self):
|
||||
|
||||
sys.argv = ['rewritemeta', '--to', 'yml', 'a']
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
||||
os.mkdir('metadata')
|
||||
with open('metadata/a.txt', 'w') as f:
|
||||
f.write('Auto Name:a')
|
||||
|
||||
def boom(*args):
|
||||
raise FDroidException(' '.join((str(x) for x in args)))
|
||||
|
||||
with mock.patch('fdroidserver.metadata.write_yaml', boom):
|
||||
with self.assertRaises(FDroidException):
|
||||
rewritemeta.main()
|
||||
|
||||
with open('metadata/a.txt') as f:
|
||||
with open('metadata/b.yml') as f:
|
||||
self.assertEqual(f.read(), textwrap.dedent('''\
|
||||
Auto Name:a'''))
|
||||
License: Unknown
|
||||
|
||||
AutoName: b
|
||||
|
||||
AutoUpdateMode: None
|
||||
UpdateCheckMode: None
|
||||
'''))
|
||||
|
||||
def test_rewrite_scenario_yml_no_ruamel(self):
|
||||
sys.argv = ['rewritemeta', 'a']
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue