Merge branch 'yaml-is-king' into 'master'

remove all code for handling txt metadata

See merge request fdroid/fdroidserver!772

CHANGELOG.md

@@ -32,6 +32,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 * build: deploying buildlogs with rsync
   ([!651](https://gitlab.com/fdroid/fdroidserver/merge_requests/651))
 
+### Removed
+* removed support for txt and json metadata
+  ([!772](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/772))
+
 ## [1.1.4] - 2019-08-15
 ### Fixed
 * include bitcoin validation regex required by fdroiddata

fdroidserver/build.py

@@ -206,14 +206,11 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
             if os.path.isfile(os.path.join('srclibs', name + '.yml')):
                 ftp.put(os.path.join('srclibs', name + '.yml'),
                         name + '.yml')
-            elif os.path.isfile(os.path.join('srclibs', name + '.txt')):
-                ftp.put(os.path.join('srclibs', name + '.txt'),
-                        name + '.txt')
             else:
                 raise BuildException("can not find metadata file for "
                                      "'{name}', please make sure it is "
                                      "present in your 'srclibs' folder."
-                                     "(supported formats: txt, yml)"
+                                     "(supported format: yml)"
                                      .format(name=name))
         # Copy the main app source code
         # (no need if it's a srclib)

fdroidserver/common.py

@@ -119,7 +119,6 @@ default_config = {
     'mvn3': "mvn",
     'gradle': os.path.join(FDROID_PATH, 'gradlew-fdroid'),
     'gradle_version_dir': os.path.join(os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), 'gradle'),
-    'accepted_formats': ['txt', 'yml'],
     'sync_from_local_copy_dir': False,
     'allow_disabled_algorithms': False,
     'per_app_repos': False,
@@ -517,7 +516,7 @@ def get_local_metadata_files():
     '''get any metadata files local to an app's source repo
 
     This tries to ignore anything that does not count as app metdata,
-    including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
+    including emacs cruft ending in ~
 
     '''
     return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')

fdroidserver/lint.py

@@ -515,16 +515,12 @@ def check_for_unsupported_metadata_files(basedir=""):
     global config
 
     return_value = False
-    formats = config['accepted_formats']
     for f in glob.glob(basedir + 'metadata/*') + glob.glob(basedir + 'metadata/.*'):
         if os.path.isdir(f):
-            exists = False
+            if not os.path.exists(f + '.yml'):
-            for t in formats:
-                exists = exists or os.path.exists(f + '.' + t)
-            if not exists:
                 print(_('"%s/" has no matching metadata file!') % f)
                 return_value = True
-        elif os.path.splitext(f)[1][1:] in formats:
+        elif os.path.splitext(f)[1][1:] == "yml":
             packageName = os.path.splitext(os.path.basename(f))[0]
             if not common.is_valid_package_name(packageName):
                 print('"' + packageName + '" is an invalid package name!\n'
@@ -532,7 +528,7 @@ def check_for_unsupported_metadata_files(basedir=""):
                 return_value = True
         else:
             print('"' + f.replace(basedir, '')
-                  + '" is not a supported file format: (' + ','.join(formats) + ')')
+                  + '" is not a supported file format (use: .yml)')
             return_value = True
 
     return return_value

fdroidserver/metadata.py

@@ -18,7 +18,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import json
 import os
 import re
 import glob
@@ -59,54 +58,6 @@ def warn_or_exception(value, cause=None):
         logging.warning(value)
 
 
-# To filter which ones should be written to the metadata files if
-# present
-app_fields = set([
-    'Disabled',
-    'AntiFeatures',
-    'Provides',  # deprecated, txt only
-    'Categories',
-    'License',
-    'Author Name',
-    'Author Email',
-    'Author Web Site',
-    'Web Site',
-    'Source Code',
-    'Issue Tracker',
-    'Translation',
-    'Changelog',
-    'Donate',
-    'FlattrID',
-    'Liberapay',
-    'LiberapayID',
-    'OpenCollective',
-    'Bitcoin',
-    'Litecoin',
-    'Name',
-    'Auto Name',
-    'Summary',
-    'Description',
-    'Requires Root',
-    'Repo Type',
-    'Repo',
-    'Binaries',
-    'Maintainer Notes',
-    'Archive Policy',
-    'Auto Update Mode',
-    'Update Check Mode',
-    'Update Check Ignore',
-    'Vercode Operation',
-    'Update Check Name',
-    'Update Check Data',
-    'Current Version',
-    'Current Version Code',
-    'No Source Since',
-    'Build',
-
-    'comments',  # For formats that don't do inline comments
-    'builds',    # For formats that do builds as a list
-])
-
 yaml_app_field_order = [
     'Disabled',
     'AntiFeatures',
@@ -266,7 +217,9 @@ def fieldtype(name):
 
 
 # In the order in which they are laid out on files
-build_flags_order = [
+build_flags = [
+    'versionName',
+    'versionCode',
     'disable',
     'commit',
     'timeout',
@@ -301,10 +254,6 @@ build_flags_order = [
     'antifeatures',
 ]
 
-# old .txt format has version name/code inline in the 'Build:' line
-# but YAML and JSON have a explicit key for them
-build_flags = ['versionName', 'versionCode'] + build_flags_order
-
 
 class Build(dict):
 
@@ -691,16 +640,6 @@ class DescriptionFormatter:
         self.html.close()
 
 
-# Parse multiple lines of description as written in a metadata file, returning
-# a single string in text format and wrapped to 80 columns.
-def description_txt(s):
-    ps = DescriptionFormatter(None)
-    for line in s.splitlines():
-        ps.parseline(line)
-    ps.end()
-    return ps.text_txt
-
-
 # Parse multiple lines of description as written in a metadata file, returning
 # a single string in wiki format. Used for the Maintainer Notes field as well,
 # because it's the same format.
@@ -718,46 +657,6 @@ def description_html(s, linkres):
     return ps.text_html
 
 
-def parse_txt_srclib(metadatapath):
-
-    thisinfo = {}
-
-    # Defaults for fields that come from metadata
-    thisinfo['RepoType'] = ''
-    thisinfo['Repo'] = ''
-    thisinfo['Subdir'] = None
-    thisinfo['Prepare'] = None
-
-    if not os.path.exists(metadatapath):
-        return thisinfo
-
-    metafile = open(metadatapath, "r")
-
-    n = 0
-    for line in metafile:
-        n += 1
-        line = line.rstrip('\r\n')
-        if not line or line.startswith("#"):
-            continue
-
-        try:
-            f, v = line.split(':', 1)
-        except ValueError:
-            warn_or_exception(_("Invalid metadata in %s:%d") % (line, n))
-
-        # collapse whitespaces in field names
-        f = f.replace(' ', '')
-
-        if f == "Subdir":
-            thisinfo[f] = v.split(',')
-        else:
-            thisinfo[f] = v
-
-    metafile.close()
-
-    return thisinfo
-
-
 def parse_yaml_srclib(metadatapath):
 
     thisinfo = {'RepoType': '',
@@ -813,7 +712,7 @@ def read_srclibs():
 
     The information read will be accessible as metadata.srclibs, which is a
     dictionary, keyed on srclib name, with the values each being a dictionary
-    in the same format as that returned by the parse_txt_srclib function.
+    in the same format as that returned by the parse_yaml_srclib function.
 
     A MetaDataException is raised if there are any problems with the srclib
     metadata.
@@ -830,10 +729,6 @@ def read_srclibs():
     if not os.path.exists(srcdir):
         os.makedirs(srcdir)
 
-    for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.txt'))):
-        srclibname = os.path.basename(metadatapath[:-4])
-        srclibs[srclibname] = parse_txt_srclib(metadatapath)
-
     for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.yml'))):
         srclibname = os.path.basename(metadatapath[:-4])
         srclibs[srclibname] = parse_yaml_srclib(metadatapath)
@@ -847,11 +742,6 @@ def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False):
     sorted based on creation time, newest first.  Most of the time,
     the newer files are the most interesting.
 
-    If there are multiple metadata files for a single appid, then the first
-    file that is parsed wins over all the others, and the rest throw an
-    exception. So the original .txt format is parsed first, at least until
-    newer formats stabilize.
-
     check_vcs is the list of appids to check for .fdroid.yml in source
 
     """
@@ -866,11 +756,7 @@ def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False):
         if not os.path.exists(basedir):
             os.makedirs(basedir)
 
-    metadatafiles = (glob.glob(os.path.join('metadata', '*.txt'))
+    metadatafiles = (glob.glob(os.path.join('metadata', '*.yml'))
-                     + glob.glob(os.path.join('metadata', '*.json'))
-                     + glob.glob(os.path.join('metadata', '*.yml'))
-                     + glob.glob('.fdroid.txt')
-                     + glob.glob('.fdroid.json')
                      + glob.glob('.fdroid.yml'))
 
     if sort_by_time:
@@ -883,8 +769,6 @@ def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False):
         metadatafiles = sorted(metadatafiles)
 
     for metadatapath in metadatafiles:
-        if metadatapath == '.fdroid.txt':
-            warn_or_exception(_('.fdroid.txt is not supported!  Convert to .fdroid.yml or .fdroid.json.'))
         appid, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
         if appid != '.fdroid' and not fdroidserver.common.is_valid_package_name(appid):
             warn_or_exception(_("{appid} from {path} is not a valid Java Package Name!")
@@ -1044,10 +928,6 @@ def parse_metadata(metadatapath, check_vcs=False, refresh=True):
     '''parse metadata file, optionally checking the git repo for metadata first'''
 
     _ignored, ext = fdroidserver.common.get_extension(metadatapath)
-    accepted = fdroidserver.common.config['accepted_formats']
-    if ext not in accepted:
-        warn_or_exception(_('"{path}" is not an accepted format, convert to: {formats}')
-                          .format(path=metadatapath, formats=', '.join(accepted)))
 
     app = App()
     app.metadatapath = metadatapath
@@ -1057,15 +937,11 @@ def parse_metadata(metadatapath, check_vcs=False, refresh=True):
     else:
         app.id = name
 
+    if ext == 'yml':
         with open(metadatapath, 'r') as mf:
-        if ext == 'txt':
-            parse_txt_metadata(mf, app)
-        elif ext == 'json':
-            parse_json_metadata(mf, app)
-        elif ext == 'yml':
             parse_yaml_metadata(mf, app)
     else:
-            warn_or_exception(_('Unknown metadata format: {path}')
+        warn_or_exception(_('Unknown metadata format: {path} (use: .yml)')
                           .format(path=metadatapath))
 
     if check_vcs and app.Repo:
@@ -1098,20 +974,6 @@ def parse_metadata(metadatapath, check_vcs=False, refresh=True):
     return app
 
 
-def parse_json_metadata(mf, app):
-
-    # fdroid metadata is only strings and booleans, no floats or ints.
-    # TODO create schema using https://pypi.python.org/pypi/jsonschema
-    jsoninfo = json.load(mf, parse_int=lambda s: s,
-                         parse_float=lambda s: s)
-    app.update(jsoninfo)
-    for f in ['Description', 'Maintainer Notes']:
-        v = app.get(f)
-        if v:
-            app[f] = '\n'.join(v)
-    return app
-
-
 def parse_yaml_metadata(mf, app):
     try:
         yamldata = yaml.load(mf, Loader=SafeLoader)
@@ -1254,7 +1116,7 @@ def write_yaml(mf, app):
                 insert_newline = True
             else:
                 if app.get(field) or field == 'Builds':
-                    # .txt calls it 'builds' internally, everywhere else its 'Builds'
+                    # .txt called it 'builds' internally, everywhere else its 'Builds'
                     if field == 'Builds':
                         if app.get('builds'):
                             cm.update({field: _builds_to_yaml(app)})
@@ -1304,363 +1166,10 @@ build_line_sep = re.compile(r'(?<!\\),')
 build_cont = re.compile(r'^[ \t]')
 
 
-def parse_txt_metadata(mf, app):
-
-    linedesc = None
-
-    def add_buildflag(p, build):
-        if not p.strip():
-            warn_or_exception(_("Empty build flag at {linedesc}")
-                              .format(linedesc=linedesc))
-        bv = p.split('=', 1)
-        if len(bv) != 2:
-            warn_or_exception(_("Invalid build flag at {line} in {linedesc}")
-                              .format(line=buildlines[0], linedesc=linedesc))
-
-        pk, pv = bv
-        pk = pk.lstrip()
-        if pk == 'update':
-            pk = 'androidupdate'  # avoid conflicting with Build(dict).update()
-        t = flagtype(pk)
-        if t == TYPE_LIST:
-            pv = split_list_values(pv)
-            build[pk] = pv
-        elif t == TYPE_STRING or t == TYPE_SCRIPT:
-            build[pk] = pv
-        elif t == TYPE_BOOL:
-            build[pk] = _decode_bool(pv)
-        elif t == TYPE_INT:
-            build[pk] = int(pv)
-
-    def parse_buildline(lines):
-        v = "".join(lines)
-        parts = [p.replace("\\,", ",") for p in re.split(build_line_sep, v)]
-        if len(parts) < 3:
-            warn_or_exception(_("Invalid build format: {value} in {name}")
-                              .format(value=v, name=mf.name))
-        build = Build()
-        build.versionName = parts[0]
-        build.versionCode = parts[1]
-        check_versionCode(build.versionCode)
-
-        if parts[2].startswith('!'):
-            # For backwards compatibility, handle old-style disabling,
-            # including attempting to extract the commit from the message
-            build.disable = parts[2][1:]
-            commit = 'unknown - see disabled'
-            index = parts[2].rfind('at ')
-            if index != -1:
-                commit = parts[2][index + 3:]
-                if commit.endswith(')'):
-                    commit = commit[:-1]
-            build.commit = commit
-        else:
-            build.commit = parts[2]
-        for p in parts[3:]:
-            add_buildflag(p, build)
-
-        return build
-
-    def check_versionCode(versionCode):
-        try:
-            int(versionCode)
-        except ValueError:
-            warn_or_exception(_('Invalid versionCode: "{versionCode}" is not an integer!')
-                              .format(versionCode=versionCode))
-
-    def add_comments(key):
-        if not curcomments:
-            return
-        app.comments[key] = list(curcomments)
-        del curcomments[:]
-
-    mode = 0
-    buildlines = []
-    multiline_lines = []
-    curcomments = []
-    build = None
-    vc_seen = set()
-
-    app.builds = []
-
-    c = 0
-    for line in mf:
-        c += 1
-        linedesc = "%s:%d" % (mf.name, c)
-        line = line.rstrip('\r\n')
-        if mode == 3:
-            if build_cont.match(line):
-                if line.endswith('\\'):
-                    buildlines.append(line[:-1].lstrip())
-                else:
-                    buildlines.append(line.lstrip())
-                    bl = ''.join(buildlines)
-                    add_buildflag(bl, build)
-                    del buildlines[:]
-            else:
-                if not build.commit and not build.disable:
-                    warn_or_exception(_("No commit specified for {versionName} in {linedesc}")
-                                      .format(versionName=build.versionName, linedesc=linedesc))
-
-                app.builds.append(build)
-                add_comments('build:' + build.versionCode)
-                mode = 0
-
-        if mode == 0:
-            if not line:
-                continue
-            if line.startswith("#"):
-                curcomments.append(line[1:].strip())
-                continue
-            try:
-                f, v = line.split(':', 1)
-            except ValueError:
-                warn_or_exception(_("Invalid metadata in: ") + linedesc)
-
-            if f not in app_fields:
-                warn_or_exception(_('Unrecognised app field: ') + f)
-
-            # Translate obsolete fields...
-            if f == 'Market Version':
-                f = 'Current Version'
-            if f == 'Market Version Code':
-                f = 'Current Version Code'
-
-            f = f.replace(' ', '')
-
-            ftype = fieldtype(f)
-            if ftype not in [TYPE_BUILD]:
-                add_comments(f)
-            if ftype == TYPE_MULTILINE:
-                mode = 1
-                if v:
-                    warn_or_exception(_("Unexpected text on same line as {field} in {linedesc}")
-                                      .format(field=f, linedesc=linedesc))
-            elif ftype == TYPE_STRING:
-                app[f] = v
-            elif ftype == TYPE_LIST:
-                app[f] = split_list_values(v)
-            elif ftype == TYPE_BUILD:
-                vv = v.split(',')
-                if len(vv) != 2:
-                    warn_or_exception(_('Build should have comma-separated '
-                                        'versionName and versionCode, '
-                                        'not "{value}", in {linedesc}')
-                                      .format(value=v, linedesc=linedesc))
-                build = Build()
-                build.versionName = vv[0]
-                build.versionCode = vv[1]
-                check_versionCode(build.versionCode)
-
-                if build.versionCode in vc_seen:
-                    warn_or_exception(_('Duplicate build recipe found for versionCode {versionCode} in {linedesc}')
-                                      .format(versionCode=build.versionCode, linedesc=linedesc))
-                vc_seen.add(build.versionCode)
-                del buildlines[:]
-                mode = 3
-            elif ftype == TYPE_OBSOLETE:
-                warn_or_exception(_("'{field}' in {linedesc} is obsolete, see docs for current fields:")
-                                  .format(field=f, linedesc=linedesc)
-                                  + '\nhttps://f-droid.org/docs/')
-            else:
-                warn_or_exception(_("Unrecognised field '{field}' in {linedesc}")
-                                  .format(field=f, linedesc=linedesc))
-        elif mode == 1:     # Multiline field
-            if line == '.':
-                mode = 0
-                app[f] = '\n'.join(multiline_lines)
-                del multiline_lines[:]
-            else:
-                multiline_lines.append(line)
-        elif mode == 2:     # Line continuation mode in Build Version
-            if line.endswith("\\"):
-                buildlines.append(line[:-1])
-            else:
-                buildlines.append(line)
-                build = parse_buildline(buildlines)
-                app.builds.append(build)
-                add_comments('build:' + app.builds[-1].versionCode)
-                mode = 0
-    add_comments(None)
-
-    # Mode at end of file should always be 0
-    if mode == 1:
-        warn_or_exception(_("{field} not terminated in {name}")
-                          .format(field=f, name=mf.name))
-    if mode == 2:
-        warn_or_exception(_("Unterminated continuation in {name}")
-                          .format(name=mf.name))
-    if mode == 3:
-        warn_or_exception(_("Unterminated build in {name}")
-                          .format(name=mf.name))
-
-    return app
-
-
-def write_plaintext_metadata(mf, app, w_comment, w_field, w_build):
-
-    def field_to_attr(f):
-        """
-        Translates human-readable field names to attribute names, e.g.
-        'Auto Name' to 'AutoName'
-        """
-        return f.replace(' ', '')
-
-    def attr_to_field(k):
-        """
-        Translates attribute names to human-readable field names, e.g.
-        'AutoName' to 'Auto Name'
-        """
-        if k in app_fields:
-            return k
-        f = re.sub(r'([a-z])([A-Z])', r'\1 \2', k)
-        return f
-
-    def w_comments(key):
-        if key not in app.comments:
-            return
-        for line in app.comments[key]:
-            w_comment(line)
-
-    def w_field_always(f, v=None):
-        key = field_to_attr(f)
-        if v is None:
-            v = app.get(key)
-        w_comments(key)
-        w_field(f, v)
-
-    def w_field_nonempty(f, v=None):
-        key = field_to_attr(f)
-        if v is None:
-            v = app.get(key)
-        w_comments(key)
-        if v:
-            w_field(f, v)
-
-    w_field_nonempty('Disabled')
-    w_field_nonempty('AntiFeatures')
-    w_field_nonempty('Provides')
-    w_field_always('Categories')
-    w_field_always('License')
-    w_field_nonempty('Author Name')
-    w_field_nonempty('Author Email')
-    w_field_nonempty('Author Web Site')
-    w_field_always('Web Site')
-    w_field_always('Source Code')
-    w_field_always('Issue Tracker')
-    w_field_nonempty('Translation')
-    w_field_nonempty('Changelog')
-    w_field_nonempty('Donate')
-    w_field_nonempty('FlattrID')
-    w_field_nonempty('LiberapayID')
-    w_field_nonempty('OpenCollective')
-    w_field_nonempty('Bitcoin')
-    w_field_nonempty('Litecoin')
-    mf.write('\n')
-    w_field_nonempty('Name')
-    w_field_nonempty('Auto Name')
-    w_field_nonempty('Summary')
-    w_field_nonempty('Description', description_txt(app.Description))
-    mf.write('\n')
-    if app.RequiresRoot:
-        w_field_always('Requires Root', 'yes')
-        mf.write('\n')
-    if app.RepoType:
-        w_field_always('Repo Type')
-        w_field_always('Repo')
-        if app.Binaries:
-            w_field_always('Binaries')
-        mf.write('\n')
-
-    for build in app.builds:
-
-        if build.versionName == "Ignore":
-            continue
-
-        w_comments('build:%s' % build.versionCode)
-        w_build(build)
-        mf.write('\n')
-
-    if app.MaintainerNotes:
-        w_field_always('Maintainer Notes', app.MaintainerNotes)
-        mf.write('\n')
-
-    w_field_nonempty('Archive Policy')
-    w_field_always('Auto Update Mode')
-    w_field_always('Update Check Mode')
-    w_field_nonempty('Update Check Ignore')
-    w_field_nonempty('Vercode Operation')
-    w_field_nonempty('Update Check Name')
-    w_field_nonempty('Update Check Data')
-    if app.CurrentVersion:
-        w_field_always('Current Version')
-        w_field_always('Current Version Code')
-    if app.NoSourceSince:
-        mf.write('\n')
-        w_field_always('No Source Since')
-    w_comments(None)
-
-
-# Write a metadata file in txt format.
-#
-# 'mf'      - Writer interface (file, StringIO, ...)
-# 'app'     - The app data
-def write_txt(mf, app):
-
-    def w_comment(line):
-        mf.write("# %s\n" % line)
-
-    def w_field(f, v):
-        t = fieldtype(f)
-        if t == TYPE_LIST:
-            v = ','.join(v)
-        elif t == TYPE_MULTILINE:
-            v = '\n' + v + '\n.'
-        mf.write("%s:%s\n" % (f, v))
-
-    def w_build(build):
-        mf.write("Build:%s,%s\n" % (build.versionName, build.versionCode))
-
-        for f in build_flags_order:
-            v = build.get(f)
-            if not v:
-                continue
-
-            t = flagtype(f)
-            if f == 'androidupdate':
-                f = 'update'  # avoid conflicting with Build(dict).update()
-            mf.write('    %s=' % f)
-            if t == TYPE_STRING or t == TYPE_INT:
-                mf.write(v)
-            elif t == TYPE_BOOL:
-                mf.write('yes')
-            elif t == TYPE_SCRIPT:
-                first = True
-                for s in v.split(' && '):
-                    if first:
-                        first = False
-                    else:
-                        mf.write(' && \\\n        ')
-                    mf.write(s.strip())
-            elif t == TYPE_LIST:
-                mf.write(','.join(v))
-
-            mf.write('\n')
-
-    write_plaintext_metadata(mf, app, w_comment, w_field, w_build)
-
-
 def write_metadata(metadatapath, app):
     _ignored, ext = fdroidserver.common.get_extension(metadatapath)
-    accepted = fdroidserver.common.config['accepted_formats']
-    if ext not in accepted:
-        warn_or_exception(_('Cannot write "{path}", not an accepted format, use: {formats}')
-                          .format(path=metadatapath, formats=', '.join(accepted)))
 
-    if ext == 'txt':
+    if ext == 'yml':
-        with open(metadatapath, 'w') as mf:
-            return write_txt(mf, app)
-    elif ext == 'yml':
         if importlib.util.find_spec('ruamel.yaml'):
             with open(metadatapath, 'w') as mf:
                 return write_yaml(mf, app)

fdroidserver/nightly.py

@@ -248,7 +248,6 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
         config += "keypass = '%s'\n" % PASSWORD
         config += "keydname = '%s'\n" % DISTINGUISHED_NAME
         config += "make_current_version_link = False\n"
-        config += "accepted_formats = ('txt', 'yml')\n"
         config += "update_stats = True\n"
         with open('config.py', 'w') as fp:
             fp.write(config)

fdroidserver/rewritemeta.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 #
 # rewritemeta.py - part of the FDroid server tools
-# This cleans up the original .txt metadata file format.
+# This cleans up the original .yml metadata file format.
 # Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
 #
 # This program is free software: you can redistribute it and/or modify
@@ -21,6 +21,8 @@ from argparse import ArgumentParser
 import os
 import logging
 import io
+import tempfile
+import shutil
 
 from . import _
 from . import common
@@ -30,9 +32,6 @@ config = None
 options = None
 
 
-SUPPORTED_FORMATS = ['txt', 'yml']
-
-
 def proper_format(app):
     s = io.StringIO()
     # TODO: currently reading entire file again, should reuse first
@@ -42,8 +41,6 @@ def proper_format(app):
     _ignored, extension = common.get_extension(app.metadatapath)
     if extension == 'yml':
         metadata.write_yaml(s, app)
-    elif extension == 'txt':
-        metadata.write_txt(s, app)
     content = s.getvalue()
     s.close()
     return content == cur_content
@@ -58,8 +55,6 @@ def main():
     common.setup_global_opts(parser)
     parser.add_argument("-l", "--list", action="store_true", default=False,
                         help=_("List files that would be reformatted"))
-    parser.add_argument("-t", "--to", default=None,
-                        help=_("Rewrite to a specific format: ") + ', '.join(SUPPORTED_FORMATS))
     parser.add_argument("appid", nargs='*', help=_("applicationId in the form APPID"))
     metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
@@ -71,28 +66,15 @@ def main():
     allapps = metadata.read_metadata(xref=True)
     apps = common.read_app_args(options.appid, allapps, False)
 
-    if options.list and options.to is not None:
-        parser.error(_("Cannot use --list and --to at the same time"))
-
-    if options.to is not None and options.to not in SUPPORTED_FORMATS:
-        parser.error(_("Unsupported metadata format, use: --to [{supported}]")
-                     .format(supported=' '.join(SUPPORTED_FORMATS)))
-
     for appid, app in apps.items():
         path = app.metadatapath
         base, ext = common.get_extension(path)
-        if not options.to and ext not in SUPPORTED_FORMATS:
+        if ext != "yml":
             logging.info(_("Ignoring {ext} file at '{path}'").format(ext=ext, path=path))
             continue
-        elif options.to is not None:
-            logging.info(_("Rewriting '{appid}' to '{path}'").format(appid=appid, path=options.to))
         else:
             logging.info(_("Rewriting '{appid}'").format(appid=appid))
 
-        to_ext = ext
-        if options.to is not None:
-            to_ext = options.to
-
         if options.list:
             if not proper_format(app):
                 print(path)
@@ -109,14 +91,12 @@ def main():
             newbuilds.append(new)
         app.builds = newbuilds
 
-        try:
+        # rewrite to temporary file before overwriting existsing
-            metadata.write_metadata(base + '.' + to_ext, app)
+        # file in case there's a bug in write_metadata
-            # remove old format metadata if there was  a format change
+        with tempfile.TemporaryDirectory() as tmpdir:
-            # and rewriting to the new format worked
+            tmp_path = os.path.join(tmpdir, os.path.basename(path))
-            if ext != to_ext:
+            metadata.write_metadata(tmp_path, app)
-                os.remove(path)
+            shutil.move(tmp_path, path)
-        finally:
-            pass
 
     logging.debug(_("Finished"))
 

fdroidserver/update.py

@@ -237,7 +237,7 @@ def update_wiki(apps, apks):
         wikidata += "=Maintainer Notes=\n"
         if app.MaintainerNotes:
             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
-        wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
+        wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.yml current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.yml history]\n".format(appid)
 
         # Get a list of all packages for this application...
         apklist = []

tests/metadata.TestCase

@@ -637,88 +637,6 @@ class MetadataTest(unittest.TestCase):
             UpdateCheckMode: None
             """))
 
-    def test_parse_txt_srclib(self):
-        fdroidserver.metadata.warnings_action = 'error'
-        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
-            with open('JSoup.txt', 'w', encoding='utf-8') as f:
-                f.write(textwrap.dedent('''\
-                    # Source details (the only mandatory fields)
-                    Repo Type:git
-                    Repo:https://github.com/jhy/jsoup.git
-
-                    # Comma-separated list of subdirs to use. The first existing subdirectory
-                    # found between those given will be used. If none is found or provided, the
-                    # root of the repo directory will be used instead.
-                    Subdir:
-
-                    # Any extra commands to prepare the source library
-                    Prepare:
-                    '''))
-            srclib = fdroidserver.metadata.parse_txt_srclib('JSoup.txt')
-            self.assertDictEqual({'Repo': 'https://github.com/jhy/jsoup.git',
-                                  'RepoType': 'git',
-                                  'Subdir': [''],
-                                  'Prepare': ''},
-                                 srclib)
-
-    def test_parse_txt_srclib_simple(self):
-        fdroidserver.metadata.warnings_action = 'error'
-        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
-            with open('simple.txt', 'w', encoding='utf-8') as f:
-                f.write(textwrap.dedent('''\
-                    # this should be simple
-                    Repo Type:git
-                    Repo:https://git.host/repo.git
-                    '''))
-            srclib = fdroidserver.metadata.parse_txt_srclib('simple.txt')
-            self.assertDictEqual({'Repo': 'https://git.host/repo.git',
-                                  'RepoType': 'git',
-                                  'Subdir': None,
-                                  'Prepare': None},
-                                 srclib)
-
-    def test_parse_txt_srclib_simple_blanks(self):
-        fdroidserver.metadata.warnings_action = 'error'
-        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
-            with open('simple.txt', 'w', encoding='utf-8') as f:
-                f.write(textwrap.dedent('''\
-                    # this should be simple
-
-                    Repo Type:git
-
-                    Repo:https://git.host/repo.git
-
-                    Subdir:
-
-                    Prepare:
-                    '''))
-            srclib = fdroidserver.metadata.parse_txt_srclib('simple.txt')
-            self.assertDictEqual({'Repo': 'https://git.host/repo.git',
-                                  'RepoType': 'git',
-                                  'Subdir': [''],
-                                  'Prepare': ''},
-                                 srclib)
-
-    def test_parse_txt_srclib_Changelog_cketti(self):
-        fdroidserver.metadata.warnings_action = 'error'
-        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
-            with open('Changelog-cketti.txt', 'w', encoding='utf-8') as f:
-                f.write(textwrap.dedent('''\
-                    Repo Type:git
-                    Repo:https://github.com/cketti/ckChangeLog
-
-                    Subdir:library,ckChangeLog/src/main
-                    Prepare:[ -f project.properties ] || echo 'source.dir=java' > ant.properties && echo -e 'android.library=true\\ntarget=android-19' > project.properties
-                    '''))
-            srclib = fdroidserver.metadata.parse_txt_srclib('Changelog-cketti.txt')
-            self.assertDictEqual({'Repo': 'https://github.com/cketti/ckChangeLog',
-                                  'RepoType': 'git',
-                                  'Subdir': ['library', 'ckChangeLog/src/main'],
-                                  'Prepare': "[ -f project.properties ] || echo 'source.dir=java' > "
-                                             "ant.properties && echo -e "
-                                             "'android.library=true\\ntarget=android-19' > project.properties"},
-                                 srclib)
-
     def test_parse_yaml_srclib_unknown_key(self):
         fdroidserver.metadata.warnings_action = 'error'
         with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
@@ -885,7 +803,7 @@ class MetadataTest(unittest.TestCase):
                     RepoType: git
                     Repo: https://git.host/repo.git
                     '''))
-            with open('srclibs/simple-wb.txt', 'w', encoding='utf-8') as f:
+            with open('srclibs/simple-wb.yml', 'w', encoding='utf-8') as f:
                 f.write(textwrap.dedent('''\
                     # this should be simple
                     RepoType: git

tests/rewritemeta.TestCase

@@ -49,20 +49,16 @@ class RewriteMetaTest(unittest.TestCase):
 
         with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
             os.mkdir('metadata')
-            with open('metadata/a.txt', 'w') as f:
+            with open('metadata/a.yml', 'w') as f:
                 f.write('AutoName: a')
             with open('metadata/b.yml', 'w') as f:
                 f.write('AutoName: b')
 
             rewritemeta.main()
 
-            with open('metadata/a.txt') as f:
+            with open('metadata/a.yml') as f:
                 self.assertEqual(f.read(), textwrap.dedent('''\
-                    Categories:
                     License: Unknown
-                    Web Site:
-                    Source Code:
-                    Issue Tracker:
 
                     AutoName: a
 
@@ -80,47 +76,6 @@ class RewriteMetaTest(unittest.TestCase):
                     UpdateCheckMode: None
                     '''))
 
-    def test_rewrite_scenario_txt_to_yml(self):
-
-        sys.argv = ['rewritemeta', '--to', 'yml', 'a']
-
-        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
-            os.mkdir('metadata')
-            with open('metadata/a.txt', 'w') as f:
-                f.write('Auto Name:a')
-
-            rewritemeta.main()
-
-            with open('metadata/a.yml') as f:
-                self.assertEqual(f.read(), textwrap.dedent('''\
-                    License: Unknown
-
-                    AutoName: a
-
-                    AutoUpdateMode: None
-                    UpdateCheckMode: None
-                    '''))
-
-    def test_rewrite_scenario_txt_to_yml_no_ruamel(self):
-
-        sys.argv = ['rewritemeta', '--to', 'yml', 'a']
-
-        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
-            os.mkdir('metadata')
-            with open('metadata/a.txt', 'w') as f:
-                f.write('Auto Name:a')
-
-            def boom(*args):
-                raise FDroidException(' '.join((str(x) for x in args)))
-
-            with mock.patch('fdroidserver.metadata.write_yaml', boom):
-                with self.assertRaises(FDroidException):
-                    rewritemeta.main()
-
-            with open('metadata/a.txt') as f:
-                self.assertEqual(f.read(), textwrap.dedent('''\
-                    Auto Name:a'''))
-
     def test_rewrite_scenario_yml_no_ruamel(self):
         sys.argv = ['rewritemeta', 'a']
         with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):