diff --git a/CHANGELOG.md b/CHANGELOG.md index de4c33a4..3acb9b4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) * build: deploying buildlogs with rsync ([!651](https://gitlab.com/fdroid/fdroidserver/merge_requests/651)) +### Removed +* removed support for txt and json metadata + ([!772](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/772)) + ## [1.1.4] - 2019-08-15 ### Fixed * include bitcoin validation regex required by fdroiddata diff --git a/fdroidserver/build.py b/fdroidserver/build.py index e8880c02..ecf466c2 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -206,14 +206,11 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): if os.path.isfile(os.path.join('srclibs', name + '.yml')): ftp.put(os.path.join('srclibs', name + '.yml'), name + '.yml') - elif os.path.isfile(os.path.join('srclibs', name + '.txt')): - ftp.put(os.path.join('srclibs', name + '.txt'), - name + '.txt') else: raise BuildException("can not find metadata file for " "'{name}', please make sure it is " "present in your 'srclibs' folder." - "(supported formats: txt, yml)" + "(supported format: yml)" .format(name=name)) # Copy the main app source code # (no need if it's a srclib) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 4a5ad875..5f057e40 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -119,7 +119,6 @@ default_config = { 'mvn3': "mvn", 'gradle': os.path.join(FDROID_PATH, 'gradlew-fdroid'), 'gradle_version_dir': os.path.join(os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), 'gradle'), - 'accepted_formats': ['txt', 'yml'], 'sync_from_local_copy_dir': False, 'allow_disabled_algorithms': False, 'per_app_repos': False, @@ -517,7 +516,7 @@ def get_local_metadata_files(): '''get any metadata files local to an app's source repo This tries to ignore anything that does not count as app metdata, - including emacs cruft ending in ~ and the .fdroid.key*pass.txt files. + including emacs cruft ending in ~ ''' return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]') diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index cf6a1119..8fafc7a0 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -515,16 +515,12 @@ def check_for_unsupported_metadata_files(basedir=""): global config return_value = False - formats = config['accepted_formats'] for f in glob.glob(basedir + 'metadata/*') + glob.glob(basedir + 'metadata/.*'): if os.path.isdir(f): - exists = False - for t in formats: - exists = exists or os.path.exists(f + '.' + t) - if not exists: + if not os.path.exists(f + '.yml'): print(_('"%s/" has no matching metadata file!') % f) return_value = True - elif os.path.splitext(f)[1][1:] in formats: + elif os.path.splitext(f)[1][1:] == "yml": packageName = os.path.splitext(os.path.basename(f))[0] if not common.is_valid_package_name(packageName): print('"' + packageName + '" is an invalid package name!\n' @@ -532,7 +528,7 @@ def check_for_unsupported_metadata_files(basedir=""): return_value = True else: print('"' + f.replace(basedir, '') - + '" is not a supported file format: (' + ','.join(formats) + ')') + + '" is not a supported file format (use: .yml)') return_value = True return return_value diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index ee58b1b4..60762076 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -18,7 +18,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import json import os import re import glob @@ -59,54 +58,6 @@ def warn_or_exception(value, cause=None): logging.warning(value) -# To filter which ones should be written to the metadata files if -# present -app_fields = set([ - 'Disabled', - 'AntiFeatures', - 'Provides', # deprecated, txt only - 'Categories', - 'License', - 'Author Name', - 'Author Email', - 'Author Web Site', - 'Web Site', - 'Source Code', - 'Issue Tracker', - 'Translation', - 'Changelog', - 'Donate', - 'FlattrID', - 'Liberapay', - 'LiberapayID', - 'OpenCollective', - 'Bitcoin', - 'Litecoin', - 'Name', - 'Auto Name', - 'Summary', - 'Description', - 'Requires Root', - 'Repo Type', - 'Repo', - 'Binaries', - 'Maintainer Notes', - 'Archive Policy', - 'Auto Update Mode', - 'Update Check Mode', - 'Update Check Ignore', - 'Vercode Operation', - 'Update Check Name', - 'Update Check Data', - 'Current Version', - 'Current Version Code', - 'No Source Since', - 'Build', - - 'comments', # For formats that don't do inline comments - 'builds', # For formats that do builds as a list -]) - yaml_app_field_order = [ 'Disabled', 'AntiFeatures', @@ -266,7 +217,9 @@ def fieldtype(name): # In the order in which they are laid out on files -build_flags_order = [ +build_flags = [ + 'versionName', + 'versionCode', 'disable', 'commit', 'timeout', @@ -301,10 +254,6 @@ build_flags_order = [ 'antifeatures', ] -# old .txt format has version name/code inline in the 'Build:' line -# but YAML and JSON have a explicit key for them -build_flags = ['versionName', 'versionCode'] + build_flags_order - class Build(dict): @@ -691,16 +640,6 @@ class DescriptionFormatter: self.html.close() -# Parse multiple lines of description as written in a metadata file, returning -# a single string in text format and wrapped to 80 columns. -def description_txt(s): - ps = DescriptionFormatter(None) - for line in s.splitlines(): - ps.parseline(line) - ps.end() - return ps.text_txt - - # Parse multiple lines of description as written in a metadata file, returning # a single string in wiki format. Used for the Maintainer Notes field as well, # because it's the same format. @@ -718,46 +657,6 @@ def description_html(s, linkres): return ps.text_html -def parse_txt_srclib(metadatapath): - - thisinfo = {} - - # Defaults for fields that come from metadata - thisinfo['RepoType'] = '' - thisinfo['Repo'] = '' - thisinfo['Subdir'] = None - thisinfo['Prepare'] = None - - if not os.path.exists(metadatapath): - return thisinfo - - metafile = open(metadatapath, "r") - - n = 0 - for line in metafile: - n += 1 - line = line.rstrip('\r\n') - if not line or line.startswith("#"): - continue - - try: - f, v = line.split(':', 1) - except ValueError: - warn_or_exception(_("Invalid metadata in %s:%d") % (line, n)) - - # collapse whitespaces in field names - f = f.replace(' ', '') - - if f == "Subdir": - thisinfo[f] = v.split(',') - else: - thisinfo[f] = v - - metafile.close() - - return thisinfo - - def parse_yaml_srclib(metadatapath): thisinfo = {'RepoType': '', @@ -813,7 +712,7 @@ def read_srclibs(): The information read will be accessible as metadata.srclibs, which is a dictionary, keyed on srclib name, with the values each being a dictionary - in the same format as that returned by the parse_txt_srclib function. + in the same format as that returned by the parse_yaml_srclib function. A MetaDataException is raised if there are any problems with the srclib metadata. @@ -830,10 +729,6 @@ def read_srclibs(): if not os.path.exists(srcdir): os.makedirs(srcdir) - for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.txt'))): - srclibname = os.path.basename(metadatapath[:-4]) - srclibs[srclibname] = parse_txt_srclib(metadatapath) - for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.yml'))): srclibname = os.path.basename(metadatapath[:-4]) srclibs[srclibname] = parse_yaml_srclib(metadatapath) @@ -847,11 +742,6 @@ def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False): sorted based on creation time, newest first. Most of the time, the newer files are the most interesting. - If there are multiple metadata files for a single appid, then the first - file that is parsed wins over all the others, and the rest throw an - exception. So the original .txt format is parsed first, at least until - newer formats stabilize. - check_vcs is the list of appids to check for .fdroid.yml in source """ @@ -866,11 +756,7 @@ def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False): if not os.path.exists(basedir): os.makedirs(basedir) - metadatafiles = (glob.glob(os.path.join('metadata', '*.txt')) - + glob.glob(os.path.join('metadata', '*.json')) - + glob.glob(os.path.join('metadata', '*.yml')) - + glob.glob('.fdroid.txt') - + glob.glob('.fdroid.json') + metadatafiles = (glob.glob(os.path.join('metadata', '*.yml')) + glob.glob('.fdroid.yml')) if sort_by_time: @@ -883,8 +769,6 @@ def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False): metadatafiles = sorted(metadatafiles) for metadatapath in metadatafiles: - if metadatapath == '.fdroid.txt': - warn_or_exception(_('.fdroid.txt is not supported! Convert to .fdroid.yml or .fdroid.json.')) appid, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath)) if appid != '.fdroid' and not fdroidserver.common.is_valid_package_name(appid): warn_or_exception(_("{appid} from {path} is not a valid Java Package Name!") @@ -1044,10 +928,6 @@ def parse_metadata(metadatapath, check_vcs=False, refresh=True): '''parse metadata file, optionally checking the git repo for metadata first''' _ignored, ext = fdroidserver.common.get_extension(metadatapath) - accepted = fdroidserver.common.config['accepted_formats'] - if ext not in accepted: - warn_or_exception(_('"{path}" is not an accepted format, convert to: {formats}') - .format(path=metadatapath, formats=', '.join(accepted))) app = App() app.metadatapath = metadatapath @@ -1057,16 +937,12 @@ def parse_metadata(metadatapath, check_vcs=False, refresh=True): else: app.id = name - with open(metadatapath, 'r') as mf: - if ext == 'txt': - parse_txt_metadata(mf, app) - elif ext == 'json': - parse_json_metadata(mf, app) - elif ext == 'yml': + if ext == 'yml': + with open(metadatapath, 'r') as mf: parse_yaml_metadata(mf, app) - else: - warn_or_exception(_('Unknown metadata format: {path}') - .format(path=metadatapath)) + else: + warn_or_exception(_('Unknown metadata format: {path} (use: .yml)') + .format(path=metadatapath)) if check_vcs and app.Repo: build_dir = fdroidserver.common.get_build_dir(app) @@ -1098,20 +974,6 @@ def parse_metadata(metadatapath, check_vcs=False, refresh=True): return app -def parse_json_metadata(mf, app): - - # fdroid metadata is only strings and booleans, no floats or ints. - # TODO create schema using https://pypi.python.org/pypi/jsonschema - jsoninfo = json.load(mf, parse_int=lambda s: s, - parse_float=lambda s: s) - app.update(jsoninfo) - for f in ['Description', 'Maintainer Notes']: - v = app.get(f) - if v: - app[f] = '\n'.join(v) - return app - - def parse_yaml_metadata(mf, app): try: yamldata = yaml.load(mf, Loader=SafeLoader) @@ -1254,7 +1116,7 @@ def write_yaml(mf, app): insert_newline = True else: if app.get(field) or field == 'Builds': - # .txt calls it 'builds' internally, everywhere else its 'Builds' + # .txt called it 'builds' internally, everywhere else its 'Builds' if field == 'Builds': if app.get('builds'): cm.update({field: _builds_to_yaml(app)}) @@ -1304,363 +1166,10 @@ build_line_sep = re.compile(r'(? ant.properties && echo -e 'android.library=true\\ntarget=android-19' > project.properties - ''')) - srclib = fdroidserver.metadata.parse_txt_srclib('Changelog-cketti.txt') - self.assertDictEqual({'Repo': 'https://github.com/cketti/ckChangeLog', - 'RepoType': 'git', - 'Subdir': ['library', 'ckChangeLog/src/main'], - 'Prepare': "[ -f project.properties ] || echo 'source.dir=java' > " - "ant.properties && echo -e " - "'android.library=true\\ntarget=android-19' > project.properties"}, - srclib) - def test_parse_yaml_srclib_unknown_key(self): fdroidserver.metadata.warnings_action = 'error' with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): @@ -885,11 +803,11 @@ class MetadataTest(unittest.TestCase): RepoType: git Repo: https://git.host/repo.git ''')) - with open('srclibs/simple-wb.txt', 'w', encoding='utf-8') as f: + with open('srclibs/simple-wb.yml', 'w', encoding='utf-8') as f: f.write(textwrap.dedent('''\ # this should be simple - Repo Type:git - Repo:https://git.host/repo.git + RepoType: git + Repo: https://git.host/repo.git Subdir: Prepare: diff --git a/tests/rewritemeta.TestCase b/tests/rewritemeta.TestCase index 664f3c78..8a7d41e4 100755 --- a/tests/rewritemeta.TestCase +++ b/tests/rewritemeta.TestCase @@ -49,48 +49,13 @@ class RewriteMetaTest(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): os.mkdir('metadata') - with open('metadata/a.txt', 'w') as f: - f.write('Auto Name:a') + with open('metadata/a.yml', 'w') as f: + f.write('AutoName: a') with open('metadata/b.yml', 'w') as f: f.write('AutoName: b') rewritemeta.main() - with open('metadata/a.txt') as f: - self.assertEqual(f.read(), textwrap.dedent('''\ - Categories: - License:Unknown - Web Site: - Source Code: - Issue Tracker: - - Auto Name:a - - Auto Update Mode:None - Update Check Mode:None - ''')) - - with open('metadata/b.yml') as f: - self.assertEqual(f.read(), textwrap.dedent('''\ - License: Unknown - - AutoName: b - - AutoUpdateMode: None - UpdateCheckMode: None - ''')) - - def test_rewrite_scenario_txt_to_yml(self): - - sys.argv = ['rewritemeta', '--to', 'yml', 'a'] - - with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): - os.mkdir('metadata') - with open('metadata/a.txt', 'w') as f: - f.write('Auto Name:a') - - rewritemeta.main() - with open('metadata/a.yml') as f: self.assertEqual(f.read(), textwrap.dedent('''\ License: Unknown @@ -101,25 +66,15 @@ class RewriteMetaTest(unittest.TestCase): UpdateCheckMode: None ''')) - def test_rewrite_scenario_txt_to_yml_no_ruamel(self): - - sys.argv = ['rewritemeta', '--to', 'yml', 'a'] - - with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): - os.mkdir('metadata') - with open('metadata/a.txt', 'w') as f: - f.write('Auto Name:a') - - def boom(*args): - raise FDroidException(' '.join((str(x) for x in args))) - - with mock.patch('fdroidserver.metadata.write_yaml', boom): - with self.assertRaises(FDroidException): - rewritemeta.main() - - with open('metadata/a.txt') as f: + with open('metadata/b.yml') as f: self.assertEqual(f.read(), textwrap.dedent('''\ - Auto Name:a''')) + License: Unknown + + AutoName: b + + AutoUpdateMode: None + UpdateCheckMode: None + ''')) def test_rewrite_scenario_yml_no_ruamel(self): sys.argv = ['rewritemeta', 'a']