From c6a97939f1254f44318d1496c21a828bb5ed1d21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Fri, 13 Dec 2019 23:51:18 +0100 Subject: [PATCH 01/11] rename parse_srclib to parese_txt_srclib + test case --- fdroidserver/metadata.py | 6 +++--- tests/metadata.TestCase | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 26bdcf22..fbf11a44 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -704,7 +704,7 @@ def description_html(s, linkres): return ps.text_html -def parse_srclib(metadatapath): +def parse_txt_srclib(metadatapath): thisinfo = {} @@ -746,7 +746,7 @@ def read_srclibs(): The information read will be accessible as metadata.srclibs, which is a dictionary, keyed on srclib name, with the values each being a dictionary - in the same format as that returned by the parse_srclib function. + in the same format as that returned by the parse_txt_srclib function. A MetaDataException is raised if there are any problems with the srclib metadata. @@ -765,7 +765,7 @@ def read_srclibs(): for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.txt'))): srclibname = os.path.basename(metadatapath[:-4]) - srclibs[srclibname] = parse_srclib(metadatapath) + srclibs[srclibname] = parse_txt_srclib(metadatapath) def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False): diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index 686301db..08d4137c 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -23,6 +23,8 @@ print('localmodule: ' + localmodule) if localmodule not in sys.path: sys.path.insert(0, localmodule) +from testcommon import TmpCwd + import fdroidserver.common import fdroidserver.metadata from fdroidserver.exception import MetaDataException @@ -568,6 +570,29 @@ class MetadataTest(unittest.TestCase): UpdateCheckMode: None """)) + def test_parse_txt_srclib(self): + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + with open('JSoup.txt', 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + # Source details (the only mandatory fields) + Repo Type:git + Repo:https://github.com/jhy/jsoup.git + + # Comma-separated list of subdirs to use. The first existing subdirectory + # found between those given will be used. If none is found or provided, the + # root of the repo directory will be used instead. + Subdir: + + # Any extra commands to prepare the source library + Prepare: + ''')) + srclib = fdroidserver.metadata.parse_txt_srclib('JSoup.txt') + self.assertDictEqual({'Repo': 'https://github.com/jhy/jsoup.git', + 'Repo Type': 'git', + 'Subdir': [''], + 'Prepare': ''}, + srclib) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From 2c86d3802855cacdab5f08a99009fa19e4e09e6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sun, 15 Dec 2019 17:27:27 +0100 Subject: [PATCH 02/11] srclib: no spaces in dict key --- fdroidserver/common.py | 4 ++-- fdroidserver/metadata.py | 5 ++++- tests/metadata.TestCase | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index bab6c3b9..59b58e34 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -773,7 +773,7 @@ def getvcs(vcstype, remote, local): def getsrclibvcs(name): if name not in fdroidserver.metadata.srclibs: raise VCSException("Missing srclib " + name) - return fdroidserver.metadata.srclibs[name]['Repo Type'] + return fdroidserver.metadata.srclibs[name]['RepoType'] class vcs: @@ -1838,7 +1838,7 @@ def getsrclib(spec, srclib_dir, subdir=None, basepath=False, sdir = os.path.join(srclib_dir, name) if not preponly: - vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir) + vcs = getvcs(srclib["RepoType"], srclib["Repo"], sdir) vcs.srclib = (name, number, sdir) if ref: vcs.gotorevision(ref, refresh) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index fbf11a44..a17e4d69 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -709,7 +709,7 @@ def parse_txt_srclib(metadatapath): thisinfo = {} # Defaults for fields that come from metadata - thisinfo['Repo Type'] = '' + thisinfo['RepoType'] = '' thisinfo['Repo'] = '' thisinfo['Subdir'] = None thisinfo['Prepare'] = None @@ -731,6 +731,9 @@ def parse_txt_srclib(metadatapath): except ValueError: warn_or_exception(_("Invalid metadata in %s:%d") % (line, n)) + # collapse whitespaces in field names + f = f.replace(' ', '') + if f == "Subdir": thisinfo[f] = v.split(',') else: diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index 08d4137c..0f3e3cfe 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -588,7 +588,7 @@ class MetadataTest(unittest.TestCase): ''')) srclib = fdroidserver.metadata.parse_txt_srclib('JSoup.txt') self.assertDictEqual({'Repo': 'https://github.com/jhy/jsoup.git', - 'Repo Type': 'git', + 'RepoType': 'git', 'Subdir': [''], 'Prepare': ''}, srclib) From 286220fe1101c7eb94e36dbdad6c471e8c5d8b6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Mon, 16 Dec 2019 19:17:46 +0100 Subject: [PATCH 03/11] srclib: add test for getsrclibvcs --- tests/common.TestCase | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/common.TestCase b/tests/common.TestCase index 268d5dd4..acd0e887 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -34,7 +34,7 @@ import fdroidserver.signindex import fdroidserver.common import fdroidserver.metadata from testcommon import TmpCwd -from fdroidserver.exception import FDroidException +from fdroidserver.exception import FDroidException, VCSException class CommonTest(unittest.TestCase): @@ -1241,6 +1241,16 @@ class CommonTest(unittest.TestCase): self.assertEqual(123, fdroidserver.common.version_code_string_to_int('0000123')) self.assertEqual(-42, fdroidserver.common.version_code_string_to_int('-42')) + def test_getsrclibvcs(self): + fdroidserver.metadata.srclibs = {'somelib': {'RepoType': 'git'}, + 'yeslib': {'RepoType': 'hg'}, + 'nolib': {'RepoType': 'git-svn'}} + self.assertEqual(fdroidserver.common.getsrclibvcs('somelib'), 'git') + self.assertEqual(fdroidserver.common.getsrclibvcs('yeslib'), 'hg') + self.assertEqual(fdroidserver.common.getsrclibvcs('nolib'), 'git-svn') + with self.assertRaises(VCSException): + fdroidserver.common.getsrclibvcs('nonexistentlib') + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From fa7885063b568feea02adf9ec4929b25a24154b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 19 Dec 2019 22:55:14 +0100 Subject: [PATCH 04/11] scrlib: add test for getsrclib --- tests/common.TestCase | 47 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/common.TestCase b/tests/common.TestCase index acd0e887..2ee7852f 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -1251,6 +1251,53 @@ class CommonTest(unittest.TestCase): with self.assertRaises(VCSException): fdroidserver.common.getsrclibvcs('nonexistentlib') + def test_getsrclib_not_found(self): + fdroidserver.common.config = {'sdk_path': '', + 'java_paths': {}} + fdroidserver.metadata.srclibs = {} + + with self.assertRaisesRegex(VCSException, 'srclib SDL not found.'): + fdroidserver.common.getsrclib('SDL@release-2.0.3', 'srclib') + + def test_getsrclib_gotorevision_raw(self): + fdroidserver.common.config = {'sdk_path': '', + 'java_paths': {}} + fdroidserver.metadata.srclibs = {'SDL': {'RepoType': 'git', + 'Repo': ''}} + + vcs = mock.Mock() + + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + os.makedirs(os.path.join('srclib', 'SDL')) + with mock.patch('fdroidserver.common.getvcs', return_value=vcs): + ret = fdroidserver.common.getsrclib('SDL', 'srclib', raw=True) + self.assertEqual(vcs.srclib, ('SDL', None, 'srclib/SDL')) + self.assertEqual(ret, vcs) + + def test_getsrclib_gotorevision_ref(self): + fdroidserver.common.config = {'sdk_path': '', + 'java_paths': {}} + fdroidserver.metadata.srclibs = {'ACRA': {'RepoType': 'git', + 'Repo': 'https://github.com/ACRA/acra.git', + 'Subdir': None, + 'Prepare': None}} + + vcs = mock.Mock() + skm = mock.Mock() + dfm = mock.Mock() + + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + os.makedirs(os.path.join('srclib', 'ACRA')) + with mock.patch('fdroidserver.common.getvcs', return_value=vcs): + with mock.patch('fdroidserver.common.remove_signing_keys', skm): + with mock.patch('fdroidserver.common.remove_debuggable_flags', dfm): + ret = fdroidserver.common.getsrclib('ACRA@acra-4.6.2', 'srclib') + self.assertEqual(vcs.srclib, ('ACRA', None, 'srclib/ACRA')) + vcs.gotorevision.assert_called_once_with('acra-4.6.2', True) + skm.assert_called_once_with('srclib/ACRA') + dfm.assert_called_once_with('srclib/ACRA') + self.assertEqual(ret, ('ACRA', None, 'srclib/ACRA')) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From ee3d8d2f1833c91acee08f9ec5d04cb182c801fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Tue, 24 Dec 2019 00:42:29 +0100 Subject: [PATCH 05/11] srclib: yml parsing + tests for yml and txt --- fdroidserver/metadata.py | 41 ++++++++++ tests/metadata.TestCase | 170 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index a17e4d69..52c00d85 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -744,6 +744,43 @@ def parse_txt_srclib(metadatapath): return thisinfo +def parse_yml_srclib(metadatapath): + + thisinfo = {'RepoType': '', + 'Repo': '', + 'Subdir': None, + 'Prepare': None} + + if not os.path.exists(metadatapath): + warn_or_exception(_("Invalid scrlib metadata: '{file}' " + "does not exist" + .format(file=metadatapath))) + return thisinfo + + with open(metadatapath, "r", encoding="utf-8") as f: + try: + data = yaml.load(f, Loader=SafeLoader) + except yaml.error.YAMLError as e: + warn_or_exception(_("Invalid srclib metadata: could not " + "parse '{file}'" + .format(file=metadatapath))) + return thisinfo + + for key in data.keys(): + if key not in thisinfo.keys(): + warn_or_exception(_("Invalid srclib metadata: unknown key " + "'{key}' in '{file}'") + .format(key=key, file=metadatapath)) + return thisinfo + else: + if key == 'Subdir': + thisinfo[key] = str(data[key] or '').split(',') + else: + thisinfo[key] = str(data[key] or '') + + return thisinfo + + def read_srclibs(): """Read all srclib metadata. @@ -770,6 +807,10 @@ def read_srclibs(): srclibname = os.path.basename(metadatapath[:-4]) srclibs[srclibname] = parse_txt_srclib(metadatapath) + for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.yml'))): + srclibname = os.path.basename(metadatapath[:-4]) + srclibs[srclibname] = parse_yml_srclib(metadatapath) + def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False): """Return a list of App instances sorted newest first diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index 0f3e3cfe..844de8ae 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -571,6 +571,7 @@ class MetadataTest(unittest.TestCase): """)) def test_parse_txt_srclib(self): + fdroidserver.metadata.warnings_action = 'error' with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): with open('JSoup.txt', 'w', encoding='utf-8') as f: f.write(textwrap.dedent('''\ @@ -593,6 +594,175 @@ class MetadataTest(unittest.TestCase): 'Prepare': ''}, srclib) + def test_parse_txt_srclib_simple(self): + fdroidserver.metadata.warnings_action = 'error' + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + with open('simple.txt', 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + # this should be simple + Repo Type:git + Repo:https://git.host/repo.git + ''')) + srclib = fdroidserver.metadata.parse_txt_srclib('simple.txt') + self.assertDictEqual({'Repo': 'https://git.host/repo.git', + 'RepoType': 'git', + 'Subdir': None, + 'Prepare': None}, + srclib) + + def test_parse_txt_srclib_simple_blanks(self): + fdroidserver.metadata.warnings_action = 'error' + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + with open('simple.txt', 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + # this should be simple + + Repo Type:git + + Repo:https://git.host/repo.git + + Subdir: + + Prepare: + ''')) + srclib = fdroidserver.metadata.parse_txt_srclib('simple.txt') + self.assertDictEqual({'Repo': 'https://git.host/repo.git', + 'RepoType': 'git', + 'Subdir': [''], + 'Prepare': ''}, + srclib) + + def test_parse_txt_srclib_Changelog_cketti(self): + fdroidserver.metadata.warnings_action = 'error' + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + with open('Changelog-cketti.txt', 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + Repo Type:git + Repo:https://github.com/cketti/ckChangeLog + + Subdir:library,ckChangeLog/src/main + Prepare:[ -f project.properties ] || echo 'source.dir=java' > ant.properties && echo -e 'android.library=true\\ntarget=android-19' > project.properties + ''')) + srclib = fdroidserver.metadata.parse_txt_srclib('Changelog-cketti.txt') + self.assertDictEqual({'Repo': 'https://github.com/cketti/ckChangeLog', + 'RepoType': 'git', + 'Subdir': ['library', 'ckChangeLog/src/main'], + 'Prepare': "[ -f project.properties ] || echo 'source.dir=java' > " + "ant.properties && echo -e " + "'android.library=true\\ntarget=android-19' > project.properties"}, + srclib) + + def test_parse_yml_srclib_unknown_key(self): + fdroidserver.metadata.warnings_action = 'error' + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + with open('test.yml', 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + RepoType: git + Repo: https://example.com/test.git + Evil: I should not be here. + ''')) + with self.assertRaisesRegex(MetaDataException, + "Invalid srclib metadata: " + "unknown key 'Evil' in " + "'test.yml'"): + fdroidserver.metadata.parse_yml_srclib('test.yml') + + def test_parse_yml_srclib_does_not_exists(self): + fdroidserver.metadata.warnings_action = 'error' + with self.assertRaisesRegex(MetaDataException, + "Invalid scrlib metadata: " + "'non/existent-test-srclib.yml' " + "does not exist"): + fdroidserver.metadata.parse_yml_srclib('non/existent-test-srclib.yml') + + def test_parse_yml_srclib_simple(self): + fdroidserver.metadata.warnings_action = 'error' + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + with open('simple.yml', 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + # this should be simple + RepoType: git + Repo: https://git.host/repo.git + ''')) + srclib = fdroidserver.metadata.parse_yml_srclib('simple.yml') + self.assertDictEqual({'Repo': 'https://git.host/repo.git', + 'RepoType': 'git', + 'Subdir': None, + 'Prepare': None}, + srclib) + + def test_parse_yml_srclib_simple_with_blanks(self): + fdroidserver.metadata.warnings_action = 'error' + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + with open('simple.yml', 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + # this should be simple + + RepoType: git + + Repo: https://git.host/repo.git + + Subdir: + + Prepare: + ''')) + srclib = fdroidserver.metadata.parse_yml_srclib('simple.yml') + self.assertDictEqual({'Repo': 'https://git.host/repo.git', + 'RepoType': 'git', + 'Subdir': [''], + 'Prepare': ''}, + srclib) + + def test_parse_yml_srclib_Changelog_cketti(self): + fdroidserver.metadata.warnings_action = 'error' + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + with open('Changelog-cketti.yml', 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + RepoType: git + Repo: https://github.com/cketti/ckChangeLog + + Subdir: library,ckChangeLog/src/main + Prepare: "[ -f project.properties ] || echo 'source.dir=java' > ant.properties && echo -e 'android.library=true\\\\ntarget=android-19' > project.properties" + ''')) + srclib = fdroidserver.metadata.parse_yml_srclib('Changelog-cketti.yml') + self.assertDictEqual({'Repo': 'https://github.com/cketti/ckChangeLog', + 'RepoType': 'git', + 'Subdir': ['library', 'ckChangeLog/src/main'], + 'Prepare': "[ -f project.properties ] || echo 'source.dir=java' > " + "ant.properties && echo -e " + "'android.library=true\\ntarget=android-19' > project.properties"}, + srclib) + + def test_read_srclibs(self): + fdroidserver.metadata.warnings_action = 'error' + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + os.mkdir('srclibs') + with open('srclibs/simple.yml', 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + RepoType: git + Repo: https://git.host/repo.git + ''')) + with open('srclibs/simple-wb.txt', 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + # this should be simple + Repo Type:git + Repo:https://git.host/repo.git + + Subdir: + Prepare: + ''')) + fdroidserver.metadata.read_srclibs() + print(fdroidserver.metadata.srclibs) + self.assertDictEqual(fdroidserver.metadata.srclibs, + {'simple-wb': {'RepoType': 'git', + 'Repo': 'https://git.host/repo.git', + 'Subdir': [''], + 'Prepare': ''}, + 'simple': {'RepoType': 'git', + 'Repo': 'https://git.host/repo.git', + 'Subdir': None, + 'Prepare': None}}) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From aa020af040e087c97eb4fa5ed85fda52bbc45217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Tue, 24 Dec 2019 08:35:27 +0100 Subject: [PATCH 06/11] srclibs: forward yml parsing error cause --- fdroidserver/metadata.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 52c00d85..d01736cf 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -42,12 +42,15 @@ srclibs = None warnings_action = None -def warn_or_exception(value): +def warn_or_exception(value, cause=None): '''output warning or Exception depending on -W''' if warnings_action == 'ignore': pass elif warnings_action == 'error': - raise MetaDataException(value) + if cause: + raise MetaDataException(value) from cause + else: + raise MetaDataException(value) else: logging.warning(value) @@ -763,14 +766,15 @@ def parse_yml_srclib(metadatapath): except yaml.error.YAMLError as e: warn_or_exception(_("Invalid srclib metadata: could not " "parse '{file}'" - .format(file=metadatapath))) + .format(file=metadatapath)), + e) return thisinfo for key in data.keys(): if key not in thisinfo.keys(): warn_or_exception(_("Invalid srclib metadata: unknown key " "'{key}' in '{file}'") - .format(key=key, file=metadatapath)) + .format(key=key, file=metadatapath)) return thisinfo else: if key == 'Subdir': From 58776da694c86a7b09c629fa04b6a7f5dc391ca1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Mon, 3 Feb 2020 23:30:23 +0100 Subject: [PATCH 07/11] get tests working --- tests/metadata.TestCase | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index 844de8ae..bc141246 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -725,16 +725,17 @@ class MetadataTest(unittest.TestCase): Prepare: "[ -f project.properties ] || echo 'source.dir=java' > ant.properties && echo -e 'android.library=true\\\\ntarget=android-19' > project.properties" ''')) srclib = fdroidserver.metadata.parse_yml_srclib('Changelog-cketti.yml') - self.assertDictEqual({'Repo': 'https://github.com/cketti/ckChangeLog', + self.assertDictEqual(srclib, + {'Repo': 'https://github.com/cketti/ckChangeLog', 'RepoType': 'git', 'Subdir': ['library', 'ckChangeLog/src/main'], 'Prepare': "[ -f project.properties ] || echo 'source.dir=java' > " "ant.properties && echo -e " - "'android.library=true\\ntarget=android-19' > project.properties"}, - srclib) + "'android.library=true\\ntarget=android-19' > project.properties"}) def test_read_srclibs(self): fdroidserver.metadata.warnings_action = 'error' + fdroidserver.metadata.srclibs = None with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): os.mkdir('srclibs') with open('srclibs/simple.yml', 'w', encoding='utf-8') as f: @@ -752,7 +753,6 @@ class MetadataTest(unittest.TestCase): Prepare: ''')) fdroidserver.metadata.read_srclibs() - print(fdroidserver.metadata.srclibs) self.assertDictEqual(fdroidserver.metadata.srclibs, {'simple-wb': {'RepoType': 'git', 'Repo': 'https://git.host/repo.git', From 1ac7d612b10094bb8ee1ad3ad92599d804fb8023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sat, 29 Feb 2020 13:42:12 +0100 Subject: [PATCH 08/11] yml srclibs: support multiline Prepare --- fdroidserver/metadata.py | 2 ++ tests/metadata.TestCase | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index d01736cf..e86faa70 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -779,6 +779,8 @@ def parse_yml_srclib(metadatapath): else: if key == 'Subdir': thisinfo[key] = str(data[key] or '').split(',') + elif key == 'Prepare' and isinstance(data[key], list): + thisinfo[key] = ' && '.join(data[key]) else: thisinfo[key] = str(data[key] or '') diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index bc141246..7e7edcf8 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -733,6 +733,43 @@ class MetadataTest(unittest.TestCase): "ant.properties && echo -e " "'android.library=true\\ntarget=android-19' > project.properties"}) + def test_read_srclibs_yml_prepare_list(self): + fdroidserver.metadata.warnings_action = 'error' + fdroidserver.metadata.srclibs = None + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + os.mkdir('srclibs') + with open('srclibs/with-list.yml', 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + # this should be simple + RepoType: git + Repo: https://git.host/repo.git + + Subdir: + Prepare: + - The Matrix is a system, Neo. + - That system is our enemy. + - But when you're inside, you look around, what do you see? + - Businessmen, teachers, lawyers, carpenters. + - The very minds of the people we are trying to save. + - But until we do, these people are still a part of that system and that makes them our enemy. + - You have to understand, most of these people are not ready to be unplugged. + - And many of them are so inert, so hopelessly dependent on the system that they will fight to protect it. + ''')) + fdroidserver.metadata.read_srclibs() + self.maxDiff = None + self.assertDictEqual(fdroidserver.metadata.srclibs, + {'with-list': {'RepoType': 'git', + 'Repo': 'https://git.host/repo.git', + 'Subdir': [''], + 'Prepare': 'The Matrix is a system, Neo. && ' + 'That system is our enemy. && ' + 'But when you\'re inside, you look around, what do you see? && ' + 'Businessmen, teachers, lawyers, carpenters. && ' + 'The very minds of the people we are trying to save. && ' + 'But until we do, these people are still a part of that system and that makes them our enemy. && ' + 'You have to understand, most of these people are not ready to be unplugged. && ' + 'And many of them are so inert, so hopelessly dependent on the system that they will fight to protect it.'}}) + def test_read_srclibs(self): fdroidserver.metadata.warnings_action = 'error' fdroidserver.metadata.srclibs = None From 5741e6930b853a97c14bb36b561c4eacd76ad414 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sat, 29 Feb 2020 15:50:06 +0100 Subject: [PATCH 09/11] yml srclibs: support Subdir as list --- fdroidserver/metadata.py | 7 ++++++- tests/metadata.TestCase | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index e86faa70..3f79db43 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -778,7 +778,12 @@ def parse_yml_srclib(metadatapath): return thisinfo else: if key == 'Subdir': - thisinfo[key] = str(data[key] or '').split(',') + if isinstance(data[key], str): + thisinfo[key] = data[key].split(',') + elif isinstance(data[key], list): + thisinfo[key] = data[key] + elif data[key] is None: + thisinfo[key] = [''] elif key == 'Prepare' and isinstance(data[key], list): thisinfo[key] = ' && '.join(data[key]) else: diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index 7e7edcf8..dedbdd3f 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -733,6 +733,44 @@ class MetadataTest(unittest.TestCase): "ant.properties && echo -e " "'android.library=true\\ntarget=android-19' > project.properties"}) + def test_read_srclibs_yml_subdir_list(self): + fdroidserver.metadata.warnings_action = 'error' + fdroidserver.metadata.srclibs = None + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + os.mkdir('srclibs') + with open('srclibs/with-list.yml', 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + # this should be simple + RepoType: git + Repo: https://git.host/repo.git + + Subdir: + - This is your last chance. + - After this, there is no turning back. + - You take the blue pill—the story ends, + - you wake up in your bed + - and believe whatever you want to believe. + - You take the red pill—you stay in Wonderland + - and I show you how deep the rabbit-hole goes. + Prepare: + There is a difference between knowing the path + and walking the path. + ''')) + fdroidserver.metadata.read_srclibs() + self.maxDiff = None + self.assertDictEqual(fdroidserver.metadata.srclibs, + {'with-list': {'RepoType': 'git', + 'Repo': 'https://git.host/repo.git', + 'Subdir': ['This is your last chance.', + 'After this, there is no turning back.', + 'You take the blue pill—the story ends,', + 'you wake up in your bed', + 'and believe whatever you want to believe.', + 'You take the red pill—you stay in Wonderland', + 'and I show you how deep the rabbit-hole goes.'], + 'Prepare': 'There is a difference between knowing the path ' + 'and walking the path.'}}) + def test_read_srclibs_yml_prepare_list(self): fdroidserver.metadata.warnings_action = 'error' fdroidserver.metadata.srclibs = None From 8344e510f6d202b2f3c987e15512dee9590c40ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 9 Apr 2020 11:41:15 +0200 Subject: [PATCH 10/11] yml srclibs: changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10cf3c6f..32ff73dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ([!656](https://gitlab.com/fdroid/fdroidserver/merge_requests/656)) * add SHA256 to filename of repo graphics ([!669](https://gitlab.com/fdroid/fdroidserver/merge_requests/669)) +* support for srclibs metadata in YAML format + ([!700](https://gitlab.com/fdroid/fdroidserver/merge_requests/700)) ### Fixed * fix build-logs dissapearing when deploying From 911e86fc874f7721ad53f3bf48618e88949c1692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 16 Apr 2020 11:32:37 +0200 Subject: [PATCH 11/11] support copying yml srclibs to buildserver vm during builds --- fdroidserver/build.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index dbcccde6..41467533 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -203,8 +203,18 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): send_dir(lib) # Copy the metadata file too... ftp.chdir(posixpath.join(homedir, 'srclibs')) - ftp.put(os.path.join('srclibs', name + '.txt'), - name + '.txt') + if os.path.isfile(os.path.join('srclibs', name + '.yml')): + ftp.put(os.path.join('srclibs', name + '.yml'), + name + '.yml') + elif os.path.isfile(os.path.join('srclibs', name + '.txt')): + ftp.put(os.path.join('srclibs', name + '.txt'), + name + '.txt') + else: + raise BuildException("can not find metadata file for " + "'{name}', please make sure it is " + "present in your 'srclibs' folder." + "(supported formats: txt, yml)" + .format(name=name)) # Copy the main app source code # (no need if it's a srclib) if (not basesrclib) and os.path.exists(build_dir):