From 17320c23f4f7be1c71abdc5f0ea36f778ff25fcb Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 30 Sep 2019 13:07:49 +0200 Subject: [PATCH 1/5] lint: don't trip up on projects with 'master' in the name https://gitlab.com/fdroid/fdroiddata/merge_requests/5557#note_223283359 --- fdroidserver/lint.py | 2 +- tests/lint.TestCase | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index 86246d36..ba6fd442 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -122,7 +122,7 @@ http_url_shorteners = [ http_checks = https_enforcings + http_url_shorteners + [ (re.compile(r'.*github\.com/[^/]+/[^/]+\.git'), _("Appending .git is not necessary")), - (re.compile(r'.*://[^/]*(github|gitlab|bitbucket|rawgit)[^/]*/([^/]+/){1,3}master'), + (re.compile(r'^https://[^/]*(github|gitlab|bitbucket|rawgit)\.[a-zA-Z]+/([^/]+/){2,3}master/'), _("Use /HEAD instead of /master to point at a file in the default branch")), ] diff --git a/tests/lint.TestCase b/tests/lint.TestCase index f5dd2c30..12285431 100755 --- a/tests/lint.TestCase +++ b/tests/lint.TestCase @@ -70,6 +70,49 @@ class LintTest(unittest.TestCase): logging.debug(warn) self.assertTrue(anywarns) + def test_source_urls(self): + config = dict() + fdroidserver.common.fill_config_defaults(config) + fdroidserver.common.config = config + fdroidserver.lint.config = config + + app = { + 'Name': 'My App', + 'Summary': 'just a placeholder', + 'Description': 'This app does all sorts of useful stuff', + } + good_urls = [ + 'https://github.com/Matteljay/mastermindy-android', + 'https://gitlab.com/origin/master', + 'https://gitlab.com/group/subgroup/masterthing', + 'https://raw.githubusercontent.com/Seva-coder/Finder/HEAD/ChangeLog.txt', + 'https://github.com/scoutant/blokish/blob/HEAD/README.md#changelog', + 'https://git.ieval.ro/?p=fonbot.git;a=blob;f=Changes;hb=HEAD', + 'https://htmlpreview.github.io/?https://github.com/YasuakiHonda/Maxima-on-Android-AS/blob/HEAD/app/src/main/assets/About_MoA/index.html', + '', + ] + + anywarns = False + for url in good_urls: + app['SourceCode'] = url + for warn in fdroidserver.lint.check_regexes(app): + anywarns = True + logging.debug(warn) + self.assertFalse(anywarns) + + bad_urls = [ + 'https://raw.githubusercontent.com/Seva-coder/Finder/master/ChangeLog.txt', + 'https://github.com/scoutant/blokish/blob/master/README.md#changelog', + ] + anywarns = False + logging.debug('bad urls:') + for url in bad_urls: + app['SourceCode'] = url + for warn in fdroidserver.lint.check_regexes(app): + anywarns = True + logging.debug(warn) + self.assertTrue(anywarns) + def test_check_app_field_types(self): config = dict() fdroidserver.common.fill_config_defaults(config) From af4e231f7dde885fd0db2a730ec419dbad3b099a Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 30 Sep 2019 13:08:58 +0200 Subject: [PATCH 2/5] lint: enforce HTTPS on GitHub and GitLab pages The always provide HTTPS, so let's enforce it. --- fdroidserver/lint.py | 6 +++++- tests/lint.TestCase | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index ba6fd442..e8881a84 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -120,7 +120,11 @@ http_url_shorteners = [ ] http_checks = https_enforcings + http_url_shorteners + [ - (re.compile(r'.*github\.com/[^/]+/[^/]+\.git'), + (re.compile(r'^(?!https?://)[^/]+'), + _("URL must start with https:// or http://")), + (re.compile(r'^http://[^.]+\.(github|gitlab)\.io/'), + _("URL must start with https://")), + (re.compile(r'^https://(github|gitlab)\.com(/[^/]+){2,3}\.git'), _("Appending .git is not necessary")), (re.compile(r'^https://[^/]*(github|gitlab|bitbucket|rawgit)\.[a-zA-Z]+/([^/]+/){2,3}master/'), _("Use /HEAD instead of /master to point at a file in the default branch")), diff --git a/tests/lint.TestCase b/tests/lint.TestCase index 12285431..65f351c0 100755 --- a/tests/lint.TestCase +++ b/tests/lint.TestCase @@ -101,8 +101,12 @@ class LintTest(unittest.TestCase): self.assertFalse(anywarns) bad_urls = [ + 'github.com/my/proj', + 'https://github.com/foo/bar.git', + 'https://gitlab.com/group/subgroup/project.git', 'https://raw.githubusercontent.com/Seva-coder/Finder/master/ChangeLog.txt', 'https://github.com/scoutant/blokish/blob/master/README.md#changelog', + 'http://htmlpreview.github.io/?https://github.com/my/project/blob/HEAD/index.html', ] anywarns = False logging.debug('bad urls:') From a8b7342e4c8d2bea119635f719c935368ae3d8df Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 30 Sep 2019 13:12:27 +0200 Subject: [PATCH 3/5] jenkins-build-all: enable deploy_process_logs for CI test fdroidserver!515 --- jenkins-build-all | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins-build-all b/jenkins-build-all index 2765ab63..34b6dfd7 100755 --- a/jenkins-build-all +++ b/jenkins-build-all @@ -82,6 +82,7 @@ else fi echo "build_server_always = True" > config.py +echo "deploy_process_logs = True" >> config.py # if the local mediawiki is available, then use it if nc -z -w1 localhost 32445; then wikiflag="--wiki" From 3801db064a4f5a9910e03a9c47b5986d03812cf9 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 3 Oct 2019 23:46:34 +0200 Subject: [PATCH 4/5] lint: improve HTTPS check It was missing some domains, so I added another rule. @IzzySoft pointed out it was redudnant, so this removes the redudant rule and fixes the original. https://gitlab.com/fdroid/fdroidserver/merge_requests/681/diffs#note_225263464 --- fdroidserver/lint.py | 6 ++---- tests/lint.TestCase | 6 ++++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index e8881a84..bc3a2388 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -33,7 +33,7 @@ options = None def enforce_https(domain): - return (re.compile(r'^[^h][^t][^t][^p][^s]://[^/]*' + re.escape(domain) + r'(/.*)?', re.IGNORECASE), + return (re.compile(r'^http://([^/]*\.)?' + re.escape(domain) + r'(/.*)?', re.IGNORECASE), domain + " URLs should always use https://") @@ -122,11 +122,9 @@ http_url_shorteners = [ http_checks = https_enforcings + http_url_shorteners + [ (re.compile(r'^(?!https?://)[^/]+'), _("URL must start with https:// or http://")), - (re.compile(r'^http://[^.]+\.(github|gitlab)\.io/'), - _("URL must start with https://")), (re.compile(r'^https://(github|gitlab)\.com(/[^/]+){2,3}\.git'), _("Appending .git is not necessary")), - (re.compile(r'^https://[^/]*(github|gitlab|bitbucket|rawgit)\.[a-zA-Z]+/([^/]+/){2,3}master/'), + (re.compile(r'^https://[^/]*(github|gitlab|bitbucket|rawgit|githubusercontent)\.[a-zA-Z]+/([^/]+/){2,3}master/'), _("Use /HEAD instead of /master to point at a file in the default branch")), ] diff --git a/tests/lint.TestCase b/tests/lint.TestCase index 65f351c0..54cfc365 100755 --- a/tests/lint.TestCase +++ b/tests/lint.TestCase @@ -102,20 +102,22 @@ class LintTest(unittest.TestCase): bad_urls = [ 'github.com/my/proj', + 'http://github.com/not/secure', 'https://github.com/foo/bar.git', 'https://gitlab.com/group/subgroup/project.git', 'https://raw.githubusercontent.com/Seva-coder/Finder/master/ChangeLog.txt', 'https://github.com/scoutant/blokish/blob/master/README.md#changelog', 'http://htmlpreview.github.io/?https://github.com/my/project/blob/HEAD/index.html', + 'http://fdroid.gitlab.io/fdroid-website', ] - anywarns = False logging.debug('bad urls:') for url in bad_urls: + anywarns = False app['SourceCode'] = url for warn in fdroidserver.lint.check_regexes(app): anywarns = True logging.debug(warn) - self.assertTrue(anywarns) + self.assertTrue(anywarns, url + " does not fail lint!") def test_check_app_field_types(self): config = dict() From 1ef4f74affb5d24038d5643000c3741a1115328a Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 4 Oct 2019 11:26:52 +0200 Subject: [PATCH 5/5] lint: include MIT-CMU, it is so close to FSF/OSI-free MIT fdroidserver!682 https://github.com/spdx/license-list-data/issues/53 * FSF lists two closely related variants as X11 or Expat, search for "MIT license" in https://www.gnu.org/licenses/license-list.html * X11 is considered free: https://directory.fsf.org/wiki/License:X11 * Expat is considered free: https://directory.fsf.org/wiki/License:Expat * It is included in Debian, so it is DFSG-free: https://metadata.ftp-master.debian.org/changelogs//main/f/flite/flite_2.1-release-3_copyright * Fedora considers it free https://fedoraproject.org/wiki/Licensing:MIT#CMU_Style --- fdroidserver/lint.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index bc3a2388..1dfc00fa 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -728,6 +728,7 @@ APPROVED_LICENSES = [ 'LiLiQ-Rplus-1.1', 'MIT', 'MIT-0', + 'MIT-CMU', 'MPL-1.0', 'MPL-1.1', 'MPL-2.0',