diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 415a39b0..8afae7b0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,12 @@ +variables: + pip: pip3 --timeout 100 --retries 10 + + test: image: registry.gitlab.com/fdroid/ci-images-server:latest script: - - pip3 install -e . + - $pip install -e . - cd tests - ./complete-ci-tests @@ -104,8 +108,8 @@ ubuntu_xenial_pip: - rm -rf env - pyvenv env - . env/bin/activate - - pip3 install --upgrade babel pip setuptools - - pip3 install -e . + - $pip install --upgrade babel pip setuptools + - $pip install -e . - ./setup.py compile_catalog - ./tests/run-tests @@ -135,13 +139,12 @@ lint_format_safety_bandit_checks: script: - apk add --no-cache bash dash ca-certificates python3 - python3 -m ensurepip - - pip3 install Babel 'bandit<1.6.0' pycodestyle pyflakes 'pylint<2.0' safety + - $pip install Babel 'bandit<1.6.0' pycodestyle pyflakes 'pylint<2.0' safety - export EXITVALUE=0 - ./hooks/pre-commit || export EXITVALUE=1 - bandit -ii - -s B110,B310,B322,B404,B408,B410,B603,B607 - -x fdroidserver/dscanner.py,docker/install_agent.py,docker/drozer.py + -s B110,B322,B404,B408,B410,B603,B607 -r $CI_PROJECT_DIR fdroid || export EXITVALUE=1 - safety check --full-report || export EXITVALUE=1 @@ -185,7 +188,7 @@ fedora_latest: which - ./setup.py compile_catalog sdist - useradd -m -c "test account" --password "fakepassword" testuser - - su testuser --login --command "cd `pwd`; pip3 install --user dist/fdroidserver-*.tar.gz" + - su testuser --login --command "cd `pwd`; $pip install --user dist/fdroidserver-*.tar.gz" - test -e ~testuser/.local/share/locale/de/LC_MESSAGES/fdroidserver.mo - wget --no-verbose -O tools.zip https://dl.google.com/android/repository/tools_r25.2.5-linux.zip - unzip -q tools.zip @@ -223,5 +226,5 @@ gradle: test -z "$CHANGED" && exit; fi - python3 -m ensurepip - - pip3 install beautifulsoup4 requests + - $pip install beautifulsoup4 requests - ./tests/gradle-release-checksums.py diff --git a/completion/bash-completion b/completion/bash-completion index 780509db..d679fbd9 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -73,7 +73,7 @@ __vercode() { line="${line#*,}" printf "${line%%,*} " fi - done < "metadata/${p}.txt" )" -- $cur ) ) + done < "metadata/${p}.yml" )" -- $cur ) ) } __complete_options() { @@ -253,7 +253,7 @@ __complete_btlog() { __complete_mirror() { opts="-v" - lopts="--archive --output-dir" + lopts="--all --archive --build-logs --pgp-signatures --src-tarballs --output-dir" __complete_options } diff --git a/examples/template.yml b/examples/template.yml index 37d72c16..72d584aa 100644 --- a/examples/template.yml +++ b/examples/template.yml @@ -2,7 +2,7 @@ AuthorName: . WebSite: '' Bitcoin: null Litecoin: null -Donation: null +Donate: null License: Unknown Categories: @@ -17,5 +17,5 @@ Summary: . Description: | . -Archive Policy: 2 versions -Requires Root: No +ArchivePolicy: 2 versions +RequiresRoot: No diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py index 8620f899..871ec98c 100644 --- a/fdroidserver/checkupdates.py +++ b/fdroidserver/checkupdates.py @@ -64,7 +64,7 @@ def check_http(app): if len(urlcode) > 0: logging.debug("...requesting {0}".format(urlcode)) req = urllib.request.Request(urlcode, None) - resp = urllib.request.urlopen(req, None, 20) + resp = urllib.request.urlopen(req, None, 20) # nosec B310 scheme is filtered above page = resp.read().decode('utf-8') m = re.search(codeex, page) @@ -77,7 +77,7 @@ def check_http(app): if urlver != '.': logging.debug("...requesting {0}".format(urlver)) req = urllib.request.Request(urlver, None) - resp = urllib.request.urlopen(req, None, 20) + resp = urllib.request.urlopen(req, None, 20) # nosec B310 scheme is filtered above page = resp.read().decode('utf-8') m = re.search(verex, page) @@ -295,7 +295,7 @@ def check_gplay(app): headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'} req = urllib.request.Request(url, None, headers) try: - resp = urllib.request.urlopen(req, None, 20) + resp = urllib.request.urlopen(req, None, 20) # nosec B310 URL base is hardcoded above page = resp.read().decode() except urllib.error.HTTPError as e: return (None, str(e.code)) @@ -358,6 +358,18 @@ def possible_subdirs(app): yield subdir +def _getappname(app): + if app.Name: + return app.Name + if app.AutoName: + return app.AutoName + return app.id + + +def _getcvname(app): + return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode) + + def fetch_autoname(app, tag): if not app.RepoType or app.UpdateCheckMode in ('None', 'Static') \ @@ -393,7 +405,7 @@ def fetch_autoname(app, tag): if new_name != app.AutoName: app.AutoName = new_name if not commitmsg: - commitmsg = "Set autoname of {0}".format(common.getappname(app)) + commitmsg = "Set autoname of {0}".format(_getappname(app)) else: logging.debug("...couldn't get autoname") @@ -472,8 +484,8 @@ def checkupdates_app(app): commitmsg = fetch_autoname(app, tag) if updating: - name = common.getappname(app) - ver = common.getcvname(app) + name = _getappname(app) + ver = _getcvname(app) logging.info('...updating to version %s' % ver) commitmsg = 'Update CV of %s to %s' % (name, ver) @@ -513,8 +525,8 @@ def checkupdates_app(app): commit = commit.replace('%c', newbuild.versionCode) newbuild.commit = commit app.builds.append(newbuild) - name = common.getappname(app) - ver = common.getcvname(app) + name = _getappname(app) + ver = _getcvname(app) commitmsg = "Update %s to %s" % (name, ver) else: logging.warning('Invalid auto update mode "' + mode + '" on ' + app.id) @@ -610,24 +622,24 @@ def main(): version, reason = check_gplay(app) if version is None: if reason == '404': - logging.info("{0} is not in the Play Store".format(common.getappname(app))) + logging.info("{0} is not in the Play Store".format(_getappname(app))) else: - logging.info("{0} encountered a problem: {1}".format(common.getappname(app), reason)) + logging.info("{0} encountered a problem: {1}".format(_getappname(app), reason)) if version is not None: stored = app.CurrentVersion if not stored: logging.info("{0} has no Current Version but has version {1} on the Play Store" - .format(common.getappname(app), version)) + .format(_getappname(app), version)) elif LooseVersion(stored) < LooseVersion(version): logging.info("{0} has version {1} on the Play Store, which is bigger than {2}" - .format(common.getappname(app), version, stored)) + .format(_getappname(app), version, stored)) else: if stored != version: logging.info("{0} has version {1} on the Play Store, which differs from {2}" - .format(common.getappname(app), version, stored)) + .format(_getappname(app), version, stored)) else: logging.info("{0} has the same version {1} on the Play Store" - .format(common.getappname(app), version)) + .format(_getappname(app), version)) update_wiki(gplaylog, None) return diff --git a/fdroidserver/common.py b/fdroidserver/common.py index f68bb88f..946c0de1 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -664,18 +664,6 @@ def getsrcname(app, build): return "%s_%s_src.tar.gz" % (app.id, build.versionCode) -def getappname(app): - if app.Name: - return app.Name - if app.AutoName: - return app.AutoName - return app.id - - -def getcvname(app): - return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode) - - def get_build_dir(app): '''get the dir that this app will be built in''' diff --git a/fdroidserver/import.py b/fdroidserver/import.py index 51713cee..b463824c 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -40,8 +40,9 @@ SETTINGS_GRADLE = re.compile(r'''include\s+['"]:([^'"]*)['"]''') # when one of these is found it's assumed that's the information we want. # Returns repotype, address, or None, reason def getrepofrompage(url): - - req = urllib.request.urlopen(url) + if not url.startswith('http'): + return (None, _('{url} does not start with "http"!'.format(url=url))) + req = urllib.request.urlopen(url) # nosec B310 non-http URLs are filtered out if req.getcode() != 200: return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode())) page = req.read().decode(req.headers.get_content_charset()) diff --git a/fdroidserver/mirror.py b/fdroidserver/mirror.py index 920c9acf..2510e3d0 100644 --- a/fdroidserver/mirror.py +++ b/fdroidserver/mirror.py @@ -46,16 +46,26 @@ def main(): parser.add_argument("url", nargs='?', help=_('Base URL to mirror, can include the index signing key ' + 'using the query string: ?fingerprint=')) + parser.add_argument("--all", action='store_true', default=False, + help=_("Mirror the full repo and archive, all file types.")) parser.add_argument("--archive", action='store_true', default=False, help=_("Also mirror the full archive section")) parser.add_argument("--build-logs", action='store_true', default=False, help=_("Include the build logs in the mirror")) + parser.add_argument("--pgp-signatures", action='store_true', default=False, + help=_("Include the PGP signature .asc files in the mirror")) parser.add_argument("--src-tarballs", action='store_true', default=False, help=_("Include the source tarballs in the mirror")) parser.add_argument("--output-dir", default=None, help=_("The directory to write the mirror to")) options = parser.parse_args() + if options.all: + options.archive = True + options.build_logs = True + options.pgp_signatures = True + options.src_tarballs = True + if options.url is None: logging.error(_('A URL is required as an argument!') + '\n') parser.print_help() @@ -152,7 +162,8 @@ def main(): if not os.path.exists(f) \ or (f.endswith('.apk') and os.path.getsize(f) != package['size']): urls.append(_append_to_url_path(section, f)) - urls.append(_append_to_url_path(section, f + '.asc')) + if options.pgp_signatures: + urls.append(_append_to_url_path(section, f + '.asc')) if options.build_logs and f.endswith('.apk'): urls.append(_append_to_url_path(section, f[:-4] + '.log.gz')) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index bc2583fc..44d4b8c0 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -714,7 +714,10 @@ def _set_localized_text_entry(app, locale, key, f): with open(f, errors='replace') as fp: text = fp.read()[:limit] if len(text) > 0: - localized[key] = text + if key in ('name', 'summary', 'video'): # hardcoded as a single line + localized[key] = text.strip('\n') + else: + localized[key] = text def _set_author_entry(app, key, f): diff --git a/makebuildserver b/makebuildserver index 9bf09d40..8195983f 100755 --- a/makebuildserver +++ b/makebuildserver @@ -362,8 +362,8 @@ CACHE_FILES = [ '1f3067073041bc44554d0efe5d402a33bc3d3c93cc39ab684f308586d732a80d'), ('https://services.gradle.org/distributions/gradle-6.0.1-bin.zip', 'd364b7098b9f2e58579a3603dc0a12a1991353ac58ed339316e6762b21efba44'), - ('https://services.gradle.org/distributions/gradle-6.1-bin.zip', - 'd0c43d14e1c70a48b82442f435d06186351a2d290d72afd5b8866f15e6d7038a'), + ('https://services.gradle.org/distributions/gradle-6.1.1-bin.zip', + '9d94e6e4a28ad328072ef6e56bce79a810494ae756751fdcedffdeaf27c093b1'), ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin', '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'), ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip', @@ -531,6 +531,10 @@ def main(): open(logfilename, 'a').close() # create blank file log_cm = vagrant.make_file_cm(logfilename) v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm) + # https://phoenhex.re/2018-03-25/not-a-vagrant-bug + os_env = os.environ.copy() + os_env['VAGRANT_DISABLE_VBOXSYMLINKCREATE'] = '1' + v.env = os_env if options.verbosity >= 2: tail = fdroidserver.tail.Tail(logfilename) diff --git a/tests/checkupdates.TestCase b/tests/checkupdates.TestCase index ab68cd8d..7772dfa5 100755 --- a/tests/checkupdates.TestCase +++ b/tests/checkupdates.TestCase @@ -19,10 +19,11 @@ if localmodule not in sys.path: import fdroidserver.checkupdates import fdroidserver.metadata +from fdroidserver.exception import FDroidException -class CommonTest(unittest.TestCase): - '''fdroidserver/common.py''' +class CheckupdatesTest(unittest.TestCase): + '''fdroidserver/checkupdates.py''' def setUp(self): logging.basicConfig(level=logging.DEBUG) @@ -123,6 +124,17 @@ class CommonTest(unittest.TestCase): self.assertEqual(vername, '1.1.9') self.assertEqual(vercode, '10109') + def test_check_http_blocks_unknown_schemes(self): + app = fdroidserver.metadata.App() + for scheme in ('file', 'ssh', 'http', ';pwn'): + app.id = scheme + faked = scheme + '://fake.url/for/testing/scheme' + app.UpdateCheckData = faked + '|ignored|' + faked + '|ignored' + app.metadatapath = 'metadata/' + app.id + '.yml' + vername, vercode = fdroidserver.checkupdates.check_http(app) + self.assertIsNone(vername) + self.assertTrue(FDroidException.__name__ in vercode) + def test_check_http_ignore(self): fdroidserver.checkupdates.options = mock.Mock() @@ -151,5 +163,5 @@ if __name__ == "__main__": (fdroidserver.common.options, args) = parser.parse_args(['--verbose']) newSuite = unittest.TestSuite() - newSuite.addTest(unittest.makeSuite(CommonTest)) + newSuite.addTest(unittest.makeSuite(CheckupdatesTest)) unittest.main(failfast=False) diff --git a/tests/config.py b/tests/config.py index 6ee05902..745fb9d6 100644 --- a/tests/config.py +++ b/tests/config.py @@ -26,8 +26,8 @@ keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=" keydname = "CN=sova, OU=F-Droid" mirrors = ( - 'https://foo.bar/fdroid', 'http://foobarfoobarfoobar.onion/fdroid', + 'https://foo.bar/fdroid', ) update_stats = True diff --git a/tests/repo/index-v1.json b/tests/repo/index-v1.json index 492b6051..32d76b7d 100644 --- a/tests/repo/index-v1.json +++ b/tests/repo/index-v1.json @@ -198,9 +198,9 @@ "description": "full description\n", "featureGraphic": "featureGraphic_GFRT5BovZsENGpJq1HqPODGWBRPWQsx25B95Ol5w_wU=.png", "icon": "icon_NJXNzMcyf-v9i5a1ElJi0j9X1LvllibCa48xXYPlOqQ=.png", - "name": "title\n", - "summary": "short description\n", - "video": "video\n" + "name": "title", + "summary": "short description", + "video": "video" } } } diff --git a/tests/update.TestCase b/tests/update.TestCase index 61658292..e3d9ef53 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -114,9 +114,9 @@ class UpdateTest(unittest.TestCase): if packageName == 'info.guardianproject.urzip': self.assertEqual(7, len(app['localized']['en-US'])) self.assertEqual('full description\n', app['localized']['en-US']['description']) - self.assertEqual('title\n', app['localized']['en-US']['name']) - self.assertEqual('short description\n', app['localized']['en-US']['summary']) - self.assertEqual('video\n', app['localized']['en-US']['video']) + self.assertEqual('title', app['localized']['en-US']['name']) + self.assertEqual('short description', app['localized']['en-US']['summary']) + self.assertEqual('video', app['localized']['en-US']['video']) self.assertEqual('icon_NJXNzMcyf-v9i5a1ElJi0j9X1LvllibCa48xXYPlOqQ=.png', app['localized']['en-US']['icon']) self.assertEqual('featureGraphic_GFRT5BovZsENGpJq1HqPODGWBRPWQsx25B95Ol5w_wU=.png', @@ -140,7 +140,7 @@ class UpdateTest(unittest.TestCase): elif packageName == 'com.nextcloud.client.dev': self.assertEqual('Nextcloud Dev', app['localized']['en-US']['name']) self.assertEqual(586, len(app['localized']['en-US']['description'])) - self.assertEqual(79, len(app['localized']['en-US']['summary'])) + self.assertEqual(78, len(app['localized']['en-US']['summary'])) elif packageName == 'eu.siacs.conversations': self.assertEqual('Conversations', app['localized']['en-US']['name'])