From ba854cbc0fe1a5a568bba715965a8bf977d18465 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 8 Dec 2020 19:44:39 +0100 Subject: [PATCH 1/6] index: fix GitLab Raw URLs with gitlab.com and recent versions GitLab seems to be moving to always having "-" as the first path segment in all the project URLs. So the URL without a "-" is now a redirect. --- .gitlab-ci.yml | 37 +++++++++++++++++++++++++++++++++++++ fdroidserver/index.py | 7 +++---- tests/index.TestCase | 20 ++++++++++++++++++++ tests/key-tricks.py | 25 +++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 4 deletions(-) create mode 100755 tests/key-tricks.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 52aafa2e..8b679c7b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -291,3 +291,40 @@ fdroid build: # each `fdroid build --on-server` run expects sudo, then uninstalls it - apt-get install sudo - fdroid build --verbose --on-server --no-tarball --latest org.fdroid.fdroid + + +# test a full update and deploy cycle to gitlab.com +servergitmirrors: + image: debian + <<: *apt-template + only: + - master@fdroid/fdroidserver + script: + - apt-get install + default-jdk-headless + git + openssh-client + openssl + python3-pip + python3-venv + rsync + wget + - python3 -m venv env + - . env/bin/activate + - export PYTHONPATH=`pwd` + - $pip install -e . + - mkdir /root/.ssh/ + - ./tests/key-tricks.py + - ssh-keyscan gitlab.com >> /root/.ssh/known_hosts + - test -d /tmp/fdroid/repo || mkdir -p /tmp/fdroid/repo + - cp tests/config.py tests/keystore.jks /tmp/fdroid/ + - cp tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/ + - cd /tmp/fdroid + - touch fdroid-icon.png + - printf "\nservergitmirrors = 'git@gitlab.com:fdroid/ci-test-servergitmirrors-repo.git'\n" >> config.py + - $PYTHONPATH/fdroid update --verbose --create-metadata + - $PYTHONPATH/fdroid deploy --verbose + - export DLURL=`grep -Eo 'https://gitlab.com/fdroid/ci-test-servergitmirrors-repo[^"]+' repo/index-v1.json` + - echo $DLURL + - wget $DLURL/index-v1.jar + - diff repo/index-v1.jar index-v1.jar diff --git a/fdroidserver/index.py b/fdroidserver/index.py index e707824d..9003ba57 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -648,7 +648,7 @@ def get_mirror_service_urls(url): ''' if url.startswith('git@'): - url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url) + url = re.sub(r'^git@([^:]+):(.+)', r'https://\1/\2', url) segments = url.split("/") @@ -676,10 +676,9 @@ def get_mirror_service_urls(url): # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder" gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder] urls.append('/'.join(gitlab_pages)) - # Gitlab Raw "https://gitlab.com/user/repo/raw/branch/folder" - gitlab_raw = segments + ['raw', branch, folder] + # GitLab Raw "https://gitlab.com/user/repo/-/raw/branch/folder" + gitlab_raw = segments + ['-', 'raw', branch, folder] urls.append('/'.join(gitlab_raw)) - return urls return urls diff --git a/tests/index.TestCase b/tests/index.TestCase index 10a5b9a6..a260114a 100755 --- a/tests/index.TestCase +++ b/tests/index.TestCase @@ -247,6 +247,26 @@ class IndexTest(unittest.TestCase): fdroidserver.common.default_config['repo_icon']))) self.assertTrue(os.path.exists(os.path.join('repo', 'index.xml'))) + def test_get_mirror_service_urls(self): + for url in [ + 'git@github.com:foo/bar', + 'git@github.com:foo/bar.git', + 'https://github.com/foo/bar', + 'https://github.com/foo/bar.git', + ]: + self.assertEqual(['https://raw.githubusercontent.com/foo/bar/master/fdroid'], + fdroidserver.index.get_mirror_service_urls(url)) + + for url in [ + 'git@gitlab.com:group/project', + 'git@gitlab.com:group/project.git', + 'https://gitlab.com/group/project', + 'https://gitlab.com/group/project.git', + ]: + self.assertEqual(['https://group.gitlab.io/project/fdroid', + 'https://gitlab.com/group/project/-/raw/master/fdroid'], + fdroidserver.index.get_mirror_service_urls(url)) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) diff --git a/tests/key-tricks.py b/tests/key-tricks.py new file mode 100755 index 00000000..b634803f --- /dev/null +++ b/tests/key-tricks.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +import os +import fdroidserver +import shutil +import sys +from fdroidserver import common, nightly + +if os.getenv('CI') is None: + print('ERROR: This can overwrite SSH keys, so it should only be run in CI') + sys.exit(1) + +os.chdir(os.path.dirname(__file__)) +config = fdroidserver.common.read_config(common.options) +nightly.PASSWORD = config['keystorepass'] +nightly.KEY_ALIAS = config['repo_keyalias'] + +privkey = nightly._ssh_key_from_debug_keystore('keystore.jks') +print('privkey', privkey) +ssh_private_key_file = os.path.join(os.getenv('HOME'), '.ssh', 'id_rsa') +if os.path.exists(ssh_private_key_file): + print('ERROR:', ssh_private_key_file, 'exists!') + sys.exit(1) +shutil.move(privkey, ssh_private_key_file) +shutil.move(privkey + '.pub', ssh_private_key_file + '.pub') From d9171f11e0f2df291317bc1a6fc92e1c10604c8e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 8 Dec 2020 09:19:25 +0100 Subject: [PATCH 2/6] update: improve logging when exiting due to bad APK file closes #851 --- fdroidserver/update.py | 5 +++-- tests/update.TestCase | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index fd5dc924..091fc23f 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1448,8 +1448,9 @@ def scan_apk_androguard(apk, apkfile): logging.error(_("Failed to get apk information, skipping {path}") .format(path=apkfile)) raise BuildException(_("Invalid APK")) - except FileNotFoundError: - logging.error(_("Could not open apk file for analysis")) + except (FileNotFoundError, zipfile.BadZipFile) as e: + logging.error(_("Could not open APK {path} for analysis: ").format(path=apkfile) + + str(e)) raise BuildException(_("Invalid APK")) apk['packageName'] = apkobject.get_package() diff --git a/tests/update.TestCase b/tests/update.TestCase index 322387be..f8760600 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -625,6 +625,20 @@ class UpdateTest(unittest.TestCase): with self.assertRaises(fdroidserver.exception.BuildException): fdroidserver.update.scan_apk('urzip-release-unsigned.apk') + def test_scan_apk_bad_zip(self): + config = dict() + fdroidserver.common.fill_config_defaults(config) + fdroidserver.common.config = config + fdroidserver.update.config = config + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + os.mkdir('repo') + apkfile = 'repo/badzip_1.apk' + with open(apkfile, 'w') as fp: + fp.write('this is not a zip file') + with self.assertRaises(fdroidserver.exception.BuildException): + fdroidserver.update.scan_apk(apkfile) + def test_process_apk(self): def _build_yaml_representer(dumper, data): From 384922118fdd820a859cab820e47c7f9b932de8f Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 22 Oct 2020 14:39:18 +0200 Subject: [PATCH 3/6] index: sanitize fingerprint arg, extract_pubkey() returns with spaces The key fingerprint should be only hex digits, everything else can be discarded. That makes it easy to use this function various fingerprint formats, including the common, human-readable forms spaces between pairs or quartets. --- fdroidserver/index.py | 7 ++++++- tests/index.TestCase | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 9003ba57..aae0eb05 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -725,7 +725,11 @@ def download_repo_index(url_str, etag=None, verify_fingerprint=True, timeout=600 def get_index_from_jar(jarfile, fingerprint=None): """Returns the data, public key, and fingerprint from index-v1.jar + :param fingerprint is the SHA-256 fingerprint of signing key. Only + hex digits count, all other chars will can be discarded. + :raises: VerificationException() if the repository could not be verified + """ logging.debug(_('Verifying index signature:')) @@ -733,7 +737,8 @@ def get_index_from_jar(jarfile, fingerprint=None): with zipfile.ZipFile(jarfile) as jar: public_key, public_key_fingerprint = get_public_key_from_jar(jar) if fingerprint is not None: - if fingerprint.upper() != public_key_fingerprint: + fingerprint = re.sub(r'[^0-9A-F]', r'', fingerprint.upper()) + if fingerprint != public_key_fingerprint: raise VerificationException(_("The repository's fingerprint does not match.")) data = json.loads(jar.read('index-v1.json').decode()) return data, public_key, public_key_fingerprint diff --git a/tests/index.TestCase b/tests/index.TestCase index a260114a..131b3a1a 100755 --- a/tests/index.TestCase +++ b/tests/index.TestCase @@ -54,6 +54,10 @@ class IndexTest(unittest.TestCase): fdroidserver.common.config = config fdroidserver.signindex.config = config + if not os.path.exists('repo/index-v1.jar'): + fdroidserver.signindex.sign_index_v1(os.path.join(self.basedir, 'repo'), + 'index-v1.json') + def test_get_public_key_from_jar_succeeds(self): source_dir = os.path.join(self.basedir, 'signindex') for f in ('testy.jar', 'guardianproject.jar'): @@ -83,6 +87,25 @@ class IndexTest(unittest.TestCase): with self.assertRaises(requests.exceptions.RequestException): fdroidserver.index.download_repo_index("http://example.org?fingerprint=nope") + def test_get_repo_key_fingerprint(self): + pubkey, fingerprint = fdroidserver.index.extract_pubkey() + data, public_key, public_key_fingerprint = \ + fdroidserver.index.get_index_from_jar('repo/index-v1.jar', fingerprint) + self.assertIsNotNone(data) + self.assertIsNotNone(public_key) + self.assertIsNotNone(public_key_fingerprint) + + def test_get_index_from_jar_with_bad_fingerprint(self): + pubkey, fingerprint = fdroidserver.index.extract_pubkey() + fingerprint = fingerprint[:-1] + 'G' + with self.assertRaises(fdroidserver.exception.VerificationException): + fdroidserver.index.get_index_from_jar('repo/index-v1.jar', fingerprint) + + def test_get_index_from_jar_with_chars_to_be_stripped(self): + fingerprint = 'NOOOO F4 9A F3 F1 1E FD DF 20 DF FD 70 F5 E3 11 7B 99 76 67 41 67 AD CA 28 0E 6B 19 32 A0 60 1B 26 F6' + data, public_key, public_key_fingerprint = \ + fdroidserver.index.get_index_from_jar('repo/index-v1.jar', fingerprint) + @patch('requests.head') def test_download_repo_index_same_etag(self, head): url = 'http://example.org?fingerprint=test' From fb628c2cb2c97e687cff31bc78b2ba226d3509d4 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 17 Nov 2020 13:50:28 +0100 Subject: [PATCH 4/6] include modified and untracked files in status JSON Ideally, an fdroid repo should be running from a clean git repo, so that all changes are tracked in git. This change is useful in seeing which changes and/or files are not in git. If there are modified files, the dirty flag will be set, so this info can help debugging that. --- fdroidserver/common.py | 4 +++ tests/common.TestCase | 62 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index a9f14c23..090ba1e4 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -746,6 +746,8 @@ def setup_status_output(start_timestamp): output['fdroiddata'] = { 'commitId': get_head_commit_id(git_repo), 'isDirty': git_repo.is_dirty(), + 'modifiedFiles': git_repo.git().ls_files(modified=True).split(), + 'untrackedFiles': git_repo.untracked_files, } fdroidserver_dir = os.path.dirname(sys.argv[0]) if os.path.isdir(os.path.join(fdroidserver_dir, '.git')): @@ -753,6 +755,8 @@ def setup_status_output(start_timestamp): output['fdroidserver'] = { 'commitId': get_head_commit_id(git_repo), 'isDirty': git_repo.is_dirty(), + 'modifiedFiles': git_repo.git().ls_files(modified=True).split(), + 'untrackedFiles': git_repo.untracked_files, } write_running_status_json(output) return output diff --git a/tests/common.TestCase b/tests/common.TestCase index 4ba80c0d..90e109ae 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -3,6 +3,7 @@ # http://www.drdobbs.com/testing/unit-testing-with-python/240165163 import difflib +import git import glob import inspect import json @@ -1550,6 +1551,67 @@ class CommonTest(unittest.TestCase): self.assertFalse(os.path.exists('config.py')) fdroidserver.common.read_config(fdroidserver.common.options) + def test_setup_status_output(self): + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + print(testdir) + os.chdir(testdir) + start_timestamp = time.gmtime() + subcommand = 'test' + + fakecmd = ['fdroid ' + subcommand, '--option'] + sys.argv = fakecmd + fdroidserver.common.config = dict() + fdroidserver.common.setup_status_output(start_timestamp) + with open(os.path.join('repo', 'status', 'running.json')) as fp: + data = json.load(fp) + self.assertFalse(os.path.exists('.git')) + self.assertFalse('fdroiddata' in data) + self.assertEqual(fakecmd, data['commandLine']) + self.assertEqual(subcommand, data['subcommand']) + + def test_setup_status_output_in_git_repo(self): + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + + logging.getLogger('git.cmd').setLevel(logging.INFO) + git_repo = git.Repo.init(testdir) + file_in_git = 'README.md' + with open(file_in_git, 'w') as fp: + fp.write('this is just a test') + git_repo.git.add(all=True) + git_repo.index.commit("update README") + + start_timestamp = time.gmtime() + fakecmd = ['fdroid test2', '--option'] + sys.argv = fakecmd + fdroidserver.common.config = dict() + fdroidserver.common.setup_status_output(start_timestamp) + with open(os.path.join('repo', 'status', 'running.json')) as fp: + data = json.load(fp) + self.assertTrue(os.path.exists('.git')) + self.assertIsNotNone(re.match(r'[0-9a-f]{40}', data['fdroiddata']['commitId']), + 'Must be a valid git SHA1 commit ID!') + self.assertFalse(data['fdroiddata']['isDirty']) + self.assertEqual(fakecmd, data['commandLine']) + + self.assertEqual([], + data['fdroiddata']['untrackedFiles']) + dirtyfile = 'dirtyfile' + with open(dirtyfile, 'w') as fp: + fp.write('this is just a test') + with open(file_in_git, 'a') as fp: + fp.write('\nappend some stuff') + self.assertEqual([], + data['fdroiddata']['modifiedFiles']) + fdroidserver.common.setup_status_output(start_timestamp) + with open(os.path.join('repo', 'status', 'running.json')) as fp: + data = json.load(fp) + self.assertTrue(data['fdroiddata']['isDirty']) + self.assertEqual([file_in_git], + data['fdroiddata']['modifiedFiles']) + self.assertEqual([dirtyfile, 'repo/status/running.json'], + data['fdroiddata']['untrackedFiles']) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From 40cd51ed5906e9c3bac8253283cb1ac805d15963 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 23 Jul 2020 13:03:51 +0200 Subject: [PATCH 5/6] build: include commit ID in build log --- fdroidserver/build.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index a4fe326b..ea5e1420 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -684,9 +684,17 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext bindir = os.path.join(root_dir, 'bin') + if os.path.isdir(os.path.join(build_dir, '.git')): + import git + commit_id = common.get_head_commit_id(git.repo.Repo(build_dir)) + else: + commit_id = build.commit + if p is not None and p.returncode != 0: - raise BuildException("Build failed for %s:%s" % (app.id, build.versionName), p.output) - logging.info("Successfully built version " + build.versionName + ' of ' + app.id) + raise BuildException("Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id), + p.output) + logging.info("Successfully built version {versionName} of {appid} from {commit_id}" + .format(versionName=build.versionName, appid=app.id, commit_id=commit_id)) omethod = build.output_method() if omethod == 'maven': From cfec25d33a97dac2d2aa0d944d79e557e571310d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 9 Dec 2020 23:10:33 +0100 Subject: [PATCH 6/6] update: tame androguard debug logs when --verbose is set --- fdroidserver/update.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 091fc23f..7261256f 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -2200,6 +2200,7 @@ def main(): config = common.read_config(options) + common.use_androguard() if not (('jarsigner' in config or 'apksigner' in config) and 'keytool' in config): raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))