Merge branch 'logging-and-two-bug-fixes' into 'master'

Logging and two bug fixes Closes #851 See merge request fdroid/fdroidserver!837
2025-11-11 09:40:28 +03:00 · 2020-12-15 07:53:52 +00:00 · 2020-12-15 07:53:52 +00:00 · 877779195f
commit 877779195f
parent 531c36f310 cfec25d33a
9 changed files with 208 additions and 9 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -291,3 +291,40 @@ fdroid build:
    # each `fdroid build --on-server` run expects sudo, then uninstalls it
    - apt-get install sudo
    - fdroid build --verbose --on-server --no-tarball --latest org.fdroid.fdroid
+
+
+# test a full update and deploy cycle to gitlab.com
+servergitmirrors:
+  image: debian
+  <<: *apt-template
+  only:
+    - master@fdroid/fdroidserver
+  script:
+    - apt-get install
+        default-jdk-headless
+        git
+        openssh-client
+        openssl
+        python3-pip
+        python3-venv
+        rsync
+        wget
+    - python3 -m venv env
+    - . env/bin/activate
+    - export PYTHONPATH=`pwd`
+    - $pip install -e .
+    - mkdir /root/.ssh/
+    - ./tests/key-tricks.py
+    - ssh-keyscan gitlab.com >> /root/.ssh/known_hosts
+    - test -d /tmp/fdroid/repo || mkdir -p /tmp/fdroid/repo
+    - cp tests/config.py tests/keystore.jks /tmp/fdroid/
+    - cp tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/
+    - cd /tmp/fdroid
+    - touch fdroid-icon.png
+    - printf "\nservergitmirrors = 'git@gitlab.com:fdroid/ci-test-servergitmirrors-repo.git'\n" >> config.py
+    - $PYTHONPATH/fdroid update --verbose --create-metadata
+    - $PYTHONPATH/fdroid deploy --verbose
+    - export DLURL=`grep -Eo 'https://gitlab.com/fdroid/ci-test-servergitmirrors-repo[^"]+' repo/index-v1.json`
+    - echo $DLURL
+    - wget $DLURL/index-v1.jar
+    - diff repo/index-v1.jar index-v1.jar
--- a/fdroidserver/build.py
+++ b/fdroidserver/build.py
@ -684,9 +684,17 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext

        bindir = os.path.join(root_dir, 'bin')

+    if os.path.isdir(os.path.join(build_dir, '.git')):
+        import git
+        commit_id = common.get_head_commit_id(git.repo.Repo(build_dir))
+    else:
+        commit_id = build.commit
+
    if p is not None and p.returncode != 0:
-        raise BuildException("Build failed for %s:%s" % (app.id, build.versionName), p.output)
-    logging.info("Successfully built version " + build.versionName + ' of ' + app.id)
+        raise BuildException("Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id),
+                             p.output)
+    logging.info("Successfully built version {versionName} of {appid} from {commit_id}"
+                 .format(versionName=build.versionName, appid=app.id, commit_id=commit_id))

    omethod = build.output_method()
    if omethod == 'maven':
--- a/fdroidserver/common.py
+++ b/fdroidserver/common.py
@ -746,6 +746,8 @@ def setup_status_output(start_timestamp):
        output['fdroiddata'] = {
            'commitId': get_head_commit_id(git_repo),
            'isDirty': git_repo.is_dirty(),
+            'modifiedFiles': git_repo.git().ls_files(modified=True).split(),
+            'untrackedFiles': git_repo.untracked_files,
        }
    fdroidserver_dir = os.path.dirname(sys.argv[0])
    if os.path.isdir(os.path.join(fdroidserver_dir, '.git')):
@ -753,6 +755,8 @@ def setup_status_output(start_timestamp):
        output['fdroidserver'] = {
            'commitId': get_head_commit_id(git_repo),
            'isDirty': git_repo.is_dirty(),
+            'modifiedFiles': git_repo.git().ls_files(modified=True).split(),
+            'untrackedFiles': git_repo.untracked_files,
        }
    write_running_status_json(output)
    return output
--- a/fdroidserver/index.py
+++ b/fdroidserver/index.py
@ -648,7 +648,7 @@ def get_mirror_service_urls(url):
    '''

    if url.startswith('git@'):
-        url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url)
+        url = re.sub(r'^git@([^:]+):(.+)', r'https://\1/\2', url)

    segments = url.split("/")

@ -676,10 +676,9 @@ def get_mirror_service_urls(url):
        # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder"
        gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder]
        urls.append('/'.join(gitlab_pages))
-        # Gitlab Raw "https://gitlab.com/user/repo/raw/branch/folder"
-        gitlab_raw = segments + ['raw', branch, folder]
+        # GitLab Raw "https://gitlab.com/user/repo/-/raw/branch/folder"
+        gitlab_raw = segments + ['-', 'raw', branch, folder]
        urls.append('/'.join(gitlab_raw))
-        return urls

    return urls

@ -726,7 +725,11 @@ def download_repo_index(url_str, etag=None, verify_fingerprint=True, timeout=600
 def get_index_from_jar(jarfile, fingerprint=None):
    """Returns the data, public key, and fingerprint from index-v1.jar

+    :param fingerprint is the SHA-256 fingerprint of signing key. Only
+           hex digits count, all other chars will can be discarded.
+
    :raises: VerificationException() if the repository could not be verified
+
    """

    logging.debug(_('Verifying index signature:'))
@ -734,7 +737,8 @@ def get_index_from_jar(jarfile, fingerprint=None):
    with zipfile.ZipFile(jarfile) as jar:
        public_key, public_key_fingerprint = get_public_key_from_jar(jar)
        if fingerprint is not None:
-            if fingerprint.upper() != public_key_fingerprint:
+            fingerprint = re.sub(r'[^0-9A-F]', r'', fingerprint.upper())
+            if fingerprint != public_key_fingerprint:
                raise VerificationException(_("The repository's fingerprint does not match."))
        data = json.loads(jar.read('index-v1.json').decode())
        return data, public_key, public_key_fingerprint
--- a/fdroidserver/update.py
+++ b/fdroidserver/update.py
@ -1448,8 +1448,9 @@ def scan_apk_androguard(apk, apkfile):
                logging.error(_("Failed to get apk information, skipping {path}")
                              .format(path=apkfile))
            raise BuildException(_("Invalid APK"))
-    except FileNotFoundError:
-        logging.error(_("Could not open apk file for analysis"))
+    except (FileNotFoundError, zipfile.BadZipFile) as e:
+        logging.error(_("Could not open APK {path} for analysis: ").format(path=apkfile)
+                      + str(e))
        raise BuildException(_("Invalid APK"))

    apk['packageName'] = apkobject.get_package()
@ -2199,6 +2200,7 @@ def main():

    config = common.read_config(options)

+    common.use_androguard()
    if not (('jarsigner' in config or 'apksigner' in config)
            and 'keytool' in config):
        raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
--- a/tests/common.TestCase
+++ b/tests/common.TestCase
@ -3,6 +3,7 @@
 # http://www.drdobbs.com/testing/unit-testing-with-python/240165163

 import difflib
+import git
 import glob
 import inspect
 import json
@ -1550,6 +1551,67 @@ class CommonTest(unittest.TestCase):
        self.assertFalse(os.path.exists('config.py'))
        fdroidserver.common.read_config(fdroidserver.common.options)

+    def test_setup_status_output(self):
+        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+        print(testdir)
+        os.chdir(testdir)
+        start_timestamp = time.gmtime()
+        subcommand = 'test'
+
+        fakecmd = ['fdroid ' + subcommand, '--option']
+        sys.argv = fakecmd
+        fdroidserver.common.config = dict()
+        fdroidserver.common.setup_status_output(start_timestamp)
+        with open(os.path.join('repo', 'status', 'running.json')) as fp:
+            data = json.load(fp)
+        self.assertFalse(os.path.exists('.git'))
+        self.assertFalse('fdroiddata' in data)
+        self.assertEqual(fakecmd, data['commandLine'])
+        self.assertEqual(subcommand, data['subcommand'])
+
+    def test_setup_status_output_in_git_repo(self):
+        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+        os.chdir(testdir)
+
+        logging.getLogger('git.cmd').setLevel(logging.INFO)
+        git_repo = git.Repo.init(testdir)
+        file_in_git = 'README.md'
+        with open(file_in_git, 'w') as fp:
+            fp.write('this is just a test')
+        git_repo.git.add(all=True)
+        git_repo.index.commit("update README")
+
+        start_timestamp = time.gmtime()
+        fakecmd = ['fdroid test2', '--option']
+        sys.argv = fakecmd
+        fdroidserver.common.config = dict()
+        fdroidserver.common.setup_status_output(start_timestamp)
+        with open(os.path.join('repo', 'status', 'running.json')) as fp:
+            data = json.load(fp)
+        self.assertTrue(os.path.exists('.git'))
+        self.assertIsNotNone(re.match(r'[0-9a-f]{40}', data['fdroiddata']['commitId']),
+                             'Must be a valid git SHA1 commit ID!')
+        self.assertFalse(data['fdroiddata']['isDirty'])
+        self.assertEqual(fakecmd, data['commandLine'])
+
+        self.assertEqual([],
+                         data['fdroiddata']['untrackedFiles'])
+        dirtyfile = 'dirtyfile'
+        with open(dirtyfile, 'w') as fp:
+            fp.write('this is just a test')
+        with open(file_in_git, 'a') as fp:
+            fp.write('\nappend some stuff')
+        self.assertEqual([],
+                         data['fdroiddata']['modifiedFiles'])
+        fdroidserver.common.setup_status_output(start_timestamp)
+        with open(os.path.join('repo', 'status', 'running.json')) as fp:
+            data = json.load(fp)
+        self.assertTrue(data['fdroiddata']['isDirty'])
+        self.assertEqual([file_in_git],
+                         data['fdroiddata']['modifiedFiles'])
+        self.assertEqual([dirtyfile, 'repo/status/running.json'],
+                         data['fdroiddata']['untrackedFiles'])
+

 if __name__ == "__main__":
    os.chdir(os.path.dirname(__file__))
--- a/tests/index.TestCase
+++ b/tests/index.TestCase
@ -54,6 +54,10 @@ class IndexTest(unittest.TestCase):
        fdroidserver.common.config = config
        fdroidserver.signindex.config = config

+        if not os.path.exists('repo/index-v1.jar'):
+            fdroidserver.signindex.sign_index_v1(os.path.join(self.basedir, 'repo'),
+                                                 'index-v1.json')
+
    def test_get_public_key_from_jar_succeeds(self):
        source_dir = os.path.join(self.basedir, 'signindex')
        for f in ('testy.jar', 'guardianproject.jar'):
@ -83,6 +87,25 @@ class IndexTest(unittest.TestCase):
        with self.assertRaises(requests.exceptions.RequestException):
            fdroidserver.index.download_repo_index("http://example.org?fingerprint=nope")

+    def test_get_repo_key_fingerprint(self):
+        pubkey, fingerprint = fdroidserver.index.extract_pubkey()
+        data, public_key, public_key_fingerprint = \
+            fdroidserver.index.get_index_from_jar('repo/index-v1.jar', fingerprint)
+        self.assertIsNotNone(data)
+        self.assertIsNotNone(public_key)
+        self.assertIsNotNone(public_key_fingerprint)
+
+    def test_get_index_from_jar_with_bad_fingerprint(self):
+        pubkey, fingerprint = fdroidserver.index.extract_pubkey()
+        fingerprint = fingerprint[:-1] + 'G'
+        with self.assertRaises(fdroidserver.exception.VerificationException):
+            fdroidserver.index.get_index_from_jar('repo/index-v1.jar', fingerprint)
+
+    def test_get_index_from_jar_with_chars_to_be_stripped(self):
+        fingerprint = 'NOOOO F4 9A F3 F1 1E FD DF 20 DF FD 70 F5 E3 11 7B 99 76 67 41 67 AD CA 28 0E 6B 19 32 A0 60 1B 26 F6'
+        data, public_key, public_key_fingerprint = \
+            fdroidserver.index.get_index_from_jar('repo/index-v1.jar', fingerprint)
+
    @patch('requests.head')
    def test_download_repo_index_same_etag(self, head):
        url = 'http://example.org?fingerprint=test'
@ -247,6 +270,26 @@ class IndexTest(unittest.TestCase):
                                                    fdroidserver.common.default_config['repo_icon'])))
        self.assertTrue(os.path.exists(os.path.join('repo', 'index.xml')))

+    def test_get_mirror_service_urls(self):
+        for url in [
+                'git@github.com:foo/bar',
+                'git@github.com:foo/bar.git',
+                'https://github.com/foo/bar',
+                'https://github.com/foo/bar.git',
+        ]:
+            self.assertEqual(['https://raw.githubusercontent.com/foo/bar/master/fdroid'],
+                             fdroidserver.index.get_mirror_service_urls(url))
+
+        for url in [
+                'git@gitlab.com:group/project',
+                'git@gitlab.com:group/project.git',
+                'https://gitlab.com/group/project',
+                'https://gitlab.com/group/project.git',
+        ]:
+            self.assertEqual(['https://group.gitlab.io/project/fdroid',
+                              'https://gitlab.com/group/project/-/raw/master/fdroid'],
+                             fdroidserver.index.get_mirror_service_urls(url))
+

 if __name__ == "__main__":
    os.chdir(os.path.dirname(__file__))
--- a/tests/key-tricks.py
+++ b/tests/key-tricks.py
@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+
+import os
+import fdroidserver
+import shutil
+import sys
+from fdroidserver import common, nightly
+
+if os.getenv('CI') is None:
+    print('ERROR: This can overwrite SSH keys, so it should only be run in CI')
+    sys.exit(1)
+
+os.chdir(os.path.dirname(__file__))
+config = fdroidserver.common.read_config(common.options)
+nightly.PASSWORD = config['keystorepass']
+nightly.KEY_ALIAS = config['repo_keyalias']
+
+privkey = nightly._ssh_key_from_debug_keystore('keystore.jks')
+print('privkey', privkey)
+ssh_private_key_file = os.path.join(os.getenv('HOME'), '.ssh', 'id_rsa')
+if os.path.exists(ssh_private_key_file):
+    print('ERROR:', ssh_private_key_file, 'exists!')
+    sys.exit(1)
+shutil.move(privkey, ssh_private_key_file)
+shutil.move(privkey + '.pub', ssh_private_key_file + '.pub')
--- a/tests/update.TestCase
+++ b/tests/update.TestCase
@ -625,6 +625,20 @@ class UpdateTest(unittest.TestCase):
        with self.assertRaises(fdroidserver.exception.BuildException):
            fdroidserver.update.scan_apk('urzip-release-unsigned.apk')

+    def test_scan_apk_bad_zip(self):
+        config = dict()
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.common.config = config
+        fdroidserver.update.config = config
+        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+        os.chdir(testdir)
+        os.mkdir('repo')
+        apkfile = 'repo/badzip_1.apk'
+        with open(apkfile, 'w') as fp:
+            fp.write('this is not a zip file')
+        with self.assertRaises(fdroidserver.exception.BuildException):
+            fdroidserver.update.scan_apk(apkfile)
+
    def test_process_apk(self):

        def _build_yaml_representer(dumper, data):