Merge branch 'logging-and-two-bug-fixes' into 'master'

Logging and two bug fixes

Closes #851

See merge request fdroid/fdroidserver!837

.gitlab-ci.yml

@@ -291,3 +291,40 @@ fdroid build:
     # each `fdroid build --on-server` run expects sudo, then uninstalls it
     - apt-get install sudo
     - fdroid build --verbose --on-server --no-tarball --latest org.fdroid.fdroid
+
+
+# test a full update and deploy cycle to gitlab.com
+servergitmirrors:
+  image: debian
+  <<: *apt-template
+  only:
+    - master@fdroid/fdroidserver
+  script:
+    - apt-get install
+        default-jdk-headless
+        git
+        openssh-client
+        openssl
+        python3-pip
+        python3-venv
+        rsync
+        wget
+    - python3 -m venv env
+    - . env/bin/activate
+    - export PYTHONPATH=`pwd`
+    - $pip install -e .
+    - mkdir /root/.ssh/
+    - ./tests/key-tricks.py
+    - ssh-keyscan gitlab.com >> /root/.ssh/known_hosts
+    - test -d /tmp/fdroid/repo || mkdir -p /tmp/fdroid/repo
+    - cp tests/config.py tests/keystore.jks /tmp/fdroid/
+    - cp tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/
+    - cd /tmp/fdroid
+    - touch fdroid-icon.png
+    - printf "\nservergitmirrors = 'git@gitlab.com:fdroid/ci-test-servergitmirrors-repo.git'\n" >> config.py
+    - $PYTHONPATH/fdroid update --verbose --create-metadata
+    - $PYTHONPATH/fdroid deploy --verbose
+    - export DLURL=`grep -Eo 'https://gitlab.com/fdroid/ci-test-servergitmirrors-repo[^"]+' repo/index-v1.json`
+    - echo $DLURL
+    - wget $DLURL/index-v1.jar
+    - diff repo/index-v1.jar index-v1.jar

fdroidserver/build.py

@@ -684,9 +684,17 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
 
         bindir = os.path.join(root_dir, 'bin')
 
+    if os.path.isdir(os.path.join(build_dir, '.git')):
+        import git
+        commit_id = common.get_head_commit_id(git.repo.Repo(build_dir))
+    else:
+        commit_id = build.commit
+
     if p is not None and p.returncode != 0:
-        raise BuildException("Build failed for %s:%s" % (app.id, build.versionName), p.output)
+        raise BuildException("Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id),
-    logging.info("Successfully built version " + build.versionName + ' of ' + app.id)
+                             p.output)
+    logging.info("Successfully built version {versionName} of {appid} from {commit_id}"
+                 .format(versionName=build.versionName, appid=app.id, commit_id=commit_id))
 
     omethod = build.output_method()
     if omethod == 'maven':

fdroidserver/common.py

@@ -746,6 +746,8 @@ def setup_status_output(start_timestamp):
         output['fdroiddata'] = {
             'commitId': get_head_commit_id(git_repo),
             'isDirty': git_repo.is_dirty(),
+            'modifiedFiles': git_repo.git().ls_files(modified=True).split(),
+            'untrackedFiles': git_repo.untracked_files,
         }
     fdroidserver_dir = os.path.dirname(sys.argv[0])
     if os.path.isdir(os.path.join(fdroidserver_dir, '.git')):
@@ -753,6 +755,8 @@ def setup_status_output(start_timestamp):
         output['fdroidserver'] = {
             'commitId': get_head_commit_id(git_repo),
             'isDirty': git_repo.is_dirty(),
+            'modifiedFiles': git_repo.git().ls_files(modified=True).split(),
+            'untrackedFiles': git_repo.untracked_files,
         }
     write_running_status_json(output)
     return output

fdroidserver/index.py

@@ -648,7 +648,7 @@ def get_mirror_service_urls(url):
     '''
 
     if url.startswith('git@'):
-        url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url)
+        url = re.sub(r'^git@([^:]+):(.+)', r'https://\1/\2', url)
 
     segments = url.split("/")
 
@@ -676,10 +676,9 @@ def get_mirror_service_urls(url):
         # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder"
         gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder]
         urls.append('/'.join(gitlab_pages))
-        # Gitlab Raw "https://gitlab.com/user/repo/raw/branch/folder"
+        # GitLab Raw "https://gitlab.com/user/repo/-/raw/branch/folder"
-        gitlab_raw = segments + ['raw', branch, folder]
+        gitlab_raw = segments + ['-', 'raw', branch, folder]
         urls.append('/'.join(gitlab_raw))
-        return urls
 
     return urls
 
@@ -726,7 +725,11 @@ def download_repo_index(url_str, etag=None, verify_fingerprint=True, timeout=600
 def get_index_from_jar(jarfile, fingerprint=None):
     """Returns the data, public key, and fingerprint from index-v1.jar
 
+    :param fingerprint is the SHA-256 fingerprint of signing key. Only
+           hex digits count, all other chars will can be discarded.
+
     :raises: VerificationException() if the repository could not be verified
+
     """
 
     logging.debug(_('Verifying index signature:'))
@@ -734,7 +737,8 @@ def get_index_from_jar(jarfile, fingerprint=None):
     with zipfile.ZipFile(jarfile) as jar:
         public_key, public_key_fingerprint = get_public_key_from_jar(jar)
         if fingerprint is not None:
-            if fingerprint.upper() != public_key_fingerprint:
+            fingerprint = re.sub(r'[^0-9A-F]', r'', fingerprint.upper())
+            if fingerprint != public_key_fingerprint:
                 raise VerificationException(_("The repository's fingerprint does not match."))
         data = json.loads(jar.read('index-v1.json').decode())
         return data, public_key, public_key_fingerprint

fdroidserver/update.py

@@ -1448,8 +1448,9 @@ def scan_apk_androguard(apk, apkfile):
                 logging.error(_("Failed to get apk information, skipping {path}")
                               .format(path=apkfile))
             raise BuildException(_("Invalid APK"))
-    except FileNotFoundError:
+    except (FileNotFoundError, zipfile.BadZipFile) as e:
-        logging.error(_("Could not open apk file for analysis"))
+        logging.error(_("Could not open APK {path} for analysis: ").format(path=apkfile)
+                      + str(e))
         raise BuildException(_("Invalid APK"))
 
     apk['packageName'] = apkobject.get_package()
@@ -2199,6 +2200,7 @@ def main():
 
     config = common.read_config(options)
 
+    common.use_androguard()
     if not (('jarsigner' in config or 'apksigner' in config)
             and 'keytool' in config):
         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))

tests/common.TestCase

@@ -3,6 +3,7 @@
 # http://www.drdobbs.com/testing/unit-testing-with-python/240165163
 
 import difflib
+import git
 import glob
 import inspect
 import json
@@ -1550,6 +1551,67 @@ class CommonTest(unittest.TestCase):
         self.assertFalse(os.path.exists('config.py'))
         fdroidserver.common.read_config(fdroidserver.common.options)
 
+    def test_setup_status_output(self):
+        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+        print(testdir)
+        os.chdir(testdir)
+        start_timestamp = time.gmtime()
+        subcommand = 'test'
+
+        fakecmd = ['fdroid ' + subcommand, '--option']
+        sys.argv = fakecmd
+        fdroidserver.common.config = dict()
+        fdroidserver.common.setup_status_output(start_timestamp)
+        with open(os.path.join('repo', 'status', 'running.json')) as fp:
+            data = json.load(fp)
+        self.assertFalse(os.path.exists('.git'))
+        self.assertFalse('fdroiddata' in data)
+        self.assertEqual(fakecmd, data['commandLine'])
+        self.assertEqual(subcommand, data['subcommand'])
+
+    def test_setup_status_output_in_git_repo(self):
+        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+        os.chdir(testdir)
+
+        logging.getLogger('git.cmd').setLevel(logging.INFO)
+        git_repo = git.Repo.init(testdir)
+        file_in_git = 'README.md'
+        with open(file_in_git, 'w') as fp:
+            fp.write('this is just a test')
+        git_repo.git.add(all=True)
+        git_repo.index.commit("update README")
+
+        start_timestamp = time.gmtime()
+        fakecmd = ['fdroid test2', '--option']
+        sys.argv = fakecmd
+        fdroidserver.common.config = dict()
+        fdroidserver.common.setup_status_output(start_timestamp)
+        with open(os.path.join('repo', 'status', 'running.json')) as fp:
+            data = json.load(fp)
+        self.assertTrue(os.path.exists('.git'))
+        self.assertIsNotNone(re.match(r'[0-9a-f]{40}', data['fdroiddata']['commitId']),
+                             'Must be a valid git SHA1 commit ID!')
+        self.assertFalse(data['fdroiddata']['isDirty'])
+        self.assertEqual(fakecmd, data['commandLine'])
+
+        self.assertEqual([],
+                         data['fdroiddata']['untrackedFiles'])
+        dirtyfile = 'dirtyfile'
+        with open(dirtyfile, 'w') as fp:
+            fp.write('this is just a test')
+        with open(file_in_git, 'a') as fp:
+            fp.write('\nappend some stuff')
+        self.assertEqual([],
+                         data['fdroiddata']['modifiedFiles'])
+        fdroidserver.common.setup_status_output(start_timestamp)
+        with open(os.path.join('repo', 'status', 'running.json')) as fp:
+            data = json.load(fp)
+        self.assertTrue(data['fdroiddata']['isDirty'])
+        self.assertEqual([file_in_git],
+                         data['fdroiddata']['modifiedFiles'])
+        self.assertEqual([dirtyfile, 'repo/status/running.json'],
+                         data['fdroiddata']['untrackedFiles'])
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

tests/index.TestCase

@@ -54,6 +54,10 @@ class IndexTest(unittest.TestCase):
         fdroidserver.common.config = config
         fdroidserver.signindex.config = config
 
+        if not os.path.exists('repo/index-v1.jar'):
+            fdroidserver.signindex.sign_index_v1(os.path.join(self.basedir, 'repo'),
+                                                 'index-v1.json')
+
     def test_get_public_key_from_jar_succeeds(self):
         source_dir = os.path.join(self.basedir, 'signindex')
         for f in ('testy.jar', 'guardianproject.jar'):
@@ -83,6 +87,25 @@ class IndexTest(unittest.TestCase):
         with self.assertRaises(requests.exceptions.RequestException):
             fdroidserver.index.download_repo_index("http://example.org?fingerprint=nope")
 
+    def test_get_repo_key_fingerprint(self):
+        pubkey, fingerprint = fdroidserver.index.extract_pubkey()
+        data, public_key, public_key_fingerprint = \
+            fdroidserver.index.get_index_from_jar('repo/index-v1.jar', fingerprint)
+        self.assertIsNotNone(data)
+        self.assertIsNotNone(public_key)
+        self.assertIsNotNone(public_key_fingerprint)
+
+    def test_get_index_from_jar_with_bad_fingerprint(self):
+        pubkey, fingerprint = fdroidserver.index.extract_pubkey()
+        fingerprint = fingerprint[:-1] + 'G'
+        with self.assertRaises(fdroidserver.exception.VerificationException):
+            fdroidserver.index.get_index_from_jar('repo/index-v1.jar', fingerprint)
+
+    def test_get_index_from_jar_with_chars_to_be_stripped(self):
+        fingerprint = 'NOOOO F4 9A F3 F1 1E FD DF 20 DF FD 70 F5 E3 11 7B 99 76 67 41 67 AD CA 28 0E 6B 19 32 A0 60 1B 26 F6'
+        data, public_key, public_key_fingerprint = \
+            fdroidserver.index.get_index_from_jar('repo/index-v1.jar', fingerprint)
+
     @patch('requests.head')
     def test_download_repo_index_same_etag(self, head):
         url = 'http://example.org?fingerprint=test'
@@ -247,6 +270,26 @@ class IndexTest(unittest.TestCase):
                                                     fdroidserver.common.default_config['repo_icon'])))
         self.assertTrue(os.path.exists(os.path.join('repo', 'index.xml')))
 
+    def test_get_mirror_service_urls(self):
+        for url in [
+                'git@github.com:foo/bar',
+                'git@github.com:foo/bar.git',
+                'https://github.com/foo/bar',
+                'https://github.com/foo/bar.git',
+        ]:
+            self.assertEqual(['https://raw.githubusercontent.com/foo/bar/master/fdroid'],
+                             fdroidserver.index.get_mirror_service_urls(url))
+
+        for url in [
+                'git@gitlab.com:group/project',
+                'git@gitlab.com:group/project.git',
+                'https://gitlab.com/group/project',
+                'https://gitlab.com/group/project.git',
+        ]:
+            self.assertEqual(['https://group.gitlab.io/project/fdroid',
+                              'https://gitlab.com/group/project/-/raw/master/fdroid'],
+                             fdroidserver.index.get_mirror_service_urls(url))
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

tests/key-tricks.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+
+import os
+import fdroidserver
+import shutil
+import sys
+from fdroidserver import common, nightly
+
+if os.getenv('CI') is None:
+    print('ERROR: This can overwrite SSH keys, so it should only be run in CI')
+    sys.exit(1)
+
+os.chdir(os.path.dirname(__file__))
+config = fdroidserver.common.read_config(common.options)
+nightly.PASSWORD = config['keystorepass']
+nightly.KEY_ALIAS = config['repo_keyalias']
+
+privkey = nightly._ssh_key_from_debug_keystore('keystore.jks')
+print('privkey', privkey)
+ssh_private_key_file = os.path.join(os.getenv('HOME'), '.ssh', 'id_rsa')
+if os.path.exists(ssh_private_key_file):
+    print('ERROR:', ssh_private_key_file, 'exists!')
+    sys.exit(1)
+shutil.move(privkey, ssh_private_key_file)
+shutil.move(privkey + '.pub', ssh_private_key_file + '.pub')

tests/update.TestCase

@@ -625,6 +625,20 @@ class UpdateTest(unittest.TestCase):
         with self.assertRaises(fdroidserver.exception.BuildException):
             fdroidserver.update.scan_apk('urzip-release-unsigned.apk')
 
+    def test_scan_apk_bad_zip(self):
+        config = dict()
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.common.config = config
+        fdroidserver.update.config = config
+        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+        os.chdir(testdir)
+        os.mkdir('repo')
+        apkfile = 'repo/badzip_1.apk'
+        with open(apkfile, 'w') as fp:
+            fp.write('this is not a zip file')
+        with self.assertRaises(fdroidserver.exception.BuildException):
+            fdroidserver.update.scan_apk(apkfile)
+
     def test_process_apk(self):
 
         def _build_yaml_representer(dumper, data):