added functions for storing/loading signer fingerprints to stats

fdroidserver/common.py

@@ -36,6 +36,7 @@ import socket
 import base64
 import zipfile
 import tempfile
+import json
 import xml.etree.ElementTree as XMLElementTree
 
 from binascii import hexlify
@@ -2552,6 +2553,34 @@ def get_certificate(certificate_file):
     return encoder.encode(cert)
 
 
+def load_stats_fdroid_signing_key_fingerprints():
+    """Load list of signing-key fingerprints stored by fdroid publish from file.
+
+    :returns: list of dictionanryies containing the singing-key fingerprints.
+    """
+    jar_file = os.path.join('stats', 'publishsigkeys.jar')
+    if not os.path.isfile(jar_file):
+        return {}
+    cmd = [config['jarsigner'], '-strict', '-verify', jar_file]
+    p = FDroidPopen(cmd, output=False)
+    if p.returncode != 4:
+        raise FDroidException("Signature validation of '{}' failed! "
+                              "Please run publish again to rebuild this file.".format(jar_file))
+
+    jar_sigkey = apk_signer_fingerprint(jar_file)
+    repo_key_sig = config.get('repo_key_sha256')
+    if repo_key_sig:
+        if jar_sigkey != repo_key_sig:
+            raise FDroidException("Signature key fingerprint of file '{}' does not match repo_key_sha256 in config.py (found fingerprint: '{}')".format(jar_file, jar_sigkey))
+    else:
+        logging.warning("repo_key_sha256 not in config.py, setting it to the signature key fingerprint of '{}'".format(jar_file))
+        config['repo_key_sha256'] = jar_sigkey
+        write_to_config(config, 'repo_key_sha256')
+
+    with zipfile.ZipFile(jar_file, 'r') as f:
+        return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
+
+
 def write_to_config(thisconfig, key, value=None, config_file=None):
     '''write a key/value to the local config.py
 

fdroidserver/publish.py

@@ -24,14 +24,17 @@ import shutil
 import glob
 import hashlib
 from argparse import ArgumentParser
+from collections import OrderedDict
 import logging
 from gettext import ngettext
+import json
+import zipfile
 
 from . import _
 from . import common
 from . import metadata
 from .common import FDroidPopen, SdkToolsPopen
-from .exception import BuildException
+from .exception import BuildException, FDroidException
 
 config = None
 options = None
@@ -49,6 +52,92 @@ def publish_source_tarball(apkfilename, unsigned_dir, output_dir):
         logging.debug('...no source tarball for %s', apkfilename)
 
 
+def key_alias(appid, resolve=False):
+    """Get the alias which which F-Droid uses to indentify the singing key
+    for this App in F-Droids keystore.
+    """
+    if config and 'keyaliases' in config and appid in config['keyaliases']:
+        # For this particular app, the key alias is overridden...
+        keyalias = config['keyaliases'][appid]
+        if keyalias.startswith('@'):
+            m = hashlib.md5()
+            m.update(keyalias[1:].encode('utf-8'))
+            keyalias = m.hexdigest()[:8]
+        return keyalias
+    else:
+        m = hashlib.md5()
+        m.update(appid.encode('utf-8'))
+        return m.hexdigest()[:8]
+
+
+def read_fingerprints_from_keystore():
+    """Obtain a dictionary containing all singning-key fingerprints which
+    are managed by F-Droid, grouped by appid.
+    """
+    env_vars = {'LC_ALL': 'C',
+                'FDROID_KEY_STORE_PASS': config['keystorepass'],
+                'FDROID_KEY_PASS': config['keypass']}
+    p = FDroidPopen([config['keytool'], '-list',
+                     '-v', '-keystore', config['keystore'],
+                     '-storepass:env', 'FDROID_KEY_STORE_PASS'],
+                    envs=env_vars, output=False)
+    if p.returncode != 0:
+        raise FDroidException('could not read keysotre {}'.format(config['keystore']))
+
+    realias = re.compile('Alias name: (?P<alias>.+)\n')
+    resha256 = re.compile('\s+SHA256: (?P<sha256>[:0-9A-F]{95})\n')
+    fps = {}
+    for block in p.output.split(('*' * 43) + '\n' + '*' * 43):
+        s_alias = realias.search(block)
+        s_sha256 = resha256.search(block)
+        if s_alias and s_sha256:
+            sigfp = s_sha256.group('sha256').replace(':', '').lower()
+            fps[s_alias.group('alias')] = sigfp
+    return fps
+
+
+def sign_sig_key_fingerprint_list(jar_file):
+    """sign the list of app-signing key fingerprints which is
+    used primaryily by fdroid update to determine which APKs
+    where built and signed by F-Droid and which ones were
+    manually added by users.
+    """
+    cmd = [config['jarsigner']]
+    cmd += '-keystore', config['keystore']
+    cmd += '-storepass:env', 'FDROID_KEY_STORE_PASS'
+    cmd += '-digestalg', 'SHA1'
+    cmd += '-sigalg', 'SHA1withRSA'
+    cmd += jar_file, config['repo_keyalias']
+    if config['keystore'] == 'NONE':
+        cmd += config['smartcardoptions']
+    else:  # smardcards never use -keypass
+        cmd += '-keypass:env', 'FDROID_KEY_PASS'
+    env_vars = {'FDROID_KEY_STORE_PASS': config['keystorepass'],
+                'FDROID_KEY_PASS': config['keypass']}
+    p = common.FDroidPopen(cmd, envs=env_vars)
+    if p.returncode != 0:
+        raise FDroidException("Failed to sign '{}'!".format(jar_file))
+
+
+def store_stats_fdroid_signing_key_fingerprints(appids, indent=None):
+    """Store list of all signing-key fingerprints for given appids to HD.
+    This list will later on be needed by fdroid update.
+    """
+    if not os.path.exists('stats'):
+        os.makedirs('stats')
+    data = OrderedDict()
+    fps = read_fingerprints_from_keystore()
+    for appid in sorted(appids):
+        alias = key_alias(appid)
+        if alias in fps:
+            data[appid] = {'signer': fps[key_alias(appid)]}
+
+    jar_file = os.path.join('stats', 'publishsigkeys.jar')
+    with zipfile.ZipFile(jar_file, 'w', zipfile.ZIP_DEFLATED) as jar:
+        jar.writestr('publishsigkeys.json', json.dumps(data, indent=indent))
+    sign_sig_key_fingerprint_list(jar_file)
+
+
 def main():
 
     global config, options

tests/publish.TestCase

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+
+#
+#  command which created the keystore used in this test case:
+#
+#  $ for ALIAS in 'repokey a163ec9b d2d51ff2 dc3b169e 78688a0f'; \
+#        do keytool -genkey -keystore dummy-keystore.jks \
+#        -alias $ALIAS -keyalg 'RSA' -keysize '2048' \
+#        -validity '10000' -storepass 123456 \
+#        -keypass 123456 -dname 'CN=test, OU=F-Droid'; done
+#
+
+import inspect
+import optparse
+import os
+import sys
+import unittest
+import tempfile
+import textwrap
+
+localmodule = os.path.realpath(
+    os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
+print('localmodule: ' + localmodule)
+if localmodule not in sys.path:
+    sys.path.insert(0, localmodule)
+
+from fdroidserver import publish
+from fdroidserver import common
+from fdroidserver.exception import FDroidException
+
+
+class PublishTest(unittest.TestCase):
+    '''fdroidserver/publish.py'''
+
+    def test_key_alias(self):
+        publish.config = {}
+        self.assertEqual('a163ec9b', publish.key_alias('com.example.app'))
+        self.assertEqual('d2d51ff2', publish.key_alias('com.example.anotherapp'))
+        self.assertEqual('dc3b169e', publish.key_alias('org.test.testy'))
+        self.assertEqual('78688a0f', publish.key_alias('org.org.org'))
+
+        publish.config = {'keyaliases': {'yep.app': '@org.org.org',
+                                         'com.example.app': '1a2b3c4d'}}
+        self.assertEqual('78688a0f', publish.key_alias('yep.app'))
+        self.assertEqual('1a2b3c4d', publish.key_alias('com.example.app'))
+
+    def test_read_fingerprints_from_keystore(self):
+        common.config = {}
+        common.fill_config_defaults(common.config)
+        publish.config = common.config
+        publish.config['keystorepass'] = '123456'
+        publish.config['keypass'] = '123456'
+        publish.config['keystore'] = 'dummy-keystore.jks'
+
+        expected = {'78688a0f': '277655a6235bc6b0ef2d824396c51ba947f5ebc738c293d887e7083ff338af82',
+                    'd2d51ff2': 'fa3f6a017541ee7fe797be084b1bcfbf92418a7589ef1f7fdeb46741b6d2e9c3',
+                    'dc3b169e': '6ae5355157a47ddcc3834a71f57f6fb5a8c2621c8e0dc739e9ddf59f865e497c',
+                    'a163ec9b': 'd34f678afbaa8f2fa6cc0edd6f0c2d1d2e2e9eb08bea521b24c740806016bff4',
+                    'repokey': 'c58460800c7b250a619c30c13b07b7359a43e5af71a4352d86c58ae18c9f6d41'}
+        result = publish.read_fingerprints_from_keystore()
+        self.maxDiff = None
+        self.assertEqual(expected, result)
+
+    def test_store_and_load_fdroid_signing_key_fingerprints(self):
+        common.config = {}
+        common.fill_config_defaults(common.config)
+        publish.config = common.config
+        publish.config['keystorepass'] = '123456'
+        publish.config['keypass'] = '123456'
+        publish.config['keystore'] = os.path.join(os.getcwd(),
+                                                  'dummy-keystore.jks')
+        publish.config['repo_keyalias'] = 'repokey'
+
+        appids = ['com.example.app',
+                  'net.unavailable',
+                  'org.test.testy',
+                  'com.example.anotherapp',
+                  'org.org.org']
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            orig_cwd = os.getcwd()
+            try:
+                os.chdir(tmpdir)
+                with open('config.py', 'w') as f:
+                    pass
+
+                publish.store_stats_fdroid_signing_key_fingerprints(appids, indent=2)
+
+                self.maxDiff = None
+                expected = {
+                    "com.example.anotherapp": {
+                        "signer": "fa3f6a017541ee7fe797be084b1bcfbf92418a7589ef1f7fdeb46741b6d2e9c3"
+                    },
+                    "com.example.app": {
+                        "signer": "d34f678afbaa8f2fa6cc0edd6f0c2d1d2e2e9eb08bea521b24c740806016bff4"
+                    },
+                    "org.org.org": {
+                        "signer": "277655a6235bc6b0ef2d824396c51ba947f5ebc738c293d887e7083ff338af82"
+                    },
+                    "org.test.testy": {
+                        "signer": "6ae5355157a47ddcc3834a71f57f6fb5a8c2621c8e0dc739e9ddf59f865e497c"
+                    }
+                }
+                self.assertEqual(expected, common.load_stats_fdroid_signing_key_fingerprints())
+
+                with open('config.py', 'r') as f:
+                    self.assertEqual(textwrap.dedent('''\
+
+                        repo_key_sha256 = "c58460800c7b250a619c30c13b07b7359a43e5af71a4352d86c58ae18c9f6d41"
+                        '''), f.read())
+            finally:
+                os.chdir(orig_cwd)
+
+    def test_store_and_load_fdroid_signing_key_fingerprints_with_missmatch(self):
+        common.config = {}
+        common.fill_config_defaults(common.config)
+        publish.config = common.config
+        publish.config['keystorepass'] = '123456'
+        publish.config['keypass'] = '123456'
+        publish.config['keystore'] = os.path.join(os.getcwd(),
+                                                  'dummy-keystore.jks')
+        publish.config['repo_keyalias'] = 'repokey'
+        publish.config['repo_key_sha256'] = 'bad bad bad bad bad bad bad bad bad bad bad bad'
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            orig_cwd = os.getcwd()
+            try:
+                os.chdir(tmpdir)
+                publish.store_stats_fdroid_signing_key_fingerprints({}, indent=2)
+                with self.assertRaises(FDroidException):
+                    common.load_stats_fdroid_signing_key_fingerprints()
+            finally:
+                os.chdir(orig_cwd)
+
+
+if __name__ == "__main__":
+    if os.path.basename(os.getcwd()) != 'tests' and os.path.isdir('tests'):
+        os.chdir('tests')
+
+    parser = optparse.OptionParser()
+    parser.add_option("-v", "--verbose", action="store_true", default=False,
+                      help="Spew out even more information than normal")
+    (common.options, args) = parser.parse_args(['--verbose'])
+
+    newSuite = unittest.TestSuite()
+    newSuite.addTest(unittest.makeSuite(PublishTest))
+    unittest.main()