Merge branch 'nightly-fixes' into 'master'

more `fdroid nightly` polishing

See merge request fdroid/fdroidserver!399

fdroidserver/common.py

@@ -1939,6 +1939,22 @@ def get_apk_id_aapt(apkfile):
                           .format(apkfilename=apkfile))
 
 
+def get_minSdkVersion_aapt(apkfile):
+    """Extract the minimum supported Android SDK from an APK using aapt
+
+    :param apkfile: path to an APK file.
+    :returns: the integer representing the SDK version
+    """
+    r = re.compile(r"^sdkVersion:'([0-9]+)'")
+    p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
+    for line in p.output.splitlines():
+        m = r.match(line)
+        if m:
+            return int(m.group(1))
+    raise FDroidException(_('Reading minSdkVersion failed: "{apkfilename}"')
+                          .format(apkfilename=apkfile))
+
+
 class PopenResult:
     def __init__(self):
         self.returncode = None
@@ -2352,7 +2368,7 @@ def apk_strip_signatures(signed_apk, strip_manifest=False):
     """
     with tempfile.TemporaryDirectory() as tmpdir:
         tmp_apk = os.path.join(tmpdir, 'tmp.apk')
-        os.rename(signed_apk, tmp_apk)
+        shutil.move(signed_apk, tmp_apk)
         with ZipFile(tmp_apk, 'r') as in_apk:
             with ZipFile(signed_apk, 'w') as out_apk:
                 for info in in_apk.infolist():
@@ -2413,6 +2429,40 @@ def apk_extract_signatures(apkpath, outdir, manifest=True):
                     out_file.write(in_apk.read(f.filename))
 
 
+def sign_apk(unsigned_path, signed_path, keyalias):
+    """Sign and zipalign an unsigned APK, then save to a new file, deleting the unsigned
+
+    android-18 (4.3) finally added support for reasonable hash
+    algorithms, like SHA-256, before then, the only options were MD5
+    and SHA1 :-/ This aims to use SHA-256 when the APK does not target
+    older Android versions, and is therefore safe to do so.
+
+    https://issuetracker.google.com/issues/36956587
+    https://android-review.googlesource.com/c/platform/libcore/+/44491
+
+    """
+
+    if get_minSdkVersion_aapt(unsigned_path) < 18:
+        signature_algorithm = ['-sigalg', 'SHA1withRSA', '-digestalg', 'SHA1']
+    else:
+        signature_algorithm = ['-sigalg', 'SHA256withRSA', '-digestalg', 'SHA256']
+
+    p = FDroidPopen([config['jarsigner'], '-keystore', config['keystore'],
+                     '-storepass:env', 'FDROID_KEY_STORE_PASS',
+                     '-keypass:env', 'FDROID_KEY_PASS']
+                    + signature_algorithm + [unsigned_path, keyalias],
+                    envs={
+                        'FDROID_KEY_STORE_PASS': config['keystorepass'],
+                        'FDROID_KEY_PASS': config['keypass'], })
+    if p.returncode != 0:
+        raise BuildException(_("Failed to sign application"), p.output)
+
+    p = SdkToolsPopen(['zipalign', '-v', '4', unsigned_path, signed_path])
+    if p.returncode != 0:
+        raise BuildException(_("Failed to zipalign application"))
+    os.remove(unsigned_path)
+
+
 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
     """Verify that two apks are the same
 

fdroidserver/nightly.py

@@ -51,7 +51,9 @@ def _ssh_key_from_debug_keystore():
     privkey = os.path.join(tmp_dir, '.privkey')
     key_pem = os.path.join(tmp_dir, '.key.pem')
     p12 = os.path.join(tmp_dir, '.keystore.p12')
-    subprocess.check_call([common.config['keytool'], '-importkeystore',
+    _config = dict()
+    common.fill_config_defaults(_config)
+    subprocess.check_call([_config['keytool'], '-importkeystore',
                            '-srckeystore', KEYSTORE_FILE, '-srcalias', KEY_ALIAS,
                            '-srcstorepass', PASSWORD, '-srckeypass', PASSWORD,
                            '-destkeystore', p12, '-destalias', KEY_ALIAS,
@@ -68,7 +70,7 @@ def _ssh_key_from_debug_keystore():
     rsakey = paramiko.RSAKey.from_private_key_file(privkey)
     fingerprint = base64.b64encode(hashlib.sha256(rsakey.asbytes()).digest()).decode('ascii').rstrip('=')
     ssh_private_key_file = os.path.join(tmp_dir, 'debug_keystore_' + fingerprint + '_id_rsa')
-    os.rename(privkey, ssh_private_key_file)
+    shutil.move(privkey, ssh_private_key_file)
 
     pub = rsakey.get_name() + ' ' + rsakey.get_base64() + ' ' + ssh_private_key_file
     with open(ssh_private_key_file + '.pub', 'w') as fp:
@@ -91,7 +93,6 @@ def main():
                         help=_("Don't use rsync checksums"))
     # TODO add --with-btlog
     options = parser.parse_args()
-    common.read_config(None)
 
     # force a tighter umask since this writes private key material
     umask = os.umask(0o077)
@@ -235,6 +236,8 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
         with open('config.py', 'w') as fp:
             fp.write(config)
         os.chmod('config.py', 0o600)
+        config = common.read_config(options)
+        common.assert_config_keystore(config)
 
         for root, dirs, files in os.walk(cibase):
             for d in ('fdroid', '.git', '.gradle'):
@@ -243,12 +246,14 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
             for f in files:
                 if f.endswith('-debug.apk'):
                     apkfilename = os.path.join(root, f)
-                    logging.debug(_('copying {apkfilename} into {path}')
+                    logging.debug(_('Striping mystery signature from {apkfilename}')
-                                  .format(apkfilename=apkfilename, path=repodir))
+                                  .format(apkfilename=apkfilename))
                     destapk = os.path.join(repodir, os.path.basename(f))
-                    shutil.copyfile(apkfilename, destapk)
+                    os.chmod(apkfilename, 0o644)
-                    shutil.copystat(apkfilename, destapk)
+                    logging.debug(_('Resigning {apkfilename} with provided debug.keystore')
-                    os.chmod(destapk, 0o644)
+                                  .format(apkfilename=os.path.basename(apkfilename)))
+                    common.apk_strip_signatures(apkfilename, strip_manifest=True)
+                    common.sign_apk(apkfilename, destapk, KEY_ALIAS)
 
         if options.verbose:
             logging.debug(_('attempting bare ssh connection to test deploy key:'))
@@ -278,8 +283,8 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
         os.makedirs(os.path.dirname(ssh_dir), exist_ok=True)
         privkey = _ssh_key_from_debug_keystore()
         ssh_private_key_file = os.path.join(ssh_dir, os.path.basename(privkey))
-        os.rename(privkey, ssh_private_key_file)
+        shutil.move(privkey, ssh_private_key_file)
-        os.rename(privkey + '.pub', ssh_private_key_file + '.pub')
+        shutil.move(privkey + '.pub', ssh_private_key_file + '.pub')
         if shutil.rmtree.avoids_symlink_attacks:
             shutil.rmtree(os.path.dirname(privkey))
 

fdroidserver/publish.py

@@ -339,6 +339,7 @@ def main():
                                                                       unsigned_dir,
                                                                       output_dir))
 
+                # TODO replace below with common.sign_apk() once it has proven stable
                 # Sign the application...
                 p = FDroidPopen([config['jarsigner'], '-keystore', config['keystore'],
                                  '-storepass:env', 'FDROID_KEY_STORE_PASS',

tests/common.TestCase

@@ -456,6 +456,29 @@ class CommonTest(unittest.TestCase):
             self.assertEqual(keytoolcertfingerprint,
                              fdroidserver.common.apk_signer_fingerprint_short(apkfile))
 
+    def test_sign_apk(self):
+        fdroidserver.common.config = None
+        config = fdroidserver.common.read_config(fdroidserver.common.options)
+        config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner')
+        config['keyalias'] = 'sova'
+        config['keystorepass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI='
+        config['keypass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI='
+        config['keystore'] = os.path.join(self.basedir, 'keystore.jks')
+        fdroidserver.common.config = config
+        fdroidserver.signindex.config = config
+
+        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+        unsigned = os.path.join(testdir, 'urzip-release-unsigned.apk')
+        signed = os.path.join(testdir, 'urzip-release.apk')
+
+        self.assertFalse(fdroidserver.common.verify_apk_signature(unsigned))
+
+        shutil.copy(os.path.join(self.basedir, 'urzip-release-unsigned.apk'), testdir)
+        fdroidserver.common.sign_apk(unsigned, signed, config['keyalias'])
+        self.assertTrue(os.path.isfile(signed))
+        self.assertFalse(os.path.isfile(unsigned))
+        self.assertTrue(fdroidserver.common.verify_apk_signature(signed))
+
     def test_get_api_id_aapt(self):
 
         config = dict()
@@ -472,6 +495,61 @@ class CommonTest(unittest.TestCase):
         with self.assertRaises(FDroidException):
             fdroidserver.common.get_apk_id_aapt('nope')
 
+    def test_get_minSdkVersion_aapt(self):
+
+        config = dict()
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.common.config = config
+        self._set_build_tools()
+        config['aapt'] = fdroidserver.common.find_sdk_tools_cmd('aapt')
+
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('bad-unicode-πÇÇ现代通用字-български-عربي1.apk')
+        self.assertEqual(4, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('org.bitbucket.tickytacky.mirrormirror_1.apk')
+        self.assertEqual(14, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('org.bitbucket.tickytacky.mirrormirror_2.apk')
+        self.assertEqual(14, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('org.bitbucket.tickytacky.mirrormirror_3.apk')
+        self.assertEqual(14, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('org.bitbucket.tickytacky.mirrormirror_4.apk')
+        self.assertEqual(14, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('org.dyndns.fules.ck_20.apk')
+        self.assertEqual(7, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('urzip.apk')
+        self.assertEqual(4, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('urzip-badcert.apk')
+        self.assertEqual(4, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('urzip-badsig.apk')
+        self.assertEqual(4, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('urzip-release.apk')
+        self.assertEqual(4, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('urzip-release-unsigned.apk')
+        self.assertEqual(4, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('repo/com.politedroid_3.apk')
+        self.assertEqual(3, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('repo/com.politedroid_4.apk')
+        self.assertEqual(3, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('repo/com.politedroid_5.apk')
+        self.assertEqual(3, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('repo/com.politedroid_6.apk')
+        self.assertEqual(14, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('repo/obb.main.oldversion_1444412523.apk')
+        self.assertEqual(4, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('repo/obb.mainpatch.current_1619_another-release-key.apk')
+        self.assertEqual(4, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('repo/obb.mainpatch.current_1619.apk')
+        self.assertEqual(4, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('repo/obb.main.twoversions_1101613.apk')
+        self.assertEqual(4, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('repo/obb.main.twoversions_1101615.apk')
+        self.assertEqual(4, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('repo/obb.main.twoversions_1101617.apk')
+        self.assertEqual(4, minSdkVersion)
+        minSdkVersion = fdroidserver.common.get_minSdkVersion_aapt('repo/urzip-; Рахма́нинов, [rɐxˈmanʲɪnəf] سيرجي_رخمانينوف 谢尔盖·.apk')
+
+        with self.assertRaises(FDroidException):
+            fdroidserver.common.get_minSdkVersion_aapt('nope')
+
     def test_apk_release_name(self):
         appid, vercode, sigfp = fdroidserver.common.apk_parse_release_filename('com.serwylo.lexica_905.apk')
         self.assertEqual(appid, 'com.serwylo.lexica')