Merge branch 'auto-detect-java-homes' into 'master'

Auto detect java homes

There are a wide variety of possible JDKs (OpenJDK, Oracle, OSX, etc.) as well as standard paths for the JDK.  This includes code to look in standard paths for various platforms and enumerate the installed JDKs (e.g. 6, 7, 8, 9).  That is then used for `JAVA[6-9]_HOME` env vars and the path for `jarsigner` and `keytool`.

See merge request !98

buildserver/cookbooks/fdroidbuild-general/recipes/default.rb

@@ -105,3 +105,11 @@ execute "set-default-java" do
   command "update-java-alternatives --set java-1.7.0-openjdk-i386"
 end
 
+# Ubuntu trusty 14.04's paramiko does not work with jessie's openssh's default settings
+# https://stackoverflow.com/questions/7286929/paramiko-incompatible-ssh-peer-no-acceptable-kex-algorithm/32691055#32691055
+execute "support-ubuntu-trusty-paramiko" do
+  only_if { node[:settings][:ubuntu_trusty] == 'true' }
+  command "echo Ciphers aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes128-ctr >> /etc/ssh/sshd_config"
+  command "echo MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-512,hmac-sha2-256,hmac-ripemd160,hmac-sha1 >> /etc/ssh/sshd_config"
+  command "echo KexAlgorithms diffie-hellman-group-exchange-sha256,diffie-hellman-group14-sha1,diffie-hellman-group-exchange-sha1 >> /etc/ssh/sshd_config"
+end

fdroidserver/common.py

@@ -62,10 +62,7 @@ default_config = {
         'r10e': "$ANDROID_NDK",
     },
     'build_tools': "23.0.2",
-    'java_paths': {
+    'java_paths': None,
-        '1.7': "/usr/lib/jvm/java-7-openjdk",
-        '1.8': None,
-    },
     'ant': "ant",
     'mvn3': "mvn",
     'gradle': 'gradle',
@@ -131,6 +128,40 @@ def fill_config_defaults(thisconfig):
             thisconfig[k] = exp
             thisconfig[k + '_orig'] = v
 
+    # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
+    if thisconfig['java_paths'] is None:
+        thisconfig['java_paths'] = dict()
+        for d in sorted(glob.glob('/usr/lib/jvm/j*[6-9]*')
+                        + glob.glob('/usr/java/jdk1.[6-9]*')
+                        + glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
+                        + glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')):
+            if os.path.islink(d):
+                continue
+            j = os.path.basename(d)
+            # the last one found will be the canonical one, so order appropriately
+            for regex in (r'1\.([6-9])\.0\.jdk',  # OSX
+                          r'jdk1\.([6-9])\.0_[0-9]+.jdk',  # OSX and Oracle tarball
+                          r'jdk([6-9])-openjdk',  # Arch
+                          r'java-1\.([6-9])\.0-.*',  # RedHat
+                          r'java-([6-9])-oracle',  # Debian WebUpd8
+                          r'jdk-([6-9])-oracle-.*',  # Debian make-jpkg
+                          r'java-([6-9])-openjdk-[^c][^o][^m].*'):  # Debian
+                m = re.match(regex, j)
+                if m:
+                    osxhome = os.path.join(d, 'Contents', 'Home')
+                    if os.path.exists(osxhome):
+                        thisconfig['java_paths'][m.group(1)] = osxhome
+                    else:
+                        thisconfig['java_paths'][m.group(1)] = d
+
+    for java_version in ('7', '8', '9'):
+        java_home = thisconfig['java_paths'][java_version]
+        jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
+        if os.path.exists(jarsigner):
+            thisconfig['jarsigner'] = jarsigner
+            thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
+            break  # Java7 is preferred, so quit if found
+
     for k in ['ndk_paths', 'java_paths']:
         d = thisconfig[k]
         for k2 in d.copy():
@@ -194,10 +225,8 @@ def read_config(opts, config_file='config.py'):
     for n in ['ANDROID_HOME', 'ANDROID_SDK']:
         env[n] = config['sdk_path']
 
-    for v in ['7', '8']:
+    for k, v in config['java_paths'].items():
-        cpath = config['java_paths']['1.%s' % v]
+        env['JAVA%s_HOME' % k] = v
-        if cpath:
-            env['JAVA%s_HOME' % v] = cpath
 
     for k in ["keystorepass", "keypass"]:
         if k in config:
@@ -1789,7 +1818,7 @@ def verify_apks(signed_apk, unsigned_apk, tmp_dir):
         for meta_inf_file in meta_inf_files:
             unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
 
-    if subprocess.call(['jarsigner', '-verify', unsigned_apk]) != 0:
+    if subprocess.call([config['jarsigner'], '-verify', unsigned_apk]) != 0:
         logging.info("...NOT verified - {0}".format(signed_apk))
         return compare_apks(signed_apk, unsigned_apk, tmp_dir)
     logging.info("...successfully verified")
@@ -1891,7 +1920,7 @@ def genkeystore(localconfig):
 
     write_password_file("keystorepass", localconfig['keystorepass'])
     write_password_file("keypass", localconfig['keypass'])
-    p = FDroidPopen(['keytool', '-genkey',
+    p = FDroidPopen([config['keytool'], '-genkey',
                      '-keystore', localconfig['keystore'],
                      '-alias', localconfig['repo_keyalias'],
                      '-keyalg', 'RSA', '-keysize', '4096',
@@ -1905,7 +1934,7 @@ def genkeystore(localconfig):
         raise BuildException("Failed to generate key", p.output)
     os.chmod(localconfig['keystore'], 0o0600)
     # now show the lovely key that was just generated
-    p = FDroidPopen(['keytool', '-list', '-v',
+    p = FDroidPopen([config['keytool'], '-list', '-v',
                      '-keystore', localconfig['keystore'],
                      '-alias', localconfig['repo_keyalias'],
                      '-storepass:file', config['keystorepassfile']])

fdroidserver/publish.py

@@ -47,6 +47,10 @@ def main():
 
     config = common.read_config(options)
 
+    if not ('jarsigner' in config and 'keytool' in config):
+        logging.critical('Java JDK not found! Install in standard location or set java_paths!')
+        sys.exit(1)
+
     log_dir = 'logs'
     if not os.path.isdir(log_dir):
         logging.info("Creating log directory")
@@ -163,12 +167,12 @@ def main():
 
             # See if we already have a key for this application, and
             # if not generate one...
-            p = FDroidPopen(['keytool', '-list',
+            p = FDroidPopen([config['keytool'], '-list',
                              '-alias', keyalias, '-keystore', config['keystore'],
                              '-storepass:file', config['keystorepassfile']])
             if p.returncode != 0:
                 logging.info("Key does not exist - generating...")
-                p = FDroidPopen(['keytool', '-genkey',
+                p = FDroidPopen([config['keytool'], '-genkey',
                                  '-keystore', config['keystore'],
                                  '-alias', keyalias,
                                  '-keyalg', 'RSA', '-keysize', '2048',
@@ -181,7 +185,7 @@ def main():
                     raise BuildException("Failed to generate key")
 
             # Sign the application...
-            p = FDroidPopen(['jarsigner', '-keystore', config['keystore'],
+            p = FDroidPopen([config['jarsigner'], '-keystore', config['keystore'],
                              '-storepass:file', config['keystorepassfile'],
                              '-keypass:file', config['keypassfile'], '-sigalg',
                              'SHA1withRSA', '-digestalg', 'SHA1',

fdroidserver/signindex.py

@@ -40,6 +40,10 @@ def main():
 
     config = common.read_config(options)
 
+    if not 'jarsigner' in config:
+        logging.critical('Java jarsigner not found! Install in standard location or set java_paths!')
+        sys.exit(1)
+
     repodirs = ['repo']
     if config['archive_older'] != 0:
         repodirs.append('archive')
@@ -53,7 +57,7 @@ def main():
         unsigned = os.path.join(output_dir, 'index_unsigned.jar')
         if os.path.exists(unsigned):
 
-            args = ['jarsigner', '-keystore', config['keystore'],
+            args = [config['jarsigner'], '-keystore', config['keystore'],
                     '-storepass:file', config['keystorepassfile'],
                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
                     unsigned, config['repo_keyalias']]

fdroidserver/update.py

@@ -367,7 +367,7 @@ def getsig(apkpath):
     cert = None
 
     # verify the jar signature is correct
-    args = ['jarsigner', '-verify', apkpath]
+    args = [config['jarsigner'], '-verify', apkpath]
     p = FDroidPopen(args)
     if p.returncode != 0:
         logging.critical(apkpath + " has a bad signature!")
@@ -711,7 +711,7 @@ def extract_pubkey():
     if 'repo_pubkey' in config:
         pubkey = unhexlify(config['repo_pubkey'])
     else:
-        p = FDroidPopen(['keytool', '-exportcert',
+        p = FDroidPopen([config['keytool'], '-exportcert',
                          '-alias', config['repo_keyalias'],
                          '-keystore', config['keystore'],
                          '-storepass:file', config['keystorepassfile']]
@@ -970,7 +970,7 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
             if os.path.exists(signed):
                 os.remove(signed)
         else:
-            args = ['jarsigner', '-keystore', config['keystore'],
+            args = [config['jarsigner'], '-keystore', config['keystore'],
                     '-storepass:file', config['keystorepassfile'],
                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
                     signed, config['repo_keyalias']]
@@ -1118,6 +1118,10 @@ def main():
 
     config = common.read_config(options)
 
+    if not ('jarsigner' in config and 'keytool' in config):
+        logging.critical('Java JDK not found! Install in standard location or set java_paths!')
+        sys.exit(1)
+
     repodirs = ['repo']
     if config['archive_older'] != 0:
         repodirs.append('archive')

makebuildserver

@@ -392,6 +392,7 @@ vagrantfile += """
         :sdk_loc => "/home/vagrant/android-sdk",
         :ndk_loc => "/home/vagrant/android-ndk",
         :debian_mirror => "%s",
+        :ubuntu_trusty => "%s",
         :user => "vagrant"
       }
     }
@@ -402,7 +403,8 @@ vagrantfile += """
     chef.add_recipe "kivy"
   end
 end
-""" % (config['debian_mirror'])
+""" % (config['debian_mirror'],
+       str('14.04' in os.uname()[3]).lower())
 
 # Check against the existing Vagrantfile, and if they differ, we need to
 # create a new box:

tests/update.TestCase

@@ -42,6 +42,10 @@ class UpdateTest(unittest.TestCase):
             return None
 
     def testGoodGetsig(self):
+        # config needed to use jarsigner and keytool
+        config = dict()
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.update.config = config
         apkfile = os.path.join(os.path.dirname(__file__), 'urzip.apk')
         sig = self.javagetsig(apkfile)
         self.assertIsNotNone(sig, "sig is None")
@@ -59,6 +63,10 @@ class UpdateTest(unittest.TestCase):
             self.assertTrue(False, 'TypeError!')
 
     def testBadGetsig(self):
+        # config needed to use jarsigner and keytool
+        config = dict()
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.update.config = config
         apkfile = os.path.join(os.path.dirname(__file__), 'urzip-badsig.apk')
         sig = self.javagetsig(apkfile)
         self.assertIsNone(sig, "sig should be None: " + str(sig))