Merge branch 'java8-and-other-fixes' into 'master'

Java8 and other fixes

This fixes the OSX travis-ci job for the Java8 update, and also two other fixes as described in the commit message.  @CiaranG I think the _/etc/profile_ approach in 7b466b6266a466a954c845968271283ad60e43ea gets us closer to the idea of having the `fdroid build` jobs run in the same environment as `vagrant ssh`.

See merge request !138

.gitignore

@@ -17,3 +17,13 @@ docs/html/
 # files generated by tests
 tmp/
 tests/repo/icons*
+
+# files used in manual testing
+/config.py
+/tmp/
+/logs/
+/metadata/
+makebuildserver.config.py
+/tests/config.py
+/tests/fdroid-icon.png
+/unsigned/

.travis.yml

@@ -12,6 +12,7 @@ matrix:
       # this doesn't actually work yet https://github.com/travis-ci/travis-ci/issues/5337
       dist: trusty
     - os: osx
+      osx_image: xcode8
       env: ANDROID_HOME=/usr/local/opt/android-sdk
 
 licenses:

fdroidserver/build.py

@@ -388,8 +388,7 @@ def build_server(app, build, vcs, build_dir, output_dir, force):
         if options.verbose:
             cmdline += ' --verbose'
         cmdline += " %s:%s" % (app.id, build.vercode)
-        cmdline = '. /etc/profile && ' + cmdline
+        chan.exec_command('bash --login -c "' + cmdline + '"')
-        chan.exec_command('bash -c "' + cmdline + '"')
         output = bytes()
         while not chan.exit_status_ready():
             while chan.recv_ready():

fdroidserver/common.py

@@ -1794,7 +1794,8 @@ def set_FDroidPopen_env(build=None):
     set up the environment variables for the build environment
 
     There is only a weak standard, the variables used by gradle, so also set
-    up the most commonly used environment variables for SDK and NDK
+    up the most commonly used environment variables for SDK and NDK.  Also, if
+    there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
     '''
     global env, orig_path
 
@@ -1806,6 +1807,15 @@ def set_FDroidPopen_env(build=None):
         for k, v in config['java_paths'].items():
             env['JAVA%s_HOME' % k] = v
 
+    missinglocale = True
+    for k, v in env.items():
+        if k == 'LANG' and v != 'C':
+            missinglocale = False
+        elif k == 'LC_ALL':
+            missinglocale = False
+    if missinglocale:
+        env['LANG'] = 'en_US.UTF-8'
+
     if build is not None:
         path = build.ndk_path()
         paths = orig_path.split(os.pathsep)

fdroidserver/update.py

@@ -323,8 +323,10 @@ def resize_icon(iconpath, density):
     if not os.path.isfile(iconpath):
         return
 
+    fp = None
     try:
-        im = Image.open(iconpath)
+        fp = open(iconpath, 'rb')
+        im = Image.open(fp)
         size = dpi_to_px(density)
 
         if any(length > size for length in im.size):
@@ -337,6 +339,10 @@ def resize_icon(iconpath, density):
     except Exception as e:
         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 
+    finally:
+        if fp:
+            fp.close()
+
 
 def resize_all_icons(repodirs):
     """Resize all icons that exceed the max size
@@ -405,6 +411,14 @@ def getsig(apkpath):
     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 
 
+def get_icon_bytes(apkzip, iconsrc):
+    '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
+    try:
+        return apkzip.read(iconsrc)
+    except KeyError:
+        return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
+
+
 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
     """Scan the apks in the given repo directory.
 
@@ -563,7 +577,7 @@ def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
 
             # Check for debuggable apks...
             if common.isApkDebuggable(apkfile, config):
-                logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
+                logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
 
             # Get the signature (or md5 of, to be precise)...
             logging.debug('Getting signature of {0}'.format(apkfile))
@@ -607,7 +621,7 @@ def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
 
                 try:
                     with open(icondest, 'wb') as f:
-                        f.write(apkzip.read(iconsrc))
+                        f.write(get_icon_bytes(apkzip, iconsrc))
                     apk['icons'][density] = iconfilename
 
                 except:
@@ -621,7 +635,7 @@ def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
                 iconpath = os.path.join(
                     get_icon_dir(repodir, '0'), iconfilename)
                 with open(iconpath, 'wb') as f:
-                    f.write(apkzip.read(iconsrc))
+                    f.write(get_icon_bytes(apkzip, iconsrc))
                 try:
                     im = Image.open(iconpath)
                     dpi = px_to_dpi(im.size[0])
@@ -657,17 +671,21 @@ def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
                     get_icon_dir(repodir, last_density), iconfilename)
                 iconpath = os.path.join(
                     get_icon_dir(repodir, density), iconfilename)
+                fp = None
                 try:
-                    im = Image.open(last_iconpath)
+                    fp = open(last_iconpath, 'rb')
+                    im = Image.open(fp)
+
+                    size = dpi_to_px(density)
+
+                    im.thumbnail((size, size), Image.ANTIALIAS)
+                    im.save(iconpath, "PNG")
+                    empty_densities.remove(density)
                 except:
-                    logging.warn("Invalid image file at %s" % last_iconpath)
+                    logging.warning("Invalid image file at %s" % last_iconpath)
-                    continue
+                finally:
-
+                    if fp:
-                size = dpi_to_px(density)
+                        fp.close()
-
-                im.thumbnail((size, size), Image.ANTIALIAS)
-                im.save(iconpath, "PNG")
-                empty_densities.remove(density)
 
             # Then just copy from the highest resolution available
             last_density = None

tests/run-tests

@@ -138,6 +138,20 @@ $fdroid readmeta
 $fdroid update
 
 
+#------------------------------------------------------------------------------#
+echo_header "copy tests/repo, generate a keystore, and update"
+
+REPOROOT=`create_test_dir`
+cd $REPOROOT
+$fdroid init
+cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $REPOROOT/
+echo "accepted_formats = ['json', 'txt', 'xml', 'yml']" >> config.py
+$fdroid update --verbose
+test -e repo/index.xml
+test -e repo/index.jar
+grep -F '<application id=' repo/index.xml > /dev/null
+
+
 #------------------------------------------------------------------------------#
 echo_header "test metadata checks"