Merge branch 'build-FPE-update-zip' into 'master'

Build Privileged Extension OTA update.zip using `fdroid build`

This allows `fdroid build` to build the OTA update ZIP file for F-Droid Privileged Extension, so that the official releases can be built and distributed via the normal F-Droid channels.   This is related to #233 

Ultimately the client will also have to be updated to allow it to handle the non-APK files.

See merge request !181

MANIFEST.in

@@ -5,13 +5,15 @@ include fdroid
 include jenkins-build
 include makebuildserver
 include buildserver/config.buildserver.py
-include buildserver/fixpaths.sh
+include buildserver/provision-android-ndk
-include buildserver/cookbooks/android-ndk/recipes/default.rb
+include buildserver/provision-android-sdk
-include buildserver/cookbooks/android-sdk/recipes/default.rb
+include buildserver/provision-apt-get-install
-include buildserver/cookbooks/fdroidbuild-general/recipes/default.rb
+include buildserver/provision-apt-proxy
-include buildserver/cookbooks/gradle/recipes/default.rb
+include buildserver/provision-gradle
-include buildserver/cookbooks/gradle/recipes/gradle
+include buildserver/provision-pip
-include buildserver/cookbooks/kivy/recipes/default.rb
+include buildserver/provision-qt-sdk
+include buildserver/provision-ubuntu-trusty-paramiko
+include buildserver/Vagrantfile
 include completion/bash-completion
 include docs/fdl.texi
 include docs/fdroid.texi

examples/config.py

@@ -86,7 +86,7 @@ The repository of older versions of applications from the main demo repository.
 # current_version_name_source = 'id'
 
 # Optionally, override home directory for gpg
-# gpghome = /home/fdroid/somewhere/else/.gnupg
+# gpghome = '/home/fdroid/somewhere/else/.gnupg'
 
 # The ID of a GPG key for making detached signatures for apks. Optional.
 # gpgkey = '1DBA2E89'

fdroidserver/build.py

@@ -412,7 +412,7 @@ def build_server(app, build, vcs, build_dir, output_dir, force):
             ftp.chdir(homedir + '/tmp')
         else:
             ftp.chdir(homedir + '/unsigned')
-        apkfile = common.getapkname(app, build)
+        apkfile = common.get_release_filename(app, build)
         tarball = common.getsrcname(app, build)
         try:
             ftp.get(apkfile, os.path.join(output_dir, apkfile))
@@ -454,6 +454,54 @@ def capitalize_intact(string):
     return string[0].upper() + string[1:]
 
 
+def get_metadata_from_apk(app, build, apkfile):
+    """get the required metadata from the built APK"""
+
+    p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
+
+    vercode = None
+    version = None
+    foundid = None
+    nativecode = None
+    for line in p.output.splitlines():
+        if line.startswith("package:"):
+            pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
+            m = pat.match(line)
+            if m:
+                foundid = m.group(1)
+            pat = re.compile(".*versionCode='([0-9]*)'.*")
+            m = pat.match(line)
+            if m:
+                vercode = m.group(1)
+            pat = re.compile(".*versionName='([^']*)'.*")
+            m = pat.match(line)
+            if m:
+                version = m.group(1)
+        elif line.startswith("native-code:"):
+            nativecode = line[12:]
+
+    # Ignore empty strings or any kind of space/newline chars that we don't
+    # care about
+    if nativecode is not None:
+        nativecode = nativecode.strip()
+        nativecode = None if not nativecode else nativecode
+
+    if build.buildjni and build.buildjni != ['no']:
+        if nativecode is None:
+            raise BuildException("Native code should have been built but none was packaged")
+    if build.novcheck:
+        vercode = build.vercode
+        version = build.version
+    if not version or not vercode:
+        raise BuildException("Could not find version information in build in output")
+    if not foundid:
+        raise BuildException("Could not find package ID in output")
+    if foundid != app.id:
+        raise BuildException("Wrong package ID - build " + foundid + " but expected " + app.id)
+
+    return vercode, version
+
+
 def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh):
     """Do a build locally."""
 
@@ -809,7 +857,7 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir,
         src = os.path.normpath(apks[0])
 
     # Make sure it's not debuggable...
-    if common.isApkDebuggable(src, config):
+    if common.isApkAndDebuggable(src, config):
         raise BuildException("APK is debuggable")
 
     # By way of a sanity check, make sure the version and version
@@ -818,64 +866,17 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir,
     if not os.path.exists(src):
         raise BuildException("Unsigned apk is not at expected location of " + src)
 
-    p = SdkToolsPopen(['aapt', 'dump', 'badging', src], output=False)
+    if common.get_file_extension(src) == 'apk':
-
+        vercode, version = get_metadata_from_apk(app, build, src)
-    vercode = None
+        if (version != build.version or vercode != build.vercode):
-    version = None
+            raise BuildException(("Unexpected version/version code in output;"
-    foundid = None
+                                  " APK: '%s' / '%s', "
-    nativecode = None
+                                  " Expected: '%s' / '%s'")
-    for line in p.output.splitlines():
+                                 % (version, str(vercode), build.version,
-        if line.startswith("package:"):
+                                    str(build.vercode)))
-            pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
+    else:
-            m = pat.match(line)
-            if m:
-                foundid = m.group(1)
-            pat = re.compile(".*versionCode='([0-9]*)'.*")
-            m = pat.match(line)
-            if m:
-                vercode = m.group(1)
-            pat = re.compile(".*versionName='([^']*)'.*")
-            m = pat.match(line)
-            if m:
-                version = m.group(1)
-        elif line.startswith("native-code:"):
-            nativecode = line[12:]
-
-    # Ignore empty strings or any kind of space/newline chars that we don't
-    # care about
-    if nativecode is not None:
-        nativecode = nativecode.strip()
-        nativecode = None if not nativecode else nativecode
-
-    if build.buildjni and build.buildjni != ['no']:
-        if nativecode is None:
-            raise BuildException("Native code should have been built but none was packaged")
-    if build.novcheck:
         vercode = build.vercode
         version = build.version
-    if not version or not vercode:
-        raise BuildException("Could not find version information in build in output")
-    if not foundid:
-        raise BuildException("Could not find package ID in output")
-    if foundid != app.id:
-        raise BuildException("Wrong package ID - build " + foundid + " but expected " + app.id)
-
-    # Some apps (e.g. Timeriffic) have had the bonkers idea of
-    # including the entire changelog in the version number. Remove
-    # it so we can compare. (TODO: might be better to remove it
-    # before we compile, in fact)
-    index = version.find(" //")
-    if index != -1:
-        version = version[:index]
-
-    if (version != build.version or
-            vercode != build.vercode):
-        raise BuildException(("Unexpected version/version code in output;"
-                              " APK: '%s' / '%s', "
-                              " Expected: '%s' / '%s'")
-                             % (version, str(vercode), build.version,
-                                str(build.vercode))
-                             )
 
     # Add information for 'fdroid verify' to be able to reproduce the build
     # environment.
@@ -892,7 +893,7 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir,
 
     # Copy the unsigned apk to our destination directory for further
     # processing (by publish.py)...
-    dest = os.path.join(output_dir, common.getapkname(app, build))
+    dest = os.path.join(output_dir, common.get_release_filename(app, build))
     shutil.copyfile(src, dest)
 
     # Move the source tarball into the output directory...
@@ -920,17 +921,17 @@ def trybuild(app, build, build_dir, output_dir, also_check_dir, srclib_dir, extl
     :returns: True if the build was done, False if it wasn't necessary.
     """
 
-    dest_apk = common.getapkname(app, build)
+    dest_file = common.get_release_filename(app, build)
 
-    dest = os.path.join(output_dir, dest_apk)
+    dest = os.path.join(output_dir, dest_file)
-    dest_repo = os.path.join(repo_dir, dest_apk)
+    dest_repo = os.path.join(repo_dir, dest_file)
 
     if not test:
         if os.path.exists(dest) or os.path.exists(dest_repo):
             return False
 
         if also_check_dir:
-            dest_also = os.path.join(also_check_dir, dest_apk)
+            dest_also = os.path.join(also_check_dir, dest_file)
             if os.path.exists(dest_also):
                 return False
 

fdroidserver/common.py

@@ -459,8 +459,11 @@ def apknameinfo(filename):
     return result
 
 
-def getapkname(app, build):
+def get_release_filename(app, build):
-    return "%s_%s.apk" % (app.id, build.vercode)
+    if build.output:
+        return "%s_%s.%s" % (app.id, build.vercode, get_file_extension(build.output))
+    else:
+        return "%s_%s.apk" % (app.id, build.vercode)
 
 
 def getsrcname(app, build):
@@ -1615,11 +1618,14 @@ def get_file_extension(filename):
     return os.path.splitext(filename)[1].lower()[1:]
 
 
-def isApkDebuggable(apkfile, config):
+def isApkAndDebuggable(apkfile, config):
-    """Returns True if the given apk file is debuggable
+    """Returns True if the given file is an APK and is debuggable
 
     :param apkfile: full path to the apk to check"""
 
+    if get_file_extension(apkfile) != 'apk':
+        return False
+
     p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
                       output=False)
     if p.returncode != 0:
@@ -2078,3 +2084,14 @@ def get_per_app_repos():
                 repos.append(d)
         break
     return repos
+
+
+def is_repo_file(filename):
+    '''Whether the file in a repo is a build product to be delivered to users'''
+    return os.path.isfile(filename) \
+        and os.path.basename(filename) not in [
+            'index.jar',
+            'index.xml',
+            'index.html',
+            'categories.txt',
+        ]

fdroidserver/gpgsign.py

@@ -50,10 +50,13 @@ def main():
             sys.exit(1)
 
         # Process any apks that are waiting to be signed...
-        for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))):
+        for f in sorted(glob.glob(os.path.join(output_dir, '*.*'))):
-
+            if common.get_file_extension(f) == 'asc':
-            apkfilename = os.path.basename(apkfile)
+                continue
-            sigfilename = apkfilename + ".asc"
+            if not common.is_repo_file(f):
+                continue
+            filename = os.path.basename(f)
+            sigfilename = filename + ".asc"
             sigpath = os.path.join(output_dir, sigfilename)
 
             if not os.path.exists(sigpath):
@@ -64,13 +67,13 @@ def main():
                     gpgargs.extend(['--homedir', config['gpghome']])
                 if 'gpgkey' in config:
                     gpgargs.extend(['--local-user', config['gpgkey']])
-                gpgargs.append(os.path.join(output_dir, apkfilename))
+                gpgargs.append(os.path.join(output_dir, filename))
                 p = FDroidPopen(gpgargs)
                 if p.returncode != 0:
                     logging.error("Signing failed.")
                     sys.exit(1)
 
-                logging.info('Signed ' + apkfilename)
+                logging.info('Signed ' + filename)
 
 
 if __name__ == "__main__":

fdroidserver/update.py

@@ -517,13 +517,11 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
     cachechanged = False
     repo_files = []
     for name in os.listdir(repodir):
-        if name in ['index.jar', 'index.xml', 'index.html', 'categories.txt', ]:
-            continue
         file_extension = common.get_file_extension(name)
         if file_extension == 'apk' or file_extension == 'obb':
             continue
         filename = os.path.join(repodir, name)
-        if not os.path.isfile(filename):
+        if not common.is_repo_file(name):
             continue
         stat = os.stat(filename)
         if stat.st_size == 0:
@@ -739,7 +737,7 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
                 apk['minSdkVersion'] = 1
 
             # Check for debuggable apks...
-            if common.isApkDebuggable(apkfile, config):
+            if common.isApkAndDebuggable(apkfile, config):
                 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
 
             # Get the signature (or md5 of, to be precise)...

tests/common.TestCase

@@ -74,7 +74,7 @@ class CommonTest(unittest.TestCase):
         testfiles.append(os.path.join(os.path.dirname(__file__), 'urzip-badsig.apk'))
         testfiles.append(os.path.join(os.path.dirname(__file__), 'urzip-badcert.apk'))
         for apkfile in testfiles:
-            debuggable = fdroidserver.common.isApkDebuggable(apkfile, config)
+            debuggable = fdroidserver.common.isApkAndDebuggable(apkfile, config)
             self.assertTrue(debuggable,
                             "debuggable APK state was not properly parsed!")
         # these are set NOT debuggable
@@ -82,7 +82,7 @@ class CommonTest(unittest.TestCase):
         testfiles.append(os.path.join(os.path.dirname(__file__), 'urzip-release.apk'))
         testfiles.append(os.path.join(os.path.dirname(__file__), 'urzip-release-unsigned.apk'))
         for apkfile in testfiles:
-            debuggable = fdroidserver.common.isApkDebuggable(apkfile, config)
+            debuggable = fdroidserver.common.isApkAndDebuggable(apkfile, config)
             self.assertFalse(debuggable,
                              "debuggable APK state was not properly parsed!")
 

tests/run-tests

@@ -139,21 +139,33 @@ $fdroid update
 
 
 #------------------------------------------------------------------------------#
-echo_header "copy tests/repo, generate a keystore, and update"
+echo_header "copy tests/repo, generate java/gpg keys, update, and gpgsign"
 
 REPOROOT=`create_test_dir`
+GNUPGHOME=$REPOROOT/gnupghome
 cd $REPOROOT
 $fdroid init
 cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $REPOROOT/
+cp -a $WORKSPACE/tests/gnupghome $GNUPGHOME
+chmod 0700 $GNUPGHOME
 echo "accepted_formats = ['json', 'txt', 'xml', 'yml']" >> config.py
 echo "install_list = 'org.adaway'" >> config.py
 echo "uninstall_list = {'com.android.vending', 'com.facebook.orca',}" >> config.py
+echo "gpghome = '$GNUPGHOME'" >> config.py
+echo "gpgkey = 'CE71F7FB'" >> config.py
 $fdroid update --verbose
 test -e repo/index.xml
 test -e repo/index.jar
 grep -F '<application id=' repo/index.xml > /dev/null
 grep -F '<install packageName=' repo/index.xml > /dev/null
 grep -F '<uninstall packageName=' repo/index.xml > /dev/null
+$fdroid gpgsign --verbose
+$fdroid gpgsign --verbose
+test -e repo/obb.mainpatch.current_1619.apk.asc
+test -e repo/obb.main.twoversions_1101617_src.tar.gz.asc
+! test -e repo/obb.mainpatch.current_1619.apk.asc.asc
+! test -e repo/obb.main.twoversions_1101617_src.tar.gz.asc.asc
+! test -e repo/index.xml.asc
 
 
 #------------------------------------------------------------------------------#

wp-fdroid/wp-fdroid.php

@@ -515,6 +515,9 @@ class FDroid
 					if($srcbuild) {
 						$out.='<br /><a href="https://f-droid.org/repo/'.$apk['srcname'].'">source tarball</a> ';
 						$out.=$this->human_readable_size(filesize($this->site_path.'/repo/'.$apk['srcname']));
+						if(file_exists($this->site_path.'/repo/'.$apk['srcname'].'.asc')) {
+							$out.=' <a href="https://f-droid.org/repo/'.$apk['srcname'].'.asc">GPG Signature</a> ';
+						}
 					}
 
 					if(isset($apk['permissions'])) {