Merge branch 'subdir-for-reprod-apks' into 'master'

put .binary.apk files into sub-directory

another shot at #509

See merge request fdroid/fdroidserver!533

fdroidserver/build.py

@@ -1026,6 +1026,7 @@ def main():
         if not os.path.isdir(output_dir):
             logging.info("Creating output directory")
             os.makedirs(output_dir)
+    binaries_dir = os.path.join(output_dir, 'binaries')
 
     if config['archive_older'] != 0:
         also_check_dir = 'archive'
@@ -1142,12 +1143,18 @@ def main():
                         # binary. We get that binary now, and save it
                         # alongside our built one in the 'unsigend'
                         # directory.
+                        if not os.path.isdir(binaries_dir):
+                            os.makedirs(binaries_dir)
+                            logging.info("Created directory for storing "
+                                         "developer supplied reference "
+                                         "binaries: '{path}'"
+                                         .format(path=binaries_dir))
                         url = app.Binaries
                         url = url.replace('%v', build.versionName)
                         url = url.replace('%c', str(build.versionCode))
                         logging.info("...retrieving " + url)
                         of = re.sub(r'.apk$', '.binary.apk', common.get_release_filename(app, build))
-                        of = os.path.join(output_dir, of)
+                        of = os.path.join(binaries_dir, of)
                         try:
                             net.download_file(url, local_filename=of)
                         except requests.exceptions.HTTPError as e:

fdroidserver/publish.py

@@ -82,7 +82,7 @@ def read_fingerprints_from_keystore():
                      '-storepass:env', 'FDROID_KEY_STORE_PASS'],
                     envs=env_vars, output=False)
     if p.returncode != 0:
-        raise FDroidException('could not read keysotre {}'.format(config['keystore']))
+        raise FDroidException('could not read keystore {}'.format(config['keystore']))
 
     realias = re.compile('Alias name: (?P<alias>.+)\n')
     resha256 = re.compile(r'\s+SHA256: (?P<sha256>[:0-9A-F]{95})\n')
@@ -178,6 +178,7 @@ def main():
     if not os.path.isdir(unsigned_dir):
         logging.warning(_("No unsigned directory - nothing to do"))
         sys.exit(1)
+    binaries_dir = os.path.join(unsigned_dir, 'binaries')
 
     if not os.path.exists(config['keystore']):
         logging.error("Config error - missing '{0}'".format(config['keystore']))
@@ -210,10 +211,6 @@ def main():
     for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))
                           + glob.glob(os.path.join(unsigned_dir, '*.zip'))):
 
-        # skip over developer supplied reference binaries for reproducible builds
-        if apkfile.endswith('.binary.apk'):
-            continue
-
         appid, vercode = common.publishednameinfo(apkfile)
         apkfilename = os.path.basename(apkfile)
         if vercodes and appid not in vercodes:
@@ -238,22 +235,30 @@ def main():
             # version if everything checks out.
             # The binary should already have been retrieved during the build
             # process.
-            srcapk = re.sub(r'.apk$', '.binary.apk', apkfile)
 
-            # Compare our unsigned one with the downloaded one...
+            srcapk = re.sub(r'\.apk$', '.binary.apk', apkfile)
-            compare_result = common.verify_apks(srcapk, apkfile, tmp_dir)
+            srcapk = srcapk.replace(unsigned_dir, binaries_dir)
-            if compare_result:
+
-                logging.error("...verification failed - publish skipped : "
+            if not os.path.isfile(srcapk):
-                              + compare_result)
+                logging.error("...reference binary missing - publish skipped: "
+                              "'{refpath}'".format(refpath=srcapk))
             else:
+                # Compare our unsigned one with the downloaded one...
+                compare_result = common.verify_apks(srcapk, apkfile, tmp_dir)
+                if compare_result:
+                    logging.error("...verification failed - publish skipped : "
+                                  "{result}".format(result=compare_result))
+                    os.remove(srcapk)
+                    logging.debug('removed developer supplied reference binary: {path}'
+                                  .format(path=srcapk))
+                else:
+                    # Success! So move the downloaded file to the repo, and remove
+                    # our built version.
+                    shutil.move(srcapk, os.path.join(output_dir, apkfilename))
+                    os.remove(apkfile)
 
-                # Success! So move the downloaded file to the repo, and remove
+                    publish_source_tarball(apkfilename, unsigned_dir, output_dir)
-                # our built version.
+                    logging.info('Published ' + apkfilename)
-                shutil.move(srcapk, os.path.join(output_dir, apkfilename))
-                os.remove(apkfile)
-
-                publish_source_tarball(apkfilename, unsigned_dir, output_dir)
-                logging.info('Published ' + apkfilename)
 
         elif apkfile.endswith('.zip'):
 

tests/publish.TestCase

@@ -14,6 +14,7 @@ import inspect
 import logging
 import optparse
 import os
+import shutil
 import sys
 import unittest
 import tempfile
@@ -133,6 +134,32 @@ class PublishTest(unittest.TestCase):
         with self.assertRaises(FDroidException):
             common.load_stats_fdroid_signing_key_fingerprints()
 
+    def test_reproducible_binaries_process(self):
+        common.config = {}
+        common.fill_config_defaults(common.config)
+        publish.config = common.config
+        publish.config['keystore'] = 'keystore.jks'
+        publish.config['repo_keyalias'] = 'sova'
+        publish.config['keystorepass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI='
+        publish.config['keypass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI='
+        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+
+        shutil.copy('keystore.jks', testdir)
+        os.mkdir(os.path.join(testdir, 'repo'))
+        metadata_dir = os.path.join(testdir, 'metadata')
+        os.mkdir(metadata_dir)
+        shutil.copy(os.path.join('metadata', 'com.politedroid.txt'), metadata_dir)
+        with open(os.path.join(metadata_dir, 'com.politedroid.txt'), 'a') as fp:
+            fp.write('\nBinaries:https://placeholder/foo%v.apk\n')
+        os.mkdir(os.path.join(testdir, 'unsigned'))
+        shutil.copy('repo/com.politedroid_6.apk', os.path.join(testdir, 'unsigned'))
+        os.mkdir(os.path.join(testdir, 'unsigned', 'binaries'))
+        shutil.copy('repo/com.politedroid_6.apk',
+                    os.path.join(testdir, 'unsigned', 'binaries', 'com.politedroid_6.binary.apk'))
+
+        os.chdir(testdir)
+        publish.main()
+
 
 if __name__ == "__main__":
     parser = optparse.OptionParser()