From cdc2147de93de3eac93158df624de29bdc39feea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 12 Jul 2018 23:52:46 +0200 Subject: [PATCH 1/5] put .binary.apk files into sub-directory --- fdroidserver/build.py | 9 ++++++++- fdroidserver/publish.py | 34 ++++++++++++++++++---------------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 6fbe925a..de5024da 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -1026,6 +1026,7 @@ def main(): if not os.path.isdir(output_dir): logging.info("Creating output directory") os.makedirs(output_dir) + binaries_dir = os.path.join(output_dir, 'binaries') if config['archive_older'] != 0: also_check_dir = 'archive' @@ -1142,12 +1143,18 @@ def main(): # binary. We get that binary now, and save it # alongside our built one in the 'unsigend' # directory. + if not os.path.isdir(binaries_dir): + os.makedirs(binaries_dir) + logging.info("Created directory for storing " + "devleoper supplied reference " + "binaries: '{path}'" + .format(path=binaries_dir)) url = app.Binaries url = url.replace('%v', build.versionName) url = url.replace('%c', str(build.versionCode)) logging.info("...retrieving " + url) of = re.sub(r'.apk$', '.binary.apk', common.get_release_filename(app, build)) - of = os.path.join(output_dir, of) + of = os.path.join(binaries_dir, of) try: net.download_file(url, local_filename=of) except requests.exceptions.HTTPError as e: diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py index 74f019c0..7c7082a7 100644 --- a/fdroidserver/publish.py +++ b/fdroidserver/publish.py @@ -178,6 +178,7 @@ def main(): if not os.path.isdir(unsigned_dir): logging.warning(_("No unsigned directory - nothing to do")) sys.exit(1) + binaries_dir = os.path.join(unsigned_dir, 'binaries') if not os.path.exists(config['keystore']): logging.error("Config error - missing '{0}'".format(config['keystore'])) @@ -210,10 +211,6 @@ def main(): for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk')) + glob.glob(os.path.join(unsigned_dir, '*.zip'))): - # skip over developer supplied reference binaries for reproducible builds - if apkfile.endswith('.binary.apk'): - continue - appid, vercode = common.publishednameinfo(apkfile) apkfilename = os.path.basename(apkfile) if vercodes and appid not in vercodes: @@ -238,22 +235,27 @@ def main(): # version if everything checks out. # The binary should already have been retrieved during the build # process. + srcapk = re.sub(r'.apk$', '.binary.apk', apkfile) + srcapk = srcapk.replace(unsigned_dir, binaries_dir) - # Compare our unsigned one with the downloaded one... - compare_result = common.verify_apks(srcapk, apkfile, tmp_dir) - if compare_result: - logging.error("...verification failed - publish skipped : " - + compare_result) + if not os.path.isfile(srcapk): + logging.error("...reference binary missing - publish skipped: " + "'{refpath}'".format(refpath=srcapk)) else: + # Compare our unsigned one with the downloaded one... + compare_result = common.verify_apks(srcapk, apkfile, tmp_dir) + if compare_result: + logging.error("...verification failed - publish skipped : " + "{result}".format(result=compare_result)) + else: + # Success! So move the downloaded file to the repo, and remove + # our built version. + shutil.move(srcapk, os.path.join(output_dir, apkfilename)) + os.remove(apkfile) - # Success! So move the downloaded file to the repo, and remove - # our built version. - shutil.move(srcapk, os.path.join(output_dir, apkfilename)) - os.remove(apkfile) - - publish_source_tarball(apkfilename, unsigned_dir, output_dir) - logging.info('Published ' + apkfilename) + publish_source_tarball(apkfilename, unsigned_dir, output_dir) + logging.info('Published ' + apkfilename) elif apkfile.endswith('.zip'): From db5ed265178c961bb3111da5f0bbbd737c84d5ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 12 Jul 2018 23:04:05 +0200 Subject: [PATCH 2/5] publish: avoid double removal of dev-supplied reference binary --- fdroidserver/publish.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py index 7c7082a7..ea9b20d3 100644 --- a/fdroidserver/publish.py +++ b/fdroidserver/publish.py @@ -248,6 +248,9 @@ def main(): if compare_result: logging.error("...verification failed - publish skipped : " "{result}".format(result=compare_result)) + os.remove(srcapk) + logging.debug('remvoed developer supplied reference binary: {path}' + .format(path=srcapk)) else: # Success! So move the downloaded file to the repo, and remove # our built version. From 68fb3d2bd9593313c2c9659040dba08286596498 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 12 Jul 2018 23:43:19 +0200 Subject: [PATCH 3/5] fix typos --- fdroidserver/build.py | 2 +- fdroidserver/publish.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index de5024da..7dbc326d 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -1146,7 +1146,7 @@ def main(): if not os.path.isdir(binaries_dir): os.makedirs(binaries_dir) logging.info("Created directory for storing " - "devleoper supplied reference " + "developer supplied reference " "binaries: '{path}'" .format(path=binaries_dir)) url = app.Binaries diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py index ea9b20d3..ee45e716 100644 --- a/fdroidserver/publish.py +++ b/fdroidserver/publish.py @@ -82,7 +82,7 @@ def read_fingerprints_from_keystore(): '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars, output=False) if p.returncode != 0: - raise FDroidException('could not read keysotre {}'.format(config['keystore'])) + raise FDroidException('could not read keystore {}'.format(config['keystore'])) realias = re.compile('Alias name: (?P.+)\n') resha256 = re.compile(r'\s+SHA256: (?P[:0-9A-F]{95})\n') @@ -249,7 +249,7 @@ def main(): logging.error("...verification failed - publish skipped : " "{result}".format(result=compare_result)) os.remove(srcapk) - logging.debug('remvoed developer supplied reference binary: {path}' + logging.debug('removed developer supplied reference binary: {path}' .format(path=srcapk)) else: # Success! So move the downloaded file to the repo, and remove From 58b81b5ca63dc9fad2cde6d739829c09a0a015c2 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 12 Jul 2018 23:44:03 +0200 Subject: [PATCH 4/5] publish: strict regex replacement for .binary.apk . matches anything \. matches the char "." --- fdroidserver/publish.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py index ee45e716..a6fb258c 100644 --- a/fdroidserver/publish.py +++ b/fdroidserver/publish.py @@ -236,7 +236,7 @@ def main(): # The binary should already have been retrieved during the build # process. - srcapk = re.sub(r'.apk$', '.binary.apk', apkfile) + srcapk = re.sub(r'\.apk$', '.binary.apk', apkfile) srcapk = srcapk.replace(unsigned_dir, binaries_dir) if not os.path.isfile(srcapk): From 64b999ca19c28392ee206c839ed2d035a3d4df29 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 12 Jul 2018 23:36:23 +0200 Subject: [PATCH 5/5] publish: test case that successfully verifies based on Binaries: This is a stupid test of the process just to exercise the most basic path. It should always succeed since it is verifying two copes of the same APK. --- tests/publish.TestCase | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/publish.TestCase b/tests/publish.TestCase index e296a48a..b02ece99 100755 --- a/tests/publish.TestCase +++ b/tests/publish.TestCase @@ -14,6 +14,7 @@ import inspect import logging import optparse import os +import shutil import sys import unittest import tempfile @@ -133,6 +134,32 @@ class PublishTest(unittest.TestCase): with self.assertRaises(FDroidException): common.load_stats_fdroid_signing_key_fingerprints() + def test_reproducible_binaries_process(self): + common.config = {} + common.fill_config_defaults(common.config) + publish.config = common.config + publish.config['keystore'] = 'keystore.jks' + publish.config['repo_keyalias'] = 'sova' + publish.config['keystorepass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=' + publish.config['keypass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=' + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + + shutil.copy('keystore.jks', testdir) + os.mkdir(os.path.join(testdir, 'repo')) + metadata_dir = os.path.join(testdir, 'metadata') + os.mkdir(metadata_dir) + shutil.copy(os.path.join('metadata', 'com.politedroid.txt'), metadata_dir) + with open(os.path.join(metadata_dir, 'com.politedroid.txt'), 'a') as fp: + fp.write('\nBinaries:https://placeholder/foo%v.apk\n') + os.mkdir(os.path.join(testdir, 'unsigned')) + shutil.copy('repo/com.politedroid_6.apk', os.path.join(testdir, 'unsigned')) + os.mkdir(os.path.join(testdir, 'unsigned', 'binaries')) + shutil.copy('repo/com.politedroid_6.apk', + os.path.join(testdir, 'unsigned', 'binaries', 'com.politedroid_6.binary.apk')) + + os.chdir(testdir) + publish.main() + if __name__ == "__main__": parser = optparse.OptionParser()