From a3e7eacc9b14a43ffde0df319d5099f04c21f19b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Tue, 4 Apr 2017 18:58:16 +0200 Subject: [PATCH 1/6] compare apk with Binaries from metadata right after building --- fdroidserver/build.py | 35 +++++++++++++++++++++++++++++++++++ fdroidserver/common.py | 9 +++++---- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index c63df9d1..e41b5d6a 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -28,6 +28,7 @@ import traceback import time import json import requests +import tempfile from configparser import ConfigParser from argparse import ArgumentParser import logging @@ -1218,8 +1219,42 @@ def main(): except requests.exceptions.HTTPError as e: raise FDroidException('downloading Binaries from %s failed' % url) from e + # Now we check weather the build can be verified to + # match the supplied binary or not. Should the + # comparison fail, we mark this build as a failure + # and remove everything from the unsigend folder. + with tempfile.TemporaryDirectory() as tmpdir: + unsigned_apk = \ + '{0}_{1}.apk'.format(appid, + build.versionCode) + unsigned_apk = os.path.join(output_dir, + unsigned_apk) + compare_result = \ + common.compare_apks(of, unsigned_apk, + tmpdir, log_dir, + skip_manual_diff=True) + if compare_result: + compare_result = compare_result.split('\n') + line_count = len(compare_result) + compare_result = compare_result[:299] + if line_count > len(compare_result): + line_difference = \ + line_count - len(compare_result) + compare_result.append('%d more lines ...' % + line_difference) + compare_result = '\n'.join(compare_result) + raise FDroidException('compared built binary ' + 'to supplied reference ' + 'binary but failed', + compare_result) + else: + logging.info('compared built binary to ' + 'supplied reference binary ' + 'successfully') + build_succeeded.append(app) wikilog = "Build succeeded" + except VCSException as vcse: reason = str(vcse).split('\n', 1)[0] if options.verbose else str(vcse) logging.error("VCS error while building app %s: %s" % ( diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 8ced40c9..181c1fd9 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -2049,7 +2049,7 @@ def verify_apk_signature(apk, jar=False): apk_badchars = re.compile('''[/ :;'"]''') -def compare_apks(apk1, apk2, tmp_dir, log_dir=None): +def compare_apks(apk1, apk2, tmp_dir, log_dir=None, skip_manual_diff=False): """Compare two apks Returns None if the apk content is the same (apart from the signing key), @@ -2101,9 +2101,10 @@ def compare_apks(apk1, apk2, tmp_dir, log_dir=None): p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False) lines = p.output.splitlines() if len(lines) != 1 or 'META-INF' not in lines[0]: - meld = find_command('meld') - if meld is not None: - p = FDroidPopen(['meld', apk1dir, apk2dir], output=False) + if not skip_manual_diff: + meld = find_command('meld') + if meld is not None: + p = FDroidPopen(['meld', apk1dir, apk2dir], output=False) return("Unexpected diff output - " + p.output) # since everything verifies, delete the comparison to keep cruft down From f4a7c190436da16f88f8cc7bcfe2ee69436fe4e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 20 Apr 2017 12:27:04 +0200 Subject: [PATCH 2/6] fix java architecture in buildserver config --- buildserver/config.buildserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildserver/config.buildserver.py b/buildserver/config.buildserver.py index e7a4f416..ac978fbe 100644 --- a/buildserver/config.buildserver.py +++ b/buildserver/config.buildserver.py @@ -9,5 +9,5 @@ ndk_paths = { } qt_sdk_path = "/home/vagrant/qt-sdk/5.7.0/5.7" java_paths = { - '8': "/usr/lib/jvm/java-8-openjdk-i386", + '8': "/usr/lib/jvm/java-8-openjdk-amd64", } From 4044e06e7b8ba495ce4dd6e694ae81bbeb4d65bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 20 Apr 2017 12:38:50 +0200 Subject: [PATCH 3/6] use verify_apks for checking build output --- fdroidserver/build.py | 5 ++--- fdroidserver/common.py | 8 ++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index e41b5d6a..9c629a0a 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -1230,9 +1230,8 @@ def main(): unsigned_apk = os.path.join(output_dir, unsigned_apk) compare_result = \ - common.compare_apks(of, unsigned_apk, - tmpdir, log_dir, - skip_manual_diff=True) + common.verify_apks(of, unsigned_apk, tmpdir, + skip_manual_diff=True) if compare_result: compare_result = compare_result.split('\n') line_count = len(compare_result) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 181c1fd9..edf5307e 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1968,7 +1968,7 @@ def place_srclib(root_dir, number, libpath): apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)') -def verify_apks(signed_apk, unsigned_apk, tmp_dir): +def verify_apks(signed_apk, unsigned_apk, tmp_dir, skip_manual_diff=False): """Verify that two apks are the same One of the inputs is signed, the other is unsigned. The signature metadata @@ -1987,6 +1987,8 @@ def verify_apks(signed_apk, unsigned_apk, tmp_dir): :param signed_apk: Path to a signed apk file :param unsigned_apk: Path to an unsigned apk file expected to match it :param tmp_dir: Path to directory for temporary files + :param skip_manual_diff: Skipping to displaying defferences between apks + with meld, kdiff, etc. :returns: None if the verification is successful, otherwise a string describing what went wrong. """ @@ -2020,7 +2022,9 @@ def verify_apks(signed_apk, unsigned_apk, tmp_dir): if not verified: logging.info("...NOT verified - {0}".format(tmp_apk)) - return compare_apks(signed_apk, tmp_apk, tmp_dir, os.path.dirname(unsigned_apk)) + return compare_apks(signed_apk, tmp_apk, tmp_dir, + os.path.dirname(unsigned_apk), + skip_manual_diff=skip_manual_diff) logging.info("...successfully verified") return None From 62af9b8e3d7021d6097372a4549b9df18e240626 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 20 Apr 2017 12:44:22 +0200 Subject: [PATCH 4/6] check meld config for manual apk diffs --- fdroidserver/build.py | 3 +-- fdroidserver/common.py | 15 +++++---------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 9c629a0a..7c4fddf3 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -1230,8 +1230,7 @@ def main(): unsigned_apk = os.path.join(output_dir, unsigned_apk) compare_result = \ - common.verify_apks(of, unsigned_apk, tmpdir, - skip_manual_diff=True) + common.verify_apks(of, unsigned_apk, tmpdir) if compare_result: compare_result = compare_result.split('\n') line_count = len(compare_result) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index edf5307e..629d1b4a 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1968,7 +1968,7 @@ def place_srclib(root_dir, number, libpath): apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)') -def verify_apks(signed_apk, unsigned_apk, tmp_dir, skip_manual_diff=False): +def verify_apks(signed_apk, unsigned_apk, tmp_dir): """Verify that two apks are the same One of the inputs is signed, the other is unsigned. The signature metadata @@ -1987,8 +1987,6 @@ def verify_apks(signed_apk, unsigned_apk, tmp_dir, skip_manual_diff=False): :param signed_apk: Path to a signed apk file :param unsigned_apk: Path to an unsigned apk file expected to match it :param tmp_dir: Path to directory for temporary files - :param skip_manual_diff: Skipping to displaying defferences between apks - with meld, kdiff, etc. :returns: None if the verification is successful, otherwise a string describing what went wrong. """ @@ -2023,8 +2021,7 @@ def verify_apks(signed_apk, unsigned_apk, tmp_dir, skip_manual_diff=False): if not verified: logging.info("...NOT verified - {0}".format(tmp_apk)) return compare_apks(signed_apk, tmp_apk, tmp_dir, - os.path.dirname(unsigned_apk), - skip_manual_diff=skip_manual_diff) + os.path.dirname(unsigned_apk)) logging.info("...successfully verified") return None @@ -2053,7 +2050,7 @@ def verify_apk_signature(apk, jar=False): apk_badchars = re.compile('''[/ :;'"]''') -def compare_apks(apk1, apk2, tmp_dir, log_dir=None, skip_manual_diff=False): +def compare_apks(apk1, apk2, tmp_dir, log_dir=None): """Compare two apks Returns None if the apk content is the same (apart from the signing key), @@ -2105,10 +2102,8 @@ def compare_apks(apk1, apk2, tmp_dir, log_dir=None, skip_manual_diff=False): p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False) lines = p.output.splitlines() if len(lines) != 1 or 'META-INF' not in lines[0]: - if not skip_manual_diff: - meld = find_command('meld') - if meld is not None: - p = FDroidPopen(['meld', apk1dir, apk2dir], output=False) + if set_command_in_config('meld'): + p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False) return("Unexpected diff output - " + p.output) # since everything verifies, delete the comparison to keep cruft down From 5b7209d88206f3a6eabeb546818c0096f4ea4a0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sat, 22 Apr 2017 12:04:32 +0200 Subject: [PATCH 5/6] unified method for naming build output --- fdroidserver/build.py | 9 ++++----- fdroidserver/dscanner.py | 2 +- fdroidserver/update.py | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 7c4fddf3..c16bd182 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -1212,7 +1212,7 @@ def main(): url = url.replace('%v', build.versionName) url = url.replace('%c', str(build.versionCode)) logging.info("...retrieving " + url) - of = "{0}_{1}.apk.binary".format(app.id, build.versionCode) + of = common.get_release_filename(app, build) + '.binary' of = os.path.join(output_dir, of) try: net.download_file(url, local_filename=of) @@ -1225,10 +1225,9 @@ def main(): # and remove everything from the unsigend folder. with tempfile.TemporaryDirectory() as tmpdir: unsigned_apk = \ - '{0}_{1}.apk'.format(appid, - build.versionCode) - unsigned_apk = os.path.join(output_dir, - unsigned_apk) + common.get_release_filename(app, build) + unsigned_apk = \ + os.path.join(output_dir, unsigned_apk) compare_result = \ common.verify_apks(of, unsigned_apk, tmpdir) if compare_result: diff --git a/fdroidserver/dscanner.py b/fdroidserver/dscanner.py index 10e98b31..a9ffa027 100644 --- a/fdroidserver/dscanner.py +++ b/fdroidserver/dscanner.py @@ -467,7 +467,7 @@ def main(): for build in app.builds: apks = [] for f in os.listdir(options.repo_path): - n = "%v_%v.apk".format(app_id, build.versionCode) + n = common.get_release_filename(app, build) if f == n: apks.append(f) for apk in sorted(apks): diff --git a/fdroidserver/update.py b/fdroidserver/update.py index a97a40bf..d08bb25b 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -319,7 +319,7 @@ def delete_disabled_builds(apps, apkcache, repodirs): for build in app['builds']: if not build.disable: continue - apkfilename = appid + '_' + str(build.versionCode) + '.apk' + apkfilename = common.get_release_filename(app, build) iconfilename = "%s.%s.png" % ( appid, build.versionCode) From fc4e40255d2a387dfba269b201a25684f0833a41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sun, 23 Apr 2017 11:33:51 +0200 Subject: [PATCH 6/6] remove build output when not reproducible --- fdroidserver/build.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index c16bd182..f0f027be 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -1231,6 +1231,10 @@ def main(): compare_result = \ common.verify_apks(of, unsigned_apk, tmpdir) if compare_result: + logging.debug('removing %s', unsigned_apk) + os.remove(unsigned_apk) + logging.debug('removing %s', of) + os.remove(of) compare_result = compare_result.split('\n') line_count = len(compare_result) compare_result = compare_result[:299]