From ee84eb762c76a5dad045a02366fd687b13e5aa1f Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 18 Mar 2021 08:22:57 +0100 Subject: [PATCH 1/5] buildserver: SharedFoldersEnableSymlinksCreate: false --- buildserver/Vagrantfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildserver/Vagrantfile b/buildserver/Vagrantfile index 392d3df6..982c0267 100644 --- a/buildserver/Vagrantfile +++ b/buildserver/Vagrantfile @@ -48,7 +48,8 @@ Vagrant.configure("2") do |config| else synced_folder_type = '9p' end - config.vm.synced_folder './', '/vagrant', type: synced_folder_type + config.vm.synced_folder './', '/vagrant', type: synced_folder_type, + SharedFoldersEnableSymlinksCreate: false else abort("No supported VM Provider found, set vm_provider in Vagrantfile.yaml!") end From 8b393189c15a09d1691569afddb377417df534fc Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 18 Mar 2021 08:22:34 +0100 Subject: [PATCH 2/5] include /etc/issue.net in status JSON to identify the base system --- fdroidserver/common.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 79c77456..179ae8d5 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -767,6 +767,10 @@ def setup_status_output(start_timestamp): 'modifiedFiles': git_repo.git().ls_files(modified=True).split(), 'untrackedFiles': git_repo.untracked_files, } + etc_issue_net = '/etc/issue.net' + if os.path.exists(etc_issue_net): + with open(etc_issue_net) as fp: + output[etc_issue_net] = fp.read(100).strip() write_running_status_json(output) return output From 9135dfbeed0fb088048661f082138ef873b32940 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 12 Mar 2021 13:11:58 +0100 Subject: [PATCH 3/5] verify: include diffoscope setup info in output JSON --- fdroidserver/verify.py | 50 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/fdroidserver/verify.py b/fdroidserver/verify.py index 664d06d4..8cb74a32 100644 --- a/fdroidserver/verify.py +++ b/fdroidserver/verify.py @@ -40,7 +40,11 @@ class hashabledict(OrderedDict): return tuple((k, self[k]) for k in sorted(self)) def __hash__(self): - return hash(self.__key()) + try: + return hash(self.__key()) + except TypeError as e: + print(self.__key()) + raise e def __eq__(self, other): return self.__key() == other.__key() @@ -64,6 +68,49 @@ class Decoder(json.JSONDecoder): return set(values), end +def _add_diffoscope_info(d): + """Add diffoscope setup metadata to provided dict under 'diffoscope' key + + The imports are broken out at stages since various versions of + diffoscope support various parts of these. + + """ + try: + import diffoscope + d['diffoscope'] = hashabledict() + d['diffoscope']['VERSION'] = diffoscope.VERSION + + from diffoscope.comparators import ComparatorManager + ComparatorManager().reload() + + from diffoscope.tools import tool_check_installed, tool_required + external_tools = sorted(tool_required.all) + external_tools = [ + tool + for tool in external_tools + if not tool_check_installed(tool) + ] + d['diffoscope']['External-Tools-Required'] = tuple(external_tools) + + from diffoscope.tools import OS_NAMES, get_current_os + from diffoscope.external_tools import EXTERNAL_TOOLS + current_os = get_current_os() + os_list = [current_os] if (current_os in OS_NAMES) else iter(OS_NAMES) + for os_ in os_list: + tools = set() + for x in external_tools: + try: + tools.add(EXTERNAL_TOOLS[x][os_]) + except KeyError: + pass + d['diffoscope']['Available-in-{}-packages'.format(OS_NAMES[os_])] = tuple(sorted(tools)) + + from diffoscope.tools import python_module_missing + d['diffoscope']['Missing-Python-Modules'] = tuple(sorted(python_module_missing.modules)) + except ImportError: + pass + + def write_json_report(url, remote_apk, unsigned_apk, compare_result): """write out the results of the verify run to JSON @@ -81,6 +128,7 @@ def write_json_report(url, remote_apk, unsigned_apk, compare_result): else: data = OrderedDict() output = hashabledict() + _add_diffoscope_info(output) output['url'] = url for key, filename in (('local', unsigned_apk), ('remote', remote_apk)): d = hashabledict() From b69b95103efafdc8559ffc93def4e7422d5bc4db Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 18 Mar 2021 09:47:19 +0100 Subject: [PATCH 4/5] add complete tests for finding apksigner; fix minor detection bug find_apksigner() was preferring the oldest valid version rather than the newest. --- fdroidserver/common.py | 5 +- tests/common.TestCase | 105 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 98 insertions(+), 12 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 179ae8d5..00d91c36 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -285,7 +285,7 @@ def regsub_file(pattern, repl, path): f.write(text) -def read_config(opts): +def read_config(opts=None): """Read the repository config The config is read from config_file, which is in the current @@ -489,14 +489,15 @@ def find_apksigner(config): continue try: if LooseVersion(f) < LooseVersion(MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION): + logging.debug("Local Android SDK only has outdated apksigner versions") return except TypeError: continue if os.path.exists(os.path.join(build_tools_path, f, 'apksigner')): apksigner = os.path.join(build_tools_path, f, 'apksigner') logging.info("Using %s " % apksigner) - # memoize result config['apksigner'] = apksigner + return def find_sdk_tools_cmd(cmd): diff --git a/tests/common.TestCase b/tests/common.TestCase index 607de389..409dceda 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -51,6 +51,13 @@ class CommonTest(unittest.TestCase): os.makedirs(self.tmpdir) os.chdir(self.basedir) fdroidserver.common.config = None + self.path = os.environ['PATH'] + self.android_home = os.environ.get('ANDROID_HOME') + + def tearDown(self): + os.environ['PATH'] = self.path + if self.android_home: + os.environ['ANDROID_HOME'] = self.android_home def test_parse_human_readable_size(self): for k, v in ((9827, 9827), (123.456, 123), ('123b', 123), ('1.2', 1), @@ -366,7 +373,6 @@ class CommonTest(unittest.TestCase): self.assertEqual(p.output, 'stdout message\n') def test_signjar(self): - fdroidserver.common.config = None config = fdroidserver.common.read_config(fdroidserver.common.options) config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner') fdroidserver.common.config = config @@ -383,9 +389,7 @@ class CommonTest(unittest.TestCase): self.assertNotEqual(open(sourcefile, 'rb').read(), open(testfile, 'rb').read()) def test_verify_apk_signature(self): - fdroidserver.common.config = None config = fdroidserver.common.read_config(fdroidserver.common.options) - config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner') fdroidserver.common.config = config self.assertTrue(fdroidserver.common.verify_apk_signature('bad-unicode-πÇÇ现代通用字-български-عربي1.apk')) @@ -407,7 +411,6 @@ class CommonTest(unittest.TestCase): self.assertFalse(fdroidserver.common.verify_apk_signature('urzip-release-unsigned.apk')) def test_verify_old_apk_signature(self): - fdroidserver.common.config = None config = fdroidserver.common.read_config(fdroidserver.common.options) config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner') fdroidserver.common.config = config @@ -425,9 +428,7 @@ class CommonTest(unittest.TestCase): self.assertFalse(fdroidserver.common.verify_old_apk_signature('urzip-release-unsigned.apk')) def test_verify_jar_signature_succeeds(self): - fdroidserver.common.config = None config = fdroidserver.common.read_config(fdroidserver.common.options) - config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner') fdroidserver.common.config = config source_dir = os.path.join(self.basedir, 'signindex') for f in ('testy.jar', 'guardianproject.jar'): @@ -435,7 +436,6 @@ class CommonTest(unittest.TestCase): fdroidserver.common.verify_jar_signature(testfile) def test_verify_jar_signature_fails(self): - fdroidserver.common.config = None config = fdroidserver.common.read_config(fdroidserver.common.options) config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner') fdroidserver.common.config = config @@ -445,9 +445,7 @@ class CommonTest(unittest.TestCase): fdroidserver.common.verify_jar_signature(testfile) def test_verify_apks(self): - fdroidserver.common.config = None config = fdroidserver.common.read_config(fdroidserver.common.options) - config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner') fdroidserver.common.config = config sourceapk = os.path.join(self.basedir, 'urzip.apk') @@ -616,8 +614,95 @@ class CommonTest(unittest.TestCase): self.assertEqual(keytoolcertfingerprint, fdroidserver.common.apk_signer_fingerprint_short(apkfile)) + def test_find_apksigner_system_package_default_path(self): + """apksigner should be automatically used from the PATH""" + usr_bin_apksigner = '/usr/bin/apksigner' + if not os.path.isfile(usr_bin_apksigner): + self.skipTest('SKIPPING since %s is not installed!' % usr_bin_apksigner) + os.environ['PATH'] = '/usr/local/bin:/usr/bin:/bin' + config = {} + fdroidserver.common.find_apksigner(config) + self.assertEqual(usr_bin_apksigner, config.get('apksigner')) + + def test_find_apksigner_config_overrides(self): + """apksigner should come from config before any auto-detection""" + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + android_home = os.path.join(testdir, 'ANDROID_HOME') + do_not_use = os.path.join(android_home, 'build-tools', '30.0.3', 'apksigner') + os.makedirs(os.path.dirname(do_not_use)) + with open(do_not_use, 'w') as fp: + fp.write('#!/bin/sh\ndate\n') + os.chmod(do_not_use, 0o0755) + apksigner = os.path.join(testdir, 'apksigner') + config = {'apksigner': apksigner} + os.environ['ANDROID_HOME'] = android_home + os.environ['PATH'] = '%s:/usr/local/bin:/usr/bin:/bin' % android_home + fdroidserver.common.find_apksigner(config) + self.assertEqual(apksigner, config.get('apksigner')) + + def test_find_apksigner_prefer_path(self): + """apksigner should come from PATH before ANDROID_HOME""" + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + + apksigner = os.path.join(testdir, 'apksigner') + with open(apksigner, 'w') as fp: + fp.write('#!/bin/sh\ndate\n') + os.chmod(apksigner, 0o0755) + + android_home = os.path.join(testdir, 'ANDROID_HOME') + do_not_use = os.path.join(android_home, 'build-tools', '30.0.3', 'apksigner') + os.makedirs(os.path.dirname(do_not_use)) + with open(do_not_use, 'w') as fp: + fp.write('#!/bin/sh\ndate\n') + os.chmod(do_not_use, 0o0755) + + config = {'sdk_path': android_home} + os.environ['ANDROID_HOME'] = android_home + os.environ['PATH'] = '%s:/usr/local/bin:/usr/bin:/bin' % os.path.dirname(apksigner) + fdroidserver.common.find_apksigner(config) + self.assertEqual(apksigner, config.get('apksigner')) + + def test_find_apksigner_prefer_newest(self): + """apksigner should be the newest available in ANDROID_HOME""" + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + android_home = os.path.join(testdir, 'ANDROID_HOME') + + apksigner = os.path.join(android_home, 'build-tools', '30.0.3', 'apksigner') + os.makedirs(os.path.dirname(apksigner)) + with open(apksigner, 'w') as fp: + fp.write('#!/bin/sh\necho 30.0.3\n') + os.chmod(apksigner, 0o0755) + + do_not_use = os.path.join(android_home, 'build-tools', '29.0.3', 'apksigner') + os.makedirs(os.path.dirname(do_not_use)) + with open(do_not_use, 'w') as fp: + fp.write('#!/bin/sh\necho 29.0.3\n') + os.chmod(do_not_use, 0o0755) + + config = {'sdk_path': android_home} + os.environ['PATH'] = '/fake/path/to/avoid/conflicts' + fdroidserver.common.find_apksigner(config) + self.assertEqual(apksigner, config.get('apksigner')) + + def test_find_apksigner_system_package_android_home(self): + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + android_home = os.getenv('ANDROID_HOME') + if not android_home or not os.path.isdir(android_home): + self.skipTest('SKIPPING since ANDROID_HOME (%s) is not a dir!' % android_home) + fdroidserver.common.config = {'sdk_path': android_home} + os.environ['PATH'] = '/fake/path/to/avoid/conflicts' + config = fdroidserver.common.read_config() + fdroidserver.common.find_apksigner(config) + self.assertEqual( + os.path.join(android_home, 'build-tools'), + os.path.dirname(os.path.dirname(config.get('apksigner'))), + ) + def test_sign_apk(self): - fdroidserver.common.config = None config = fdroidserver.common.read_config(fdroidserver.common.options) if 'apksigner' not in config: self.skipTest('SKIPPING test_sign_apk, apksigner not installed!') From f811d44f0632825de046e017dc2e79bc15671d4a Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 23 Mar 2021 11:58:58 +0100 Subject: [PATCH 5/5] jenkins-build-all: use $WORKSPACE var to find correct file to modify --- jenkins-build-all | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jenkins-build-all b/jenkins-build-all index cbac12c2..9e9c76e5 100755 --- a/jenkins-build-all +++ b/jenkins-build-all @@ -96,7 +96,8 @@ else fi # force global timeout to 6 hours -sed -Ei 's,^(\s+endtime\s*=\s*time\.time\(\))\s*.*,\1 + 6 * 60 * 60 # 6 hours,' fdroidserver/build.py +sed -Ei 's,^(\s+endtime\s*=\s*time\.time\(\))\s*.*,\1 + 6 * 60 * 60 # 6 hours,' \ + $WORKSPACE/fdroidserver/build.py $WORKSPACE/fdroid build --verbose --latest --no-tarball --all $wikiflag