From d46d9574b463fc99d6be5e4ddab1e6eb7cd50de9 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 29 Nov 2017 11:21:34 +0100 Subject: [PATCH 01/10] update: use KnownApks dates to check system clock on offline machines KnownApks provides a reliable source of a relatively recent date. --- fdroidserver/common.py | 20 +++++++++++++++++++- fdroidserver/update.py | 15 ++------------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 89c1a1d5..332aaae3 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -40,7 +40,7 @@ import json import xml.etree.ElementTree as XMLElementTree from binascii import hexlify -from datetime import datetime +from datetime import datetime, timedelta from distutils.version import LooseVersion from queue import Queue from zipfile import ZipFile @@ -1716,6 +1716,23 @@ def natural_key(s): return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)] +def check_system_clock(dt_obj, path): + """Check if system clock is updated based on provided date + + If an APK has files newer than the system time, suggest updating + the system clock. This is useful for offline systems, used for + signing, which do not have another source of clock sync info. It + has to be more than 24 hours newer because ZIP/APK files do not + store timezone info + + """ + checkdt = dt_obj - timedelta(1) + if datetime.today() < checkdt: + logging.warning(_('System clock is older than date in {path}!').format(path=path) + + '\n' + _('Set clock to that time using:') + '\n' + + 'sudo date -s "' + str(dt_obj) + '"') + + class KnownApks: """permanent store of existing APKs with the date they were added @@ -1744,6 +1761,7 @@ class KnownApks: date = datetime.strptime(t[-1], '%Y-%m-%d') filename = line[0:line.rfind(appid) - 1] self.apks[filename] = (appid, date) + check_system_clock(date, self.path) self.changed = False def writeifchanged(self): diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 2019063c..61026cab 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -29,7 +29,7 @@ import zipfile import hashlib import pickle import time -from datetime import datetime, timedelta +from datetime import datetime from argparse import ArgumentParser import collections @@ -1297,22 +1297,11 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal apkzip = zipfile.ZipFile(apkfile, 'r') - # if an APK has files newer than the system time, suggest updating - # the system clock. This is useful for offline systems, used for - # signing, which do not have another source of clock sync info. It - # has to be more than 24 hours newer because ZIP/APK files do not - # store timezone info manifest = apkzip.getinfo('AndroidManifest.xml') if manifest.date_time[1] == 0: # month can't be zero logging.debug(_('AndroidManifest.xml has no date')) else: - dt_obj = datetime(*manifest.date_time) - checkdt = dt_obj - timedelta(1) - if datetime.today() < checkdt: - logging.warning('System clock is older than manifest in: ' - + apkfilename - + '\nSet clock to that time using:\n' - + 'sudo date -s "' + str(dt_obj) + '"') + common.check_system_clock(datetime(*manifest.date_time), apkfilename) # extract icons from APK zip file iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode']) From c98740a7d2014325e9858df1090e25ea5ccb45a0 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 27 Nov 2017 22:24:59 +0100 Subject: [PATCH 02/10] nightly: include 'qrcode' as dependency --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 1552a636..a382b254 100755 --- a/setup.py +++ b/setup.py @@ -89,6 +89,7 @@ setup(name='fdroidserver', 'pyasn1-modules', 'python-vagrant', 'PyYAML', + 'qrcode', 'ruamel.yaml >= 0.13', 'requests >= 2.5.2, != 2.11.0, != 2.12.2, != 2.18.0', 'docker-py >= 1.9, < 2.0', From 6f97be128fbad07c0339ce94f70ae118be67f6e5 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 27 Nov 2017 22:29:11 +0100 Subject: [PATCH 03/10] nightly: fix QR code repo icon generation --- fdroidserver/nightly.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 3834f79e..9be69a43 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -186,10 +186,10 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, mirror_git_repo.git.add(all=True) mirror_git_repo.index.commit("update README") - icon_path = os.path.join(repo_basedir, 'icon.png') + icon_path = os.path.join(git_mirror_path, 'icon.png') try: import qrcode - img = qrcode.make('Some data here') + img = qrcode.make(repo_url) with open(icon_path, 'wb') as fp: fp.write(img) except Exception: @@ -197,6 +197,7 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, shutil.copy(exampleicon, icon_path) mirror_git_repo.git.add(all=True) mirror_git_repo.index.commit("update repo/website icon") + shutil.copy(icon_path, repo_basedir) os.chdir(repo_basedir) common.local_rsync(options, git_mirror_repodir + '/', 'repo/') From f01b6af57f4c491175c2a646c3f0ed4533efe639 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 27 Nov 2017 22:35:57 +0100 Subject: [PATCH 04/10] nightly: automatically create and manage app metadata This also allows the developer to edit the metadata in the *-nightly git repo to customize it. closes #421 --- fdroidserver/nightly.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 9be69a43..454616e4 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -155,6 +155,7 @@ def main(): repo_url = repo_base + '/repo' git_mirror_path = os.path.join(repo_basedir, 'git-mirror') git_mirror_repodir = os.path.join(git_mirror_path, 'fdroid', 'repo') + git_mirror_metadatadir = os.path.join(git_mirror_path, 'fdroid', 'metadata') if not os.path.isdir(git_mirror_repodir): logging.debug(_('cloning {url}').format(url=clone_url)) try: @@ -200,7 +201,10 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, shutil.copy(icon_path, repo_basedir) os.chdir(repo_basedir) - common.local_rsync(options, git_mirror_repodir + '/', 'repo/') + if os.path.isdir(git_mirror_repodir): + common.local_rsync(options, git_mirror_repodir + '/', 'repo/') + if os.path.isdir(git_mirror_metadatadir): + common.local_rsync(options, git_mirror_metadatadir + '/', 'metadata/') ssh_private_key_file = _ssh_key_from_debug_keystore() # this is needed for GitPython to find the SSH key @@ -255,7 +259,11 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, except subprocess.CalledProcessError: pass - subprocess.check_call(['fdroid', 'update', '--rename-apks', '--verbose'], cwd=repo_basedir) + subprocess.check_call(['fdroid', 'update', '--rename-apks', '--create-metadata', '--verbose'], + cwd=repo_basedir) + common.local_rsync(options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/') + mirror_git_repo.git.add(all=True) + mirror_git_repo.index.commit("update app metadata") try: subprocess.check_call(['fdroid', 'server', 'update', '--verbose'], cwd=repo_basedir) except subprocess.CalledProcessError: From 19af92c9829223be4f284ef3ba9afdb8a6b09f77 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 24 Nov 2017 09:21:28 +0100 Subject: [PATCH 05/10] buildserver: include all Android SDK licenses in their exact format I tried to clone the files I got from `sdkmanager --licenses`, byte for byte. --- buildserver/provision-android-sdk | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/buildserver/provision-android-sdk b/buildserver/provision-android-sdk index b79c2e4a..7cd96e0c 100644 --- a/buildserver/provision-android-sdk +++ b/buildserver/provision-android-sdk @@ -74,12 +74,25 @@ y EOH mkdir -p $ANDROID_HOME/licenses/ + cat << EOF > $ANDROID_HOME/licenses/android-sdk-license 8933bad161af4178b1185d1a37fbf41ea5269c55 + d56f5187479451eabf01fb78af6dfcb131a6481e EOF -echo -e "\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license + +cat < $ANDROID_HOME/licenses/android-sdk-preview-license + +84831b9409646a918e30573bab4c9c91346d8abd +EOF + +cat < $ANDROID_HOME/licenses/android-sdk-preview-license-old +79120722343a6f314e0719f863036c702b0e6b2a + +84831b9409646a918e30573bab4c9c91346d8abd +EOF + echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout;1.0.1" echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout-solver;1.0.1" echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout;1.0.2" From 2b6825ccfdfd26828883d8bb83055fe376223e3b Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 28 Nov 2017 10:39:35 +0100 Subject: [PATCH 06/10] build: set open file limit based on how many apps are being processed When running `fdroid build --all` on a buildserver with thousands of apps, it was frequently hitting the open file limit. This increases the open file limit based on how many apps are being process. It is doubled to provide a margin of safety. There are probably open file leaks which ideally would be fixed, but this is also useful to make things more resilient to all the random stuff apps include in their build systems. --- fdroidserver/build.py | 14 ++++++++++++++ jenkins-build-all | 1 - 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 740d7f2c..16901e78 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -23,6 +23,7 @@ import shutil import glob import subprocess import re +import resource import tarfile import traceback import time @@ -1120,6 +1121,19 @@ def main(): if not apps: raise FDroidException("No apps to process.") + # make sure enough open files are allowed to process everything + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if len(apps) > soft: + try: + soft = len(apps) * 2 + if soft > hard: + soft = hard + resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) + logging.debug(_('Set open file limit to {integer}') + .format(integer=soft)) + except (OSError, ValueError) as e: + logging.warning(_('Setting open file limit failed: ') + str(e)) + if options.latest: for app in apps.values(): for build in reversed(app.builds): diff --git a/jenkins-build-all b/jenkins-build-all index 2abac58c..d41d920f 100755 --- a/jenkins-build-all +++ b/jenkins-build-all @@ -31,7 +31,6 @@ else echo "No virtualization is used." fi sudo /bin/chmod -R a+rX /var/lib/libvirt/images -ulimit -n 2048 echo 'maximum allowed number of open file descriptors: ' `ulimit -n` ls -ld /var/lib/libvirt/images ls -l /var/lib/libvirt/images || echo no access From e2bbeb50832b666439ba78ceca60cc661bd43e42 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 28 Nov 2017 10:40:11 +0100 Subject: [PATCH 07/10] common: document read_pkg_args() and read_app_args() It took me a long time to figure out how `fdroid build --all` builds the whole list of apps... --- fdroidserver/common.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 332aaae3..a23ae2e9 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -444,17 +444,16 @@ def get_local_metadata_files(): return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]') -def read_pkg_args(args, allow_vercodes=False): +def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False): """ - :param args: arguments in the form of multiple appid:[vc] strings + :param appids: arguments in the form of multiple appid:[vc] strings :returns: a dictionary with the set of vercodes specified for each package """ - vercodes = {} - if not args: + if not appid_versionCode_pairs: return vercodes - for p in args: + for p in appid_versionCode_pairs: if allow_vercodes and ':' in p: package, vercode = p.split(':') else: @@ -468,13 +467,17 @@ def read_pkg_args(args, allow_vercodes=False): return vercodes -def read_app_args(args, allapps, allow_vercodes=False): - """ - On top of what read_pkg_args does, this returns the whole app metadata, but - limiting the builds list to the builds matching the vercodes specified. +def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False): + """Build a list of App instances for processing + + On top of what read_pkg_args does, this returns the whole app + metadata, but limiting the builds list to the builds matching the + appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then + all App and Build instances are returned. + """ - vercodes = read_pkg_args(args, allow_vercodes) + vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes) if not vercodes: return allapps From 7389947cc33cbb64979a8ff70ec7abd7e343e166 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 28 Nov 2017 12:04:07 +0100 Subject: [PATCH 08/10] init: handle case where keystore.jks is present but config.py is not --- fdroidserver/init.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 9fdb5836..9d03e0b9 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -178,6 +178,7 @@ def main(): + '" does not exist, creating a new keystore there.') common.write_to_config(test_config, 'keystore', keystore) repo_keyalias = None + keydname = None if options.repo_keyalias: repo_keyalias = options.repo_keyalias common.write_to_config(test_config, 'repo_keyalias', repo_keyalias) @@ -211,7 +212,16 @@ def main(): flags=re.MULTILINE) with open('opensc-fdroid.cfg', 'w') as f: f.write(opensc_fdroid) - elif not os.path.exists(keystore): + elif os.path.exists(keystore): + to_set = ['keystorepass', 'keypass', 'repo_keyalias', 'keydname'] + if repo_keyalias: + to_set.remove('repo_keyalias') + if keydname: + to_set.remove('keydname') + logging.warning('\n' + _('Using existing keystore "{path}"').format(path=keystore) + + '\n' + _('Now set these in config.py:') + ' ' + + ', '.join(to_set) + '\n') + else: password = common.genpassword() c = dict(test_config) c['keystorepass'] = password From b1bab817391308964bef544465fae5c496825902 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 29 Nov 2017 13:24:19 +0100 Subject: [PATCH 09/10] server: prevent crash when uploading to virustotal --- fdroidserver/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/server.py b/fdroidserver/server.py index bd5a4426..004a6068 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -513,7 +513,7 @@ def upload_to_virustotal(repo_section, vt_apikey): with open(outputfilename, 'w') as fp: json.dump(response, fp, indent=2, sort_keys=True) - if response.get('positives') > 0: + if response.get('positives', 0) > 0: logging.warning(repofilename + ' has been flagged by virustotal ' + str(response['positives']) + ' times:' + '\n\t' + response['permalink']) From 27d332c8ae7ec3216a6c5a84a968d06eea16499c Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 29 Nov 2017 20:04:18 +0100 Subject: [PATCH 10/10] vmtools: in KVM mode, only call sudo hack when file is not readable --- fdroidserver/vmtools.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index aae46c75..aab7eb0d 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -29,6 +29,8 @@ import textwrap from .common import FDroidException from logging import getLogger +from fdroidserver import _ + logger = getLogger('fdroidserver-vmtools') @@ -383,7 +385,9 @@ class LibvirtBuildVm(FDroidBuildVm): vol = storagePool.storageVolLookupByName(self.srvname + '.img') imagepath = vol.path() # TODO use a libvirt storage pool to ensure the img file is readable - _check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']) + if not os.access(imagepath, os.R_OK): + logger.warning(_('Cannot read "{path}"!').format(path=imagepath)) + _check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']) shutil.copy2(imagepath, 'box.img') _check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img']) img_info_raw = _check_output(['qemu-img', 'info', '--output=json', 'box.img'])