diff --git a/examples/config.py b/examples/config.py index 1ed0de51..e46116db 100644 --- a/examples/config.py +++ b/examples/config.py @@ -173,10 +173,14 @@ The repository of older versions of applications from the main demo repository. # 'bar.info:/var/www/fdroid', # } -# Uncomment this option if you want to logs of builds and other processes to -# your repository server(s). Logs get published to all servers configured in -# 'serverwebroot'. The name scheme is: .../repo/$APPID_$VERCODE.log.gz -# Only logs from build-jobs running inside a buildserver VM are supported. +# When running fdroid processes on a remote server, it is possible to +# publish extra information about the status. Each fdroid sub-command +# can create repo/status/running.json when it starts, then a +# repo/status/.json when it completes. The builds logs +# and other processes will also get published, if they are running in +# a buildserver VM. The build logs name scheme is: +# .../repo/$APPID_$VERCODE.log.gz. These files are also pushed to all +# servers configured in 'serverwebroot'. # # deploy_process_logs = True diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 6d8a1ce8..733af34a 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -83,6 +83,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): buildserverid = subprocess.check_output(['vagrant', 'ssh', '-c', 'cat /home/vagrant/buildserverid'], cwd='builder').strip().decode() + status_output['buildserverid'] = buildserverid logging.debug(_('Fetched buildserverid from VM: {buildserverid}') .format(buildserverid=buildserverid)) except Exception as e: @@ -912,6 +913,7 @@ config = None buildserverid = None fdroidserverid = None start_timestamp = time.gmtime() +status_output = None timeout_event = threading.Event() @@ -978,6 +980,8 @@ def main(): else: also_check_dir = None + status_output = common.setup_status_output(start_timestamp) + repo_dir = 'repo' build_dir = 'build' @@ -1029,6 +1033,8 @@ def main(): # Build applications... failed_apps = {} build_succeeded = [] + status_output['failedBuilds'] = failed_apps + status_output['successfulBuilds'] = build_succeeded # Only build for 36 hours, then stop gracefully. endtime = time.time() + 36 * 60 * 60 max_build_time_reached = False @@ -1201,10 +1207,12 @@ def main(): except Exception as e: logging.error("Error while attempting to publish build log: %s" % e) + common.write_running_status_json(status_output) if timer: timer.cancel() # kill the watchdog timer if max_build_time_reached: + status_output['maxBuildTimeReached'] = True logging.info("Stopping after global build timeout...") break @@ -1263,6 +1271,8 @@ def main(): newpage = site.Pages['build'] newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect') + common.write_status_json(status_output, options.pretty) + # hack to ensure this exits, even is some threads are still running common.force_exit() diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py index 881f9f48..52f1ceb4 100644 --- a/fdroidserver/checkupdates.py +++ b/fdroidserver/checkupdates.py @@ -543,6 +543,18 @@ def checkupdates_app(app): raise FDroidException("Git commit failed") +def status_update_json(processed, failed): + """Output a JSON file with metadata about this run""" + + logging.debug(_('Outputting JSON')) + output = common.setup_status_output(start_timestamp) + if processed: + output['processed'] = processed + if failed: + output['failed'] = failed + common.write_status_json(output) + + def update_wiki(gplaylog, locallog): if config.get('wiki_server') and config.get('wiki_path'): try: @@ -644,6 +656,8 @@ def main(): return locallog = '' + processed = [] + failed = dict() for appid, app in apps.items(): if options.autoonly and app.AutoUpdateMode in ('None', 'Static'): @@ -656,13 +670,15 @@ def main(): try: checkupdates_app(app) + processed.append(appid) except Exception as e: msg = _("...checkupdate failed for {appid} : {error}").format(appid=appid, error=e) logging.error(msg) locallog += msg + '\n' + failed[appid] = str(e) update_wiki(None, locallog) - + status_update_json(processed, failed) logging.info(_("Finished")) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index a3e5108f..3697fec4 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -20,6 +20,7 @@ # common.py is imported by all modules, so do not import third-party # libraries here as they will become a requirement for all commands. +import git import io import os import sys @@ -47,7 +48,7 @@ except ImportError: import xml.etree.ElementTree as XMLElementTree # nosec this is a fallback only from binascii import hexlify -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from distutils.version import LooseVersion from queue import Queue from zipfile import ZipFile @@ -587,17 +588,13 @@ def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False): def get_extension(filename): + """get name and extension of filename, with extension always lower case""" base, ext = os.path.splitext(filename) if not ext: return base, '' return base, ext.lower()[1:] -def has_extension(filename, ext): - _ignored, f_ext = get_extension(filename) - return ext == f_ext - - publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$") @@ -674,6 +671,66 @@ def get_build_dir(app): return os.path.join('build', app.id) +class Encoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, set): + return sorted(obj) + return super().default(obj) + + +def setup_status_output(start_timestamp): + """Create the common output dictionary for public status updates""" + output = { + 'commandLine': sys.argv, + 'startTimestamp': int(time.mktime(start_timestamp) * 1000), + 'subcommand': sys.argv[0].split()[1], + } + if os.path.isdir('.git'): + git_repo = git.repo.Repo(os.getcwd()) + output['fdroiddata'] = { + 'commitId': get_head_commit_id(git_repo), + 'isDirty': git_repo.is_dirty(), + } + fdroidserver_dir = os.path.dirname(sys.argv[0]) + if os.path.isdir(os.path.join(fdroidserver_dir, '.git')): + git_repo = git.repo.Repo(fdroidserver_dir) + output['fdroidserver'] = { + 'commitId': get_head_commit_id(git_repo), + 'isDirty': git_repo.is_dirty(), + } + write_running_status_json(output) + return output + + +def write_running_status_json(output): + write_status_json(output, pretty=True, name='running') + + +def write_status_json(output, pretty=False, name=None): + """Write status out as JSON, and rsync it to the repo server""" + status_dir = os.path.join('repo', 'status') + if not os.path.exists(status_dir): + os.mkdir(status_dir) + if not name: + output['endTimestamp'] = int(datetime.now(timezone.utc).timestamp() * 1000) + name = sys.argv[0].split()[1] # fdroid subcommand + path = os.path.join(status_dir, name + '.json') + with open(path, 'w') as fp: + if pretty: + json.dump(output, fp, sort_keys=True, cls=Encoder, indent=2) + else: + json.dump(output, fp, sort_keys=True, cls=Encoder, separators=(',', ':')) + rsync_status_file_to_repo(path, repo_subdir='status') + + +def get_head_commit_id(git_repo): + """Get git commit ID for HEAD as a str + + repo.head.commit.binsha is a bytearray stored in a str + """ + return hexlify(bytearray(git_repo.head.commit.binsha)).decode() + + def setup_vcs(app): '''checkout code from VCS and return instance of vcs and the build dir''' build_dir = get_build_dir(app) @@ -1316,7 +1373,7 @@ def manifest_paths(app_dir, flavours): def fetch_real_name(app_dir, flavours): '''Retrieve the package name. Returns the name, or None if not found.''' for path in manifest_paths(app_dir, flavours): - if not has_extension(path, 'xml') or not os.path.isfile(path): + if not path.endswith('.xml') or not os.path.isfile(path): continue logging.debug("fetch_real_name: Checking manifest at " + path) xml = parse_xml(path) @@ -1808,11 +1865,11 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= for path in manifest_paths(root_dir, flavours): if not os.path.isfile(path): continue - if has_extension(path, 'xml'): + if path.endswith('.xml'): regsub_file(r'android:versionName="[^"]*"', r'android:versionName="%s"' % build.versionName, path) - elif has_extension(path, 'gradle'): + elif path.endswith('.gradle'): regsub_file(r"""(\s*)versionName[\s'"=]+.*""", r"""\1versionName '%s'""" % build.versionName, path) @@ -1822,11 +1879,11 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= for path in manifest_paths(root_dir, flavours): if not os.path.isfile(path): continue - if has_extension(path, 'xml'): + if path.endswith('.xml'): regsub_file(r'android:versionCode="[^"]*"', r'android:versionCode="%s"' % build.versionCode, path) - elif has_extension(path, 'gradle'): + elif path.endswith('.gradle'): regsub_file(r'versionCode[ =]+[0-9]+', r'versionCode %s' % build.versionCode, path) @@ -3300,11 +3357,6 @@ def deploy_build_log_with_rsync(appid, vercode, log_content): be decoded as 'utf-8') """ - # check if deploying logs is enabled in config - if not config.get('deploy_process_logs', False): - logging.debug(_('skip deploying full build logs: not enabled in config')) - return - if not log_content: logging.warning(_('skip deploying full build logs: log content is empty')) return @@ -3322,13 +3374,17 @@ def deploy_build_log_with_rsync(appid, vercode, log_content): f.write(bytes(log_content, 'utf-8')) else: f.write(log_content) + rsync_status_file_to_repo(log_gz_path) - # TODO: sign compressed log file, if a signing key is configured + +def rsync_status_file_to_repo(path, repo_subdir=None): + """Copy a build log or status JSON to the repo using rsync""" + + if not config.get('deploy_process_logs', False): + logging.debug(_('skip deploying full build logs: not enabled in config')) + return for webroot in config.get('serverwebroot', []): - dest_path = os.path.join(webroot, "repo") - if not dest_path.endswith('/'): - dest_path += '/' # make sure rsync knows this is a directory cmd = ['rsync', '--archive', '--delete-after', @@ -3339,15 +3395,21 @@ def deploy_build_log_with_rsync(appid, vercode, log_content): cmd += ['--quiet'] if 'identity_file' in config: cmd += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']] - cmd += [log_gz_path, dest_path] - # TODO: also deploy signature file if present + dest_path = os.path.join(webroot, "repo") + if repo_subdir is not None: + dest_path = os.path.join(dest_path, repo_subdir) + if not dest_path.endswith('/'): + dest_path += '/' # make sure rsync knows this is a directory + cmd += [path, dest_path] retcode = subprocess.call(cmd) if retcode: - logging.warning(_("failed deploying build logs to '{path}'").format(path=webroot)) + logging.error(_('process log deploy {path} to {dest} failed!') + .format(path=path, dest=webroot)) else: - logging.info(_("deployed build logs to '{path}'").format(path=webroot)) + logging.debug(_('deployed process log {path} to {dest}') + .format(path=path, dest=webroot)) def get_per_app_repos(): diff --git a/fdroidserver/gpgsign.py b/fdroidserver/gpgsign.py index b942a21b..a224c3f7 100644 --- a/fdroidserver/gpgsign.py +++ b/fdroidserver/gpgsign.py @@ -20,6 +20,7 @@ import os import glob from argparse import ArgumentParser import logging +import time from . import _ from . import common @@ -28,6 +29,17 @@ from .exception import FDroidException config = None options = None +start_timestamp = time.gmtime() + + +def status_update_json(signed): + """Output a JSON file with metadata about this run""" + + logging.debug(_('Outputting JSON')) + output = common.setup_status_output(start_timestamp) + if signed: + output['signed'] = signed + common.write_status_json(output) def main(): @@ -45,6 +57,7 @@ def main(): if config['archive_older'] != 0: repodirs.append('archive') + signed = [] for output_dir in repodirs: if not os.path.isdir(output_dir): raise FDroidException(_("Missing output directory") + " '" + output_dir + "'") @@ -72,7 +85,9 @@ def main(): if p.returncode != 0: raise FDroidException("Signing failed.") + signed.append(filename) logging.info('Signed ' + filename) + status_update_json(signed) if __name__ == "__main__": diff --git a/fdroidserver/import.py b/fdroidserver/import.py index 44d6ec32..42ba55e3 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -17,12 +17,13 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import binascii +import git import glob import json import os import re import shutil +import sys import urllib.parse import urllib.request import yaml @@ -230,7 +231,7 @@ def main(): apps = metadata.read_metadata() app = None - build_dir = None + tmp_importer_dir = None local_metadata_files = common.get_local_metadata_files() if local_metadata_files != []: @@ -241,35 +242,34 @@ def main(): app = metadata.App() app.AutoName = os.path.basename(os.getcwd()) app.RepoType = 'git' - app.UpdateCheckMode = "Tags" if os.path.exists('build.gradle') or os.path.exists('build.gradle.kts'): build.gradle = ['yes'] - import git - repo = git.repo.Repo(os.getcwd()) # git repo - for remote in git.Remote.iter_items(repo): + git_repo = git.repo.Repo(os.getcwd()) + for remote in git.Remote.iter_items(git_repo): if remote.name == 'origin': - url = repo.remotes.origin.url + url = git_repo.remotes.origin.url if url.startswith('https://git'): # github, gitlab app.SourceCode = url.rstrip('.git') app.Repo = url break - # repo.head.commit.binsha is a bytearray stored in a str - build.commit = binascii.hexlify(bytearray(repo.head.commit.binsha)) write_local_file = True elif options.url: app = get_app_from_url(options.url) - build_dir = clone_to_tmp_dir(app) - build.commit = '?' + tmp_importer_dir = clone_to_tmp_dir(app) + git_repo = git.repo.Repo(tmp_importer_dir) build.disable = 'Generated by import.py - check/set version fields and commit id' write_local_file = False else: raise FDroidException("Specify project url.") + app.UpdateCheckMode = 'Tags' + build.commit = common.get_head_commit_id(git_repo) + # Extract some information... - paths = get_all_gradle_and_manifests(build_dir) - subdir = get_gradle_subdir(build_dir, paths) + paths = get_all_gradle_and_manifests(tmp_importer_dir) + subdir = get_gradle_subdir(tmp_importer_dir, paths) if paths: versionName, versionCode, package = common.parse_androidmanifests(paths, app) if not package: @@ -303,8 +303,8 @@ def main(): or os.path.exists(os.path.join(subdir, 'build.gradle')): build.gradle = ['yes'] - package_json = os.path.join(build_dir, 'package.json') # react-native - pubspec_yaml = os.path.join(build_dir, 'pubspec.yaml') # flutter + package_json = os.path.join(tmp_importer_dir, 'package.json') # react-native + pubspec_yaml = os.path.join(tmp_importer_dir, 'pubspec.yaml') # flutter if os.path.exists(package_json): build.sudo = ['apt-get install npm', 'npm install -g react-native-cli'] build.init = ['npm install'] @@ -314,7 +314,7 @@ def main(): app.License = data.get('license', app.License) app.Description = data.get('description', app.Description) app.WebSite = data.get('homepage', app.WebSite) - app_json = os.path.join(build_dir, 'app.json') + app_json = os.path.join(tmp_importer_dir, 'app.json') if os.path.exists(app_json): with open(app_json) as fp: data = json.load(fp) @@ -343,8 +343,13 @@ def main(): # Keep the repo directory to save bandwidth... if not os.path.exists('build'): os.mkdir('build') - if build_dir is not None: - shutil.move(build_dir, os.path.join('build', package)) + build_dir = os.path.join('build', package) + if os.path.exists(build_dir): + logging.warning(_('{path} already exists, ignoring import results!') + .format(path=build_dir)) + sys.exit(1) + elif tmp_importer_dir is not None: + shutil.move(tmp_importer_dir, build_dir) with open('build/.fdroidvcs-' + package, 'w') as f: f.write(app.RepoType + ' ' + app.Repo) diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py index 1369d177..d69e3656 100644 --- a/fdroidserver/publish.py +++ b/fdroidserver/publish.py @@ -28,6 +28,7 @@ from collections import OrderedDict import logging from gettext import ngettext import json +import time import zipfile from . import _ @@ -38,6 +39,7 @@ from .exception import BuildException, FDroidException config = None options = None +start_timestamp = time.gmtime() def publish_source_tarball(apkfilename, unsigned_dir, output_dir): @@ -138,6 +140,20 @@ def store_stats_fdroid_signing_key_fingerprints(appids, indent=None): sign_sig_key_fingerprint_list(jar_file) +def status_update_json(newKeyAliases, generatedKeys, signedApks): + """Output a JSON file with metadata about this run""" + + logging.debug(_('Outputting JSON')) + output = common.setup_status_output(start_timestamp) + if newKeyAliases: + output['newKeyAliases'] = newKeyAliases + if generatedKeys: + output['generatedKeys'] = generatedKeys + if signedApks: + output['signedApks'] = signedApks + common.write_status_json(output) + + def main(): global config, options @@ -195,6 +211,9 @@ def main(): # collisions, and refuse to do any publishing if that's the case... allapps = metadata.read_metadata() vercodes = common.read_pkg_args(options.appid, True) + signed_apks = dict() + new_key_aliases = [] + generated_keys = dict() allaliases = [] for appid in allapps: m = hashlib.md5() # nosec just used to generate a keyalias @@ -314,6 +333,7 @@ def main(): m = hashlib.md5() # nosec just used to generate a keyalias m.update(appid.encode('utf-8')) keyalias = m.hexdigest()[:8] + new_key_aliases.append(keyalias) logging.info("Key alias: " + keyalias) # See if we already have a key for this application, and @@ -336,6 +356,9 @@ def main(): '-dname', config['keydname']], envs=env_vars) if p.returncode != 0: raise BuildException("Failed to generate key", p.output) + if appid not in generated_keys: + generated_keys[appid] = set() + generated_keys[appid].add(appid) signed_apk_path = os.path.join(output_dir, apkfilename) if os.path.exists(signed_apk_path): @@ -353,6 +376,9 @@ def main(): apkfile, keyalias], envs=env_vars) if p.returncode != 0: raise BuildException(_("Failed to sign application"), p.output) + if appid not in signed_apks: + signed_apks[appid] = [] + signed_apks[appid].append(apkfile) # Zipalign it... common._zipalign(apkfile, os.path.join(output_dir, apkfilename)) @@ -362,6 +388,7 @@ def main(): logging.info('Published ' + apkfilename) store_stats_fdroid_signing_key_fingerprints(allapps.keys()) + status_update_json(new_key_aliases, generated_keys, signed_apks) logging.info('published list signing-key fingerprints') diff --git a/fdroidserver/server.py b/fdroidserver/server.py index d00dd2da..c01c1f26 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -809,6 +809,7 @@ def main(): if config.get('wiki_server') and config.get('wiki_path'): update_wiki() + common.write_status_json(common.setup_status_output(start_timestamp)) sys.exit(0) diff --git a/fdroidserver/signindex.py b/fdroidserver/signindex.py index cbd55239..1f02d0f9 100644 --- a/fdroidserver/signindex.py +++ b/fdroidserver/signindex.py @@ -17,6 +17,7 @@ # along with this program. If not, see . import os +import time import zipfile from argparse import ArgumentParser import logging @@ -27,6 +28,7 @@ from .exception import FDroidException config = None options = None +start_timestamp = time.gmtime() def sign_jar(jar): @@ -75,6 +77,16 @@ def sign_index_v1(repodir, json_name): sign_jar(jar_file) +def status_update_json(signed): + """Output a JSON file with metadata about this run""" + + logging.debug(_('Outputting JSON')) + output = common.setup_status_output(start_timestamp) + if signed: + output['signed'] = signed + common.write_status_json(output) + + def main(): global config, options @@ -94,7 +106,7 @@ def main(): if config['archive_older'] != 0: repodirs.append('archive') - signed = 0 + signed = [] for output_dir in repodirs: if not os.path.isdir(output_dir): raise FDroidException("Missing output directory '" + output_dir + "'") @@ -102,9 +114,10 @@ def main(): unsigned = os.path.join(output_dir, 'index_unsigned.jar') if os.path.exists(unsigned): sign_jar(unsigned) - os.rename(unsigned, os.path.join(output_dir, 'index.jar')) + index_jar = os.path.join(output_dir, 'index.jar') + os.rename(unsigned, index_jar) logging.info('Signed index in ' + output_dir) - signed += 1 + signed.append(index_jar) json_name = 'index-v1.json' index_file = os.path.join(output_dir, json_name) @@ -112,10 +125,11 @@ def main(): sign_index_v1(output_dir, json_name) os.remove(index_file) logging.info('Signed ' + index_file) - signed += 1 + signed.append(index_file) - if signed == 0: + if not signed: logging.info(_("Nothing to do")) + status_update_json(signed) if __name__ == "__main__": diff --git a/fdroidserver/update.py b/fdroidserver/update.py index ae4a4bfa..13947570 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -121,6 +121,57 @@ def disabled_algorithms_allowed(): return options.allow_disabled_algorithms or config['allow_disabled_algorithms'] +def status_update_json(apps, sortedids, apks): + """Output a JSON file with metadata about this `fdroid update` run + + :param apps: fully populated list of all applications + :param apks: all to be published apks + + """ + + logging.debug(_('Outputting JSON')) + output = common.setup_status_output(start_timestamp) + output['antiFeatures'] = dict() + output['disabled'] = [] + output['failedBuilds'] = dict() + output['noPackages'] = [] + + for appid in sortedids: + app = apps[appid] + for af in app.get('AntiFeatures', []): + antiFeatures = output['antiFeatures'] # JSON camelCase + if af not in antiFeatures: + antiFeatures[af] = dict() + if appid not in antiFeatures[af]: + antiFeatures[af]['apps'] = set() + antiFeatures[af]['apps'].add(appid) + + apklist = [] + for apk in apks: + if apk['packageName'] == appid: + apklist.append(apk) + builds = app.get('builds', []) + validapks = 0 + for build in builds: + if not build.get('disable'): + builtit = False + for apk in apklist: + if apk['versionCode'] == int(build.versionCode): + builtit = True + validapks += 1 + break + if not builtit: + failedBuilds = output['failedBuilds'] + if appid not in failedBuilds: + failedBuilds[appid] = [] + failedBuilds[appid].append(build.versionCode) + if validapks == 0: + output['noPackages'].append(appid) + if app.get('Disabled'): + output['disabled'].append(appid) + common.write_status_json(output, options.pretty) + + def update_wiki(apps, sortedids, apks): """Update the wiki @@ -2200,6 +2251,7 @@ def main(): # Update the wiki... if options.wiki: update_wiki(apps, sortedids, apks + archapks) + status_update_json(apps, sortedids, apks + archapks) logging.info(_("Finished")) diff --git a/fdroidserver/verify.py b/fdroidserver/verify.py index 8025b054..bd1433d2 100644 --- a/fdroidserver/verify.py +++ b/fdroidserver/verify.py @@ -64,13 +64,6 @@ class Decoder(json.JSONDecoder): return set(values), end -class Encoder(json.JSONEncoder): - def default(self, obj): - if isinstance(obj, set): - return sorted(obj) - return super().default(obj) - - def write_json_report(url, remote_apk, unsigned_apk, compare_result): """write out the results of the verify run to JSON @@ -118,7 +111,7 @@ def write_json_report(url, remote_apk, unsigned_apk, compare_result): data['packages'][packageName] = set() data['packages'][packageName].add(output) with open(jsonfile, 'w') as fp: - json.dump(data, fp, cls=Encoder, sort_keys=True) + json.dump(data, fp, cls=common.Encoder, sort_keys=True) def main(): diff --git a/jenkins-test b/jenkins-test index d6ce9659..945d5d37 100755 --- a/jenkins-test +++ b/jenkins-test @@ -55,6 +55,7 @@ fi gpg --import $GNUPGHOME/secring.gpg echo "build_server_always = True" >> config.py +echo "deploy_process_logs = True" >> config.py echo "make_current_version_link = False" >> config.py echo "gpghome = '$GNUPGHOME'" >> config.py echo "gpgkey = 'CE71F7FB'" >> config.py @@ -66,6 +67,7 @@ test -d repo || mkdir repo test -d archive || mkdir archive # when everything is copied over to run on SIGN machine ../fdroid publish + ../fdroid gpgsign # when everything is copied over to run on BUILD machine, # which does not have a keyring, only a cached pubkey diff --git a/tests/common.TestCase b/tests/common.TestCase index 799455f0..856b71e7 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -5,13 +5,16 @@ import difflib import glob import inspect +import json import logging import optparse import os import re import shutil +import subprocess import sys import tempfile +import time import unittest import textwrap import yaml @@ -1131,6 +1134,55 @@ class CommonTest(unittest.TestCase): with gzip.open(expected_log_path, 'r') as f: self.assertEqual(f.read(), mocklogcontent) + def test_deploy_status_json(self): + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + + fakesubcommand = 'fakesubcommand' + fake_timestamp = 1234567890 + fakeserver = 'example.com:/var/www/fbot/' + expected_dir = os.path.join(testdir, fakeserver.replace(':', ''), 'repo', 'status') + + fdroidserver.common.options = mock.Mock() + fdroidserver.common.config = {} + fdroidserver.common.config['serverwebroot'] = [fakeserver] + fdroidserver.common.config['identity_file'] = 'ssh/id_rsa' + + def assert_subprocess_call(cmd): + dest_path = os.path.join(testdir, cmd[-1].replace(':', '')) + if not os.path.exists(dest_path): + os.makedirs(dest_path) + return subprocess.run(cmd[:-1] + [dest_path]).returncode + + with mock.patch('subprocess.call', side_effect=assert_subprocess_call): + with mock.patch.object(sys, 'argv', ['fdroid ' + fakesubcommand]): + output = fdroidserver.common.setup_status_output(time.localtime(fake_timestamp)) + self.assertFalse(os.path.exists(os.path.join(expected_dir, 'running.json'))) + with mock.patch.object(sys, 'argv', ['fdroid ' + fakesubcommand]): + fdroidserver.common.write_status_json(output) + self.assertFalse(os.path.exists(os.path.join(expected_dir, fakesubcommand + '.json'))) + + fdroidserver.common.config['deploy_process_logs'] = True + + output = fdroidserver.common.setup_status_output(time.localtime(fake_timestamp)) + expected_path = os.path.join(expected_dir, 'running.json') + self.assertTrue(os.path.isfile(expected_path)) + with open(expected_path) as fp: + data = json.load(fp) + self.assertEqual(fake_timestamp * 1000, data['startTimestamp']) + self.assertFalse('endTimestamp' in data) + + testvalue = 'asdfasd' + output['testvalue'] = testvalue + + fdroidserver.common.write_status_json(output) + expected_path = os.path.join(expected_dir, fakesubcommand + '.json') + self.assertTrue(os.path.isfile(expected_path)) + with open(expected_path) as fp: + data = json.load(fp) + self.assertEqual(fake_timestamp * 1000, data['startTimestamp']) + self.assertTrue('endTimestamp' in data) + self.assertEqual(testvalue, output.get('testvalue')) + def test_string_is_integer(self): self.assertTrue(fdroidserver.common.string_is_integer('0x10')) self.assertTrue(fdroidserver.common.string_is_integer('010')) diff --git a/tests/publish.TestCase b/tests/publish.TestCase index 2e658c1e..ac00d6d9 100755 --- a/tests/publish.TestCase +++ b/tests/publish.TestCase @@ -19,6 +19,7 @@ import sys import unittest import tempfile import textwrap +from unittest import mock localmodule = os.path.realpath( os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) @@ -158,7 +159,8 @@ class PublishTest(unittest.TestCase): os.path.join(testdir, 'unsigned', 'binaries', 'com.politedroid_6.binary.apk')) os.chdir(testdir) - publish.main() + with mock.patch.object(sys, 'argv', ['fdroid fakesubcommand']): + publish.main() if __name__ == "__main__":