diff --git a/examples/config.py b/examples/config.py index 4fcba228..dab12df1 100644 --- a/examples/config.py +++ b/examples/config.py @@ -163,6 +163,13 @@ The repository of older versions of applications from the main demo repository. # 'bar.info:/var/www/fdroid', # } +# Uncomment this option if you want to logs of builds and other processes to +# your repository server(s). Logs get published to all servers configured in +# 'serverwebroot'. For builds, only logs from build-jobs running inside a +# buildserver VM are supported. +# +# deploy_process_logs = True + # The full URL to a git remote repository. You can include # multiple servers to mirror to by wrapping the whole thing in {} or [], and # including the servergitmirrors strings in a comma-separated list. diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 73e590ac..9d951765 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -75,6 +75,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): sshinfo = vmtools.get_clean_builder('builder', options.reset_server) + output = None try: if not buildserverid: buildserverid = subprocess.check_output(['vagrant', 'ssh', '-c', @@ -279,6 +280,13 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): vm = vmtools.get_build_vm('builder') vm.suspend() + # deploy logfile to repository web server + if output: + common.publish_build_log_with_rsync(app.id, build.versionCode, output) + else: + logging.debug('skip publishing full build logs: ' + 'no output present') + def force_gradle_build_tools(build_dir, build_tools): for root, dirs, files in os.walk(build_dir): @@ -1137,7 +1145,7 @@ def main(): raise FDroidException( 'Downloading Binaries from %s failed. %s' % (url, e)) - # Now we check weather the build can be verified to + # Now we check whether the build can be verified to # match the supplied binary or not. Should the # comparison fail, we mark this build as a failure # and remove everything from the unsigend folder. diff --git a/fdroidserver/common.py b/fdroidserver/common.py index ec22d4ab..4584a433 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -25,6 +25,7 @@ import os import sys import re import ast +import gzip import shutil import glob import stat @@ -100,6 +101,7 @@ default_config = { 'per_app_repos': False, 'make_current_version_link': True, 'current_version_name_source': 'Name', + 'deploy_process_logs': False, 'update_stats': False, 'stats_ignore': [], 'stats_server': None, @@ -3070,6 +3072,70 @@ def local_rsync(options, fromdir, todir): raise FDroidException() +def deploy_build_log_with_rsync(appid, vercode, log_content, + timestamp=int(time.time())): + """Upload build log of one individual app build to an fdroid repository. + + :param appid: package name for dientifying to which app this log belongs. + :param vercode: version of the app to which this build belongs. + :param log_content: Content of the log which is about to be posted. + Should be either a string or bytes. (bytes will + be decoded as 'utf-8') + :param timestamp: timestamp for avoiding logfile name collisions. + """ + + # check if deploying logs is enabled in config + if not config.get('deploy_process_logs', False): + logging.debug(_('skip deploying full build logs: not enabled in config')) + return + + if not log_content: + logging.warning(_('skip deploying full build logs: log content is empty')) + return + + if not (isinstance(timestamp, int) or isinstance(timestamp, float)): + raise ValueError(_("supplied timestamp value '{timestamp}' is not a unix timestamp" + .format(timestamp=timestamp))) + + with tempfile.TemporaryDirectory() as tmpdir: + # gzip compress log file + log_gz_path = os.path.join( + tmpdir, '{pkg}_{ver}_{ts}.log.gz'.format(pkg=appid, + ver=vercode, + ts=int(timestamp))) + with gzip.open(log_gz_path, 'wb') as f: + if isinstance(log_content, str): + f.write(bytes(log_content, 'utf-8')) + else: + f.write(log_content) + + # TODO: sign compressed log file, if a signing key is configured + + for webroot in config.get('serverwebroot', []): + dest_path = os.path.join(webroot, "buildlogs") + if not dest_path.endswith('/'): + dest_path += '/' # make sure rsync knows this is a directory + cmd = ['rsync', + '--archive', + '--delete-after', + '--safe-links'] + if options.verbose: + cmd += ['--verbose'] + if options.quiet: + cmd += ['--quiet'] + if 'identity_file' in config: + cmd += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']] + cmd += [log_gz_path, dest_path] + + # TODO: also deploy signature file if present + + retcode = subprocess.call(cmd) + if retcode: + logging.warning(_("failed deploying build logs to '{path}'").format(path=webroot)) + else: + logging.info(_("deployeded build logs to '{path}'").format(path=webroot)) + + def get_per_app_repos(): '''per-app repos are dirs named with the packageName of a single app''' diff --git a/tests/common.TestCase b/tests/common.TestCase index 68b288ec..f5db4f3f 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -13,7 +13,9 @@ import tempfile import unittest import textwrap import yaml +import gzip from zipfile import ZipFile +from unittest import mock localmodule = os.path.realpath( @@ -659,12 +661,11 @@ class CommonTest(unittest.TestCase): self.assertEqual('b30bb971af0d134866e158ec748fcd553df97c150f58b0a963190bbafbeb0868', sig) def test_parse_androidmanifests(self): - source_files_dir = os.path.join(os.path.dirname(__file__), 'source-files') app = fdroidserver.metadata.App() app.id = 'org.fdroid.fdroid' paths = [ - os.path.join(source_files_dir, 'fdroid', 'fdroidclient', 'AndroidManifest.xml'), - os.path.join(source_files_dir, 'fdroid', 'fdroidclient', 'build.gradle'), + os.path.join('source-files', 'fdroid', 'fdroidclient', 'AndroidManifest.xml'), + os.path.join('source-files', 'fdroid', 'fdroidclient', 'build.gradle'), ] for path in paths: self.assertTrue(os.path.isfile(path)) @@ -672,16 +673,14 @@ class CommonTest(unittest.TestCase): fdroidserver.common.parse_androidmanifests(paths, app)) def test_parse_androidmanifests_with_flavor(self): - source_files_dir = os.path.join(os.path.dirname(__file__), 'source-files') - app = fdroidserver.metadata.App() build = fdroidserver.metadata.Build() build.gradle = ['devVersion'] app.builds = [build] app.id = 'org.fdroid.fdroid.dev' paths = [ - os.path.join(source_files_dir, 'fdroid', 'fdroidclient', 'AndroidManifest.xml'), - os.path.join(source_files_dir, 'fdroid', 'fdroidclient', 'build.gradle'), + os.path.join('source-files', 'fdroid', 'fdroidclient', 'AndroidManifest.xml'), + os.path.join('source-files', 'fdroid', 'fdroidclient', 'build.gradle'), ] for path in paths: self.assertTrue(os.path.isfile(path)) @@ -694,7 +693,7 @@ class CommonTest(unittest.TestCase): app.builds = [build] app.id = 'eu.siacs.conversations' paths = [ - os.path.join(source_files_dir, 'eu.siacs.conversations', 'build.gradle'), + os.path.join('source-files', 'eu.siacs.conversations', 'build.gradle'), ] for path in paths: self.assertTrue(os.path.isfile(path)) @@ -707,7 +706,7 @@ class CommonTest(unittest.TestCase): app.builds = [build] app.id = 'com.nextcloud.client' paths = [ - os.path.join(source_files_dir, 'com.nextcloud.client', 'build.gradle'), + os.path.join('source-files', 'com.nextcloud.client', 'build.gradle'), ] for path in paths: self.assertTrue(os.path.isfile(path)) @@ -720,7 +719,7 @@ class CommonTest(unittest.TestCase): app.builds = [build] app.id = 'com.nextcloud.android.beta' paths = [ - os.path.join(source_files_dir, 'com.nextcloud.client', 'build.gradle'), + os.path.join('source-files', 'com.nextcloud.client', 'build.gradle'), ] for path in paths: self.assertTrue(os.path.isfile(path)) @@ -733,7 +732,7 @@ class CommonTest(unittest.TestCase): app.builds = [build] app.id = 'at.bitfire.davdroid' paths = [ - os.path.join(source_files_dir, 'at.bitfire.davdroid', 'build.gradle'), + os.path.join('source-files', 'at.bitfire.davdroid', 'build.gradle'), ] for path in paths: self.assertTrue(os.path.isfile(path)) @@ -746,7 +745,7 @@ class CommonTest(unittest.TestCase): app.builds = [build] app.id = 'com.kunzisoft.fdroidtest.applicationidsuffix.libre' paths = [ - os.path.join(source_files_dir, 'com.kunzisoft.testcase', 'build.gradle'), + os.path.join('source-files', 'com.kunzisoft.testcase', 'build.gradle'), ] for path in paths: self.assertTrue(os.path.isfile(path)) @@ -759,7 +758,7 @@ class CommonTest(unittest.TestCase): app.builds = [build] app.id = 'com.kunzisoft.fdroidtest.applicationidsuffix.pro' paths = [ - os.path.join(source_files_dir, 'com.kunzisoft.testcase', 'build.gradle'), + os.path.join('source-files', 'com.kunzisoft.testcase', 'build.gradle'), ] for path in paths: self.assertTrue(os.path.isfile(path)) @@ -772,7 +771,7 @@ class CommonTest(unittest.TestCase): app.builds = [build] app.id = 'com.kunzisoft.fdroidtest.applicationidsuffix' paths = [ - os.path.join(source_files_dir, 'com.kunzisoft.testcase', 'build.gradle'), + os.path.join('source-files', 'com.kunzisoft.testcase', 'build.gradle'), ] for path in paths: self.assertTrue(os.path.isfile(path)) @@ -792,8 +791,70 @@ class CommonTest(unittest.TestCase): with self.assertRaises(SyntaxError): fdroidserver.common.calculate_math_string('1-1 # no comment') + def test_deploy_build_log_with_rsync_with_id_file(self): + + mocklogcontent = bytes(textwrap.dedent("""\ + build started + building... + build completed + profit!"""), 'utf-8') + + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.verbose = False + fdroidserver.common.options.quiet = False + fdroidserver.common.config = {} + fdroidserver.common.config['serverwebroot'] = [ + 'example.com:/var/www/fdroid/repo/', + 'example.com:/var/www/fdroid/archive/'] + fdroidserver.common.config['deploy_process_logs'] = True + fdroidserver.common.config['identity_file'] = 'ssh/id_rsa' + + assert_subprocess_call_iteration = 0 + + def assert_subprocess_call(cmd): + nonlocal assert_subprocess_call_iteration + logging.debug(cmd) + if assert_subprocess_call_iteration == 0: + self.assertListEqual(['rsync', + '--archive', + '--delete-after', + '--safe-links', + '-e', + 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ssh/id_rsa', + cmd[6], + 'example.com:/var/www/fdroid/repo/buildlogs'], + cmd) + self.assertTrue(cmd[6].endswith('/com.example.app_4711_1.log.gz')) + with gzip.open(cmd[6], 'r') as f: + self.assertTrue(f.read(), mocklogcontent) + elif assert_subprocess_call_iteration == 1: + self.assertListEqual(['rsync', + '--archive', + '--delete-after', + '--safe-links', + '-e', + 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ssh/id_rsa', + cmd[6], + 'example.com:/var/www/fdroid/archive/buildlogs'], + cmd) + self.assertTrue(cmd[6].endswith('/com.example.app_4711_1.log.gz')) + with gzip.open(cmd[6], 'r') as f: + self.assertTrue(f.read(), mocklogcontent) + else: + self.fail('unexpected subprocess.call invocation ({})' + .format(assert_subprocess_call_iteration)) + assert_subprocess_call_iteration += 1 + return 0 + + with mock.patch('subprocess.call', + side_effect=assert_subprocess_call): + fdroidserver.common.deploy_build_log_with_rsync( + 'com.example.app', '4711', mocklogcontent, 1.1) + if __name__ == "__main__": + os.chdir(os.path.dirname(__file__)) + parser = optparse.OptionParser() parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") diff --git a/tests/server.TestCase b/tests/server.TestCase new file mode 100755 index 00000000..5d058153 --- /dev/null +++ b/tests/server.TestCase @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 + +import inspect +import logging +import optparse +import os +import sys +import tempfile +import unittest +from unittest import mock + +localmodule = os.path.realpath( + os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) +if localmodule not in sys.path: + sys.path.insert(0, localmodule) + +import fdroidserver.common +import fdroidserver.server +from testcommon import TmpCwd + + +class ServerTest(unittest.TestCase): + '''fdroidserver/server.py''' + + def setUp(self): + logging.basicConfig(level=logging.DEBUG) + self.basedir = os.path.join(localmodule, 'tests') + + fdroidserver.server.options = mock.Mock() + fdroidserver.server.config = {} + + def test_update_serverwebroot_make_cur_version_link(self): + + # setup parameters for this test run + fdroidserver.server.options.no_chcksum = True + fdroidserver.server.options.identity_file = None + fdroidserver.server.options.verbose = False + fdroidserver.server.options.quiet = True + fdroidserver.server.options.identity_file = None + fdroidserver.server.config['make_current_version_link'] = True + serverwebroot = "example.com:/var/www/fdroid" + repo_section = 'repo' + + # setup function for asserting subprocess.call invocations + call_iteration = 0 + + def update_server_webroot_call(cmd): + nonlocal call_iteration + if call_iteration == 0: + self.assertListEqual(cmd, ['rsync', + '--archive', + '--delete-after', + '--safe-links', + '--quiet', + '--exclude', 'repo/index.xml', + '--exclude', 'repo/index.jar', + '--exclude', 'repo/index-v1.jar', + 'repo', + 'example.com:/var/www/fdroid']) + elif call_iteration == 1: + self.assertListEqual(cmd, ['rsync', + '--archive', + '--delete-after', + '--safe-links', + '--quiet', + 'repo', + serverwebroot]) + elif call_iteration == 2: + self.assertListEqual(cmd, ['rsync', + '--archive', + '--delete-after', + '--safe-links', + '--quiet', + 'Sym.apk', + 'Sym.apk.asc', + 'Sym.apk.sig', + 'example.com:/var/www/fdroid']) + else: + self.fail('unexpected subprocess.call invocation') + call_iteration += 1 + return 0 + + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + os.mkdir('repo') + os.symlink('repo/com.example.sym.apk', 'Sym.apk') + os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc') + os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig') + with mock.patch('subprocess.call', side_effect=update_server_webroot_call): + fdroidserver.server.update_serverwebroot(serverwebroot, + repo_section) + self.assertEqual(call_iteration, 3, 'expected 3 invocations of subprocess.call') + + def test_update_serverwebroot_with_id_file(self): + + # setup parameters for this test run + fdroidserver.server.options.no_chcksum = False + fdroidserver.server.options.verbose = True + fdroidserver.server.options.quiet = False + fdroidserver.server.options.identity_file = None + fdroidserver.server.config['identity_file'] = './id_rsa' + fdroidserver.server.config['make_current_version_link'] = False + serverwebroot = "example.com:/var/www/fdroid" + repo_section = 'archive' + + # setup function for asserting subprocess.call invocations + call_iteration = 0 + + def update_server_webroot_call(cmd): + nonlocal call_iteration + if call_iteration == 0: + self.assertListEqual(cmd, ['rsync', + '--archive', + '--delete-after', + '--safe-links', + '--verbose', + '-e', + 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + + fdroidserver.server.config['identity_file'], + '--exclude', 'archive/index.xml', + '--exclude', 'archive/index.jar', + '--exclude', 'archive/index-v1.jar', + 'archive', + serverwebroot]) + elif call_iteration == 1: + self.assertListEqual(cmd, ['rsync', + '--archive', + '--delete-after', + '--safe-links', + '--verbose', + '-e', + 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + + fdroidserver.server.config['identity_file'], + 'archive', + serverwebroot]) + else: + self.fail('unexpected subprocess.call invocation') + call_iteration += 1 + return 0 + + with mock.patch('subprocess.call', side_effect=update_server_webroot_call): + fdroidserver.server.update_serverwebroot(serverwebroot, + repo_section) + self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call') + + +if __name__ == "__main__": + os.chdir(os.path.dirname(__file__)) + + parser = optparse.OptionParser() + parser.add_option("-v", "--verbose", action="store_true", default=False, + help="Spew out even more information than normal") + (fdroidserver.common.options, args) = parser.parse_args(['--verbose']) + + newSuite = unittest.TestSuite() + newSuite.addTest(unittest.makeSuite(ServerTest)) + unittest.main(failfast=False)