From aa86e5633896ef1bf671c078618f51e18559a6a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Tue, 26 Jun 2018 10:53:42 +0200 Subject: [PATCH 1/9] updated buildserver provisions to stretch --- buildserver/provision-apt-get-install | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index 8702c7de..dc10206c 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -22,8 +22,8 @@ fi sed -i "s,http://ftp.uk.debian.org/debian/,${debian_mirror},g" /etc/apt/sources.list -if grep --quiet jessie /etc/apt/sources.list; then - echo "deb $debian_mirror jessie-backports main" > /etc/apt/sources.list.d/backports.list +if grep --quiet stretch /etc/apt/sources.list; then + echo "deb $debian_mirror stretch-backports main" > /etc/apt/sources.list.d/stretch-backports.list echo "deb $debian_mirror testing main" > /etc/apt/sources.list.d/testing.list printf "Package: *\nPin: release o=Debian,a=testing\nPin-Priority: -300\n" > /etc/apt/preferences.d/debian-testing fi @@ -45,16 +45,16 @@ packages=" autopoint bison bzr - ca-certificates-java/jessie-backports - cmake/jessie-backports + ca-certificates-java + cmake curl expect faketime flex - gettext/jessie-backports - gettext-base/jessie-backports - git-core/jessie-backports - git-svn/jessie-backports + gettext + gettext-base + git-core + git-svn gperf graphviz imagemagick @@ -75,7 +75,7 @@ packages=" libtool-bin/testing make maven - mercurial/jessie-backports + mercurial nasm nodejs openjdk-8-jre-headless @@ -90,13 +90,13 @@ packages=" python-magic python-setuptools python3-defusedxml - python3-git/jessie-backports - python3-gitdb/jessie-backports + python3-git + python3-gitdb python3-gnupg python3-pyasn1 python3-pyasn1-modules python3-requests - python3-smmap/jessie-backports + python3-smmap python3-yaml python3-ruamel.yaml quilt From a5ef08f57d64859c4e24442800b1e8fd1607fdf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Tue, 26 Jun 2018 11:25:07 +0200 Subject: [PATCH 2/9] more general regex for updating debian repo urls in buildserver provisions --- buildserver/provision-apt-get-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index dc10206c..19cc91eb 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -20,7 +20,7 @@ if echo $debian_mirror | grep '^https' 2>&1 > /dev/null; then apt-get -y install apt-transport-https fi -sed -i "s,http://ftp.uk.debian.org/debian/,${debian_mirror},g" /etc/apt/sources.list +sed -Ei "s,^deb [a-z]+://\S+,deb ${debian_mirror}," /etc/apt/sources.list if grep --quiet stretch /etc/apt/sources.list; then echo "deb $debian_mirror stretch-backports main" > /etc/apt/sources.list.d/stretch-backports.list From c005d8c5f462f050100d37fb9b9fb41e59b8b172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 27 Jun 2018 01:05:45 +0200 Subject: [PATCH 3/9] more detailed error message and handling when fetching buildserverid in build.py --- fdroidserver/build.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 2ecb1de9..c2854126 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -67,8 +67,8 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): try: paramiko - except NameError: - raise BuildException("Paramiko is required to use the buildserver") + except NameError as e: + raise BuildException("Paramiko is required to use the buildserver") from e if options.verbose: logging.getLogger("paramiko").setLevel(logging.INFO) else: @@ -79,11 +79,18 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): output = None try: if not buildserverid: - buildserverid = subprocess.check_output(['vagrant', 'ssh', '-c', - 'cat /home/vagrant/buildserverid'], - cwd='builder').strip().decode() - logging.debug(_('Fetched buildserverid from VM: {buildserverid}') - .format(buildserverid=buildserverid)) + try: + buildserverid = subprocess.check_output(['vagrant', 'ssh', '-c', + 'cat /home/vagrant/buildserverid'], + cwd='builder').strip().decode() + logging.debug(_('Fetched buildserverid from VM: {buildserverid}') + .format(buildserverid=buildserverid)) + except Exception as e: + if type(buildserverid) is not str or not re.match('^[0-9a-f]{40}$', buildserverid): + logging.info(subprocess.check_output(['vagrant', 'status'], cwd="builder")) + raise FDroidException("Could not obtain buildserverid from buldserver VM. " + "(stored inside the buildserver VM at '/home/vagrant/buildserverid') " + "Please reset your buildserver, the setup VM is broken.") from e # Open SSH connection... logging.info("Connecting to virtual machine...") @@ -1160,7 +1167,7 @@ def main(): net.download_file(url, local_filename=of) except requests.exceptions.HTTPError as e: raise FDroidException( - 'Downloading Binaries from %s failed. %s' % (url, e)) + 'Downloading Binaries from %s failed.' % url) from e # Now we check whether the build can be verified to # match the supplied binary or not. Should the From 5e4eb294c5ed0f73b38ee4c8c5fded635969c66e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sat, 28 Jul 2018 16:48:15 +0200 Subject: [PATCH 4/9] switch to fdroid-stretch64 basebox; remove baseboxurl form makebuildserver --- examples/makebuildserver.config.py | 11 +------- makebuildserver | 44 ++++++++++++++---------------- 2 files changed, 22 insertions(+), 33 deletions(-) diff --git a/examples/makebuildserver.config.py b/examples/makebuildserver.config.py index 3c243899..ba373858 100644 --- a/examples/makebuildserver.config.py +++ b/examples/makebuildserver.config.py @@ -3,16 +3,7 @@ # You may want to alter these before running ./makebuildserver # Name of the base box to use -# basebox = "jessie64" - -# Location where testing32.box can be found, if you don't already have -# it. For security reasons, it's recommended that you make your own -# in a secure environment using trusted media (see the manual) but -# you can use this default if you like... -# baseboxurl = "https://f-droid.org/jessie64.box" -# -# or if you have a cached local copy, you can use that first: -# baseboxurl = ["file:///home/fdroid/fdroidserver/cache/jessie64.box", "https://f-droid.org/jessie64.box"] +# basebox = "fdroid-stretch64" # In the process of setting up the build server, many gigs of files # are downloaded (Android SDK components, gradle, etc). These are diff --git a/makebuildserver b/makebuildserver index 6fc3de23..2411afb1 100755 --- a/makebuildserver +++ b/makebuildserver @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os -import pathlib import re import requests import stat @@ -56,10 +55,7 @@ cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver') logger.debug('cachedir set to: %s', cachedir) config = { - 'basebox': 'jessie64', - 'baseboxurl': [ - pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri() - ], + 'basebox': 'fdroid-stretch64', 'debian_mirror': 'http://http.debian.net/debian/', 'apt_package_cache': False, 'copy_caches_from_host': False, @@ -535,23 +531,25 @@ def main(): with open(vf, 'w', encoding='utf-8') as f: yaml.dump(config, f) - if config['vm_provider'] == 'libvirt': - available_providers = [x.provider for x in v.box_list() if x.name == config['basebox']] - found_basebox = len(available_providers) > 0 - needs_mutate = 'libvirt' not in available_providers - if not found_basebox: - if isinstance(config['baseboxurl'], str): - baseboxurl = config['baseboxurl'] - else: - baseboxurl = config['baseboxurl'][0] - logger.info('Adding %s from %s', config['basebox'], baseboxurl) - v.box_add(config['basebox'], baseboxurl) - needs_mutate = True - if needs_mutate: - logger.info('Converting %s to libvirt format', config['basebox']) - v._call_vagrant_command(['mutate', config['basebox'], 'libvirt']) - logger.info('Removing virtualbox format copy of %s', config['basebox']) - v.box_remove(config['basebox'], 'virtualbox') + # Check if selected provider is supported + if config['vm_provider'] not in ['libvirt', 'virtualbox']: + logger.critical("Currently selected VM provider '{vm_provider}' " + "is not supported. (please choose from: " + "virtualbox, libvirt)" + .format(vm_provider=config['cm_provider'])) + sys.exit(1) + # Check if selected basebox is available + available_boxes_by_provider = [x.name for x in v.box_list() if x.provider == config['vm_provider']] + if config['basebox'] not in available_boxes_by_provider: + logger.critical("Vagrant box '{basebox}' not available " + "for '{vm_provider}' VM provider. " + "Please make sure it's added to vagrant. " + "(If you need a basebox to begin with, " + "here is how we're bootstrapping it: " + "https://gitlab.com/fdroid/basebox)" + .format(vm_provider=config['vm_provider'], + basebox=config['basebox'])) + sys.exit(1) logger.info("Configuring build server VM") debug_log_vagrant_vm(serverdir, config) @@ -559,7 +557,7 @@ def main(): v.up(provision=True) except subprocess.CalledProcessError: debug_log_vagrant_vm(serverdir, config) - logger.error("'vagrant up' failed, is the base box missing?") + logger.error("'vagrant up' failed.") sys.exit(1) if config['copy_caches_from_host']: From 9136080defec2e230ba3238ec10dc222e6731752 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sun, 29 Jul 2018 13:43:48 +0200 Subject: [PATCH 5/9] update default mirror to deb.debian.org --- makebuildserver | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makebuildserver b/makebuildserver index 2411afb1..9ecc9a4b 100755 --- a/makebuildserver +++ b/makebuildserver @@ -56,7 +56,7 @@ logger.debug('cachedir set to: %s', cachedir) config = { 'basebox': 'fdroid-stretch64', - 'debian_mirror': 'http://http.debian.net/debian/', + 'debian_mirror': 'http://deb.debian.org/debian/', 'apt_package_cache': False, 'copy_caches_from_host': False, 'boot_timeout': 600, From 98806d7c012df40aa26cc9ffd5ead3fe5fc7d1d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Tue, 21 Aug 2018 12:55:31 +0200 Subject: [PATCH 6/9] configure headless openjdk to run without gtk accessability dependencies --- buildserver/provision-apt-get-install | 3 +++ 1 file changed, 3 insertions(+) diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index 19cc91eb..a4f14c0b 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -120,3 +120,6 @@ apt-get install --yes $packages highestjava=`update-java-alternatives --list | sort -n | tail -1 | cut -d ' ' -f 1` update-java-alternatives --set $highestjava + +# configure headless openjdk to work without gtk accessability dependencies +sed -i -e 's@\(assistive_technologies=org.GNOME.Accessibility.AtkWrapper\)@#\1@' /etc/java-8-openjdk/accessibility.properties From 6d0797eb5e81af29389fed05840892adf676f148 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 13 Sep 2018 11:08:51 +0200 Subject: [PATCH 7/9] allow using remote boxes from vagrant cloud --- makebuildserver | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makebuildserver b/makebuildserver index 9ecc9a4b..e681300a 100755 --- a/makebuildserver +++ b/makebuildserver @@ -540,7 +540,7 @@ def main(): sys.exit(1) # Check if selected basebox is available available_boxes_by_provider = [x.name for x in v.box_list() if x.provider == config['vm_provider']] - if config['basebox'] not in available_boxes_by_provider: + if '/' not in config['basebox'] and config['basebox'] not in available_boxes_by_provider: logger.critical("Vagrant box '{basebox}' not available " "for '{vm_provider}' VM provider. " "Please make sure it's added to vagrant. " From fd7cc2c9540ebb80c394dc7fd6fcf66494b29dd0 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Sat, 28 Jul 2018 16:48:15 +0200 Subject: [PATCH 8/9] document how to avoid downloading from vagrant cloud --- examples/makebuildserver.config.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/makebuildserver.config.py b/examples/makebuildserver.config.py index ba373858..7706c487 100644 --- a/examples/makebuildserver.config.py +++ b/examples/makebuildserver.config.py @@ -2,7 +2,11 @@ # # You may want to alter these before running ./makebuildserver -# Name of the base box to use +# Name of the Vagrant basebox to use, by default it will be downloaded +# from Vagrant Cloud. For release builds setup, generate the basebox +# locally using https://gitlab.com/fdroid/basebox, add it to Vagrant, +# then set this to the local basebox name. +# # basebox = "fdroid-stretch64" # In the process of setting up the build server, many gigs of files From 6f889a08d76d03c872bb4c5e7414162aad204b8c Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 11 Sep 2018 12:57:31 +0200 Subject: [PATCH 9/9] jenkins: switch to locally generated stretch64 basebox --- jenkins-setup-build-environment | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/jenkins-setup-build-environment b/jenkins-setup-build-environment index cab721c6..906f0aff 100755 --- a/jenkins-setup-build-environment +++ b/jenkins-setup-build-environment @@ -32,12 +32,7 @@ trap cleanup_all INT TERM EXIT set -e set -x -# make sure we have the vagrant box image cached test -e ~/.cache/fdroidserver || mkdir -p ~/.cache/fdroidserver -cd ~/.cache/fdroidserver -wget --tries=1 --timeout=5 --continue https://f-droid.org/jessie64.box || true -echo "de3e3c4a9c13e8c015e30edeea0f583b195d1ee8ff9ad4814e933bbfb560200f jessie64.box" > jessie64.box.sha256 -sha256sum -c jessie64.box.sha256 # redirect homes to be in the git repo, so they'll get cleaned and reset export XDG_CONFIG_HOME=$WORKSPACE @@ -61,6 +56,8 @@ virsh -c qemu:///system vol-delete --pool default \ /var/lib/libvirt/images/jessie64_vagrant_box_image_0.img || true rm -rf "$WORKSPACE"/../*/.testfiles +vagrant box add --force fdroid-stretch64 ~/.cache/fdroidserver/fdroid-stretch64-libvirt.box + cd $WORKSPACE echo "debian_mirror = 'https://deb.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py