diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index 8702c7de..a4f14c0b 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -20,10 +20,10 @@ if echo $debian_mirror | grep '^https' 2>&1 > /dev/null; then apt-get -y install apt-transport-https fi -sed -i "s,http://ftp.uk.debian.org/debian/,${debian_mirror},g" /etc/apt/sources.list +sed -Ei "s,^deb [a-z]+://\S+,deb ${debian_mirror}," /etc/apt/sources.list -if grep --quiet jessie /etc/apt/sources.list; then - echo "deb $debian_mirror jessie-backports main" > /etc/apt/sources.list.d/backports.list +if grep --quiet stretch /etc/apt/sources.list; then + echo "deb $debian_mirror stretch-backports main" > /etc/apt/sources.list.d/stretch-backports.list echo "deb $debian_mirror testing main" > /etc/apt/sources.list.d/testing.list printf "Package: *\nPin: release o=Debian,a=testing\nPin-Priority: -300\n" > /etc/apt/preferences.d/debian-testing fi @@ -45,16 +45,16 @@ packages=" autopoint bison bzr - ca-certificates-java/jessie-backports - cmake/jessie-backports + ca-certificates-java + cmake curl expect faketime flex - gettext/jessie-backports - gettext-base/jessie-backports - git-core/jessie-backports - git-svn/jessie-backports + gettext + gettext-base + git-core + git-svn gperf graphviz imagemagick @@ -75,7 +75,7 @@ packages=" libtool-bin/testing make maven - mercurial/jessie-backports + mercurial nasm nodejs openjdk-8-jre-headless @@ -90,13 +90,13 @@ packages=" python-magic python-setuptools python3-defusedxml - python3-git/jessie-backports - python3-gitdb/jessie-backports + python3-git + python3-gitdb python3-gnupg python3-pyasn1 python3-pyasn1-modules python3-requests - python3-smmap/jessie-backports + python3-smmap python3-yaml python3-ruamel.yaml quilt @@ -120,3 +120,6 @@ apt-get install --yes $packages highestjava=`update-java-alternatives --list | sort -n | tail -1 | cut -d ' ' -f 1` update-java-alternatives --set $highestjava + +# configure headless openjdk to work without gtk accessability dependencies +sed -i -e 's@\(assistive_technologies=org.GNOME.Accessibility.AtkWrapper\)@#\1@' /etc/java-8-openjdk/accessibility.properties diff --git a/examples/makebuildserver.config.py b/examples/makebuildserver.config.py index 3c243899..7706c487 100644 --- a/examples/makebuildserver.config.py +++ b/examples/makebuildserver.config.py @@ -2,17 +2,12 @@ # # You may want to alter these before running ./makebuildserver -# Name of the base box to use -# basebox = "jessie64" - -# Location where testing32.box can be found, if you don't already have -# it. For security reasons, it's recommended that you make your own -# in a secure environment using trusted media (see the manual) but -# you can use this default if you like... -# baseboxurl = "https://f-droid.org/jessie64.box" +# Name of the Vagrant basebox to use, by default it will be downloaded +# from Vagrant Cloud. For release builds setup, generate the basebox +# locally using https://gitlab.com/fdroid/basebox, add it to Vagrant, +# then set this to the local basebox name. # -# or if you have a cached local copy, you can use that first: -# baseboxurl = ["file:///home/fdroid/fdroidserver/cache/jessie64.box", "https://f-droid.org/jessie64.box"] +# basebox = "fdroid-stretch64" # In the process of setting up the build server, many gigs of files # are downloaded (Android SDK components, gradle, etc). These are diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 2ecb1de9..c2854126 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -67,8 +67,8 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): try: paramiko - except NameError: - raise BuildException("Paramiko is required to use the buildserver") + except NameError as e: + raise BuildException("Paramiko is required to use the buildserver") from e if options.verbose: logging.getLogger("paramiko").setLevel(logging.INFO) else: @@ -79,11 +79,18 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): output = None try: if not buildserverid: - buildserverid = subprocess.check_output(['vagrant', 'ssh', '-c', - 'cat /home/vagrant/buildserverid'], - cwd='builder').strip().decode() - logging.debug(_('Fetched buildserverid from VM: {buildserverid}') - .format(buildserverid=buildserverid)) + try: + buildserverid = subprocess.check_output(['vagrant', 'ssh', '-c', + 'cat /home/vagrant/buildserverid'], + cwd='builder').strip().decode() + logging.debug(_('Fetched buildserverid from VM: {buildserverid}') + .format(buildserverid=buildserverid)) + except Exception as e: + if type(buildserverid) is not str or not re.match('^[0-9a-f]{40}$', buildserverid): + logging.info(subprocess.check_output(['vagrant', 'status'], cwd="builder")) + raise FDroidException("Could not obtain buildserverid from buldserver VM. " + "(stored inside the buildserver VM at '/home/vagrant/buildserverid') " + "Please reset your buildserver, the setup VM is broken.") from e # Open SSH connection... logging.info("Connecting to virtual machine...") @@ -1160,7 +1167,7 @@ def main(): net.download_file(url, local_filename=of) except requests.exceptions.HTTPError as e: raise FDroidException( - 'Downloading Binaries from %s failed. %s' % (url, e)) + 'Downloading Binaries from %s failed.' % url) from e # Now we check whether the build can be verified to # match the supplied binary or not. Should the diff --git a/jenkins-setup-build-environment b/jenkins-setup-build-environment index cab721c6..906f0aff 100755 --- a/jenkins-setup-build-environment +++ b/jenkins-setup-build-environment @@ -32,12 +32,7 @@ trap cleanup_all INT TERM EXIT set -e set -x -# make sure we have the vagrant box image cached test -e ~/.cache/fdroidserver || mkdir -p ~/.cache/fdroidserver -cd ~/.cache/fdroidserver -wget --tries=1 --timeout=5 --continue https://f-droid.org/jessie64.box || true -echo "de3e3c4a9c13e8c015e30edeea0f583b195d1ee8ff9ad4814e933bbfb560200f jessie64.box" > jessie64.box.sha256 -sha256sum -c jessie64.box.sha256 # redirect homes to be in the git repo, so they'll get cleaned and reset export XDG_CONFIG_HOME=$WORKSPACE @@ -61,6 +56,8 @@ virsh -c qemu:///system vol-delete --pool default \ /var/lib/libvirt/images/jessie64_vagrant_box_image_0.img || true rm -rf "$WORKSPACE"/../*/.testfiles +vagrant box add --force fdroid-stretch64 ~/.cache/fdroidserver/fdroid-stretch64-libvirt.box + cd $WORKSPACE echo "debian_mirror = 'https://deb.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py diff --git a/makebuildserver b/makebuildserver index 6fc3de23..e681300a 100755 --- a/makebuildserver +++ b/makebuildserver @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os -import pathlib import re import requests import stat @@ -56,11 +55,8 @@ cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver') logger.debug('cachedir set to: %s', cachedir) config = { - 'basebox': 'jessie64', - 'baseboxurl': [ - pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri() - ], - 'debian_mirror': 'http://http.debian.net/debian/', + 'basebox': 'fdroid-stretch64', + 'debian_mirror': 'http://deb.debian.org/debian/', 'apt_package_cache': False, 'copy_caches_from_host': False, 'boot_timeout': 600, @@ -535,23 +531,25 @@ def main(): with open(vf, 'w', encoding='utf-8') as f: yaml.dump(config, f) - if config['vm_provider'] == 'libvirt': - available_providers = [x.provider for x in v.box_list() if x.name == config['basebox']] - found_basebox = len(available_providers) > 0 - needs_mutate = 'libvirt' not in available_providers - if not found_basebox: - if isinstance(config['baseboxurl'], str): - baseboxurl = config['baseboxurl'] - else: - baseboxurl = config['baseboxurl'][0] - logger.info('Adding %s from %s', config['basebox'], baseboxurl) - v.box_add(config['basebox'], baseboxurl) - needs_mutate = True - if needs_mutate: - logger.info('Converting %s to libvirt format', config['basebox']) - v._call_vagrant_command(['mutate', config['basebox'], 'libvirt']) - logger.info('Removing virtualbox format copy of %s', config['basebox']) - v.box_remove(config['basebox'], 'virtualbox') + # Check if selected provider is supported + if config['vm_provider'] not in ['libvirt', 'virtualbox']: + logger.critical("Currently selected VM provider '{vm_provider}' " + "is not supported. (please choose from: " + "virtualbox, libvirt)" + .format(vm_provider=config['cm_provider'])) + sys.exit(1) + # Check if selected basebox is available + available_boxes_by_provider = [x.name for x in v.box_list() if x.provider == config['vm_provider']] + if '/' not in config['basebox'] and config['basebox'] not in available_boxes_by_provider: + logger.critical("Vagrant box '{basebox}' not available " + "for '{vm_provider}' VM provider. " + "Please make sure it's added to vagrant. " + "(If you need a basebox to begin with, " + "here is how we're bootstrapping it: " + "https://gitlab.com/fdroid/basebox)" + .format(vm_provider=config['vm_provider'], + basebox=config['basebox'])) + sys.exit(1) logger.info("Configuring build server VM") debug_log_vagrant_vm(serverdir, config) @@ -559,7 +557,7 @@ def main(): v.up(provision=True) except subprocess.CalledProcessError: debug_log_vagrant_vm(serverdir, config) - logger.error("'vagrant up' failed, is the base box missing?") + logger.error("'vagrant up' failed.") sys.exit(1) if config['copy_caches_from_host']: