Merge branch '482-upgrade-buildserver-to-stretch' into 'master'

upgrade buildserver basebox to stretch

Closes #514 and #482

See merge request fdroid/fdroidserver!514

buildserver/provision-apt-get-install

@@ -20,10 +20,10 @@ if echo $debian_mirror | grep '^https' 2>&1 > /dev/null; then
     apt-get -y install apt-transport-https
 fi
 
-sed -i "s,http://ftp.uk.debian.org/debian/,${debian_mirror},g" /etc/apt/sources.list
+sed -Ei "s,^deb [a-z]+://\S+,deb ${debian_mirror}," /etc/apt/sources.list
 
-if grep --quiet jessie /etc/apt/sources.list; then
+if grep --quiet stretch /etc/apt/sources.list; then
-    echo "deb $debian_mirror jessie-backports main" > /etc/apt/sources.list.d/backports.list
+    echo "deb $debian_mirror stretch-backports main" > /etc/apt/sources.list.d/stretch-backports.list
     echo "deb $debian_mirror testing main" > /etc/apt/sources.list.d/testing.list
     printf "Package: *\nPin: release o=Debian,a=testing\nPin-Priority: -300\n" > /etc/apt/preferences.d/debian-testing
 fi
@@ -45,16 +45,16 @@ packages="
  autopoint
  bison
  bzr
- ca-certificates-java/jessie-backports
+ ca-certificates-java
- cmake/jessie-backports
+ cmake
  curl
  expect
  faketime
  flex
- gettext/jessie-backports
+ gettext
- gettext-base/jessie-backports
+ gettext-base
- git-core/jessie-backports
+ git-core
- git-svn/jessie-backports
+ git-svn
  gperf
  graphviz
  imagemagick
@@ -75,7 +75,7 @@ packages="
  libtool-bin/testing
  make
  maven
- mercurial/jessie-backports
+ mercurial
  nasm
  nodejs
  openjdk-8-jre-headless
@@ -90,13 +90,13 @@ packages="
  python-magic
  python-setuptools
  python3-defusedxml
- python3-git/jessie-backports
+ python3-git
- python3-gitdb/jessie-backports
+ python3-gitdb
  python3-gnupg
  python3-pyasn1
  python3-pyasn1-modules
  python3-requests
- python3-smmap/jessie-backports
+ python3-smmap
  python3-yaml
  python3-ruamel.yaml
  quilt
@@ -120,3 +120,6 @@ apt-get install --yes $packages
 
 highestjava=`update-java-alternatives --list | sort -n | tail -1 | cut -d ' ' -f 1`
 update-java-alternatives --set $highestjava
+
+# configure headless openjdk to work without gtk accessability dependencies
+sed -i -e 's@\(assistive_technologies=org.GNOME.Accessibility.AtkWrapper\)@#\1@' /etc/java-8-openjdk/accessibility.properties

examples/makebuildserver.config.py

@@ -2,17 +2,12 @@
 #
 # You may want to alter these before running ./makebuildserver
 
-# Name of the base box to use
+# Name of the Vagrant basebox to use, by default it will be downloaded
-# basebox = "jessie64"
+# from Vagrant Cloud.  For release builds setup, generate the basebox
-
+# locally using https://gitlab.com/fdroid/basebox, add it to Vagrant,
-# Location where testing32.box can be found, if you don't already have
+# then set this to the local basebox name.
-# it. For security reasons, it's recommended that you make your own
-# in a secure environment using trusted media (see the manual) but
-# you can use this default if you like...
-# baseboxurl = "https://f-droid.org/jessie64.box"
 #
-# or if you have a cached local copy, you can use that first:
+# basebox = "fdroid-stretch64"
-# baseboxurl = ["file:///home/fdroid/fdroidserver/cache/jessie64.box", "https://f-droid.org/jessie64.box"]
 
 # In the process of setting up the build server, many gigs of files
 # are downloaded (Android SDK components, gradle, etc).  These are

fdroidserver/build.py

@@ -67,8 +67,8 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
 
     try:
         paramiko
-    except NameError:
+    except NameError as e:
-        raise BuildException("Paramiko is required to use the buildserver")
+        raise BuildException("Paramiko is required to use the buildserver") from e
     if options.verbose:
         logging.getLogger("paramiko").setLevel(logging.INFO)
     else:
@@ -79,11 +79,18 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
     output = None
     try:
         if not buildserverid:
-            buildserverid = subprocess.check_output(['vagrant', 'ssh', '-c',
+            try:
-                                                     'cat /home/vagrant/buildserverid'],
+                buildserverid = subprocess.check_output(['vagrant', 'ssh', '-c',
-                                                    cwd='builder').strip().decode()
+                                                         'cat /home/vagrant/buildserverid'],
-            logging.debug(_('Fetched buildserverid from VM: {buildserverid}')
+                                                        cwd='builder').strip().decode()
-                          .format(buildserverid=buildserverid))
+                logging.debug(_('Fetched buildserverid from VM: {buildserverid}')
+                              .format(buildserverid=buildserverid))
+            except Exception as e:
+                if type(buildserverid) is not str or not re.match('^[0-9a-f]{40}$', buildserverid):
+                    logging.info(subprocess.check_output(['vagrant', 'status'], cwd="builder"))
+                    raise FDroidException("Could not obtain buildserverid from buldserver VM. "
+                                          "(stored inside the buildserver VM at '/home/vagrant/buildserverid') "
+                                          "Please reset your buildserver, the setup VM is broken.") from e
 
         # Open SSH connection...
         logging.info("Connecting to virtual machine...")
@@ -1160,7 +1167,7 @@ def main():
                             net.download_file(url, local_filename=of)
                         except requests.exceptions.HTTPError as e:
                             raise FDroidException(
-                                'Downloading Binaries from %s failed. %s' % (url, e))
+                                'Downloading Binaries from %s failed.' % url) from e
 
                         # Now we check whether the build can be verified to
                         # match the supplied binary or not. Should the

jenkins-setup-build-environment

@@ -32,12 +32,7 @@ trap cleanup_all INT TERM EXIT
 set -e
 set -x
 
-# make sure we have the vagrant box image cached
 test -e ~/.cache/fdroidserver || mkdir -p ~/.cache/fdroidserver
-cd ~/.cache/fdroidserver
-wget --tries=1 --timeout=5 --continue https://f-droid.org/jessie64.box || true
-echo "de3e3c4a9c13e8c015e30edeea0f583b195d1ee8ff9ad4814e933bbfb560200f  jessie64.box" > jessie64.box.sha256
-sha256sum -c jessie64.box.sha256
 
 # redirect homes to be in the git repo, so they'll get cleaned and reset
 export XDG_CONFIG_HOME=$WORKSPACE
@@ -61,6 +56,8 @@ virsh -c qemu:///system vol-delete --pool default \
       /var/lib/libvirt/images/jessie64_vagrant_box_image_0.img || true
 rm -rf "$WORKSPACE"/../*/.testfiles
 
+vagrant box add --force fdroid-stretch64 ~/.cache/fdroidserver/fdroid-stretch64-libvirt.box
+
 cd $WORKSPACE
 echo "debian_mirror = 'https://deb.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py
 echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py

makebuildserver

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import os
-import pathlib
 import re
 import requests
 import stat
@@ -56,11 +55,8 @@ cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
 logger.debug('cachedir set to: %s', cachedir)
 
 config = {
-    'basebox': 'jessie64',
+    'basebox': 'fdroid-stretch64',
-    'baseboxurl': [
+    'debian_mirror': 'http://deb.debian.org/debian/',
-        pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri()
-    ],
-    'debian_mirror': 'http://http.debian.net/debian/',
     'apt_package_cache': False,
     'copy_caches_from_host': False,
     'boot_timeout': 600,
@@ -535,23 +531,25 @@ def main():
         with open(vf, 'w', encoding='utf-8') as f:
             yaml.dump(config, f)
 
-    if config['vm_provider'] == 'libvirt':
+    # Check if selected provider is supported
-        available_providers = [x.provider for x in v.box_list() if x.name == config['basebox']]
+    if config['vm_provider'] not in ['libvirt', 'virtualbox']:
-        found_basebox = len(available_providers) > 0
+        logger.critical("Currently selected VM provider '{vm_provider}' "
-        needs_mutate = 'libvirt' not in available_providers
+                        "is not supported. (please choose from: "
-        if not found_basebox:
+                        "virtualbox, libvirt)"
-            if isinstance(config['baseboxurl'], str):
+                        .format(vm_provider=config['cm_provider']))
-                baseboxurl = config['baseboxurl']
+        sys.exit(1)
-            else:
+    # Check if selected basebox is available
-                baseboxurl = config['baseboxurl'][0]
+    available_boxes_by_provider = [x.name for x in v.box_list() if x.provider == config['vm_provider']]
-            logger.info('Adding %s from %s', config['basebox'], baseboxurl)
+    if '/' not in config['basebox'] and config['basebox'] not in available_boxes_by_provider:
-            v.box_add(config['basebox'], baseboxurl)
+        logger.critical("Vagrant box '{basebox}' not available "
-            needs_mutate = True
+                        "for '{vm_provider}' VM provider. "
-        if needs_mutate:
+                        "Please make sure it's added to vagrant. "
-            logger.info('Converting %s to libvirt format', config['basebox'])
+                        "(If you need a basebox to begin with, "
-            v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
+                        "here is how we're bootstrapping it: "
-            logger.info('Removing virtualbox format copy of %s', config['basebox'])
+                        "https://gitlab.com/fdroid/basebox)"
-            v.box_remove(config['basebox'], 'virtualbox')
+                        .format(vm_provider=config['vm_provider'],
+                                basebox=config['basebox']))
+        sys.exit(1)
 
     logger.info("Configuring build server VM")
     debug_log_vagrant_vm(serverdir, config)
@@ -559,7 +557,7 @@ def main():
         v.up(provision=True)
     except subprocess.CalledProcessError:
         debug_log_vagrant_vm(serverdir, config)
-        logger.error("'vagrant up' failed, is the base box missing?")
+        logger.error("'vagrant up' failed.")
         sys.exit(1)
 
     if config['copy_caches_from_host']: