Merge branch 'jenkins-makebuildserver-works' into 'master'

Jenkins ./makebuildserver works!

These commits get `./makebuildserver` running in a Jenkins job. It builds the whole setup from scratch every time, including the vagrant and virtualbox setups.

Also, this modifies the git pre-commit hook to only run on the files being committed when called by git.

See merge request !95

examples/makebuildserver.config.py

@@ -21,13 +21,20 @@
 #
 # cachedir = 'buildserver/cache'
 
+# A big part of creating a new instance is downloading packages from Debian.
+# This setups up a folder in ~/.cache/fdroidserver to cache the downloaded
+# packages when rebuilding the build server from scratch.  This requires
+# that virtualbox-guest-utils is installed.
+#
+# apt_package_cache = True
+
 # To specify which Debian mirror the build server VM should use, by
 # default it uses http.debian.net, which auto-detects which is the
 # best mirror to use.
 #
 # debian_mirror = 'http://ftp.uk.debian.org/debian/'
 
-# The amount of RAM the build server will have
+# The amount of RAM the build server will have (default: 1024)
 # memory = 3584
 
 # The number of CPUs the build server will have

hooks/pre-commit

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 # Simple pre-commit hook to check that there are no errors in the fdroidserver
 # source files.
@@ -6,11 +6,45 @@
 # Redirect output to stderr.
 exec 1>&2
 
-PY_FILES="fdroid makebuildserver setup.py examples/*.py buildserver/*.py fdroidserver/*.py"
+files=`git diff-index --cached HEAD 2>&1 | sed 's/^:.*     //' | uniq | cut -b100-500`
-PY_TEST_FILES="tests/*.TestCase"
+if [ -z $files ]; then
-SH_FILES="hooks/pre-commit"
+    PY_FILES="fdroid makebuildserver setup.py examples/*.py buildserver/*.py fdroidserver/*.py"
-BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion"
+    PY_TEST_FILES="tests/*.TestCase"
-RB_FILES="buildserver/cookbooks/*/recipes/*.rb"
+    SH_FILES="hooks/pre-commit"
+    BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion"
+    RB_FILES="buildserver/cookbooks/*/recipes/*.rb"
+else
+    # if actually committing right now, then only run on the files
+    # that are going to be committed at this moment
+    PY_FILES=
+    PY_TEST_FILES=
+    SH_FILES=
+    BASH_FILES=
+    RB_FILES=
+
+    for f in $files; do
+        case $f in
+            *.py)
+                PY_FILES+=" $f"
+                ;;
+            *.TestCase)
+                PY_TEST_FILES+=" $f"
+                ;;
+            *.rb)
+                RB_FILES+=" $f"
+                ;;
+            *)
+                if head -1 $f | grep '^#!/bin/sh' > /dev/null 2>&1; then
+                    SH_FILES+=" $f"
+                elif head -1 $f | grep '^#!/bin/bash' > /dev/null 2>&1; then
+                    BASH_FILES+=" $f"
+                elif head -1 $f | grep '^#!.*python' > /dev/null 2>&1; then
+                    PY_FILES+=" $f"
+                fi
+                ;;
+        esac
+    done
+fi
 
 # In the default configuration, the checks E123, E133, E226, E241 and E242 are
 # ignored because they are not rules unanimously accepted
@@ -49,22 +83,27 @@ else
 	err "pep8 is not installed!"
 fi
 
-if ! $PYFLAKES $PY_FILES $PY_TEST_FILES; then
+if [ ! -z $PY_FILES $PY_TEST_FILES ]; then
+    if ! $PYFLAKES $PY_FILES $PY_TEST_FILES; then
 	err "pyflakes tests failed!"
+    fi
 fi
 
-if ! $PEP8 --ignore=$PEP8_IGNORE $PY_FILES; then
+if [ ! -z $PY_FILES ]; then
+    if ! $PEP8 --ignore=$PEP8_IGNORE $PY_FILES; then
 	err "pep8 tests failed!"
+    fi
 fi
 
 # The tests use a little hack in order to cleanly import the fdroidserver
 # package locally like a regular package.  pep8 doesn't see that, so this
 # makes pep8 skip E402 on the test files that need that hack.
-if ! $PEP8 --ignore=$PEP8_IGNORE,E402 $PY_TEST_FILES; then
+if [ ! -z $PY_TEST_FILES ]; then
+    if ! $PEP8 --ignore=$PEP8_IGNORE,E402 $PY_TEST_FILES; then
 	err "pep8 tests failed!"
+    fi
 fi
 
-
 for f in $SH_FILES; do
 	if ! dash -n $f; then
 		err "dash tests failed!"

jenkins-build-makebuildserver

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+if [ `dirname $0` != "." ]; then
+    echo "only run this script like ./`basename $0`"
+    exit
+fi
+
+if [ -z $WORKSPACE ]; then
+    WORKSPACE=`pwd`
+fi
+
+# make sure that no VirtualBox processes are left running
+cleanup_all() {
+    echo "$(date -u) - cleanup in progress..."
+    ps auxww|grep VBox
+    cd $WORKSPACE/buildserver
+    vagrant halt || true
+    sleep 5
+    killall VBoxHeadless || true
+    sleep 5
+    killall -9 VBoxHeadless || true
+    echo "$(date -u) - cleanup done."
+}
+trap cleanup_all INT TERM EXIT
+
+set -e
+set -x
+
+# make sure we have the vagrant box image cached
+test -e ~/.cache/fdroidserver || mkdir -p ~/.cache/fdroidserver
+cd ~/.cache/fdroidserver
+wget --tries=1 --timeout=5 --continue https://f-droid.org/jessie32.box || true
+echo "ff6b0c0bebcb742783becbc51a9dfff5a2a0a839bfcbfd0288dcd3113f33e533  jessie32.box" > jessie32.box.sha256
+sha256sum -c jessie32.box.sha256
+
+# redirect homes to be in the git repo, so they'll get cleaned and reset
+export XDG_CONFIG_HOME=$WORKSPACE
+export VBOX_USER_HOME=$WORKSPACE/VirtualBox
+mkdir $VBOX_USER_HOME
+VBoxManage setproperty machinefolder $WORKSPACE/virtualbox.d
+VBoxManage setproperty logginglevel debug
+export VAGRANT_HOME=$WORKSPACE/vagrant.d
+mkdir $VAGRANT_HOME
+
+cd $WORKSPACE
+echo "debian_mirror = 'http://ftp.uk.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py
+echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py
+echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py
+./makebuildserver
+
+# this can be handled in the jenkins job, or here:
+if [ ! -e fdroiddata ]; then
+    git clone --depth 1 --branch master --single-branch \
+        https://gitlab.com/fdroid/fdroiddata.git fdroiddata
+fi
+cd fdroiddata
+echo "build_server_always = True" > config.py
+../fdroid build info.guardianproject.checkey

makebuildserver

@@ -54,10 +54,11 @@ config = {
         'https://f-droid.org/jessie32.box',
     ],
     'debian_mirror': 'http://http.debian.net/debian/',
+    'apt_package_cache': False,
     'boot_timeout': 600,
     'cachedir': cachedir,
     'cpus': 1,
-    'memory': 3584,
+    'memory': 1024,
 }
 
 # load config file, if present
@@ -367,6 +368,21 @@ if 'aptproxy' in config and config['aptproxy']:
   config.vm.provision :shell, :inline => 'sudo echo "Acquire::http {{ Proxy \\"{0}\\"; }};" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update'
 """.format(config['aptproxy'])
 
+# buildserver/ is shared to the VM's /vagrant by default so the old default
+# does not need a custom mount
+if cachedir != 'buildserver/cache':
+    vagrantfile += """
+  config.vm.synced_folder '{0}', '/vagrant/cache'
+""".format(cachedir)
+
+# cache .deb packages on the host via a mount trick
+if config['apt_package_cache']:
+    aptcachedir = cachedir + '/apt/archives'
+    vagrantfile += """
+  config.vm.synced_folder "{0}", "/var/cache/apt/archives",
+    owner: 'root', group: 'root', create: true
+""".format(aptcachedir)
+
 vagrantfile += """
   config.vm.provision :chef_solo do |chef|
     chef.cookbooks_path = "cookbooks"