From 7c30569aed884efd4f64a3881dd2e941ccf5d200 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 3 Feb 2016 14:48:40 +0100 Subject: [PATCH 1/6] make git pre-commit hook only test files to be committed Right now, the git pre-commit hook is pretty annoying to work with when development since it tests every single file. That means notes, incomplete stuff, etc. will be run through the tests. So all of the files need to be clean in order to commit even a single trivial fix. This commit changes it so that the pre-commit hook is only run on the files staged to be committed. --- hooks/pre-commit | 59 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/hooks/pre-commit b/hooks/pre-commit index 4cc15059..d33c7018 100755 --- a/hooks/pre-commit +++ b/hooks/pre-commit @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Simple pre-commit hook to check that there are no errors in the fdroidserver # source files. @@ -6,11 +6,45 @@ # Redirect output to stderr. exec 1>&2 -PY_FILES="fdroid makebuildserver setup.py examples/*.py buildserver/*.py fdroidserver/*.py" -PY_TEST_FILES="tests/*.TestCase" -SH_FILES="hooks/pre-commit" -BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion" -RB_FILES="buildserver/cookbooks/*/recipes/*.rb" +files=`git diff-index --cached HEAD 2>&1 | sed 's/^:.* //' | uniq | cut -b100-500` +if [ -z $files ]; then + PY_FILES="fdroid makebuildserver setup.py examples/*.py buildserver/*.py fdroidserver/*.py" + PY_TEST_FILES="tests/*.TestCase" + SH_FILES="hooks/pre-commit" + BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion" + RB_FILES="buildserver/cookbooks/*/recipes/*.rb" +else + # if actually committing right now, then only run on the files + # that are going to be committed at this moment + PY_FILES= + PY_TEST_FILES= + SH_FILES= + BASH_FILES= + RB_FILES= + + for f in $files; do + case $f in + *.py) + PY_FILES+=" $f" + ;; + *.TestCase) + PY_TEST_FILES+=" $f" + ;; + *.rb) + RB_FILES+=" $f" + ;; + *) + if head -1 $f | grep '^#!/bin/sh' > /dev/null 2>&1; then + SH_FILES+=" $f" + elif head -1 $f | grep '^#!/bin/bash' > /dev/null 2>&1; then + BASH_FILES+=" $f" + elif head -1 $f | grep '^#!.*python' > /dev/null 2>&1; then + PY_FILES+=" $f" + fi + ;; + esac + done +fi # In the default configuration, the checks E123, E133, E226, E241 and E242 are # ignored because they are not rules unanimously accepted @@ -49,22 +83,27 @@ else err "pep8 is not installed!" fi -if ! $PYFLAKES $PY_FILES $PY_TEST_FILES; then +if [ ! -z $PY_FILES $PY_TEST_FILES ]; then + if ! $PYFLAKES $PY_FILES $PY_TEST_FILES; then err "pyflakes tests failed!" + fi fi -if ! $PEP8 --ignore=$PEP8_IGNORE $PY_FILES; then +if [ ! -z $PY_FILES ]; then + if ! $PEP8 --ignore=$PEP8_IGNORE $PY_FILES; then err "pep8 tests failed!" + fi fi # The tests use a little hack in order to cleanly import the fdroidserver # package locally like a regular package. pep8 doesn't see that, so this # makes pep8 skip E402 on the test files that need that hack. -if ! $PEP8 --ignore=$PEP8_IGNORE,E402 $PY_TEST_FILES; then +if [ ! -z $PY_TEST_FILES ]; then + if ! $PEP8 --ignore=$PEP8_IGNORE,E402 $PY_TEST_FILES; then err "pep8 tests failed!" + fi fi - for f in $SH_FILES; do if ! dash -n $f; then err "dash tests failed!" From b6a5978b11856a46599708e0c2cfe12ef1861f87 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 26 Jan 2016 22:06:10 +0100 Subject: [PATCH 2/6] makebuildserver: make chef find the custom cache locations --- makebuildserver | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/makebuildserver b/makebuildserver index aeb5ebbc..77c6063c 100755 --- a/makebuildserver +++ b/makebuildserver @@ -367,6 +367,13 @@ if 'aptproxy' in config and config['aptproxy']: config.vm.provision :shell, :inline => 'sudo echo "Acquire::http {{ Proxy \\"{0}\\"; }};" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update' """.format(config['aptproxy']) +# buildserver/ is shared to the VM's /vagrant by default so the old default +# does not need a custom mount +if cachedir != 'buildserver/cache': + vagrantfile += """ + config.vm.synced_folder '{0}', '/vagrant/cache' +""".format(cachedir) + vagrantfile += """ config.vm.provision :chef_solo do |chef| chef.cookbooks_path = "cookbooks" From dd93505fceb3d4a8293a0b153f2e09f9483f1247 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 26 Jan 2016 11:21:21 +0100 Subject: [PATCH 3/6] allow setting up an apt package cache for the build server setup This creates a cache folder which will store the apt cache from the VM. --- examples/makebuildserver.config.py | 7 +++++++ makebuildserver | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/examples/makebuildserver.config.py b/examples/makebuildserver.config.py index cc536844..4747dc88 100644 --- a/examples/makebuildserver.config.py +++ b/examples/makebuildserver.config.py @@ -21,6 +21,13 @@ # # cachedir = 'buildserver/cache' +# A big part of creating a new instance is downloading packages from Debian. +# This setups up a folder in ~/.cache/fdroidserver to cache the downloaded +# packages when rebuilding the build server from scratch. This requires +# that virtualbox-guest-utils is installed. +# +# apt_package_cache = True + # To specify which Debian mirror the build server VM should use, by # default it uses http.debian.net, which auto-detects which is the # best mirror to use. diff --git a/makebuildserver b/makebuildserver index 77c6063c..6f476625 100755 --- a/makebuildserver +++ b/makebuildserver @@ -54,6 +54,7 @@ config = { 'https://f-droid.org/jessie32.box', ], 'debian_mirror': 'http://http.debian.net/debian/', + 'apt_package_cache': False, 'boot_timeout': 600, 'cachedir': cachedir, 'cpus': 1, @@ -374,6 +375,14 @@ if cachedir != 'buildserver/cache': config.vm.synced_folder '{0}', '/vagrant/cache' """.format(cachedir) +# cache .deb packages on the host via a mount trick +if config['apt_package_cache']: + aptcachedir = cachedir + '/apt/archives' + vagrantfile += """ + config.vm.synced_folder "{0}", "/var/cache/apt/archives", + owner: 'root', group: 'root', create: true +""".format(aptcachedir) + vagrantfile += """ config.vm.provision :chef_solo do |chef| chef.cookbooks_path = "cookbooks" From 09daa5eee0dd0ccb4e6b7a1daac37733fcb332d5 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 28 Jan 2016 13:00:18 +0100 Subject: [PATCH 4/6] makebuildserver: default memory to 1024MB, so it runs on normal machines 4 gigs is still a common amount of RAM these days for laptops, if the VM takes almost all of that, it makes the machine drag to almost a halt. Most apps build fine in 1gig of RAM, indeed that's the default for most CI instances, like travis-ci and gitlab-ci. --- examples/makebuildserver.config.py | 2 +- makebuildserver | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/makebuildserver.config.py b/examples/makebuildserver.config.py index 4747dc88..9eae743c 100644 --- a/examples/makebuildserver.config.py +++ b/examples/makebuildserver.config.py @@ -34,7 +34,7 @@ # # debian_mirror = 'http://ftp.uk.debian.org/debian/' -# The amount of RAM the build server will have +# The amount of RAM the build server will have (default: 1024) # memory = 3584 # The number of CPUs the build server will have diff --git a/makebuildserver b/makebuildserver index 6f476625..0ec3eebe 100755 --- a/makebuildserver +++ b/makebuildserver @@ -58,7 +58,7 @@ config = { 'boot_timeout': 600, 'cachedir': cachedir, 'cpus': 1, - 'memory': 3584, + 'memory': 1024, } # load config file, if present From 2d2ac66a26f9a53dfb699429302d900876377926 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 3 Feb 2016 22:00:48 +0100 Subject: [PATCH 5/6] script to run ./makebuildserver on Jenkins instances This is being used to run it on both Guardian Project's and Debian's Jenkins instances. --- jenkins-build-makebuildserver | 50 +++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100755 jenkins-build-makebuildserver diff --git a/jenkins-build-makebuildserver b/jenkins-build-makebuildserver new file mode 100755 index 00000000..5b9c4228 --- /dev/null +++ b/jenkins-build-makebuildserver @@ -0,0 +1,50 @@ +#!/bin/bash + +if [ `dirname $0` != "." ]; then + echo "only run this script like ./`basename $0`" + exit +fi + +if [ -z $WORKSPACE ]; then + WORKSPACE=`pwd` +fi + +# make sure that no VirtualBox processes are left running +cleanup_all() { + echo "$(date -u) - cleanup in progress..." + ps auxww|grep VBox + cd $WORKSPACE/buildserver + vagrant halt || true + sleep 5 + killall VBoxHeadless || true + sleep 5 + killall -9 VBoxHeadless || true + echo "$(date -u) - cleanup done." +} +trap cleanup_all INT TERM EXIT + +set -e +set -x + +# make sure we have the vagrant box image cached +test -e ~/.cache/fdroidserver || mkdir -p ~/.cache/fdroidserver +cd ~/.cache/fdroidserver +wget --tries=1 --timeout=5 --continue https://f-droid.org/jessie32.box || true +echo "ff6b0c0bebcb742783becbc51a9dfff5a2a0a839bfcbfd0288dcd3113f33e533 jessie32.box" > jessie32.box.sha256 +sha256sum -c jessie32.box.sha256 + +# redirect homes to be in the git repo, so they'll get cleaned and reset +export XDG_CONFIG_HOME=$WORKSPACE +export VBOX_USER_HOME=$WORKSPACE/VirtualBox +mkdir $VBOX_USER_HOME +VBoxManage setproperty machinefolder $WORKSPACE/virtualbox.d +VBoxManage setproperty logginglevel debug +export VAGRANT_HOME=$WORKSPACE/vagrant.d +mkdir $VAGRANT_HOME + +cd $WORKSPACE +echo "debian_mirror = 'http://ftp.uk.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py +echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py +echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py +./makebuildserver + From 2ea38fa7e3eb18a8d8e1d898a2fbbccd09240c43 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 3 Feb 2016 22:02:50 +0100 Subject: [PATCH 6/6] test ./makebuildserver by running a build in jenkins --- jenkins-build-makebuildserver | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/jenkins-build-makebuildserver b/jenkins-build-makebuildserver index 5b9c4228..76a13a25 100755 --- a/jenkins-build-makebuildserver +++ b/jenkins-build-makebuildserver @@ -48,3 +48,11 @@ echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py ./makebuildserver +# this can be handled in the jenkins job, or here: +if [ ! -e fdroiddata ]; then + git clone --depth 1 --branch master --single-branch \ + https://gitlab.com/fdroid/fdroiddata.git fdroiddata +fi +cd fdroiddata +echo "build_server_always = True" > config.py +../fdroid build info.guardianproject.checkey