diff --git a/examples/makebuildserver.config.py b/examples/makebuildserver.config.py index cc536844..9eae743c 100644 --- a/examples/makebuildserver.config.py +++ b/examples/makebuildserver.config.py @@ -21,13 +21,20 @@ # # cachedir = 'buildserver/cache' +# A big part of creating a new instance is downloading packages from Debian. +# This setups up a folder in ~/.cache/fdroidserver to cache the downloaded +# packages when rebuilding the build server from scratch. This requires +# that virtualbox-guest-utils is installed. +# +# apt_package_cache = True + # To specify which Debian mirror the build server VM should use, by # default it uses http.debian.net, which auto-detects which is the # best mirror to use. # # debian_mirror = 'http://ftp.uk.debian.org/debian/' -# The amount of RAM the build server will have +# The amount of RAM the build server will have (default: 1024) # memory = 3584 # The number of CPUs the build server will have diff --git a/hooks/pre-commit b/hooks/pre-commit index 4cc15059..d33c7018 100755 --- a/hooks/pre-commit +++ b/hooks/pre-commit @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Simple pre-commit hook to check that there are no errors in the fdroidserver # source files. @@ -6,11 +6,45 @@ # Redirect output to stderr. exec 1>&2 -PY_FILES="fdroid makebuildserver setup.py examples/*.py buildserver/*.py fdroidserver/*.py" -PY_TEST_FILES="tests/*.TestCase" -SH_FILES="hooks/pre-commit" -BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion" -RB_FILES="buildserver/cookbooks/*/recipes/*.rb" +files=`git diff-index --cached HEAD 2>&1 | sed 's/^:.* //' | uniq | cut -b100-500` +if [ -z $files ]; then + PY_FILES="fdroid makebuildserver setup.py examples/*.py buildserver/*.py fdroidserver/*.py" + PY_TEST_FILES="tests/*.TestCase" + SH_FILES="hooks/pre-commit" + BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion" + RB_FILES="buildserver/cookbooks/*/recipes/*.rb" +else + # if actually committing right now, then only run on the files + # that are going to be committed at this moment + PY_FILES= + PY_TEST_FILES= + SH_FILES= + BASH_FILES= + RB_FILES= + + for f in $files; do + case $f in + *.py) + PY_FILES+=" $f" + ;; + *.TestCase) + PY_TEST_FILES+=" $f" + ;; + *.rb) + RB_FILES+=" $f" + ;; + *) + if head -1 $f | grep '^#!/bin/sh' > /dev/null 2>&1; then + SH_FILES+=" $f" + elif head -1 $f | grep '^#!/bin/bash' > /dev/null 2>&1; then + BASH_FILES+=" $f" + elif head -1 $f | grep '^#!.*python' > /dev/null 2>&1; then + PY_FILES+=" $f" + fi + ;; + esac + done +fi # In the default configuration, the checks E123, E133, E226, E241 and E242 are # ignored because they are not rules unanimously accepted @@ -49,22 +83,27 @@ else err "pep8 is not installed!" fi -if ! $PYFLAKES $PY_FILES $PY_TEST_FILES; then +if [ ! -z $PY_FILES $PY_TEST_FILES ]; then + if ! $PYFLAKES $PY_FILES $PY_TEST_FILES; then err "pyflakes tests failed!" + fi fi -if ! $PEP8 --ignore=$PEP8_IGNORE $PY_FILES; then +if [ ! -z $PY_FILES ]; then + if ! $PEP8 --ignore=$PEP8_IGNORE $PY_FILES; then err "pep8 tests failed!" + fi fi # The tests use a little hack in order to cleanly import the fdroidserver # package locally like a regular package. pep8 doesn't see that, so this # makes pep8 skip E402 on the test files that need that hack. -if ! $PEP8 --ignore=$PEP8_IGNORE,E402 $PY_TEST_FILES; then +if [ ! -z $PY_TEST_FILES ]; then + if ! $PEP8 --ignore=$PEP8_IGNORE,E402 $PY_TEST_FILES; then err "pep8 tests failed!" + fi fi - for f in $SH_FILES; do if ! dash -n $f; then err "dash tests failed!" diff --git a/jenkins-build-makebuildserver b/jenkins-build-makebuildserver new file mode 100755 index 00000000..76a13a25 --- /dev/null +++ b/jenkins-build-makebuildserver @@ -0,0 +1,58 @@ +#!/bin/bash + +if [ `dirname $0` != "." ]; then + echo "only run this script like ./`basename $0`" + exit +fi + +if [ -z $WORKSPACE ]; then + WORKSPACE=`pwd` +fi + +# make sure that no VirtualBox processes are left running +cleanup_all() { + echo "$(date -u) - cleanup in progress..." + ps auxww|grep VBox + cd $WORKSPACE/buildserver + vagrant halt || true + sleep 5 + killall VBoxHeadless || true + sleep 5 + killall -9 VBoxHeadless || true + echo "$(date -u) - cleanup done." +} +trap cleanup_all INT TERM EXIT + +set -e +set -x + +# make sure we have the vagrant box image cached +test -e ~/.cache/fdroidserver || mkdir -p ~/.cache/fdroidserver +cd ~/.cache/fdroidserver +wget --tries=1 --timeout=5 --continue https://f-droid.org/jessie32.box || true +echo "ff6b0c0bebcb742783becbc51a9dfff5a2a0a839bfcbfd0288dcd3113f33e533 jessie32.box" > jessie32.box.sha256 +sha256sum -c jessie32.box.sha256 + +# redirect homes to be in the git repo, so they'll get cleaned and reset +export XDG_CONFIG_HOME=$WORKSPACE +export VBOX_USER_HOME=$WORKSPACE/VirtualBox +mkdir $VBOX_USER_HOME +VBoxManage setproperty machinefolder $WORKSPACE/virtualbox.d +VBoxManage setproperty logginglevel debug +export VAGRANT_HOME=$WORKSPACE/vagrant.d +mkdir $VAGRANT_HOME + +cd $WORKSPACE +echo "debian_mirror = 'http://ftp.uk.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py +echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py +echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py +./makebuildserver + +# this can be handled in the jenkins job, or here: +if [ ! -e fdroiddata ]; then + git clone --depth 1 --branch master --single-branch \ + https://gitlab.com/fdroid/fdroiddata.git fdroiddata +fi +cd fdroiddata +echo "build_server_always = True" > config.py +../fdroid build info.guardianproject.checkey diff --git a/makebuildserver b/makebuildserver index aeb5ebbc..0ec3eebe 100755 --- a/makebuildserver +++ b/makebuildserver @@ -54,10 +54,11 @@ config = { 'https://f-droid.org/jessie32.box', ], 'debian_mirror': 'http://http.debian.net/debian/', + 'apt_package_cache': False, 'boot_timeout': 600, 'cachedir': cachedir, 'cpus': 1, - 'memory': 3584, + 'memory': 1024, } # load config file, if present @@ -367,6 +368,21 @@ if 'aptproxy' in config and config['aptproxy']: config.vm.provision :shell, :inline => 'sudo echo "Acquire::http {{ Proxy \\"{0}\\"; }};" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update' """.format(config['aptproxy']) +# buildserver/ is shared to the VM's /vagrant by default so the old default +# does not need a custom mount +if cachedir != 'buildserver/cache': + vagrantfile += """ + config.vm.synced_folder '{0}', '/vagrant/cache' +""".format(cachedir) + +# cache .deb packages on the host via a mount trick +if config['apt_package_cache']: + aptcachedir = cachedir + '/apt/archives' + vagrantfile += """ + config.vm.synced_folder "{0}", "/var/cache/apt/archives", + owner: 'root', group: 'root', create: true +""".format(aptcachedir) + vagrantfile += """ config.vm.provision :chef_solo do |chef| chef.cookbooks_path = "cookbooks"