From dc2f53b48c854b40204ff9fbc3cd99381ec4a79c Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 30 Aug 2016 21:23:40 +0200 Subject: [PATCH 1/6] buildserver: vagrant-cachier conflicts with custom apt cache The technique where /var/cache/apt is mounted as a shared folder conflicts with vagrant-cachier's workings. Therefore, ignore vagrant-cachier if the user selects ./makebuildserver's custom apt cache. The shared folder way has the advantage for CI builds of storing the cache outside of VAGRANT_HOME, which is set to be in the git project. That gets wiped by `git clean -fdx` on each CI build. --- buildserver/Vagrantfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildserver/Vagrantfile b/buildserver/Vagrantfile index e586e05c..a7b1745f 100644 --- a/buildserver/Vagrantfile +++ b/buildserver/Vagrantfile @@ -4,7 +4,8 @@ configfile = YAML.load_file('Vagrantfile.yaml') Vagrant.configure("2") do |config| - if Vagrant.has_plugin?("vagrant-cachier") + # these two caching methods conflict, so only use one at a time + if Vagrant.has_plugin?("vagrant-cachier") and not configfile.has_key? "aptcachedir" config.cache.scope = :box config.cache.auto_detect = false config.cache.enable :apt From b717271c6147731b5be611aed482a1757d6f9b0b Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 30 Aug 2016 22:56:06 +0200 Subject: [PATCH 2/6] buildserver: force a known-good version of chef This forces the release channel and version of chef-solo to install on the guest VM. I was getting really massive, odd stacktraces without specifying this, and chef is only used for Kivy now anyway. --- buildserver/Vagrantfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildserver/Vagrantfile b/buildserver/Vagrantfile index a7b1745f..3e04fbc1 100644 --- a/buildserver/Vagrantfile +++ b/buildserver/Vagrantfile @@ -49,6 +49,8 @@ Vagrant.configure("2") do |config| args: [configfile['debian_mirror']] config.vm.provision :chef_solo do |chef| + chef.channel = "stable" + chef.version = "12.10.24" chef.cookbooks_path = "cookbooks" chef.log_level = :debug chef.add_recipe "kivy" From 73947092fa8dad585498943564291d4e0aa93627 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 7 Jul 2016 21:44:55 +0200 Subject: [PATCH 3/6] buildserver: test build of Checkey and old F-Droid The buildserver should still be able to build old versions of apps, and Checkey is a verified, reproducible build. --- jenkins-build-makebuildserver | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/jenkins-build-makebuildserver b/jenkins-build-makebuildserver index 120ce798..b78c4adf 100755 --- a/jenkins-build-makebuildserver +++ b/jenkins-build-makebuildserver @@ -63,12 +63,8 @@ echo "build_server_always = True" > config.py # if it can't build fdroid, then its really broken ../fdroid build --verbose --stop --latest org.fdroid.fdroid # Gradle, JNI, preassemble -../fdroid build --stop org.adaway:55 -# Maven -#../fdroid build --stop org.quantumbadger.redreader:55 -# Custom build (make) -#../fdroid build --stop com.amaze.filemanager:29 +../fdroid build --verbose --stop org.adaway:55 # Uses verification -#../fdroid build --stop info.guardianproject.checkey:101 -# Gradle with retrolambda (JDK7 and JDK8) -#../fdroid build --stop com.moez.QKSMS:124 +../fdroid build --verbose --stop info.guardianproject.checkey:101 +# building old versions should still work +../fdroid build --verbose --stop org.fdroid.fdroid:96150 From 7b64bdcf0b241f8398c854c96dde1f90caf38601 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 8 Sep 2016 21:47:42 +0200 Subject: [PATCH 4/6] buildserver: only specify lib*-dev to future proof package list Using libssl-dev will work on all releases of Debian, but Debian/stretch does not have libssl1.0.0. --- buildserver/provision-apt-get-install | 1 - 1 file changed, 1 deletion(-) diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index 86cf7de7..deeb0c66 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -50,7 +50,6 @@ packages=" librsvg2-bin libsaxonb-java libssl-dev - libssl1.0.0 libstdc++6:i386 libtool libtool-bin From 91c1e8313f44e94e594befe094c687fdbca010af Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 6 Sep 2016 14:33:13 +0200 Subject: [PATCH 5/6] buildserver: download apt package first to increase reliability This does not have the careful result rechecking that chef has, when it installs each package in the list one at a time. So to help with failures caused by a package failing to download, first try downloading all the package, then run the install. The install pass will try to download any missing packages. Really, this should use ansible or perhaps chef again since those include lots of tricks around this stuff. --- buildserver/provision-apt-get-install | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index deeb0c66..09909050 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -8,6 +8,9 @@ debian_mirror=$1 sed -i "s,http://ftp.uk.debian.org/debian/,${debian_mirror},g" /etc/apt/sources.list +printf 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";\n' \ + > /etc/apt/apt.conf.d/99no-install-recommends + if grep --quiet jessie /etc/apt/sources.list; then echo "deb $debian_mirror jessie-backports main" > /etc/apt/sources.list.d/backports.list fi @@ -15,6 +18,7 @@ fi dpkg --add-architecture i386 apt-get -y update +apt-get -y upgrade --download-only apt-get -y upgrade packages=" @@ -87,8 +91,8 @@ packages=" zip zlib1g:i386 " - -apt-get install --yes --no-install-recommends $packages +apt-get install --yes $packages --download-only +apt-get install --yes $packages highestjava=`update-java-alternatives --list | sort -n | tail -1 | cut -d ' ' -f 1` update-java-alternatives --set $highestjava From 14de399bb7114b824062f2e6618efaf09254588d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 12 Sep 2016 13:05:05 +0200 Subject: [PATCH 6/6] buildserver: remove Kivy, unused since 2013 and out of date This is the last thing using Chef, which adds a lot of time to the time it takes to fully provision the buildserver. This slows down development on the things we are actually using, like running all builds on jenkins.debian.net. #210 #165 --- buildserver/Vagrantfile | 11 ------ buildserver/cookbooks/kivy/recipes/default.rb | 34 ------------------- buildserver/fixpaths.sh | 25 -------------- hooks/pre-commit | 2 +- makebuildserver | 2 -- 5 files changed, 1 insertion(+), 73 deletions(-) delete mode 100644 buildserver/cookbooks/kivy/recipes/default.rb delete mode 100644 buildserver/fixpaths.sh diff --git a/buildserver/Vagrantfile b/buildserver/Vagrantfile index 3e04fbc1..51db27d3 100644 --- a/buildserver/Vagrantfile +++ b/buildserver/Vagrantfile @@ -23,8 +23,6 @@ Vagrant.configure("2") do |config| config.vm.boot_timeout = configfile['boot_timeout'] - config.vm.provision :shell, :path => "fixpaths.sh" - if configfile.has_key? "aptproxy" config.vm.provision :shell, path: "provision-apt-proxy", args: [configfile["aptproxy"]] @@ -47,15 +45,6 @@ Vagrant.configure("2") do |config| args: ["/home/vagrant/android-sdk"] config.vm.provision "shell", path: "provision-apt-get-install", args: [configfile['debian_mirror']] - - config.vm.provision :chef_solo do |chef| - chef.channel = "stable" - chef.version = "12.10.24" - chef.cookbooks_path = "cookbooks" - chef.log_level = :debug - chef.add_recipe "kivy" - end - config.vm.provision "shell", path: "provision-android-sdk" config.vm.provision "shell", path: "provision-android-ndk", args: ["/home/vagrant/android-ndk"] diff --git a/buildserver/cookbooks/kivy/recipes/default.rb b/buildserver/cookbooks/kivy/recipes/default.rb deleted file mode 100644 index 42699103..00000000 --- a/buildserver/cookbooks/kivy/recipes/default.rb +++ /dev/null @@ -1,34 +0,0 @@ - -%w{cython python-pygame python-pip python-virtualenv python-opengl python-gst0.10 python-enchant libgl1-mesa-dev libgles2-mesa-dev}.each do |pkg| - package pkg do - action :install - end -end - -script "install-kivy" do - cwd "/tmp" - interpreter "bash" - code " - tar xf /vagrant/cache/Kivy-1.7.2.tar.gz - cd Kivy-1.7.2 - python setup.py install - cd .. - rm -rf Kivy* - " - not_if "python -c 'import kivy'" -end - -script "install-p4a" do - cwd "/home/vagrant" - interpreter "bash" - code " - git clone https://github.com/kivy/python-for-android - chown -R vagrant:vagrant python-for-android - cd python-for-android - git checkout ca369d774e2 - " - not_if "test -d /home/vagrant/python-for-android" -end - - - diff --git a/buildserver/fixpaths.sh b/buildserver/fixpaths.sh deleted file mode 100644 index b55fcdf8..00000000 --- a/buildserver/fixpaths.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh - -echo $0 - -fixit() -{ - #Fix sudoers so the PATH gets passed through, otherwise chef - #provisioning doesn't work. - if [ -z "$1" ]; then - export EDITOR=$0 && sudo -E visudo - else - echo "Fix sudoers" - echo "Defaults exempt_group=admin" >> $1 - fi - #Stick the gems bin onto root's path as well. - sudo echo "PATH=$PATH:/var/lib/gems/1.8/bin" >>/root/.bashrc - # Restart sudo so it gets the changes straight away - sudo /etc/init.d/sudo restart -} - -sudo grep "exempt_group" /etc/sudoers -q -if [ "$?" -eq "1" ]; then - fixit -fi - diff --git a/hooks/pre-commit b/hooks/pre-commit index 45a5b052..e1449f6c 100755 --- a/hooks/pre-commit +++ b/hooks/pre-commit @@ -12,7 +12,7 @@ if [ -z "$files" ]; then PY_TEST_FILES="tests/*.TestCase" SH_FILES="hooks/pre-commit" BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion buildserver/provision-*" - RB_FILES="buildserver/cookbooks/*/recipes/*.rb buildserver/Vagrantfile" + RB_FILES="buildserver/Vagrantfile" else # if actually committing right now, then only run on the files # that are going to be committed at this moment diff --git a/makebuildserver b/makebuildserver index 9ecfa711..492ef6e1 100755 --- a/makebuildserver +++ b/makebuildserver @@ -243,8 +243,6 @@ cachefiles = [ 'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'), ('https://services.gradle.org/distributions/gradle-3.0-bin.zip', '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'), - ('https://pypi.python.org/packages/source/K/Kivy/Kivy-1.7.2.tar.gz', - '0485e2ef97b5086df886eb01f8303cb542183d2d71a159466f99ad6c8a1d03f1'), ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin', '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'), ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',