From 2daf00abb77d86f65efdcdc781f0dbd8e75b359c Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 4 Jul 2016 17:14:06 +0200 Subject: [PATCH 01/12] buildserver: remove last vestige of 32-bit buildserver --- makebuildserver | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/makebuildserver b/makebuildserver index ccc987c6..48634f6e 100755 --- a/makebuildserver +++ b/makebuildserver @@ -49,7 +49,6 @@ options, args = parser.parse_args() # set up default config cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver') config = { - 'arch64': True, 'basebox': 'jessie64', # TODO in py3, convert this to pathlib.Path(absolute_path_string).as_uri() 'baseboxurl': [ @@ -231,25 +230,14 @@ cachefiles = [ '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'), ('https://pypi.python.org/packages/source/K/Kivy/Kivy-1.7.2.tar.gz', '0485e2ef97b5086df886eb01f8303cb542183d2d71a159466f99ad6c8a1d03f1'), + ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin', + '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'), + ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2', + '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'), + ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2', + 'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'), ] -if config['arch64']: - cachefiles.extend([ - ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin', - '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'), - ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2', - '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'), - ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2', - 'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1')]) -else: - cachefiles.extend([ - ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86.bin', - '92b07d25aaad9b341a7f2b2a62402d508e948bf2dea3ee7b65a6aeb18bca7df5'), - ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86.tar.bz2', - '748104b829dd12afb2fdb3044634963abb24cdb0aad3b26030abe2e9e65bfc81'), - ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86-legacy-toolchains.tar.bz2', - '606aadf815ae28cc7b0154996247c70d609f111b14e44bcbcd6cad4c87fefb6f')]) - def sha256_for_file(path): with open(path, 'rb') as f: From e44f6380e903a321f78eb272cc8df5d93094abc7 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 4 Jul 2016 16:34:11 +0200 Subject: [PATCH 02/12] buildserver: test builds against fdroid and adaway The other apps are too flaky on gpjenkins right now, and that's our only box for running full buildserver tests. Once we get the buildserver tests running on jenkins.debian.net, then we can add a bunch more apps to the test script. gpjenkins is an extra locked down box, so that's why the builds are flaky: gradle and maven downloads regularly fail because they are blocked. --- jenkins-build-makebuildserver | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/jenkins-build-makebuildserver b/jenkins-build-makebuildserver index 7f1cdf95..120ce798 100755 --- a/jenkins-build-makebuildserver +++ b/jenkins-build-makebuildserver @@ -60,13 +60,15 @@ fi cd fdroiddata echo "build_server_always = True" > config.py +# if it can't build fdroid, then its really broken +../fdroid build --verbose --stop --latest org.fdroid.fdroid # Gradle, JNI, preassemble ../fdroid build --stop org.adaway:55 # Maven -../fdroid build --stop org.quantumbadger.redreader:55 +#../fdroid build --stop org.quantumbadger.redreader:55 # Custom build (make) -../fdroid build --stop com.amaze.filemanager:29 +#../fdroid build --stop com.amaze.filemanager:29 # Uses verification -../fdroid build --stop info.guardianproject.checkey:101 +#../fdroid build --stop info.guardianproject.checkey:101 # Gradle with retrolambda (JDK7 and JDK8) -../fdroid build --stop com.moez.QKSMS:124 +#../fdroid build --stop com.moez.QKSMS:124 From 2227cc6d1ab3341b9ce4c78bda5d428ac99a7778 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 21 Jun 2016 18:15:40 +0200 Subject: [PATCH 03/12] buildserver: download and cache latest platform-tools and m2repository Also, by having our own checksums for all the bits means that we can safely add support for local mirrors, like a bunch in China, for example: http://mirrors.neusoft.edu.cn/android/repository/ --- makebuildserver | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/makebuildserver b/makebuildserver index 48634f6e..b4a890f4 100755 --- a/makebuildserver +++ b/makebuildserver @@ -88,8 +88,10 @@ if not os.path.exists(cachedir): cachefiles = [ ('https://dl.google.com/android/repository/tools_r25.1.7-linux.zip', '3ca053600a86a5a64d5571edfbb1dad27f2bda3bfd2d38e2fe54322610b1ef0b'), - ('https://dl.google.com/android/repository/android_m2repository_r32.zip', - 'a6a8d7ffb153161f26d5fdebfa9aa1c9c84b29c62851fffff2cdfad9e094b13b'), + ('https://dl.google.com/android/repository/platform-tools_r24-linux.zip', + '076368b337d042d163364594dda63b7e778835f636fafb2c8af4d4a604175c32'), + ('https://dl.google.com/android/repository/android_m2repository_r33.zip', + 'be9bb4a27aeefb1c9adb0cade8771f764447c4cbde74426303db2ac6bde1879c'), ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip', '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'), ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip', From 588e6e55343c112a2632b0acd547f1545c25da7a Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 21 Jun 2016 18:12:09 +0200 Subject: [PATCH 04/12] buildserver: android --silent hides errors, so remove --silent seem to prevent `android update sdk` from exiting with an error, so its kind of useless. I just wanted it to suppress the verbose logs. --- buildserver/provision-android-sdk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildserver/provision-android-sdk b/buildserver/provision-android-sdk index 907b2489..ac5ab590 100644 --- a/buildserver/provision-android-sdk +++ b/buildserver/provision-android-sdk @@ -23,7 +23,7 @@ fi cd /vagrant/cache -# make hard links for `android update sdk` to use and delete +# make links for `android update sdk` to use and delete for f in android_*.zip android-[0-9]*.zip platform-[0-9]*.zip build-tools_r*-linux.zip; do rm -f ${ANDROID_HOME}/temp/$f ln -s /vagrant/cache/$f ${ANDROID_HOME}/temp/ @@ -42,7 +42,7 @@ for f in `ls -1 build-tools*.zip`; do cached=,build-tools-${ver}${cached} done -${ANDROID_HOME}/tools/android --silent update sdk --no-ui --all \ +${ANDROID_HOME}/tools/android update sdk --no-ui --all \ --filter platform-tools,extra-android-m2repository${cached} > /dev/null < Date: Wed, 22 Jun 2016 08:06:07 +0200 Subject: [PATCH 05/12] buildserver: enable debug log for provision-android-sdk --- buildserver/provision-android-sdk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildserver/provision-android-sdk b/buildserver/provision-android-sdk index ac5ab590..82e4ed43 100644 --- a/buildserver/provision-android-sdk +++ b/buildserver/provision-android-sdk @@ -2,6 +2,7 @@ # set -e +set -x if [ -z $ANDROID_HOME ]; then echo "ANDROID_HOME env var must be set!" @@ -43,7 +44,7 @@ for f in `ls -1 build-tools*.zip`; do done ${ANDROID_HOME}/tools/android update sdk --no-ui --all \ - --filter platform-tools,extra-android-m2repository${cached} > /dev/null < Date: Mon, 4 Jul 2016 13:23:25 +0200 Subject: [PATCH 06/12] buildserver: move trusty/paramiko hack to its own shell script This is part of the effort to remove moving parts from the whole build server setup. Why wrap shell scripts in ruby and chef if we can just directly run a shell script? --- .../fdroidbuild-general/recipes/default.rb | 9 --------- buildserver/provision-ubuntu-trusty-paramiko | 16 ++++++++++++++++ makebuildserver | 10 +++++++--- 3 files changed, 23 insertions(+), 12 deletions(-) create mode 100644 buildserver/provision-ubuntu-trusty-paramiko diff --git a/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb b/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb index 9ea5f508..3e0ace87 100644 --- a/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb +++ b/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb @@ -118,12 +118,3 @@ else command "update-java-alternatives --set java-1.8.0-openjdk-i386" end end - -# Ubuntu trusty 14.04's paramiko does not work with jessie's openssh's default settings -# https://stackoverflow.com/questions/7286929/paramiko-incompatible-ssh-peer-no-acceptable-kex-algorithm/32691055#32691055 -execute "support-ubuntu-trusty-paramiko" do - only_if { node[:settings][:ubuntu_trusty] == 'true' } - command "echo Ciphers aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes128-ctr >> /etc/ssh/sshd_config" - command "echo MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-512,hmac-sha2-256,hmac-ripemd160,hmac-sha1 >> /etc/ssh/sshd_config" - command "echo KexAlgorithms diffie-hellman-group-exchange-sha256,diffie-hellman-group14-sha1,diffie-hellman-group-exchange-sha1 >> /etc/ssh/sshd_config" -end diff --git a/buildserver/provision-ubuntu-trusty-paramiko b/buildserver/provision-ubuntu-trusty-paramiko new file mode 100644 index 00000000..81a3cd23 --- /dev/null +++ b/buildserver/provision-ubuntu-trusty-paramiko @@ -0,0 +1,16 @@ +#!/bin/bash + +# Ubuntu trusty 14.04's paramiko does not work with jessie's openssh's default settings +# https://stackoverflow.com/questions/7286929/paramiko-incompatible-ssh-peer-no-acceptable-kex-algorithm/32691055#32691055 + +if ! grep --quiet ^Ciphers /etc/ssh/sshd_config; then + echo Ciphers aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes128-ctr >> /etc/ssh/sshd_config +fi + +if ! grep --quiet ^MACs /etc/ssh/sshd_config; then + echo MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-512,hmac-sha2-256,hmac-ripemd160,hmac-sha1 >> /etc/ssh/sshd_config +fi + +if ! grep --quiet ^KexAlgorithms /etc/ssh/sshd_config; then + echo KexAlgorithms diffie-hellman-group-exchange-sha256,diffie-hellman-group14-sha1,diffie-hellman-group-exchange-sha1 >> /etc/ssh/sshd_config +fi diff --git a/makebuildserver b/makebuildserver index b4a890f4..b4c900ef 100755 --- a/makebuildserver +++ b/makebuildserver @@ -384,7 +384,6 @@ vagrantfile += """ chef.json = { :settings => { :debian_mirror => "%s", - :ubuntu_trusty => "%s", :user => "vagrant" } } @@ -399,9 +398,14 @@ vagrantfile += """ config.vm.provision "file", source: "gradle", destination: "/opt/gradle/bin/gradle" + # let Ubuntu/trusty's paramiko work with the VM instance + if `uname -v`.include? "14.04" + config.vm.provision "shell", path: "provision-ubuntu-trusty-paramiko" + end + end -""" % (config['debian_mirror'], - str('14.04' in os.uname()[3]).lower()) +""" % config['debian_mirror'] + # Check against the existing Vagrantfile, and if they differ, we need to # create a new box: From aafad6b909f3486fbcd4ddf02bca706fe9e1f283 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 4 Jul 2016 13:52:19 +0200 Subject: [PATCH 07/12] buildserver: move apt setup to a shell script This makes it so there is only a single `apt-get install` command run, instead of one command per-package like with the chef script. It also adds `apt-get upgrade` to make sure that the base box is fully up-to-date. --- .../fdroidbuild-general/recipes/default.rb | 115 ------------------ buildserver/cookbooks/kivy/recipes/default.rb | 2 - buildserver/provision-apt-get-install | 90 ++++++++++++++ makebuildserver | 10 +- 4 files changed, 93 insertions(+), 124 deletions(-) create mode 100644 buildserver/provision-apt-get-install diff --git a/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb b/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb index 3e0ace87..c93722f2 100644 --- a/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb +++ b/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb @@ -1,120 +1,5 @@ -user = node[:settings][:user] -debian_mirror = node[:settings][:debian_mirror] - -execute 'set_debian_mirror' do - command "sed -i 's,http://ftp.uk.debian.org/debian/,#{debian_mirror},g' /etc/apt/sources.list" -end - -execute "jessie_backports" do - command "echo 'deb #{debian_mirror} jessie-backports main' > /etc/apt/sources.list.d/backports.list" - only_if "grep jessie /etc/apt/sources.list" -end - -if node['kernel']['machine'] == "x86_64" - execute "archi386" do - command "dpkg --add-architecture i386" - end -end - -execute "apt-get-update" do - command "apt-get update" -end - -%w{ - ant - ant-contrib - autoconf - autoconf2.13 - automake1.11 - autopoint - bison - bzr - cmake - curl - expect - faketime - flex - gettext - git-core - git-svn - gperf - graphviz - imagemagick - inkscape - javacc - libarchive-zip-perl - libexpat1-dev - libglib2.0-dev - liblzma-dev - librsvg2-bin - libsaxonb-java - libssl-dev - libssl1.0.0 - libtool - libtool-bin - make - maven - }.each do |pkg| - package pkg do - action :install - end -end - -%w{ - mercurial - nasm - openjdk-8-jdk-headless - optipng - p7zip - pandoc - perlmagick - pkg-config - python-gnupg - python-magic - python-setuptools - python3-gnupg - python3-requests - python3-yaml - qt5-default - qtbase5-dev - quilt - realpath - scons - subversion - swig - texinfo - transfig - unzip - vorbis-tools - xsltproc - yasm - zip - }.each do |pkg| - package pkg do - action :install - end -end - -if node['kernel']['machine'] == "x86_64" - %w{libstdc++6:i386 libgcc1:i386 zlib1g:i386 libncurses5:i386}.each do |pkg| - package pkg do - action :install - end - end -end - easy_install_package "compare-locales" do options "-U" action :install end - -if node['kernel']['machine'] == "x86_64" - execute "set-default-java" do - command "update-java-alternatives --set java-1.8.0-openjdk-amd64" - end -else - execute "set-default-java" do - command "update-java-alternatives --set java-1.8.0-openjdk-i386" - end -end diff --git a/buildserver/cookbooks/kivy/recipes/default.rb b/buildserver/cookbooks/kivy/recipes/default.rb index 9b8a1caa..42699103 100644 --- a/buildserver/cookbooks/kivy/recipes/default.rb +++ b/buildserver/cookbooks/kivy/recipes/default.rb @@ -1,6 +1,4 @@ -user = node[:settings][:user] - %w{cython python-pygame python-pip python-virtualenv python-opengl python-gst0.10 python-enchant libgl1-mesa-dev libgles2-mesa-dev}.each do |pkg| package pkg do action :install diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install new file mode 100644 index 00000000..276d1787 --- /dev/null +++ b/buildserver/provision-apt-get-install @@ -0,0 +1,90 @@ +#!/bin/bash + +set -e +set -x + +debian_mirror=$1 + +sed -i "s,http://ftp.uk.debian.org/debian/,${debian_mirror},g" /etc/apt/sources.list + +if grep --quiet jessie /etc/apt/sources.list; then + echo "deb $debian_mirror jessie-backports main" > /etc/apt/sources.list.d/backports.list +fi + +dpkg --add-architecture i386 + +apt-get -y update +apt-get -y upgrade + +packages=" + ant + ant-contrib + autoconf + autoconf2.13 + automake1.11 + autopoint + bison + bzr + cmake + curl + expect + faketime + flex + gettext + git-core + git-svn + gperf + graphviz + imagemagick + inkscape + javacc + libarchive-zip-perl + libexpat1-dev + libgcc1:i386 + libglib2.0-dev + liblzma-dev + libncurses5:i386 + librsvg2-bin + libsaxonb-java + libssl-dev + libssl1.0.0 + libstdc++6:i386 + libtool + libtool-bin + make + maven + mercurial + nasm + openjdk-8-jdk-headless + optipng + p7zip + pandoc + perlmagick + pkg-config + python-gnupg + python-magic + python-setuptools + python3-gnupg + python3-requests + python3-yaml + qt5-default + qtbase5-dev + quilt + realpath + scons + subversion + swig + texinfo + transfig + unzip + vorbis-tools + xsltproc + yasm + zip + zlib1g:i386 +" + +apt-get install --yes --no-install-recommends $packages + +highestjava=`update-java-alternatives --list | sort -n | tail -1 | cut -d ' ' -f 1` +update-java-alternatives --set $highestjava diff --git a/makebuildserver b/makebuildserver index b4c900ef..0b1da733 100755 --- a/makebuildserver +++ b/makebuildserver @@ -377,16 +377,12 @@ vagrantfile += """ config.vm.provision "shell", path: "setup-env-vars", args: ["/home/vagrant/android-sdk"] + config.vm.provision "shell", path: "provision-apt-get-install", + args: ["{0}"] config.vm.provision :chef_solo do |chef| chef.cookbooks_path = "cookbooks" chef.log_level = :debug - chef.json = { - :settings => { - :debian_mirror => "%s", - :user => "vagrant" - } - } chef.add_recipe "fdroidbuild-general" chef.add_recipe "kivy" end @@ -404,7 +400,7 @@ vagrantfile += """ end end -""" % config['debian_mirror'] +""".format(config['debian_mirror']) # Check against the existing Vagrantfile, and if they differ, we need to From 0171dd60fd2fd0e7d908624172875a852abbe3c1 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 4 Jul 2016 14:08:54 +0200 Subject: [PATCH 08/12] buildserver: use pip instead of easy_install for caching easy_install does not provide any download caching, while pip does. This also moves the python module installing a shell script that takes python packages as args. That will allow for future uses like allowing app metadata to include pip modules that they need. --- .../fdroidbuild-general/recipes/default.rb | 5 ----- buildserver/provision-pip | 13 +++++++++++++ makebuildserver | 3 ++- 3 files changed, 15 insertions(+), 6 deletions(-) delete mode 100644 buildserver/cookbooks/fdroidbuild-general/recipes/default.rb create mode 100644 buildserver/provision-pip diff --git a/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb b/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb deleted file mode 100644 index c93722f2..00000000 --- a/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb +++ /dev/null @@ -1,5 +0,0 @@ - -easy_install_package "compare-locales" do - options "-U" - action :install -end diff --git a/buildserver/provision-pip b/buildserver/provision-pip new file mode 100644 index 00000000..654bc633 --- /dev/null +++ b/buildserver/provision-pip @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e +set -x + +# cache pypi downloads +if [ -z $PIP_DOWNLOAD_CACHE ]; then + export PIP_DOWNLOAD_CACHE=$HOME/.pip_download_cache +fi + +apt-get install --yes --no-install-recommends python-pip + +pip install --upgrade $@ diff --git a/makebuildserver b/makebuildserver index 0b1da733..0a0e0d43 100755 --- a/makebuildserver +++ b/makebuildserver @@ -383,13 +383,14 @@ vagrantfile += """ config.vm.provision :chef_solo do |chef| chef.cookbooks_path = "cookbooks" chef.log_level = :debug - chef.add_recipe "fdroidbuild-general" chef.add_recipe "kivy" end config.vm.provision "shell", path: "provision-android-sdk" config.vm.provision "shell", path: "provision-android-ndk", args: ["/home/vagrant/android-ndk"] + config.vm.provision "shell", path: "provision-pip", + args: ["compare-locales"] config.vm.provision "shell", path: "provision-gradle" config.vm.provision "file", source: "gradle", destination: "/opt/gradle/bin/gradle" From 4e787cc7502c2d7ebc68d56d07dc4eaadebd285e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 4 Jul 2016 14:20:42 +0200 Subject: [PATCH 09/12] buildserver: make provision scripts output name to log --- buildserver/fixpaths.sh | 2 ++ buildserver/provision-android-ndk | 1 + buildserver/provision-android-sdk | 1 + buildserver/provision-apt-get-install | 1 + buildserver/provision-pip | 1 + buildserver/provision-ubuntu-trusty-paramiko | 3 +++ 6 files changed, 9 insertions(+) diff --git a/buildserver/fixpaths.sh b/buildserver/fixpaths.sh index eb8a81fb..b55fcdf8 100644 --- a/buildserver/fixpaths.sh +++ b/buildserver/fixpaths.sh @@ -1,5 +1,7 @@ #!/bin/sh +echo $0 + fixit() { #Fix sudoers so the PATH gets passed through, otherwise chef diff --git a/buildserver/provision-android-ndk b/buildserver/provision-android-ndk index 9d4a54c0..4ce42cc2 100644 --- a/buildserver/provision-android-ndk +++ b/buildserver/provision-android-ndk @@ -1,6 +1,7 @@ #!/bin/bash # +echo $0 set -e NDK_BASE=$1 diff --git a/buildserver/provision-android-sdk b/buildserver/provision-android-sdk index 82e4ed43..31bb95b2 100644 --- a/buildserver/provision-android-sdk +++ b/buildserver/provision-android-sdk @@ -1,6 +1,7 @@ #!/bin/bash # +echo $0 set -e set -x diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index 276d1787..1085c0a1 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -1,5 +1,6 @@ #!/bin/bash +echo $0 set -e set -x diff --git a/buildserver/provision-pip b/buildserver/provision-pip index 654bc633..b0984dec 100644 --- a/buildserver/provision-pip +++ b/buildserver/provision-pip @@ -1,5 +1,6 @@ #!/bin/bash +echo $0 set -e set -x diff --git a/buildserver/provision-ubuntu-trusty-paramiko b/buildserver/provision-ubuntu-trusty-paramiko index 81a3cd23..88c046e4 100644 --- a/buildserver/provision-ubuntu-trusty-paramiko +++ b/buildserver/provision-ubuntu-trusty-paramiko @@ -1,5 +1,8 @@ #!/bin/bash +echo $0 +set -e + # Ubuntu trusty 14.04's paramiko does not work with jessie's openssh's default settings # https://stackoverflow.com/questions/7286929/paramiko-incompatible-ssh-peer-no-acceptable-kex-algorithm/32691055#32691055 From 2e1ec71404a9f9577eb68971fff025d9f6036101 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 4 Jul 2016 20:22:00 +0200 Subject: [PATCH 10/12] buildserver: send config to vagrant via YAML file Python can easily output dicts as YAML, and a Vagrantfile is a ruby script, which can easily read YAML. Going this route means that Vagrantfile can ultimately be committed to git, and the configuration will happen all via Python dicts output as YAML. That makes it drastically easier to follow the code, and to make modifications. --- .gitignore | 1 + buildserver/provision-apt-proxy | 11 +++++ makebuildserver | 79 +++++++++++++++------------------ 3 files changed, 49 insertions(+), 42 deletions(-) create mode 100644 buildserver/provision-apt-proxy diff --git a/.gitignore b/.gitignore index 8bff77fa..cd84eae1 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ tmp/ tests/repo/icons* # files used in manual testing +/buildserver/Vagrantfile.yaml /config.py /tmp/ /logs/ diff --git a/buildserver/provision-apt-proxy b/buildserver/provision-apt-proxy new file mode 100644 index 00000000..ec9a5ee8 --- /dev/null +++ b/buildserver/provision-apt-proxy @@ -0,0 +1,11 @@ +#!/bin/bash + +echo $0 +set -e + +rm -f /etc/apt/apt.conf.d/02proxy +echo "Acquire::ftp::Proxy \"$1\";" >> /etc/apt/apt.conf.d/02proxy +echo "Acquire::http::Proxy \"$1\";" >> /etc/apt/apt.conf.d/02proxy +echo "Acquire::https::Proxy \"$1\";" >> /etc/apt/apt.conf.d/02proxy + +apt-get update diff --git a/makebuildserver b/makebuildserver index 0a0e0d43..15a29868 100755 --- a/makebuildserver +++ b/makebuildserver @@ -6,6 +6,7 @@ import sys import subprocess import time import hashlib +import yaml from clint.textui import progress from optparse import OptionParser @@ -61,6 +62,7 @@ config = { 'cachedir': cachedir, 'cpus': 1, 'memory': 1024, + 'hwvirtex': 'off', } # load config file, if present @@ -85,6 +87,9 @@ cachedir = config['cachedir'] if not os.path.exists(cachedir): os.makedirs(cachedir, 0o755) +if config['apt_package_cache']: + config['aptcachedir'] = cachedir + '/apt/archives' + cachefiles = [ ('https://dl.google.com/android/repository/tools_r25.1.7-linux.zip', '3ca053600a86a5a64d5571edfbb1dad27f2bda3bfd2d38e2fe54322610b1ef0b'), @@ -302,29 +307,29 @@ for srcurl, shasum in cachefiles: print("Invalid shasum of '" + v + "' detected for " + local_filename) sys.exit(1) -# allow specifying a list/tuple that includes cached local copy -if type(config['baseboxurl']) in (list, tuple) or config['baseboxurl'][0] in ('(', '['): - baseboxurl = config['baseboxurl'] -else: - baseboxurl = '"{0}"'.format(config['baseboxurl']) - # use VirtualBox software virtualization if hardware is not available, # like if this is being run in kvm or some other VM platform, like # http://jenkins.debian.net, the values are 'on' or 'off' -hwvirtex = 'off' if sys.platform.startswith('darwin'): # all < 10 year old Macs work, and OSX servers as VM host are very # rare, but this could also be auto-detected if someone codes it - hwvirtex = 'on' + config['hwvirtex'] = 'on' elif os.path.exists('/proc/cpuinfo'): with open('/proc/cpuinfo') as f: contents = f.read() if 'vmx' in contents or 'svm' in contents: - hwvirtex = 'on' + config['hwvirtex'] = 'on' + +del(config['__builtins__']) # added by compile/exec +with open(os.path.join(serverdir, 'Vagrantfile.yaml'), 'w') as f: + yaml.dump(config, f) # Generate an appropriate Vagrantfile for the buildserver, based on our # settings... vagrantfile = """ +require 'yaml' +configfile = YAML.load_file('Vagrantfile.yaml') + Vagrant.configure("2") do |config| if Vagrant.has_plugin?("vagrant-cachier") @@ -334,51 +339,41 @@ Vagrant.configure("2") do |config| config.cache.enable :chef end - config.vm.box = "{0}" - config.vm.box_url = {1} + config.vm.box = configfile['basebox'] + config.vm.box_url = configfile['baseboxurl'] config.vm.provider "virtualbox" do |v| - v.customize ["modifyvm", :id, "--memory", "{2}"] - v.customize ["modifyvm", :id, "--cpus", "{3}"] - v.customize ["modifyvm", :id, "--hwvirtex", "{4}"] + v.customize ["modifyvm", :id, "--memory", configfile['memory']] + v.customize ["modifyvm", :id, "--cpus", configfile['cpus']] + v.customize ["modifyvm", :id, "--hwvirtex", configfile['hwvirtex']] end - config.vm.boot_timeout = {5} + config.vm.boot_timeout = configfile['boot_timeout'] config.vm.provision :shell, :path => "fixpaths.sh" -""".format(config['basebox'], - baseboxurl, - config['memory'], - config.get('cpus', 1), - hwvirtex, - config['boot_timeout']) -if 'aptproxy' in config and config['aptproxy']: - vagrantfile += """ - config.vm.provision :shell, :inline => 'sudo echo "Acquire::http {{ Proxy \\"{0}\\"; }};" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update' -""".format(config['aptproxy']) -# buildserver/ is shared to the VM's /vagrant by default so the old default -# does not need a custom mount -if cachedir != 'buildserver/cache': - vagrantfile += """ - config.vm.synced_folder '{0}', '/vagrant/cache', - owner: 'root', group: 'root', create: true -""".format(cachedir) + if configfile.has_key? "aptproxy" + config.vm.provision :shell, path: "provision-apt-proxy", + args: [configfile["aptproxy"]] + end -# cache .deb packages on the host via a mount trick -if config['apt_package_cache']: - aptcachedir = cachedir + '/apt/archives' - vagrantfile += """ - config.vm.synced_folder "{0}", "/var/cache/apt/archives", - owner: 'root', group: 'root', create: true -""".format(aptcachedir) + # buildserver/ is shared to the VM's /vagrant by default so the old + # default does not need a custom mount + if configfile["cachedir"] != "buildserver/cache" + config.vm.synced_folder configfile["cachedir"], '/vagrant/cache', + owner: 'root', group: 'root', create: true + end -vagrantfile += """ + # cache .deb packages on the host via a mount trick + if configfile.has_key? "aptcachedir" + config.vm.synced_folder configfile["aptcachedir"], "/var/cache/apt/archives", + owner: 'root', group: 'root', create: true + end config.vm.provision "shell", path: "setup-env-vars", args: ["/home/vagrant/android-sdk"] config.vm.provision "shell", path: "provision-apt-get-install", - args: ["{0}"] + args: [configfile['debian_mirror']] config.vm.provision :chef_solo do |chef| chef.cookbooks_path = "cookbooks" @@ -401,7 +396,7 @@ vagrantfile += """ end end -""".format(config['debian_mirror']) +""" # Check against the existing Vagrantfile, and if they differ, we need to From d4c6fffb301ca1caa8af7d7e9fcba96350fe1354 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 4 Jul 2016 20:49:41 +0200 Subject: [PATCH 11/12] buildserver: buildserver/Vagrantfile is configed by .yaml file Vagrantfile is now committed and not changed between configurations. It is configed by translating the python config file's dict to a YAML file, which Vagrantfile now loads and uses. This makes it a lot easier for vagrant users and python programmers to understand, and hopefully makes it easier to maintain and test with. --- .gitignore | 1 - buildserver/.gitignore | 2 +- buildserver/Vagrantfile | 70 +++++++++++++++++++++++++++++ hooks/pre-commit | 2 +- makebuildserver | 97 ++++------------------------------------- 5 files changed, 81 insertions(+), 91 deletions(-) create mode 100644 buildserver/Vagrantfile diff --git a/.gitignore b/.gitignore index cd84eae1..8bff77fa 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,6 @@ tmp/ tests/repo/icons* # files used in manual testing -/buildserver/Vagrantfile.yaml /config.py /tmp/ /logs/ diff --git a/buildserver/.gitignore b/buildserver/.gitignore index 4a3901ed..d8165d61 100644 --- a/buildserver/.gitignore +++ b/buildserver/.gitignore @@ -1,4 +1,4 @@ .vagrant up.log cache/ -Vagrantfile +Vagrantfile.yaml diff --git a/buildserver/Vagrantfile b/buildserver/Vagrantfile new file mode 100644 index 00000000..e586e05c --- /dev/null +++ b/buildserver/Vagrantfile @@ -0,0 +1,70 @@ + +require 'yaml' +configfile = YAML.load_file('Vagrantfile.yaml') + +Vagrant.configure("2") do |config| + + if Vagrant.has_plugin?("vagrant-cachier") + config.cache.scope = :box + config.cache.auto_detect = false + config.cache.enable :apt + config.cache.enable :chef + end + + config.vm.box = configfile['basebox'] + config.vm.box_url = configfile['baseboxurl'] + + config.vm.provider "virtualbox" do |v| + v.customize ["modifyvm", :id, "--memory", configfile['memory']] + v.customize ["modifyvm", :id, "--cpus", configfile['cpus']] + v.customize ["modifyvm", :id, "--hwvirtex", configfile['hwvirtex']] + end + + config.vm.boot_timeout = configfile['boot_timeout'] + + config.vm.provision :shell, :path => "fixpaths.sh" + + if configfile.has_key? "aptproxy" + config.vm.provision :shell, path: "provision-apt-proxy", + args: [configfile["aptproxy"]] + end + + # buildserver/ is shared to the VM's /vagrant by default so the old + # default does not need a custom mount + if configfile["cachedir"] != "buildserver/cache" + config.vm.synced_folder configfile["cachedir"], '/vagrant/cache', + owner: 'root', group: 'root', create: true + end + + # cache .deb packages on the host via a mount trick + if configfile.has_key? "aptcachedir" + config.vm.synced_folder configfile["aptcachedir"], "/var/cache/apt/archives", + owner: 'root', group: 'root', create: true + end + + config.vm.provision "shell", path: "setup-env-vars", + args: ["/home/vagrant/android-sdk"] + config.vm.provision "shell", path: "provision-apt-get-install", + args: [configfile['debian_mirror']] + + config.vm.provision :chef_solo do |chef| + chef.cookbooks_path = "cookbooks" + chef.log_level = :debug + chef.add_recipe "kivy" + end + + config.vm.provision "shell", path: "provision-android-sdk" + config.vm.provision "shell", path: "provision-android-ndk", + args: ["/home/vagrant/android-ndk"] + config.vm.provision "shell", path: "provision-pip", + args: ["compare-locales"] + config.vm.provision "shell", path: "provision-gradle" + config.vm.provision "file", source: "gradle", + destination: "/opt/gradle/bin/gradle" + + # let Ubuntu/trusty's paramiko work with the VM instance + if `uname -v`.include? "14.04" + config.vm.provision "shell", path: "provision-ubuntu-trusty-paramiko" + end + +end diff --git a/hooks/pre-commit b/hooks/pre-commit index 9f0dcb77..1929ee91 100755 --- a/hooks/pre-commit +++ b/hooks/pre-commit @@ -12,7 +12,7 @@ if [ -z "$files" ]; then PY_TEST_FILES="tests/*.TestCase" SH_FILES="hooks/pre-commit" BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion buildserver/provision-*" - RB_FILES="buildserver/cookbooks/*/recipes/*.rb" + RB_FILES="buildserver/cookbooks/*/recipes/*.rb buildserver/Vagrantfile" else # if actually committing right now, then only run on the files # that are going to be committed at this moment diff --git a/makebuildserver b/makebuildserver index 15a29868..62fee88d 100755 --- a/makebuildserver +++ b/makebuildserver @@ -71,6 +71,7 @@ if os.path.exists('makebuildserver.config.py'): elif os.path.exists('makebs.config.py'): # this is the old name for the config file exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config) +del(config['__builtins__']) # added by compile/exec if not os.path.exists('makebuildserver') or not os.path.exists(serverdir): print('This must be run from the correct directory!') @@ -320,94 +321,15 @@ elif os.path.exists('/proc/cpuinfo'): if 'vmx' in contents or 'svm' in contents: config['hwvirtex'] = 'on' -del(config['__builtins__']) # added by compile/exec -with open(os.path.join(serverdir, 'Vagrantfile.yaml'), 'w') as f: - yaml.dump(config, f) - -# Generate an appropriate Vagrantfile for the buildserver, based on our -# settings... -vagrantfile = """ -require 'yaml' -configfile = YAML.load_file('Vagrantfile.yaml') - -Vagrant.configure("2") do |config| - - if Vagrant.has_plugin?("vagrant-cachier") - config.cache.scope = :box - config.cache.auto_detect = false - config.cache.enable :apt - config.cache.enable :chef - end - - config.vm.box = configfile['basebox'] - config.vm.box_url = configfile['baseboxurl'] - - config.vm.provider "virtualbox" do |v| - v.customize ["modifyvm", :id, "--memory", configfile['memory']] - v.customize ["modifyvm", :id, "--cpus", configfile['cpus']] - v.customize ["modifyvm", :id, "--hwvirtex", configfile['hwvirtex']] - end - - config.vm.boot_timeout = configfile['boot_timeout'] - - config.vm.provision :shell, :path => "fixpaths.sh" - - if configfile.has_key? "aptproxy" - config.vm.provision :shell, path: "provision-apt-proxy", - args: [configfile["aptproxy"]] - end - - # buildserver/ is shared to the VM's /vagrant by default so the old - # default does not need a custom mount - if configfile["cachedir"] != "buildserver/cache" - config.vm.synced_folder configfile["cachedir"], '/vagrant/cache', - owner: 'root', group: 'root', create: true - end - - # cache .deb packages on the host via a mount trick - if configfile.has_key? "aptcachedir" - config.vm.synced_folder configfile["aptcachedir"], "/var/cache/apt/archives", - owner: 'root', group: 'root', create: true - end - - config.vm.provision "shell", path: "setup-env-vars", - args: ["/home/vagrant/android-sdk"] - config.vm.provision "shell", path: "provision-apt-get-install", - args: [configfile['debian_mirror']] - - config.vm.provision :chef_solo do |chef| - chef.cookbooks_path = "cookbooks" - chef.log_level = :debug - chef.add_recipe "kivy" - end - - config.vm.provision "shell", path: "provision-android-sdk" - config.vm.provision "shell", path: "provision-android-ndk", - args: ["/home/vagrant/android-ndk"] - config.vm.provision "shell", path: "provision-pip", - args: ["compare-locales"] - config.vm.provision "shell", path: "provision-gradle" - config.vm.provision "file", source: "gradle", - destination: "/opt/gradle/bin/gradle" - - # let Ubuntu/trusty's paramiko work with the VM instance - if `uname -v`.include? "14.04" - config.vm.provision "shell", path: "provision-ubuntu-trusty-paramiko" - end - -end -""" - - -# Check against the existing Vagrantfile, and if they differ, we need to -# create a new box: -vf = os.path.join(serverdir, 'Vagrantfile') +# Check against the existing Vagrantfile.yaml, and if they differ, we +# need to create a new box: +vf = os.path.join(serverdir, 'Vagrantfile.yaml') writevf = True if os.path.exists(vf): vagrant(['halt'], serverdir) - with open(vf, 'r') as f: - oldvf = f.read() - if oldvf != vagrantfile: + with open(vf, 'r', encoding='utf-8') as f: + oldconfig = yaml.load(f) + if config != oldconfig: print("Server configuration has changed, rebuild from scratch is required") vagrant(['destroy', '-f'], serverdir) else: @@ -416,9 +338,8 @@ if os.path.exists(vf): else: print("No existing server - building from scratch") if writevf: - with open(vf, 'w') as f: - f.write(vagrantfile) - + with open(vf, 'w', encoding='utf-8') as f: + yaml.dump(config, f) print("Configuring build server VM") returncode, out = vagrant(['up', '--provision'], serverdir, printout=True) From 8b53ae0ad0ac8858713a520ef5c2e06139a5c364 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 6 Jul 2016 22:16:26 +0200 Subject: [PATCH 12/12] buildserver: delete corrupt files from the cache In order to make CI and other automation easier, delete any corrupt files from the cache if they fail the SHA-256 check. https://jenkins.debian.net/view/reproducible/job/reproducible_setup_fdroid_build_environment_profitbricks3/112/console --- makebuildserver | 1 + 1 file changed, 1 insertion(+) diff --git a/makebuildserver b/makebuildserver index 62fee88d..173ca747 100755 --- a/makebuildserver +++ b/makebuildserver @@ -306,6 +306,7 @@ for srcurl, shasum in cachefiles: print("\t...shasum verified for " + local_filename) else: print("Invalid shasum of '" + v + "' detected for " + local_filename) + os.remove(local_filename) sys.exit(1) # use VirtualBox software virtualization if hardware is not available,