diff --git a/buildserver/cookbooks/android-ndk/recipes/default.rb b/buildserver/cookbooks/android-ndk/recipes/default.rb deleted file mode 100644 index bc704479..00000000 --- a/buildserver/cookbooks/android-ndk/recipes/default.rb +++ /dev/null @@ -1,50 +0,0 @@ - -ndk_loc = node[:settings][:ndk_loc] -user = node[:settings][:user] - -script "setup-android-ndk" do - timeout 14400 - interpreter "bash" - user node[:settings][:user] - cwd "/tmp" - code " - mkdir -p #{ndk_loc} - " -end - -script "setup-android-ndk-r9b" do - timeout 14400 - interpreter "bash" - user node[:settings][:user] - cwd "/tmp" - code " - if [ `uname -m` == 'x86_64' ] ; then - SUFFIX='_64' - else - SUFFIX='' - fi - tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX.tar.bz2 - tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX-legacy-toolchains.tar.bz2 - mv android-ndk-r9b #{ndk_loc}/r9b - " - not_if "test -d #{ndk_loc}/r9b" -end - -script "setup-android-ndk-r10e" do - timeout 14400 - interpreter "bash" - user node[:settings][:user] - cwd "/tmp" - code " - if [ `uname -m` == 'x86_64' ] ; then - SUFFIX='_64' - else - SUFFIX='' - fi - chmod u+x /vagrant/cache/android-ndk-r10e-linux-x86$SUFFIX.bin - /vagrant/cache/android-ndk-r10e-linux-x86$SUFFIX.bin x - mv android-ndk-r10e #{ndk_loc}/r10e - " - not_if "test -d #{ndk_loc}/r10e" -end - diff --git a/buildserver/cookbooks/android-sdk/recipes/default.rb b/buildserver/cookbooks/android-sdk/recipes/default.rb deleted file mode 100644 index d5a87170..00000000 --- a/buildserver/cookbooks/android-sdk/recipes/default.rb +++ /dev/null @@ -1,79 +0,0 @@ - -sdk_loc = node[:settings][:sdk_loc] -user = node[:settings][:user] - -script "setup-android-sdk" do - timeout 14400 - interpreter "bash" - user user - cwd "/tmp" - code " - tools=`ls -1 /vagrant/cache/tools_*.zip | sort -n | tail -1` - unzip $tools - mkdir #{sdk_loc} - mkdir #{sdk_loc}/platforms - mkdir #{sdk_loc}/build-tools - mv tools #{sdk_loc}/ - " - not_if "test -d #{sdk_loc}" -end - -execute "add-android-sdk-path" do - user user - path = "#{sdk_loc}/tools:#{sdk_loc}/platform-tools" - command "echo \"export PATH=\\$PATH:#{path} #PATH-SDK\" >> /home/#{user}/.bsenv" - not_if "grep PATH-SDK /home/#{user}/.bsenv" -end - -script "add_android_packages" do - interpreter "bash" - user user - code " - #{sdk_loc}/tools/android update sdk --no-ui --all --filter platform-tools,extra-android-m2repository <> /home/#{user}/.bashrc" - not_if "grep bsenv /home/#{user}/.bashrc" -end - execute "set-default-java" do command "update-java-alternatives --set java-1.7.0-openjdk-i386" end diff --git a/buildserver/cookbooks/gradle/recipes/default.rb b/buildserver/cookbooks/gradle/recipes/default.rb deleted file mode 100644 index 74dba2c4..00000000 --- a/buildserver/cookbooks/gradle/recipes/default.rb +++ /dev/null @@ -1,48 +0,0 @@ - -user = node[:settings][:user] - -gradle_script = IO.read(File.join( - File.expand_path(File.dirname(__FILE__)), "gradle")) - -script "add-gradle-bindir" do - cwd "/tmp" - interpreter "bash" - code "mkdir -p /opt/gradle/bin" - not_if "test -d /opt/gradle/bin" -end - -script "add-gradle-verdir" do - cwd "/tmp" - interpreter "bash" - code "mkdir -p /opt/gradle/versions" - not_if "test -d /opt/gradle/versions" -end - -%w{1.4 1.6 1.7 1.8 1.9 1.10 1.11 1.12 2.1 2.2.1 2.3 2.4 2.5 2.6 2.7 2.8 2.9 2.10 2.11 2.12 2.13}.each do |ver| - script "install-gradle-#{ver}" do - cwd "/tmp" - interpreter "bash" - code " - unzip /vagrant/cache/gradle-#{ver}-bin.zip - mv gradle-#{ver} /opt/gradle/versions/#{ver} - " - not_if "test -d /opt/gradle/versions/#{ver}" - end -end - -script "add-gradle-wrapper" do - cwd "/tmp" - interpreter "bash" - code " - cat << \"EOF\" > /opt/gradle/bin/gradle -#{gradle_script} -EOF - chmod a+x /opt/gradle/bin/gradle - " -end - -execute "add-android-ndk-path" do - user user - command "echo \"export PATH=\\$PATH:/opt/gradle/bin #PATH-GRADLE\" >> /home/#{user}/.bsenv" - not_if "grep PATH-GRADLE /home/#{user}/.bsenv" -end diff --git a/buildserver/cookbooks/gradle/recipes/gradle b/buildserver/gradle similarity index 100% rename from buildserver/cookbooks/gradle/recipes/gradle rename to buildserver/gradle diff --git a/buildserver/provision-android-ndk b/buildserver/provision-android-ndk new file mode 100644 index 00000000..3c9b0a6a --- /dev/null +++ b/buildserver/provision-android-ndk @@ -0,0 +1,36 @@ +#!/bin/bash +# + +set -e + +if [ -z $ANDROID_NDK_HOME ]; then + echo "ANDROID_NDK_HOME env var must be set!" + exit 1 +fi + +test -e $ANDROID_NDK_HOME || mkdir -p $ANDROID_NDK_HOME +cd $ANDROID_NDK_HOME + +if [ ! -e $ANDROID_NDK_HOME/r9b ]; then + if [ `uname -m` == 'x86_64' ] ; then + SUFFIX='_64' + else + SUFFIX='' + fi + tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX.tar.bz2 + tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX-legacy-toolchains.tar.bz2 + mv android-ndk-r9b r9b +fi + +if [ ! -e $ANDROID_NDK_HOME/r10e ]; then + if [ `uname -m` == 'x86_64' ] ; then + SUFFIX='_64' + else + SUFFIX='' + fi + 7zr x /vagrant/cache/android-ndk-r10e-linux-x86$SUFFIX.bin > /dev/null + mv android-ndk-r10e r10e +fi + +chmod -R a+rX $ANDROID_NDK_HOME/ +find $ANDROID_NDK_HOME/ -type f -executable -print0 | xargs -0 chmod a+x diff --git a/buildserver/provision-android-sdk b/buildserver/provision-android-sdk new file mode 100644 index 00000000..5560ecb1 --- /dev/null +++ b/buildserver/provision-android-sdk @@ -0,0 +1,53 @@ +#!/bin/bash +# + +set -e + +if [ -z $ANDROID_HOME ]; then + echo "ANDROID_HOME env var must be set!" + exit 1 +fi + +# TODO remove the rm, this should work with an existing ANDROID_HOME +if [ ! -x $ANDROID_HOME/tools/android ]; then + rm -rf $ANDROID_HOME + mkdir ${ANDROID_HOME} + mkdir ${ANDROID_HOME}/temp + mkdir ${ANDROID_HOME}/platforms + mkdir ${ANDROID_HOME}/build-tools + cd $ANDROID_HOME + + tools=`ls -1 /vagrant/cache/tools_*.zip | sort -n | tail -1` + unzip -qq $tools +fi + +cd /vagrant/cache + +# make hard links for `android update sdk` to use and delete +for f in android_*.zip android-[0-9]*.zip platform-[0-9]*.zip build-tools_r*-linux.zip; do + rm -f ${ANDROID_HOME}/temp/$f + ln -s /vagrant/cache/$f ${ANDROID_HOME}/temp/ +done + +# install all cached platforms +cached="" +for f in `ls -1 android-[0-9]*.zip platform-[0-9]*.zip`; do + sdk=`unzip -c $f "*/build.prop" | sed -n 's,^ro.build.version.sdk=,,p'` + cached=,android-${sdk}${cached} +done + +# install all cached build-tools +for f in `ls -1 build-tools*.zip`; do + ver=`unzip -c $f "*/source.properties" | sed -n 's,^Pkg.Revision=,,p'` + cached=,build-tools-${ver}${cached} +done + +${ANDROID_HOME}/tools/android --silent update sdk --no-ui --all \ + --filter platform-tools,extra-android-m2repository${cached} < $bsenv + +echo export ANDROID_HOME=$1 >> $bsenv +echo export ANDROID_NDK_HOME=$2 >> $bsenv +echo 'export PATH=$PATH:${ANDROID_HOME}/tools:${ANDROID_HOME}/platform-tools:/opt/gradle/bin' >> $bsenv + +chmod 0644 $bsenv diff --git a/fdroidserver/build.py b/fdroidserver/build.py index a387d13b..8376cadf 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -388,7 +388,7 @@ def build_server(app, build, vcs, build_dir, output_dir, force): if options.verbose: cmdline += ' --verbose' cmdline += " %s:%s" % (app.id, build.vercode) - chan.exec_command('bash -c ". ~/.bsenv && ' + cmdline + '"') + chan.exec_command('bash -c "' + cmdline + '"') output = bytes() while not chan.exit_status_ready(): while chan.recv_ready(): diff --git a/fdroidserver/update.py b/fdroidserver/update.py index cd75449d..7cd18f41 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -44,6 +44,8 @@ from . import metadata from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen from .metadata import MetaDataException +METADATA_VERSION = 16 + screen_densities = ['640', '480', '320', '240', '160', '120'] all_screen_densities = ['0'] + screen_densities @@ -811,7 +813,7 @@ def make_index(apps, sortedids, apks, repodir, archive, categories): for mirror in config.get('mirrors', []): addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel) - repoel.setAttribute("version", "16") + repoel.setAttribute("version", str(METADATA_VERSION)) repoel.setAttribute("timestamp", str(int(time.time()))) nosigningkey = False @@ -1224,6 +1226,8 @@ def main(): if not options.clean and os.path.exists(apkcachefile): with open(apkcachefile, 'rb') as cf: apkcache = pickle.load(cf, encoding='utf-8') + if apkcache.get("METADATA_VERSION") != METADATA_VERSION: + apkcache = {} else: apkcache = {} @@ -1364,6 +1368,7 @@ def main(): f.write(data) if cachechanged: + apkcache["METADATA_VERSION"] = METADATA_VERSION with open(apkcachefile, 'wb') as cf: pickle.dump(apkcache, cf) diff --git a/hooks/pre-commit b/hooks/pre-commit index 0f083d00..9f0dcb77 100755 --- a/hooks/pre-commit +++ b/hooks/pre-commit @@ -11,7 +11,7 @@ if [ -z "$files" ]; then PY_FILES="fdroid makebuildserver setup.py examples/*.py buildserver/*.py fdroidserver/*.py" PY_TEST_FILES="tests/*.TestCase" SH_FILES="hooks/pre-commit" - BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion" + BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion buildserver/provision-*" RB_FILES="buildserver/cookbooks/*/recipes/*.rb" else # if actually committing right now, then only run on the files @@ -23,6 +23,7 @@ else RB_FILES= for f in $files; do + test -e $f || continue case $f in *.py) PY_FILES+=" $f" diff --git a/makebuildserver b/makebuildserver index cff8b64e..0f5cb86e 100755 --- a/makebuildserver +++ b/makebuildserver @@ -89,6 +89,8 @@ if not os.path.exists(cachedir): cachefiles = [ ('https://dl.google.com/android/repository/tools_r25.1.7-linux.zip', '3ca053600a86a5a64d5571edfbb1dad27f2bda3bfd2d38e2fe54322610b1ef0b'), + ('https://dl.google.com/android/repository/android_m2repository_r32.zip', + 'a6a8d7ffb153161f26d5fdebfa9aa1c9c84b29c62851fffff2cdfad9e094b13b'), ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip', '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'), ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip', @@ -97,19 +99,19 @@ cachefiles = [ 'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'), ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip', 'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'), - ('https://dl.google.com/android/repository/android-2.1_r03-linux.zip', + ('https://dl.google.com/android/repository/android-2.1_r03.zip', 'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'), - ('https://dl.google.com/android/repository/android-2.2_r03-linux.zip', + ('https://dl.google.com/android/repository/android-2.2_r03.zip', '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'), - ('https://dl.google.com/android/repository/android-2.3.1_r02-linux.zip', + ('https://dl.google.com/android/repository/android-2.3.1_r02.zip', 'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'), - ('https://dl.google.com/android/repository/android-2.3.3_r02-linux.zip', + ('https://dl.google.com/android/repository/android-2.3.3_r02.zip', '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'), - ('https://dl.google.com/android/repository/android-3.0_r02-linux.zip', + ('https://dl.google.com/android/repository/android-3.0_r02.zip', '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'), - ('https://dl.google.com/android/repository/android-3.1_r03-linux.zip', + ('https://dl.google.com/android/repository/android-3.1_r03.zip', '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'), - ('https://dl.google.com/android/repository/android-3.2_r01-linux.zip', + ('https://dl.google.com/android/repository/android-3.2_r01.zip', 'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'), ('https://dl.google.com/android/repository/android-14_r04.zip', 'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'), @@ -361,7 +363,8 @@ if 'aptproxy' in config and config['aptproxy']: # does not need a custom mount if cachedir != 'buildserver/cache': vagrantfile += """ - config.vm.synced_folder '{0}', '/vagrant/cache' + config.vm.synced_folder '{0}', '/vagrant/cache', + owner: 'root', group: 'root', create: true """.format(cachedir) # cache .deb packages on the host via a mount trick @@ -373,24 +376,32 @@ if config['apt_package_cache']: """.format(aptcachedir) vagrantfile += """ + + config.vm.provision "shell" do |shell| + shell.path = "setup-env-vars" + shell.args = ["/home/vagrant/android-sdk", "/home/vagrant/android-ndk"] + end + config.vm.provision :chef_solo do |chef| chef.cookbooks_path = "cookbooks" chef.log_level = :debug chef.json = { :settings => { - :sdk_loc => "/home/vagrant/android-sdk", - :ndk_loc => "/home/vagrant/android-ndk", :debian_mirror => "%s", :ubuntu_trusty => "%s", :user => "vagrant" } } chef.add_recipe "fdroidbuild-general" - chef.add_recipe "android-sdk" - chef.add_recipe "android-ndk" - chef.add_recipe "gradle" chef.add_recipe "kivy" end + + config.vm.provision "shell", path: "provision-android-sdk" + config.vm.provision "shell", path: "provision-android-ndk" + config.vm.provision "shell", path: "provision-gradle" + config.vm.provision "file", source: "gradle", + destination: "/opt/gradle/bin/gradle" + end """ % (config['debian_mirror'], str('14.04' in os.uname()[3]).lower())