mirror of
https://github.com/f-droid/fdroidserver.git
synced 2025-11-04 14:30:30 +03:00
Much better build server provisioning
Including selection of correct ndk architecture, caching and performance improvements, better configurability, etc.
This commit is contained in:
Ciaran Gultnieks
parent
0985a6d4f4
commit
cb16a63409
5 changed files with 124 additions and 53 deletions
2
buildserver/.gitignore
vendored
2
buildserver/.gitignore
vendored
|
|
@ -1,2 +1,4 @@
|
|||
.vagrant
|
||||
up.log
|
||||
cache/
|
||||
Vagrantfile
|
||||
|
|
|
|||
26
buildserver/Vagrantfile
vendored
26
buildserver/Vagrantfile
vendored
|
|
@ -1,26 +0,0 @@
|
|||
Vagrant::Config.run do |config|
|
||||
|
||||
config.vm.box = "raring64"
|
||||
config.vm.box_url = "/shares/software/OS and Boot/raring64.box"
|
||||
|
||||
config.vm.customize ["modifyvm", :id, "--memory", "3584"]
|
||||
|
||||
config.vm.provision :shell, :path => "fixpaths.sh"
|
||||
# Set apt proxy - remove, or adjust this, accordingly!
|
||||
config.vm.provision :shell, :inline => 'sudo echo "Acquire::http { Proxy \"http://192.168.0.19:8000\"; };" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update'
|
||||
|
||||
config.vm.provision :chef_solo do |chef|
|
||||
chef.cookbooks_path = "cookbooks"
|
||||
chef.log_level = :debug
|
||||
chef.json = {
|
||||
:settings => {
|
||||
:sdk_loc => "/home/vagrant/android-sdk",
|
||||
:ndk_loc => "/home/vagrant/android-ndk",
|
||||
:user => "vagrant"
|
||||
}
|
||||
}
|
||||
chef.add_recipe "fdroidbuild-general"
|
||||
chef.add_recipe "android-sdk"
|
||||
chef.add_recipe "android-ndk"
|
||||
end
|
||||
end
|
||||
|
|
@ -7,7 +7,12 @@ script "setup-android-ndk" do
|
|||
user node[:settings][:user]
|
||||
cwd "/tmp"
|
||||
code "
|
||||
tar jxvf /vagrant/cache/android-ndk-r8e-linux-x64.tar.bz2
|
||||
if [ `uname -m` == 'x86_64' ] ; then
|
||||
SUFFIX = '_64'
|
||||
else
|
||||
SUFFIX = ''
|
||||
fi
|
||||
tar jxvf /vagrant/cache/android-ndk-r8e-linux-x86$SUFFIX.tar.bz2
|
||||
mv android-ndk-r8e #{ndk_loc}
|
||||
"
|
||||
not_if do
|
||||
|
|
|
|||
|
|
@ -37,10 +37,16 @@ end
|
|||
user user
|
||||
cwd "/tmp"
|
||||
code "
|
||||
#{sdk_loc}/tools/android update sdk --no-ui -a -t #{sdk} <<X
|
||||
if [ -f /vagrant/cache/platforms/#{sdk}.tar.gz ] ; then
|
||||
echo Installing from cache
|
||||
tar -C #{sdk_loc}/platforms -z -x -f /vagrant/cache/platforms/#{sdk}.tar.gz
|
||||
else
|
||||
echo Installing via 'android'
|
||||
#{sdk_loc}/tools/android update sdk --no-ui -a -t #{sdk} <<X
|
||||
y
|
||||
|
||||
X
|
||||
fi
|
||||
"
|
||||
not_if "test -d #{sdk_loc}/platforms/#{sdk}"
|
||||
end
|
||||
|
|
@ -54,10 +60,16 @@ end
|
|||
user user
|
||||
cwd "/tmp"
|
||||
code "
|
||||
#{sdk_loc}/tools/android update sdk --no-ui -a -t #{sdk} <<X
|
||||
if [ -f /vagrant/cache/add-ons/#{sdk}.tar.gz ] ; then
|
||||
echo Installing from cache
|
||||
tar -C #{sdk_loc}/add-ons -z -x -f /vagrant/cache/add-ons/#{sdk}.tar.gz
|
||||
else
|
||||
echo Installing via 'android'
|
||||
#{sdk_loc}/tools/android update sdk --no-ui -a -t #{sdk} <<X
|
||||
y
|
||||
|
||||
X
|
||||
fi
|
||||
"
|
||||
|
||||
not_if "test -d #{sdk_loc}/add-ons/#{sdk}"
|
||||
|
|
|
|||
|
|
@ -5,11 +5,40 @@ import sys
|
|||
import subprocess
|
||||
import time
|
||||
|
||||
def vagrant(params, cwd=None):
|
||||
|
||||
# Settings, which need to be moved elsewhere ultimately...
|
||||
basebox = "raring32"
|
||||
baseboxurl = "/shares/software/OS and Boot/raring32.box"
|
||||
memory = 3584
|
||||
aptproxy = "http://192.168.0.19:8000"
|
||||
arch64 = False
|
||||
|
||||
|
||||
|
||||
def vagrant(params, cwd=None, printout=False):
|
||||
"""Run vagrant.
|
||||
|
||||
:param: list of parameters to pass to vagrant
|
||||
:cwd: directory to run in, or None for current directory
|
||||
:printout: True to print output in realtime, False to just
|
||||
return it
|
||||
:returns: (ret, out) where ret is the return code, and out
|
||||
is the stdout (and stderr) from vagrant
|
||||
"""
|
||||
p = subprocess.Popen(['vagrant'] + params, cwd=cwd,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
out, err = p.communicate()
|
||||
return (p.returncode, out, err)
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
out = ''
|
||||
if printout:
|
||||
while True:
|
||||
line = p.stdout.readline()
|
||||
if len(line) == 0:
|
||||
break
|
||||
print line,
|
||||
out += line
|
||||
p.wait()
|
||||
else:
|
||||
out = p.communicate()[0]
|
||||
return (p.returncode, out)
|
||||
|
||||
boxfile = 'buildserver.box'
|
||||
serverdir = 'buildserver'
|
||||
|
|
@ -28,35 +57,84 @@ if not os.path.exists(cachedir):
|
|||
os.mkdir(cachedir)
|
||||
cachefiles = [
|
||||
('android-sdk_r21.0.1-linux.tgz',
|
||||
'http://dl.google.com/android/android-sdk_r21.0.1-linux.tgz',
|
||||
'cookbooks/recipes/android-sdk/default.rb'),
|
||||
('android-ndk-r8e-linux-x64.tar.bz2',
|
||||
'http://dl.google.com/android/ndk/android-ndk-r8e-linux-x64.tar.bz2',
|
||||
'cookbooks/recipes/android-ndk/default.rb')
|
||||
]
|
||||
'http://dl.google.com/android/android-sdk_r21.0.1-linux.tgz')]
|
||||
if arch64:
|
||||
cachefiles.extend([
|
||||
('android-ndk-r8e-linux-x86_64.tar.bz2',
|
||||
'http://dl.google.com/android/ndk/android-ndk-r8e-linux-x86_64.tar.bz2')])
|
||||
else:
|
||||
cachefiles.extend([
|
||||
('android-ndk-r8e-linux-x86.tar.bz2',
|
||||
'http://dl.google.com/android/ndk/android-ndk-r8e-linux-x86.tar.bz2')])
|
||||
wanted = []
|
||||
for f, src, check in cachefiles:
|
||||
if subprocess.call('grep ' + f + ' ' + check) != 0:
|
||||
print "Cache mismatch - " + f + " is not mentioned in " + check
|
||||
sys.exit(1)
|
||||
for f, src in cachefiles:
|
||||
if not os.path.exists(os.path.join(cachedir, f)):
|
||||
print "Downloading " + f + " to cache"
|
||||
if subprocess.call('wget ' + src, cwd=cachedir) != 0:
|
||||
if subprocess.call(['wget', src], cwd=cachedir) != 0:
|
||||
print "...download of " + f + " failed."
|
||||
sys.exit(1)
|
||||
wanted.append(f)
|
||||
for f in os.listdir(cachedir):
|
||||
if not f in wanted:
|
||||
print "Removing unwanted cache file " + f
|
||||
os.remove(os.path.join(cachedir, f))
|
||||
|
||||
|
||||
vagrant(['halt'], serverdir)
|
||||
# Generate an appropriate Vagrantfile for the buildserver, based on our
|
||||
# settings...
|
||||
vagrantfile = """
|
||||
Vagrant::Config.run do |config|
|
||||
|
||||
config.vm.box = "{0}"
|
||||
config.vm.box_url = "{1}"
|
||||
|
||||
config.vm.customize ["modifyvm", :id, "--memory", "{2}"]
|
||||
|
||||
config.vm.provision :shell, :path => "fixpaths.sh"
|
||||
""".format(basebox, baseboxurl, memory)
|
||||
if aptproxy:
|
||||
vagrantfile += """
|
||||
config.vm.provision :shell, :inline => 'sudo echo "Acquire::http {{ Proxy \\"{0}\\"; }};" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update'
|
||||
""".format(aptproxy)
|
||||
vagrantfile += """
|
||||
config.vm.provision :chef_solo do |chef|
|
||||
chef.cookbooks_path = "cookbooks"
|
||||
chef.log_level = :debug
|
||||
chef.json = {
|
||||
:settings => {
|
||||
:sdk_loc => "/home/vagrant/android-sdk",
|
||||
:ndk_loc => "/home/vagrant/android-ndk",
|
||||
:user => "vagrant"
|
||||
}
|
||||
}
|
||||
chef.add_recipe "fdroidbuild-general"
|
||||
chef.add_recipe "android-sdk"
|
||||
chef.add_recipe "android-ndk"
|
||||
end
|
||||
end
|
||||
"""
|
||||
|
||||
# Check against the existing Vagrantfile, and if they differ, we need to
|
||||
# create a new box:
|
||||
vf = os.path.join(serverdir, 'Vagrantfile')
|
||||
writevf = True
|
||||
if os.path.exists(vf):
|
||||
vagrant(['halt'], serverdir)
|
||||
with open(vf, 'r') as f:
|
||||
oldvf = f.read()
|
||||
if oldvf != vagrantfile:
|
||||
print "Server configuration has changed, rebuild from scratch is required"
|
||||
vagrant(['destroy', '-f'], serverdir)
|
||||
else:
|
||||
print "Re-provisioning existing server"
|
||||
writevf = False
|
||||
else:
|
||||
print "No existing server - building from scratch"
|
||||
if writevf:
|
||||
with open(vf, 'w') as f:
|
||||
f.write(vagrantfile)
|
||||
|
||||
|
||||
print "Configuring build server VM"
|
||||
returncode, out, err = vagrant(['up'], serverdir)
|
||||
returncode, out = vagrant(['up'], serverdir, printout=True)
|
||||
with open(os.path.join(serverdir, 'up.log'), 'w') as log:
|
||||
log.write('==stdout==\n' + out + '\n\n')
|
||||
log.write('==stderr==\n' + err + '\n\n')
|
||||
log.write(out)
|
||||
if returncode != 0:
|
||||
print "Failed to configure server"
|
||||
sys.exit(1)
|
||||
|
|
@ -67,7 +145,7 @@ print "Waiting for build server VM to be finished"
|
|||
ready = False
|
||||
while not ready:
|
||||
time.sleep(2)
|
||||
returncode, out, err = vagrant(['status'], serverdir)
|
||||
returncode, out = vagrant(['status'], serverdir)
|
||||
if returncode != 0:
|
||||
print "Error while checking status"
|
||||
sys.exit(1)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue