mirrors/fdroidserver
1
0
Fork 0
mirror of https://github.com/f-droid/fdroidserver.git synced 2025-11-04 06:30:27 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Work in progress on integrating build server

Browse source
This commit is contained in:
Ciaran Gultnieks 2012-02-03 16:01:35 +00:00
parent d6e390afd6
commit 498e7d3c5f
14 changed files with 334 additions and 116 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -6,3 +6,4 @@ tmp/
build_*/
*~
*.pyc
buildserver.box

17
README.buildserver Normal file
View file

@ -0,0 +1,17 @@
Setting up a build server:
1. Install VirtualBox, vagrant and vagrant-snap
2. In the buildserver directory, run 'vagrant up'. Will take a long time.
3. Log in with 'vagrant ssh'
4. Check it all looks ok, then 'sudo shutdown -h now'
5. Back in the main directory, run 'VBoxManage listvms' look for
buildserver_xxxx
6. Run 'vagrant package --base buildserver_xxxx --output buildserver.box'.
Will take a while.
7. You should now have a new 'buildserver.box'
You should now be able to use the --server option on build.py and builds will
take place in the clean, secure, isolated environment of a fresh virtual
machine for each app built.

307
build.py
View file

@ -24,6 +24,7 @@ import subprocess
import re
import zipfile
import tarfile
import traceback
from xml.dom.minidom import Document
from optparse import OptionParser
@ -44,8 +45,18 @@ parser.add_option("-s", "--stop", action="store_true", default=False,
help="Make the build stop on exceptions")
parser.add_option("-t", "--test", action="store_true", default=False,
help="Test mode - put output in the tmp directory only.")
parser.add_option("--server", action="store_true", default=False,
help="Use build server")
parser.add_option("--on-server", action="store_true", default=False,
help="Specify that we're running on the build server")
parser.add_option("-f", "--force", action="store_true", default=False,
help="Force build of disabled app. Only allowed in test mode.")
(options, args) = parser.parse_args()
if options.force and not options.test:
print "Force is only allowed in test mode"
sys.exit(1)
# Get all apps...
apps = common.read_metadata(options.verbose)
@ -84,7 +95,7 @@ for app in apps:
if options.package and options.package != app['id']:
# Silent skip...
pass
elif app['Disabled']:
elif app['Disabled'] and not options.force:
if options.verbose:
print "Skipping %s: disabled" % app['id']
elif (not app['builds']) or app['Repo Type'] =='' or len(app['builds']) == 0:
@ -107,7 +118,7 @@ for app in apps:
dest_repo = os.path.join(repo_dir, app['id'] + '_' +
thisbuild['vercode'] + '.apk')
if os.path.exists(dest) or os.path.exists(dest_repo):
if os.path.exists(dest) or (not options.test and os.path.exists(dest_repo)):
if options.verbose:
print "..version " + thisbuild['version'] + " already exists"
elif thisbuild['commit'].startswith('!'):
@ -121,131 +132,199 @@ for app in apps:
mstart = 'Building version '
print mstart + thisbuild['version'] + ' of ' + app['id']
# Prepare the source code...
root_dir = common.prepare_source(vcs, app, thisbuild,
build_dir, extlib_dir, sdk_path, ndk_path,
javacc_path, not refreshed_source)
refreshed_source = True
if options.server:
# Scan before building...
buildprobs = common.scan_source(build_dir)
if len(buildprobs) > 0:
print 'Scanner found ' + str(len(buildprobs)) + ' problems:'
for problem in buildprobs:
print '...' + problem
raise BuildException("Can't build due to " +
str(len(buildprobs)) + " scanned problems")
import paramiko
# Build the source tarball right before we build the release...
tarname = app['id'] + '_' + thisbuild['vercode'] + '_src'
tarball = tarfile.open(os.path.join(tmp_dir,
tarname + '.tar.gz'), "w:gz")
def tarexc(f):
if f in ['.svn', '.git', '.hg', '.bzr']:
return True
return False
tarball.add(build_dir, tarname, exclude=tarexc)
tarball.close()
# Start up the virtual maachine...
if subprocess.call(['vagrant', 'up'], cwd='builder') != 0:
# Not a very helpful message yet!
raise BuildException("Failed to set up build server")
# Get SSH configuration settings for us to connect...
subprocess.call('vagrant ssh-config >sshconfig',
cwd='builder', shell=True)
vagranthost = 'default' # Host in ssh config file
# Build native stuff if required...
if thisbuild.get('buildjni', 'no') == 'yes':
ndkbuild = os.path.join(ndk_path, "ndk-build")
p = subprocess.Popen([ndkbuild], cwd=root_dir,
stdout=subprocess.PIPE)
output = p.communicate()[0]
# Load and parse the SSH config...
sshconfig = paramiko.SSHConfig()
sshconfig.parse('builder/sshconfig')
sshconfig = sshconfig.lookup(vagranthost)
# Open SSH connection...
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AcceptPolicy)
ssh.connect(sshconfig['HostName'], username=sshconfig['Username'],
port=sshconfig['Port'], timeout=10, look_for_keys=False,
key_filename=sshconfig['IdentityFile'])
# Get an SFTP connection...
ftp = ssh.open_sftp()
ftp.get_channel().settimeout(15)
# Put all the necessary files in place...
ftp.chdir('/home/vagrant')
ftp.put('build.py', 'build.py')
ftp.put('common.py', 'common.py')
ftp.put('config.buildserver.py', 'config.py')
ftp.mkdir('build')
ftp.chdir('build')
ftp.mkdir('extlib')
def send_dir(path):
lastdir = path
for r, d, f in os.walk(base):
while lastdir != os.path.commonprefix([lastdir, root]):
ftp.chdir('..')
lastdir = os.path.split(lastdir)[0]
lastdir = r
for ff in f:
ftp.put(os.path.join(r, ff), ff)
ftp.send_dir(app['id'])
# TODO: send relevant extlib directories too
ftp.chdir('/home/vagrant')
# Execute the build script...
ssh.exec_command('python build.py --on-server -p ' +
app['id'])
# Retrieve the built files...
apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk'
tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz'
ftp.chdir('unsigned')
ftp.get(apkfile, os.path.join(output_dir, apkfile))
ftp.get(tarball, os.path.join(output_dir, tarball))
# Get rid of the virtual machine...
if subprocess.call(['vagrant', 'destroy'], cwd='builder') != 0:
# Not a very helpful message yet!
raise BuildException("Failed to destroy")
else:
# Prepare the source code...
root_dir = common.prepare_source(vcs, app, thisbuild,
build_dir, extlib_dir, sdk_path, ndk_path,
javacc_path, not refreshed_source)
refreshed_source = True
# Scan before building...
buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
if len(buildprobs) > 0:
print 'Scanner found ' + str(len(buildprobs)) + ' problems:'
for problem in buildprobs:
print '...' + problem
raise BuildException("Can't build due to " +
str(len(buildprobs)) + " scanned problems")
# Build the source tarball right before we build the release...
tarname = app['id'] + '_' + thisbuild['vercode'] + '_src'
tarball = tarfile.open(os.path.join(tmp_dir,
tarname + '.tar.gz'), "w:gz")
def tarexc(f):
if f in ['.svn', '.git', '.hg', '.bzr']:
return True
return False
tarball.add(build_dir, tarname, exclude=tarexc)
tarball.close()
# Build native stuff if required...
if thisbuild.get('buildjni', 'no') == 'yes':
ndkbuild = os.path.join(ndk_path, "ndk-build")
p = subprocess.Popen([ndkbuild], cwd=root_dir,
stdout=subprocess.PIPE)
output = p.communicate()[0]
if p.returncode != 0:
print output
raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version']))
elif options.verbose:
print output
# Build the release...
if thisbuild.has_key('maven'):
p = subprocess.Popen(['mvn', 'clean', 'install',
'-Dandroid.sdk.path=' + sdk_path],
cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
else:
if thisbuild.has_key('antcommand'):
antcommand = thisbuild['antcommand']
else:
antcommand = 'release'
p = subprocess.Popen(['ant', antcommand], cwd=root_dir,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, error = p.communicate()
if p.returncode != 0:
print output
raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version']))
raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip())
elif options.verbose:
print output
print "Build successful"
# Build the release...
if thisbuild.has_key('maven'):
p = subprocess.Popen(['mvn', 'clean', 'install',
'-Dandroid.sdk.path=' + sdk_path],
cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
else:
if thisbuild.has_key('antcommand'):
antcommand = thisbuild['antcommand']
# Find the apk name in the output...
if thisbuild.has_key('bindir'):
bindir = os.path.join(build_dir, thisbuild['bindir'])
else:
antcommand = 'release'
p = subprocess.Popen(['ant', antcommand], cwd=root_dir,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, error = p.communicate()
if p.returncode != 0:
raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip())
elif options.verbose:
print output
print "Build successful"
# Find the apk name in the output...
if thisbuild.has_key('bindir'):
bindir = os.path.join(build_dir, thisbuild['bindir'])
else:
bindir = os.path.join(root_dir, 'bin')
if thisbuild.get('initfun', 'no') == "yes":
# Special case (again!) for funambol...
src = ("funambol-android-sync-client-" +
thisbuild['version'] + "-unsigned.apk")
src = os.path.join(bindir, src)
elif thisbuild.has_key('maven'):
src = re.match(r".*^\[INFO\] Installing /.*/([^/]*)\.apk",
output, re.S|re.M).group(1)
src = os.path.join(bindir, src) + '.apk'
bindir = os.path.join(root_dir, 'bin')
if thisbuild.get('initfun', 'no') == "yes":
# Special case (again!) for funambol...
src = ("funambol-android-sync-client-" +
thisbuild['version'] + "-unsigned.apk")
src = os.path.join(bindir, src)
elif thisbuild.has_key('maven'):
src = re.match(r".*^\[INFO\] Installing /.*/([^/]*)\.apk",
output, re.S|re.M).group(1)
src = os.path.join(bindir, src) + '.apk'
#[INFO] Installing /home/ciaran/fdroidserver/tmp/mainline/application/target/callerid-1.0-SNAPSHOT.apk
else:
src = re.match(r".*^.*Creating (\S+) for release.*$.*", output,
re.S|re.M).group(1)
src = os.path.join(bindir, src)
else:
src = re.match(r".*^.*Creating (\S+) for release.*$.*", output,
re.S|re.M).group(1)
src = os.path.join(bindir, src)
# By way of a sanity check, make sure the version and version
# code in our new apk match what we expect...
print "Checking " + src
p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools',
'aapt'),
'dump', 'badging', src],
stdout=subprocess.PIPE)
output = p.communicate()[0]
if thisbuild.get('novcheck', 'no') == "yes":
vercode = thisbuild['vercode']
version = thisbuild['version']
else:
vercode = None
version = None
for line in output.splitlines():
if line.startswith("package:"):
pat = re.compile(".*versionCode='([0-9]*)'.*")
vercode = re.match(pat, line).group(1)
pat = re.compile(".*versionName='([^']*)'.*")
version = re.match(pat, line).group(1)
if version == None or vercode == None:
raise BuildException("Could not find version information in build in output")
# By way of a sanity check, make sure the version and version
# code in our new apk match what we expect...
print "Checking " + src
p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools',
'aapt'),
'dump', 'badging', src],
stdout=subprocess.PIPE)
output = p.communicate()[0]
if thisbuild.get('novcheck', 'no') == "yes":
vercode = thisbuild['vercode']
version = thisbuild['version']
else:
vercode = None
version = None
for line in output.splitlines():
if line.startswith("package:"):
pat = re.compile(".*versionCode='([0-9]*)'.*")
vercode = re.match(pat, line).group(1)
pat = re.compile(".*versionName='([^']*)'.*")
version = re.match(pat, line).group(1)
if version == None or vercode == None:
raise BuildException("Could not find version information in build in output")
# Some apps (e.g. Timeriffic) have had the bonkers idea of
# including the entire changelog in the version number. Remove
# it so we can compare. (TODO: might be better to remove it
# before we compile, in fact)
index = version.find(" //")
if index != -1:
version = version[:index]
# Some apps (e.g. Timeriffic) have had the bonkers idea of
# including the entire changelog in the version number. Remove
# it so we can compare. (TODO: might be better to remove it
# before we compile, in fact)
index = version.find(" //")
if index != -1:
version = version[:index]
if (version != thisbuild['version'] or
vercode != thisbuild['vercode']):
raise BuildException(("Unexpected version/version code in output"
"APK: %s / %s"
"Expected: %s / %s")
% (version, str(vercode), thisbuild['version'], str(thisbuild['vercode']))
)
if (version != thisbuild['version'] or
vercode != thisbuild['vercode']):
raise BuildException(("Unexpected version/version code in output"
"APK: %s / %s"
"Expected: %s / %s")
% (version, str(vercode), thisbuild['version'], str(thisbuild['vercode']))
)
# Copy the unsigned apk to our destination directory for further
# processing (by publish.py)...
shutil.copyfile(src, dest)
# Copy the unsigned apk to our destination directory for further
# processing (by publish.py)...
shutil.copyfile(src, dest)
# Move the source tarball into the output directory...
if output_dir != tmp_dir:
tarfilename = tarname + '.tar.gz'
shutil.move(os.path.join(tmp_dir, tarfilename),
os.path.join(output_dir, tarfilename))
# Move the source tarball into the output directory...
if output_dir != tmp_dir:
tarfilename = tarname + '.tar.gz'
shutil.move(os.path.join(tmp_dir, tarfilename),
os.path.join(output_dir, tarfilename))
build_succeeded.append(app)
except BuildException as be:
@ -264,7 +343,7 @@ for app in apps:
except Exception as e:
if options.stop:
raise
print "Could not build app %s due to unknown error: %s" % (app['id'], e)
print "Could not build app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
failed_apps[app['id']] = e
for app in build_succeeded:

1
builder/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
sshconfig

8
builder/Vagrantfile vendored Normal file
View file

@ -0,0 +1,8 @@
Vagrant::Config.run do |config|
config.vm.box = "buildserver"
config.vm.box_url = "../buildserver.box"
config.vm.customize ["modifyvm", :id, "--memory", "768"]
end

1
buildserver/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
.vagrant

26
buildserver/Vagrantfile vendored Normal file
View file

@ -0,0 +1,26 @@
Vagrant::Config.run do |config|
config.vm.box = "debian6-32"
config.vm.box_url = "/shares/software/OS and Boot/debian6-32.box"
config.vm.customize ["modifyvm", :id, "--memory", "1024"]
config.vm.provision :shell, :path => "fixpaths.sh"
# Set apt proxy - remove, or adjust this, accordingly!
config.vm.provision :shell, :inline => 'sudo echo "Acquire::http { Proxy \"http://thurlow:3142\"; };" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update'
config.vm.provision :chef_solo do |chef|
chef.cookbooks_path = "cookbooks"
chef.log_level = :debug
chef.json = {
:settings => {
:sdk_loc => "/home/vagrant/android-sdk",
:ndk_loc => "/home/vagrant/android-ndk",
:user => "vagrant"
}
}
chef.add_recipe "android-sdk"
chef.add_recipe "android-ndk"
chef.add_recipe "fdroidbuild-general"
end
end

17
buildserver/cookbooks/android-ndk/recipes/default.rb Normal file
View file

@ -0,0 +1,17 @@
ndk_loc = node[:settings][:ndk_loc]
script "setup-android-ndk" do
interpreter "bash"
user node[:settings][:user]
cwd "/tmp"
code "
wget http://dl.google.com/android/ndk/android-ndk-r7-linux-x86.tar.bz2
tar jxvf android-ndk-r7-linux-x86.tar.bz2
mv android-ndk-r7 #{ndk_loc}
"
not_if do
File.exists?("#{ndk_loc}")
end
end

34
buildserver/cookbooks/android-sdk/recipes/default.rb Normal file
View file

@ -0,0 +1,34 @@
%w{openjdk-6-jdk}.each do |pkg|
package pkg do
action :install
end
end
sdk_loc = node[:settings][:sdk_loc]
user = node[:settings][:user]
script "setup-android-sdk" do
interpreter "bash"
user user
cwd "/tmp"
code "
wget http://dl.google.com/android/android-sdk_r16-linux.tgz
tar zxvf android-sdk_r16-linux.tgz
mv android-sdk-linux #{sdk_loc}
rm android-sdk_r16-linux.tgz
#{sdk_loc}/tools/android update sdk --no-ui -t platform-tool
#{sdk_loc}/tools/android update sdk --no-ui -t platform
#{sdk_loc}/tools/android update sdk --no-ui -t tool,platform-tool
"
not_if do
File.exists?("#{sdk_loc}")
end
end
execute "add-android-sdk-path" do
user user
path = "#{sdk_loc}/tools:#{sdk_loc}/platform-tools"
command "echo \"export PATH=\\$PATH:#{path}\" >> /home/#{user}/.bashrc"
not_if "grep #{sdk_loc} /home/#{user}/.bashrc"
end

7
buildserver/cookbooks/fdroidbuild-general/recipes/default.rb Normal file
View file

@ -0,0 +1,7 @@
%w{ant ant-contrib maven2 javacc python}.each do |pkg|
package pkg do
action :install
end
end

23
buildserver/fixpaths.sh Normal file
View file

@ -0,0 +1,23 @@
#!/bin/sh
fixit()
{
#Fix sudoers so the PATH gets passed through, otherwise chef
#provisioning doesn't work.
if [ -z "$1" ]; then
export EDITOR=$0 && sudo -E visudo
else
echo "Fix sudoers"
echo "Defaults exempt_group=admin" >> $1
fi
#Stick the gems bin onto root's path as well.
sudo echo "PATH=$PATH:/var/lib/gems/1.8/bin" >>/root/.bashrc
# Restart sudo so it gets the changes straight away
sudo /etc/init.d/sudo restart
}
sudo grep "exempt_group" /etc/sudoers -q
if [ "$?" -eq "1" ]; then
fixit
fi

2
common.py
View file

@ -892,7 +892,7 @@ def prepare_source(vcs, app, build, build_dir, extlib_dir, sdk_path, ndk_path, j
# Scan the source code in the given directory (and all subdirectories)
# and return a list of potential problems.
def scan_source(source_dir):
def scan_source(build_dir, root_dir, thisbuild):
problems = []

4
config.buildserver.py Normal file
View file

@ -0,0 +1,4 @@
aapt_path = "/home/vagrant/android-sdk/platform-tools/aapt"
sdk_path = "/home/vagrant/android-sdk"
ndk_path = "/home/vagrant/android-ndk"
javacc_path = "/usr/share/java"

2
scanner.py
View file

@ -93,7 +93,7 @@ for app in apps:
refreshed_source = True
# Do the scan...
buildprobs = common.scan_source(build_dir)
buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
for problem in buildprobs:
problems.append(problem +
' in ' + app['id'] + ' ' + thisbuild['version'])