Integrated latest build server stuff (except snapshots)

builder/.gitignore

@@ -1,3 +0,0 @@
-sshconfig
-.vagrant
-*.log

builder/Vagrantfile

@@ -1,7 +0,0 @@
-Vagrant::Config.run do |config|
-
-  config.vm.box = "buildserver"
-
-  config.vm.customize ["modifyvm", :id, "--memory", "768"]
-
-end

buildserver/Vagrantfile

@@ -1,7 +1,7 @@
 Vagrant::Config.run do |config|
 
-  config.vm.box = "debian6-32"
-  config.vm.box_url = "/shares/software/OS and Boot/debian6-32.box"
+  config.vm.box = "precise32"
+  config.vm.box_url = "/shares/software/OS and Boot/precise32.box"
 
   config.vm.customize ["modifyvm", :id, "--memory", "1024"]
 

buildserver/cookbooks/android-sdk/recipes/default.rb

@@ -17,8 +17,18 @@ script "setup-android-sdk" do
     mv android-sdk-linux #{sdk_loc}
     rm android-sdk_r16-linux.tgz
     #{sdk_loc}/tools/android update sdk --no-ui -t platform-tool
-    #{sdk_loc}/tools/android update sdk --no-ui -t platform
-    #{sdk_loc}/tools/android update sdk --no-ui -t tool,platform-tool
+    #{sdk_loc}/tools/android update sdk --no-ui -t tool
+    #{sdk_loc}/tools/android update sdk --no-ui -t android-3
+    #{sdk_loc}/tools/android update sdk --no-ui -t android-4
+    #{sdk_loc}/tools/android update sdk --no-ui -t android-7
+    #{sdk_loc}/tools/android update sdk --no-ui -t android-8
+    #{sdk_loc}/tools/android update sdk --no-ui -t android-10
+    #{sdk_loc}/tools/android update sdk --no-ui -t android-11
+    #{sdk_loc}/tools/android update sdk --no-ui -t android-13
+    #{sdk_loc}/tools/android update sdk --no-ui -t android-14
+    #{sdk_loc}/tools/android update sdk --no-ui -t android-15
+    #{sdk_loc}/tools/android update sdk --no-ui -t android-16
+    #{sdk_loc}/tools/android update sdk --no-ui -t addon-google_apis-google-16
   "
   not_if do
     File.exists?("#{sdk_loc}")

buildserver/cookbooks/fdroidbuild-general/recipes/default.rb

@@ -1,5 +1,5 @@
 
-%w{ant ant-contrib maven2 javacc python git-core mercurial subversion bzr}.each do |pkg|
+%w{ant ant-contrib maven javacc python git-core mercurial subversion bzr git-svn make perlmagick}.each do |pkg|
   package pkg do
     action :install
   end

docs/fdroid.texi

@@ -979,7 +979,9 @@ the ability to execute anything) for other applications in the repository.
 @end enumerate
 
 Through complete isolation, the repurcussions are at least limited to the
-application in question.
+application in question. Not only is the build environment fresh for each
+build, and thrown away afterwards, but it is also isolated from the signing
+environment.
 
 Aside from security issues, there are some applications which have strange
 requirements such as custom versions of the NDK. It would be impractical (or
@@ -994,14 +996,14 @@ Some things may not work properly yet. Talk to CiaranG if you're trying to use
 this and have problems.
 
 In addition to the basic setup previously described, you will also need
-a Vagrant-compatible Debian Squeeze base box called 'debian6-32'. You can
-create one of these for yourself from standard Debian installation media, as
+a Vagrant-compatible Ubuntu Precise base box called 'precise32'. You can
+create one of these for yourself from standard Ubuntu installation media, as
 the specification for what's required to be Vagrant-compatible is very well
 defined. This is the sensible and secure way to do it, since you know what's
-in it. If you insist on taking a shortcut, ask CiaranG for his on the forum
-or in IRC.
+in it. If you insist on taking a shortcut, ask CiaranG about it on IRC.
 
-With this base box installed, you can then do:
+With this base box installed, you can then go to the @code{fdroidserver}
+directory and run this:
 
 @example
 ./makebuildserver.sh

fdroidserver/build.py

@@ -36,13 +36,13 @@ from common import VCSException
 def build_server(app, thisbuild, build_dir, output_dir):
     """Do a build on the build server."""
 
-    import paramiko
+    import ssh
 
     # Destroy the builder vm if it already exists...
     # TODO: need to integrate the snapshot stuff so it doesn't have to
     # keep wasting time doing this unnecessarily.
     if os.path.exists(os.path.join('builder', '.vagrant')):
-        if subprocess.call(['vagrant', 'destroy'], cwd='builder') != 0:
+        if subprocess.call(['vagrant', 'destroy', '-f'], cwd='builder') != 0:
             raise BuildException("Failed to destroy build server")
 
     # Start up the virtual maachine...
@@ -55,65 +55,114 @@ def build_server(app, thisbuild, build_dir, output_dir):
     vagranthost = 'default' # Host in ssh config file
 
     # Load and parse the SSH config...
-    sshconfig = paramiko.SSHConfig()
+    sshconfig = ssh.SSHConfig()
     sshf = open('builder/sshconfig', 'r')
     sshconfig.parse(sshf)
     sshf.close()
     sshconfig = sshconfig.lookup(vagranthost)
 
     # Open SSH connection...
-    ssh = paramiko.SSHClient()
-    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-    print sshconfig
-    ssh.connect(sshconfig['hostname'], username=sshconfig['user'],
+    sshs = ssh.SSHClient()
+    sshs.set_missing_host_key_policy(ssh.AutoAddPolicy())
+    sshs.connect(sshconfig['hostname'], username=sshconfig['user'],
         port=int(sshconfig['port']), timeout=10, look_for_keys=False,
         key_filename=sshconfig['identityfile'])
 
     # Get an SFTP connection...
-    ftp = ssh.open_sftp()
+    ftp = sshs.open_sftp()
     ftp.get_channel().settimeout(15)
 
     # Put all the necessary files in place...
     ftp.chdir('/home/vagrant')
-    ftp.put('build.py', 'build.py')
-    ftp.put('common.py', 'common.py')
-    ftp.put('config.buildserver.py', 'config.py')
+
+    # Helper to copy the contents of a directory to the server...
+    def send_dir(path):
+        root = os.path.dirname(path)
+        main = os.path.basename(path)
+        ftp.mkdir(main)
+        for r, d, f in os.walk(path):
+            rr = os.path.relpath(r, root)
+            ftp.chdir(rr)
+            for dd in d:
+                ftp.mkdir(dd)
+            for ff in f:
+                ftp.put(os.path.join(root, rr, ff), ff)
+            for i in range(len(rr.split('/'))):
+                ftp.chdir('..')
+        ftp.chdir('..')
+
+    serverpath = os.path.abspath(os.path.dirname(__file__))
+    ftp.put(os.path.join(serverpath, 'build.py'), 'build.py')
+    ftp.put(os.path.join(serverpath, 'common.py'), 'common.py')
+    ftp.put(os.path.join(serverpath, '..', 'config.buildserver.py'), 'config.py')
+
+    # Copy the metadata - just the file for this app...
     ftp.mkdir('metadata')
     ftp.chdir('metadata')
     ftp.put(os.path.join('metadata', app['id'] + '.txt'),
             app['id'] + '.txt')
     ftp.chdir('..')
+    # Create the build directory...
     ftp.mkdir('build')
     ftp.chdir('build')
     ftp.mkdir('extlib')
-    ftp.mkdir(app['id'])
-    ftp.chdir('..')
-    def send_dir(path):
-        lastdir = path
-        for r, d, f in os.walk(path):
-            ftp.chdir(r)
-            for dd in d:
-                ftp.mkdir(dd)
-            for ff in f:
-                ftp.put(os.path.join(r, ff), ff)
-            for i in range(len(r.split('/'))):
-                ftp.chdir('..')
+    # Copy the main app source code
+    if os.path.exists(build_dir):
         send_dir(build_dir)
-    # TODO: send relevant extlib and srclib directories too
+    # Copy any extlibs that are required...
+    if thisbuild.has_key('extlibs'):
+        ftp.chdir('build')
+        ftp.chdir('extlib')
+        for lib in thisbuild['extlibs'].split(';'):
+            lp = lib.split('/')
+            for d in lp[:-1]:
+                ftp.mkdir(d)
+                ftp.chdir(d)
+            ftp.put(os.path.join('build/extlib', lib), lp[-1])
+            for _ in lp[:-1]:
+                ftp.chdir('..')
+        ftp.chdir('..')
+        ftp.chdir('..')
+    # Copy any srclibs that are required...
+    if thisbuild.has_key('srclibs'):
+        ftp.chdir('build')
+        ftp.chdir('extlib')
+        for lib in thisbuild['srclibs'].split(';'):
+            lp = lib.split('@').split('/')
+            for d in lp[:-1]:
+                ftp.mkdir(d)
+                ftp.chdir(d)
+            ftp.put(os.path.join('build/extlib', lib), lp[-1])
+            for _ in lp[:-1]:
+                ftp.chdir('..')
+        ftp.chdir('..')
+        ftp.chdir('..')
 
     # Execute the build script...
-    ssh.exec_command('python build.py --on-server -p ' +
+    chan = sshs.get_transport().open_session()
+    stdoutf = chan.makefile('rb')
+    stderrf = chan.makefile_stderr('rb')
+    chan.exec_command('python build.py --on-server -p ' +
             app['id'] + ' --vercode ' + thisbuild['vercode'])
+    returncode = chan.recv_exit_status()
+    output = stdoutf.read()
+    error = stderrf.read()
+    if returncode != 0:
+        raise BuildException("Build.py failed on server for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip())
 
     # Retrieve the built files...
+    ftp.chdir('/home/vagrant/unsigned')
     apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk'
     tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz'
-    ftp.chdir('/home/vagrant/unsigned')
+    try:
         ftp.get(apkfile, os.path.join(output_dir, apkfile))
         ftp.get(tarball, os.path.join(output_dir, tarball))
+    except:
+        raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip())
+    ftp.close()
 
     # Get rid of the virtual machine...
-    if subprocess.call(['vagrant', 'destroy'], cwd='builder') != 0:
+    if subprocess.call(['vagrant', 'destroy', '-f'], cwd='builder') != 0:
         # Not a very helpful message yet!
         raise BuildException("Failed to destroy")
 
@@ -304,6 +353,10 @@ def trybuild(app, thisbuild, build_dir, output_dir, extlib_dir, tmp_dir,
     print "Building version " + thisbuild['version'] + ' of ' + app['id']
 
     if server:
+        # When using server mode, still keep a local cache of the repo, by
+        # grabbing the source now.
+        vcs.gotorevision(thisbuild['commit'])
+
         build_server(app, thisbuild, build_dir, output_dir)
     else:
         build_local(app, thisbuild, vcs, build_dir, output_dir, extlib_dir, tmp_dir, install, force, verbose)
@@ -396,6 +449,17 @@ def main():
         os.makedirs(build_dir)
     extlib_dir = os.path.join(build_dir, 'extlib')
 
+    # Create the 'builder' vagrant directory if we don't have it...
+    if options.server:
+        if not os.path.exists('builder'):
+            os.mkdir('builder')
+            vf = open('builder/Vagrantfile', 'w')
+            vf.write('Vagrant::Config.run do |config|\n')
+            vf.write('config.vm.box = "buildserver"\n')
+            vf.write('config.vm.customize ["modifyvm", :id, "--memory", "768"]\n')
+            vf.write('end\n')
+            vf.close()
+
     # Filter apps and build versions according to command-line options, etc...
     if options.package:
         apps = [app for app in apps if app['id'] == options.package]

makebuildserver.sh

@@ -3,6 +3,7 @@ set -e
 rm -f buildserver.box
 cd buildserver
 vagrant up
+sleep 5
 vagrant ssh -c "sudo shutdown -h now"
 cd ..
 # Just to wait until it's shut down!