#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# build.py - part of the FDroid server tools
# Copyright (C) 2010-11, Ciaran Gultnieks, ciaran@ciarang.com
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
import sys
import os
import shutil
import subprocess
import re
import zipfile
import tarfile
import md5
import shlex
from xml.dom.minidom import Document
from optparse import OptionParser
import common
from common import BuildException
from common import VCSException
#Read configuration...
execfile('config.py')
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-p", "--package", default=None,
help="Build only the specified package")
(options, args) = parser.parse_args()
# Get all apps...
apps = common.read_metadata(options.verbose)
failed_apps = {}
build_succeeded = []
output_dir = 'repo'
if not os.path.isdir(output_dir):
print "Creating output directory"
os.makedirs(output_dir)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
print "Creating temporary directory"
os.makedirs(tmp_dir)
build_dir = 'build'
if not os.path.isdir(build_dir):
print "Creating build directory"
os.makedirs(build_dir)
for app in apps:
if app['disabled']:
print "Skipping %s: disabled" % app['id']
elif not app['builds']:
print "Skipping %s: no builds specified" % app['id']
if (app['disabled'] is None and app['repo'] != ''
and app['repotype'] != '' and (options.package is None or
options.package == app['id']) and len(app['builds']) > 0):
print "Processing " + app['id']
build_dir = 'build/' + app['id']
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['repotype'], app['repo'], build_dir)
refreshed_source = False
for thisbuild in app['builds']:
try:
dest = os.path.join(output_dir, app['id'] + '_' +
thisbuild['vercode'] + '.apk')
dest_unsigned = os.path.join(tmp_dir, app['id'] + '_' +
thisbuild['vercode'] + '_unsigned.apk')
if os.path.exists(dest):
print "..version " + thisbuild['version'] + " already exists"
elif thisbuild['commit'].startswith('!'):
print ("..skipping version " + thisbuild['version'] + " - " +
thisbuild['commit'][1:])
else:
print "..building version " + thisbuild['version']
if not refreshed_source:
vcs.refreshlocal()
refreshed_source = True
# Optionally, the actual app source can be in a subdirectory...
if thisbuild.has_key('subdir'):
root_dir = os.path.join(build_dir, thisbuild['subdir'])
else:
root_dir = build_dir
# Get a working copy of the right revision...
if options.verbose:
print "Resetting repository to " + thisbuild['commit']
vcs.reset(thisbuild['commit'])
# Initialise submodules if requred...
if thisbuild.get('submodules', 'no') == 'yes':
vcs.initsubmodules()
# Generate (or update) the ant build file, build.xml...
if (thisbuild.get('update', 'yes') == 'yes' and
not thisbuild.has_key('maven')):
parms = [os.path.join(sdk_path, 'tools', 'android'),
'update', 'project', '-p', '.']
parms.append('--subprojects')
if thisbuild.has_key('target'):
parms.append('-t')
parms.append(thisbuild['target'])
if subprocess.call(parms, cwd=root_dir) != 0:
raise BuildException("Failed to update project")
# If the app has ant set up to sign the release, we need to switch
# that off, because we want the unsigned apk...
for propfile in ('build.properties', 'default.properties'):
if os.path.exists(os.path.join(root_dir, propfile)):
if subprocess.call(['sed','-i','s/^key.store/#/',
propfile], cwd=root_dir) !=0:
raise BuildException("Failed to amend %s" % propfile)
# Update the local.properties file...
locprops = os.path.join(root_dir, 'local.properties')
if os.path.exists(locprops):
f = open(locprops, 'r')
props = f.read()
f.close()
# Fix old-fashioned 'sdk-location' by copying
# from sdk.dir, if necessary...
if thisbuild.get('oldsdkloc', 'no') == "yes":
sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
re.S|re.M).group(1)
props += "\nsdk-location=" + sdkloc + "\n"
# Add ndk location...
props+= "\nndk.dir=" + ndk_path + "\n"
# Add java.encoding if necessary...
if thisbuild.has_key('encoding'):
props += "\njava.encoding=" + thisbuild['encoding'] + "\n"
f = open(locprops, 'w')
f.write(props)
f.close()
# Insert version code and number into the manifest if necessary...
if thisbuild.has_key('insertversion'):
if subprocess.call(['sed','-i','s/' + thisbuild['insertversion'] +
'/' + thisbuild['version'] +'/g',
'AndroidManifest.xml'], cwd=root_dir) !=0:
raise BuildException("Failed to amend manifest")
if thisbuild.has_key('insertvercode'):
if subprocess.call(['sed','-i','s/' + thisbuild['insertvercode'] +
'/' + thisbuild['vercode'] +'/g',
'AndroidManifest.xml'], cwd=root_dir) !=0:
raise BuildException("Failed to amend manifest")
# Delete unwanted file...
if thisbuild.has_key('rm'):
os.remove(os.path.join(build_dir, thisbuild['rm']))
# Fix apostrophes translation files if necessary...
if thisbuild.get('fixapos', 'no') == 'yes':
for root, dirs, files in os.walk(os.path.join(root_dir,'res')):
for filename in files:
if filename.endswith('.xml'):
if subprocess.call(['sed','-i','s@' +
r"\([^\\]\)'@\1\\'" +
'@g',
os.path.join(root, filename)]) != 0:
raise BuildException("Failed to amend " + filename)
# Fix translation files if necessary...
if thisbuild.get('fixtrans', 'no') == 'yes':
for root, dirs, files in os.walk(os.path.join(root_dir,'res')):
for filename in files:
if filename.endswith('.xml'):
f = open(os.path.join(root, filename))
changed = False
outlines = []
for line in f:
num = 1
index = 0
oldline = line
while True:
index = line.find("%", index)
if index == -1:
break
next = line[index+1:index+2]
if next == "s" or next == "d":
line = (line[:index+1] +
str(num) + "$" +
line[index+1:])
num += 1
index += 3
else:
index += 1
# We only want to insert the positional arguments
# when there is more than one argument...
if oldline != line:
if num > 2:
changed = True
else:
line = oldline
outlines.append(line)
f.close()
if changed:
f = open(os.path.join(root, filename), 'w')
f.writelines(outlines)
f.close()
# Run a pre-build command if one is required...
if thisbuild.has_key('prebuild'):
if subprocess.call(thisbuild['prebuild'],
cwd=root_dir, shell=True) != 0:
raise BuildException("Error running pre-build command")
# Apply patches if any
if 'patch' in thisbuild:
for patch in thisbuild['patch'].split(';'):
print "Applying " + patch
patch_path = os.path.join('metadata', app['id'], patch)
if subprocess.call(['patch', '-p1',
'-i', os.path.abspath(patch_path)], cwd=build_dir) != 0:
raise BuildException("Failed to apply patch %s" % patch_path)
# Special case init functions for funambol...
if thisbuild.get('initfun', 'no') == "yes":
if subprocess.call(['sed','-i','s@' +
'' +
'@' +
'' +
'' +
'' +
'' +
'' +
'@g',
'build.xml'], cwd=root_dir) !=0:
raise BuildException("Failed to amend build.xml")
if subprocess.call(['sed','-i','s@' +
'\${user.home}/funambol/build/android/build.properties' +
'@' +
'build.properties' +
'@g',
'build.xml'], cwd=root_dir) !=0:
raise BuildException("Failed to amend build.xml")
buildxml = os.path.join(root_dir, 'build.xml')
f = open(buildxml, 'r')
xml = f.read()
f.close()
xmlout = ""
mode = 0
for line in xml.splitlines():
if mode == 0:
if line.find("jarsigner") != -1:
mode = 1
else:
xmlout += line + "\n"
else:
if line.find("/exec") != -1:
mode += 1
if mode == 3:
mode =0
f = open(buildxml, 'w')
f.write(xmlout)
f.close()
if subprocess.call(['sed','-i','s@' +
'platforms/android-2.0' +
'@' +
'platforms/android-8' +
'@g',
'build.xml'], cwd=root_dir) !=0:
raise BuildException("Failed to amend build.xml")
shutil.copyfile(
os.path.join(root_dir, "build.properties.example"),
os.path.join(root_dir, "build.properties"))
if subprocess.call(['sed','-i','s@' +
'javacchome=.*'+
'@' +
'javacchome=' + javacc_path +
'@g',
'build.properties'], cwd=root_dir) !=0:
raise BuildException("Failed to amend build.properties")
if subprocess.call(['sed','-i','s@' +
'sdk-folder=.*'+
'@' +
'sdk-folder=' + sdk_path +
'@g',
'build.properties'], cwd=root_dir) !=0:
raise BuildException("Failed to amend build.properties")
if subprocess.call(['sed','-i','s@' +
'android.sdk.version.*'+
'@' +
'android.sdk.version=2.0' +
'@g',
'build.properties'], cwd=root_dir) !=0:
raise BuildException("Failed to amend build.properties")
# Build the source tarball right before we build the release...
tarname = app['id'] + '_' + thisbuild['vercode'] + '_src'
tarball = tarfile.open(os.path.join(output_dir,
tarname + '.tar.gz'), "w:gz")
tarball.add(build_dir, tarname)
tarball.close()
# Build native stuff if required...
if thisbuild.get('buildjni', 'no') == 'yes':
ndkbuild = os.path.join(ndk_path, "ndk-build")
p = subprocess.Popen([ndkbuild], cwd=root_dir,
stdout=subprocess.PIPE)
output = p.communicate()[0]
if p.returncode != 0:
print output
raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version']))
elif options.verbose:
print output
# Build the release...
if thisbuild.has_key('maven'):
p = subprocess.Popen(['mvn', 'clean', 'install',
'-Dandroid.sdk.path=' + sdk_path],
cwd=root_dir, stdout=subprocess.PIPE)
else:
if thisbuild.has_key('antcommand'):
antcommand = thisbuild['antcommand']
else:
antcommand = 'release'
p = subprocess.Popen(['ant', antcommand], cwd=root_dir,
stdout=subprocess.PIPE)
output = p.communicate()[0]
if p.returncode != 0:
print output
raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']))
elif options.verbose:
print output
print "Build successful"
# Find the apk name in the output...
if thisbuild.has_key('bindir'):
bindir = os.path.join(build_dir, thisbuild['bindir'])
else:
bindir = os.path.join(root_dir, 'bin')
if thisbuild.get('initfun', 'no') == "yes":
# Special case (again!) for funambol...
src = ("funambol-android-sync-client-" +
thisbuild['version'] + "-unsigned.apk")
src = os.path.join(bindir, src)
elif thisbuild.has_key('maven'):
src = re.match(r".*^\[INFO\] Installing /.*/([^/]*)\.apk",
output, re.S|re.M).group(1)
src = os.path.join(bindir, src) + '.apk'
#[INFO] Installing /home/ciaran/fdroidserver/tmp/mainline/application/target/callerid-1.0-SNAPSHOT.apk
else:
src = re.match(r".*^.*Creating (\S+) for release.*$.*", output,
re.S|re.M).group(1)
src = os.path.join(bindir, src)
# By way of a sanity check, make sure the version and version
# code in our new apk match what we expect...
print "Checking " + src
p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools',
'aapt'),
'dump', 'badging', src],
stdout=subprocess.PIPE)
output = p.communicate()[0]
if thisbuild.get('novcheck', 'no') == "yes":
vercode = thisbuild['vercode']
version = thisbuild['version']
else:
vercode = None
version = None
for line in output.splitlines():
if line.startswith("package:"):
pat = re.compile(".*versionCode='([0-9]*)'.*")
vercode = re.match(pat, line).group(1)
pat = re.compile(".*versionName='([^']*)'.*")
version = re.match(pat, line).group(1)
if version == None or vercode == None:
raise BuildException("Could not find version information in build in output")
# Some apps (e.g. Timeriffic) have had the bonkers idea of
# including the entire changelog in the version number. Remove
# it so we can compare. (TODO: might be better to remove it
# before we compile, in fact)
index = version.find(" //")
if index != -1:
version = version[:index]
if (version != thisbuild['version'] or
vercode != thisbuild['vercode']):
raise BuildException(("Unexpected version/version code in output"
"APK: %s / %s"
"Expected: %s / %s")
) % (version, str(vercode), thisbuild['version'], str(thisbuild['vercode']))
# Copy the unsigned apk to our temp directory for further
# processing...
shutil.copyfile(src, dest_unsigned)
# Figure out the key alias name we'll use. Only the first 8
# characters are significant, so we'll use the first 8 from
# the MD5 of the app's ID and hope there are no collisions.
# If a collision does occur later, we're going to have to
# come up with a new alogrithm, AND rename all existing keys
# in the keystore!
if keyaliases.has_key(app['id']):
# For this particular app, the key alias is overridden...
keyalias = keyaliases[app['id']]
else:
m = md5.new()
m.update(app['id'])
keyalias = m.hexdigest()[:8]
print "Key alias: " + keyalias
# See if we already have a key for this application, and
# if not generate one...
p = subprocess.Popen(['keytool', '-list',
'-alias', keyalias, '-keystore', keystore,
'-storepass', keystorepass], stdout=subprocess.PIPE)
output = p.communicate()[0]
if p.returncode !=0:
print "Key does not exist - generating..."
p = subprocess.Popen(['keytool', '-genkey',
'-keystore', keystore, '-alias', keyalias,
'-keyalg', 'RSA', '-keysize', '2048',
'-validity', '10000',
'-storepass', keystorepass, '-keypass', keypass,
'-dname', keydname], stdout=subprocess.PIPE)
output = p.communicate()[0]
print output
if p.returncode != 0:
raise BuildException("Failed to generate key")
# Sign the application...
p = subprocess.Popen(['jarsigner', '-keystore', keystore,
'-storepass', keystorepass, '-keypass', keypass,
dest_unsigned, keyalias], stdout=subprocess.PIPE)
output = p.communicate()[0]
print output
if p.returncode != 0:
raise BuildException("Failed to sign application")
# Zipalign it...
p = subprocess.Popen([os.path.join(sdk_path,'tools','zipalign'),
'-v', '4', dest_unsigned, dest],
stdout=subprocess.PIPE)
output = p.communicate()[0]
print output
if p.returncode != 0:
raise BuildException("Failed to align application")
os.remove(dest_unsigned)
except BuildException as be:
print "Could not build app %s due to BuildException: %s" % (app['id'], be)
failed_apps[app['id']] = be
except VCSException as vcse:
print "VCS error while building app %s: %s" % (app['id'], vcse)
failed_apps[app['id']] = vcse
except Exception as e:
print "Could not build app %s due to unknown error: %s" % (app['id'], e)
failed_apps[app['id']] = e
build_succeeded.append(app)
for fa in failed_apps:
print "Build for app %s failed: %s" % (fa, failed_apps[fa])
for app in build_succeeded:
print "success: %s" % (app['id'])
print "Finished."