diff --git a/build.py b/build.py index f42b1191..a7b09139 100755 --- a/build.py +++ b/build.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python # -*- coding: utf-8 -*- # # build.py - part of the FDroid server tools @@ -30,6 +30,8 @@ from xml.dom.minidom import Document from optparse import OptionParser import common +from common import BuildException +from common import VCSException #Read configuration... execfile('config.py') @@ -46,13 +48,23 @@ parser.add_option("-p", "--package", default=None, # Get all apps... apps = common.read_metadata(options.verbose) -output_dir = "repo" -tmp_dir = "tmp" -if not os.path.isdir('tmp'): - os.makedirs('tmp') +failed_apps = {} +build_succeeded = [] -if not os.path.isdir('build'): - os.makedirs('build') +output_dir = 'repo' +if not os.path.isdir(output_dir): + print "Creating output directory" + os.makedirs(output_dir) + +tmp_dir = 'tmp' +if not os.path.isdir(tmp_dir): + print "Creating temporary directory" + os.makedirs(tmp_dir) + +build_dir = 'build' +if not os.path.isdir(build_dir): + print "Creating build directory" + os.makedirs(build_dir) for app in apps: @@ -76,415 +88,413 @@ for app in apps: for thisbuild in app['builds']: + try: + dest = os.path.join(output_dir, app['id'] + '_' + + thisbuild['vercode'] + '.apk') + dest_unsigned = os.path.join(tmp_dir, app['id'] + '_' + + thisbuild['vercode'] + '_unsigned.apk') - dest = os.path.join(output_dir, app['id'] + '_' + - thisbuild['vercode'] + '.apk') - dest_unsigned = os.path.join(tmp_dir, app['id'] + '_' + - thisbuild['vercode'] + '_unsigned.apk') - - if os.path.exists(dest): - print "..version " + thisbuild['version'] + " already exists" - elif thisbuild['commit'].startswith('!'): - print ("..skipping version " + thisbuild['version'] + " - " + - thisbuild['commit'][1:]) - else: - print "..building version " + thisbuild['version'] - - if not refreshed_source: - vcs.refreshlocal() - refreshed_source = True - - # Optionally, the actual app source can be in a subdirectory... - if thisbuild.has_key('subdir'): - root_dir = os.path.join(build_dir, thisbuild['subdir']) + if os.path.exists(dest): + print "..version " + thisbuild['version'] + " already exists" + elif thisbuild['commit'].startswith('!'): + print ("..skipping version " + thisbuild['version'] + " - " + + thisbuild['commit'][1:]) else: - root_dir = build_dir + print "..building version " + thisbuild['version'] - # Get a working copy of the right revision... - if options.verbose: - print "Resetting repository to " + thisbuild['commit'] - vcs.reset(thisbuild['commit']) + if not refreshed_source: + vcs.refreshlocal() + refreshed_source = True - # Initialise submodules if requred... - if thisbuild.get('submodules', 'no') == 'yes': - vcs.initsubmodules() + # Optionally, the actual app source can be in a subdirectory... + if thisbuild.has_key('subdir'): + root_dir = os.path.join(build_dir, thisbuild['subdir']) + else: + root_dir = build_dir - # Generate (or update) the ant build file, build.xml... - if (thisbuild.get('update', 'yes') == 'yes' and - not thisbuild.has_key('maven')): - parms = [os.path.join(sdk_path, 'tools', 'android'), - 'update', 'project', '-p', '.'] - parms.append('--subprojects') - if thisbuild.has_key('target'): - parms.append('-t') - parms.append(thisbuild['target']) - if subprocess.call(parms, cwd=root_dir) != 0: - print "Failed to update project" - sys.exit(1) + # Get a working copy of the right revision... + if options.verbose: + print "Resetting repository to " + thisbuild['commit'] + vcs.reset(thisbuild['commit']) - # If the app has ant set up to sign the release, we need to switch - # that off, because we want the unsigned apk... - for propfile in ('build.properties', 'default.properties'): - if os.path.exists(os.path.join(root_dir, propfile)): - if subprocess.call(['sed','-i','s/^key.store/#/', - propfile], cwd=root_dir) !=0: - print "Failed to amend %s" % propfile - sys.exit(1) + # Initialise submodules if requred... + if thisbuild.get('submodules', 'no') == 'yes': + vcs.initsubmodules() - # Update the local.properties file... - locprops = os.path.join(root_dir, 'local.properties') - if os.path.exists(locprops): - f = open(locprops, 'r') - props = f.read() - f.close() - # Fix old-fashioned 'sdk-location' by copying - # from sdk.dir, if necessary... - if thisbuild.get('oldsdkloc', 'no') == "yes": - sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props, - re.S|re.M).group(1) - props += "\nsdk-location=" + sdkloc + "\n" - # Add ndk location... - props+= "\nndk.dir=" + ndk_path + "\n" - # Add java.encoding if necessary... - if thisbuild.has_key('encoding'): - props += "\njava.encoding=" + thisbuild['encoding'] + "\n" - f = open(locprops, 'w') - f.write(props) - f.close() + # Generate (or update) the ant build file, build.xml... + if (thisbuild.get('update', 'yes') == 'yes' and + not thisbuild.has_key('maven')): + parms = [os.path.join(sdk_path, 'tools', 'android'), + 'update', 'project', '-p', '.'] + parms.append('--subprojects') + if thisbuild.has_key('target'): + parms.append('-t') + parms.append(thisbuild['target']) + if subprocess.call(parms, cwd=root_dir) != 0: + raise BuildException("Failed to update project") - # Insert version code and number into the manifest if necessary... - if thisbuild.has_key('insertversion'): - if subprocess.call(['sed','-i','s/' + thisbuild['insertversion'] + - '/' + thisbuild['version'] +'/g', - 'AndroidManifest.xml'], cwd=root_dir) !=0: - print "Failed to amend manifest" - sys.exit(1) - if thisbuild.has_key('insertvercode'): - if subprocess.call(['sed','-i','s/' + thisbuild['insertvercode'] + - '/' + thisbuild['vercode'] +'/g', - 'AndroidManifest.xml'], cwd=root_dir) !=0: - print "Failed to amend manifest" - sys.exit(1) + # If the app has ant set up to sign the release, we need to switch + # that off, because we want the unsigned apk... + for propfile in ('build.properties', 'default.properties'): + if os.path.exists(os.path.join(root_dir, propfile)): + if subprocess.call(['sed','-i','s/^key.store/#/', + propfile], cwd=root_dir) !=0: + raise BuildException("Failed to amend %s" % propfile) - # Delete unwanted file... - if thisbuild.has_key('rm'): - os.remove(os.path.join(build_dir, thisbuild['rm'])) + # Update the local.properties file... + locprops = os.path.join(root_dir, 'local.properties') + if os.path.exists(locprops): + f = open(locprops, 'r') + props = f.read() + f.close() + # Fix old-fashioned 'sdk-location' by copying + # from sdk.dir, if necessary... + if thisbuild.get('oldsdkloc', 'no') == "yes": + sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props, + re.S|re.M).group(1) + props += "\nsdk-location=" + sdkloc + "\n" + # Add ndk location... + props+= "\nndk.dir=" + ndk_path + "\n" + # Add java.encoding if necessary... + if thisbuild.has_key('encoding'): + props += "\njava.encoding=" + thisbuild['encoding'] + "\n" + f = open(locprops, 'w') + f.write(props) + f.close() - # Fix apostrophes translation files if necessary... - if thisbuild.get('fixapos', 'no') == 'yes': - for root, dirs, files in os.walk(os.path.join(root_dir,'res')): - for filename in files: - if filename.endswith('.xml'): - if subprocess.call(['sed','-i','s@' + - r"\([^\\]\)'@\1\\'" + - '@g', - os.path.join(root, filename)]) != 0: - print "Failed to amend " + filename - sys.exit(1) + # Insert version code and number into the manifest if necessary... + if thisbuild.has_key('insertversion'): + if subprocess.call(['sed','-i','s/' + thisbuild['insertversion'] + + '/' + thisbuild['version'] +'/g', + 'AndroidManifest.xml'], cwd=root_dir) !=0: + raise BuildException("Failed to amend manifest") + if thisbuild.has_key('insertvercode'): + if subprocess.call(['sed','-i','s/' + thisbuild['insertvercode'] + + '/' + thisbuild['vercode'] +'/g', + 'AndroidManifest.xml'], cwd=root_dir) !=0: + raise BuildException("Failed to amend manifest") - # Fix translation files if necessary... - if thisbuild.get('fixtrans', 'no') == 'yes': - for root, dirs, files in os.walk(os.path.join(root_dir,'res')): - for filename in files: - if filename.endswith('.xml'): - f = open(os.path.join(root, filename)) - changed = False - outlines = [] - for line in f: - num = 1 - index = 0 - oldline = line - while True: - index = line.find("%", index) - if index == -1: - break - next = line[index+1:index+2] - if next == "s" or next == "d": - line = (line[:index+1] + - str(num) + "$" + - line[index+1:]) - num += 1 - index += 3 - else: - index += 1 - # We only want to insert the positional arguments - # when there is more than one argument... - if oldline != line: - if num > 2: - changed = True - else: - line = oldline - outlines.append(line) - f.close() - if changed: - f = open(os.path.join(root, filename), 'w') - f.writelines(outlines) + # Delete unwanted file... + if thisbuild.has_key('rm'): + os.remove(os.path.join(build_dir, thisbuild['rm'])) + + # Fix apostrophes translation files if necessary... + if thisbuild.get('fixapos', 'no') == 'yes': + for root, dirs, files in os.walk(os.path.join(root_dir,'res')): + for filename in files: + if filename.endswith('.xml'): + if subprocess.call(['sed','-i','s@' + + r"\([^\\]\)'@\1\\'" + + '@g', + os.path.join(root, filename)]) != 0: + raise BuildException("Failed to amend " + filename) + + # Fix translation files if necessary... + if thisbuild.get('fixtrans', 'no') == 'yes': + for root, dirs, files in os.walk(os.path.join(root_dir,'res')): + for filename in files: + if filename.endswith('.xml'): + f = open(os.path.join(root, filename)) + changed = False + outlines = [] + for line in f: + num = 1 + index = 0 + oldline = line + while True: + index = line.find("%", index) + if index == -1: + break + next = line[index+1:index+2] + if next == "s" or next == "d": + line = (line[:index+1] + + str(num) + "$" + + line[index+1:]) + num += 1 + index += 3 + else: + index += 1 + # We only want to insert the positional arguments + # when there is more than one argument... + if oldline != line: + if num > 2: + changed = True + else: + line = oldline + outlines.append(line) f.close() + if changed: + f = open(os.path.join(root, filename), 'w') + f.writelines(outlines) + f.close() - # Run a pre-build command if one is required... - if thisbuild.has_key('prebuild'): - if subprocess.call(thisbuild['prebuild'], - cwd=root_dir, shell=True) != 0: - print "Error running pre-build command" - sys.exit(1) + # Run a pre-build command if one is required... + if thisbuild.has_key('prebuild'): + if subprocess.call(thisbuild['prebuild'], + cwd=root_dir, shell=True) != 0: + raise BuildException("Error running pre-build command") - # Apply patches if any - if 'patch' in thisbuild: - for patch in thisbuild['patch'].split(';'): - print "Applying " + patch - patch_path = os.path.join('metadata', app['id'], patch) - if subprocess.call(['patch', '-p1', - '-i', os.path.abspath(patch_path)], cwd=build_dir) != 0: - print "Failed to apply patch %s" % patch_path - sys.exit(1) + # Apply patches if any + if 'patch' in thisbuild: + for patch in thisbuild['patch'].split(';'): + print "Applying " + patch + patch_path = os.path.join('metadata', app['id'], patch) + if subprocess.call(['patch', '-p1', + '-i', os.path.abspath(patch_path)], cwd=build_dir) != 0: + raise BuildException("Failed to apply patch %s" % patch_path) - # Special case init functions for funambol... - if thisbuild.get('initfun', 'no') == "yes": + # Special case init functions for funambol... + if thisbuild.get('initfun', 'no') == "yes": - if subprocess.call(['sed','-i','s@' + - '' + - '@' + - '' + - '' + - '' + - '' + - '' + - '@g', - 'build.xml'], cwd=root_dir) !=0: - print "Failed to amend build.xml" - sys.exit(1) + if subprocess.call(['sed','-i','s@' + + '' + + '@' + + '' + + '' + + '' + + '' + + '' + + '@g', + 'build.xml'], cwd=root_dir) !=0: + raise BuildException("Failed to amend build.xml") - if subprocess.call(['sed','-i','s@' + - '\${user.home}/funambol/build/android/build.properties' + - '@' + - 'build.properties' + - '@g', - 'build.xml'], cwd=root_dir) !=0: - print "Failed to amend build.xml" - sys.exit(1) + if subprocess.call(['sed','-i','s@' + + '\${user.home}/funambol/build/android/build.properties' + + '@' + + 'build.properties' + + '@g', + 'build.xml'], cwd=root_dir) !=0: + raise BuildException("Failed to amend build.xml") - buildxml = os.path.join(root_dir, 'build.xml') - f = open(buildxml, 'r') - xml = f.read() - f.close() - xmlout = "" - mode = 0 - for line in xml.splitlines(): - if mode == 0: - if line.find("jarsigner") != -1: - mode = 1 + buildxml = os.path.join(root_dir, 'build.xml') + f = open(buildxml, 'r') + xml = f.read() + f.close() + xmlout = "" + mode = 0 + for line in xml.splitlines(): + if mode == 0: + if line.find("jarsigner") != -1: + mode = 1 + else: + xmlout += line + "\n" else: - xmlout += line + "\n" + if line.find("/exec") != -1: + mode += 1 + if mode == 3: + mode =0 + f = open(buildxml, 'w') + f.write(xmlout) + f.close() + + if subprocess.call(['sed','-i','s@' + + 'platforms/android-2.0' + + '@' + + 'platforms/android-8' + + '@g', + 'build.xml'], cwd=root_dir) !=0: + raise BuildException("Failed to amend build.xml") + + shutil.copyfile( + os.path.join(root_dir, "build.properties.example"), + os.path.join(root_dir, "build.properties")) + + if subprocess.call(['sed','-i','s@' + + 'javacchome=.*'+ + '@' + + 'javacchome=' + javacc_path + + '@g', + 'build.properties'], cwd=root_dir) !=0: + raise BuildException("Failed to amend build.properties") + + if subprocess.call(['sed','-i','s@' + + 'sdk-folder=.*'+ + '@' + + 'sdk-folder=' + sdk_path + + '@g', + 'build.properties'], cwd=root_dir) !=0: + raise BuildException("Failed to amend build.properties") + + if subprocess.call(['sed','-i','s@' + + 'android.sdk.version.*'+ + '@' + + 'android.sdk.version=2.0' + + '@g', + 'build.properties'], cwd=root_dir) !=0: + raise BuildException("Failed to amend build.properties") + + + # Build the source tarball right before we build the release... + tarname = app['id'] + '_' + thisbuild['vercode'] + '_src' + tarball = tarfile.open(os.path.join(output_dir, + tarname + '.tar.gz'), "w:gz") + tarball.add(build_dir, tarname) + tarball.close() + + # Build native stuff if required... + if thisbuild.get('buildjni', 'no') == 'yes': + ndkbuild = os.path.join(ndk_path, "ndk-build") + p = subprocess.Popen([ndkbuild], cwd=root_dir, + stdout=subprocess.PIPE) + output = p.communicate()[0] + if p.returncode != 0: + print output + raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version'])) + elif options.verbose: + print output + + # Build the release... + if thisbuild.has_key('maven'): + p = subprocess.Popen(['mvn', 'clean', 'install', + '-Dandroid.sdk.path=' + sdk_path], + cwd=root_dir, stdout=subprocess.PIPE) + else: + if thisbuild.has_key('antcommand'): + antcommand = thisbuild['antcommand'] else: - if line.find("/exec") != -1: - mode += 1 - if mode == 3: - mode =0 - f = open(buildxml, 'w') - f.write(xmlout) - f.close() - - if subprocess.call(['sed','-i','s@' + - 'platforms/android-2.0' + - '@' + - 'platforms/android-8' + - '@g', - 'build.xml'], cwd=root_dir) !=0: - print "Failed to amend build.xml" - sys.exit(1) - - shutil.copyfile( - os.path.join(root_dir, "build.properties.example"), - os.path.join(root_dir, "build.properties")) - - if subprocess.call(['sed','-i','s@' + - 'javacchome=.*'+ - '@' + - 'javacchome=' + javacc_path + - '@g', - 'build.properties'], cwd=root_dir) !=0: - print "Failed to amend build.properties" - sys.exit(1) - - if subprocess.call(['sed','-i','s@' + - 'sdk-folder=.*'+ - '@' + - 'sdk-folder=' + sdk_path + - '@g', - 'build.properties'], cwd=root_dir) !=0: - print "Failed to amend build.properties" - sys.exit(1) - - if subprocess.call(['sed','-i','s@' + - 'android.sdk.version.*'+ - '@' + - 'android.sdk.version=2.0' + - '@g', - 'build.properties'], cwd=root_dir) !=0: - print "Failed to amend build.properties" - sys.exit(1) - - - # Build the source tarball right before we build the release... - tarname = app['id'] + '_' + thisbuild['vercode'] + '_src' - tarball = tarfile.open(os.path.join(output_dir, - tarname + '.tar.gz'), "w:gz") - tarball.add(build_dir, tarname) - tarball.close() - - # Build native stuff if required... - if thisbuild.get('buildjni', 'no') == 'yes': - ndkbuild = os.path.join(ndk_path, "ndk-build") - p = subprocess.Popen([ndkbuild], cwd=root_dir, - stdout=subprocess.PIPE) + antcommand = 'release' + p = subprocess.Popen(['ant', antcommand], cwd=root_dir, + stdout=subprocess.PIPE) output = p.communicate()[0] if p.returncode != 0: print output - print "NDK build failed for %s:%s" % (app['id'], thisbuild['version']) - sys.exit(1) + raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version'])) elif options.verbose: print output + print "Build successful" - # Build the release... - if thisbuild.has_key('maven'): - p = subprocess.Popen(['mvn', 'clean', 'install', - '-Dandroid.sdk.path=' + sdk_path], - cwd=root_dir, stdout=subprocess.PIPE) - else: - if thisbuild.has_key('antcommand'): - antcommand = thisbuild['antcommand'] + # Find the apk name in the output... + if thisbuild.has_key('bindir'): + bindir = os.path.join(build_dir, thisbuild['bindir']) else: - antcommand = 'release' - p = subprocess.Popen(['ant', antcommand], cwd=root_dir, - stdout=subprocess.PIPE) - output = p.communicate()[0] - if p.returncode != 0: - print output - print "Build failed for %s:%s" % (app['id'], thisbuild['version']) - sys.exit(1) - elif options.verbose: - print output - print "Build successful" - - # Find the apk name in the output... - if thisbuild.has_key('bindir'): - bindir = os.path.join(build_dir, thisbuild['bindir']) - else: - bindir = os.path.join(root_dir, 'bin') - if thisbuild.get('initfun', 'no') == "yes": - # Special case (again!) for funambol... - src = ("funambol-android-sync-client-" + - thisbuild['version'] + "-unsigned.apk") - src = os.path.join(bindir, src) - elif thisbuild.has_key('maven'): - src = re.match(r".*^\[INFO\] Installing /.*/([^/]*)\.apk", - output, re.S|re.M).group(1) - src = os.path.join(bindir, src) + '.apk' + bindir = os.path.join(root_dir, 'bin') + if thisbuild.get('initfun', 'no') == "yes": + # Special case (again!) for funambol... + src = ("funambol-android-sync-client-" + + thisbuild['version'] + "-unsigned.apk") + src = os.path.join(bindir, src) + elif thisbuild.has_key('maven'): + src = re.match(r".*^\[INFO\] Installing /.*/([^/]*)\.apk", + output, re.S|re.M).group(1) + src = os.path.join(bindir, src) + '.apk' #[INFO] Installing /home/ciaran/fdroidserver/tmp/mainline/application/target/callerid-1.0-SNAPSHOT.apk - else: - src = re.match(r".*^.*Creating (\S+) for release.*$.*", output, - re.S|re.M).group(1) - src = os.path.join(bindir, src) + else: + src = re.match(r".*^.*Creating (\S+) for release.*$.*", output, + re.S|re.M).group(1) + src = os.path.join(bindir, src) - # By way of a sanity check, make sure the version and version - # code in our new apk match what we expect... - print "Checking " + src - p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools', - 'aapt'), - 'dump', 'badging', src], - stdout=subprocess.PIPE) - output = p.communicate()[0] - if thisbuild.get('novcheck', 'no') == "yes": - vercode = thisbuild['vercode'] - version = thisbuild['version'] - else: - vercode = None - version = None - for line in output.splitlines(): - if line.startswith("package:"): - pat = re.compile(".*versionCode='([0-9]*)'.*") - vercode = re.match(pat, line).group(1) - pat = re.compile(".*versionName='([^']*)'.*") - version = re.match(pat, line).group(1) - if version == None or vercode == None: - print "Could not find version information in build in output" - sys.exit(1) + # By way of a sanity check, make sure the version and version + # code in our new apk match what we expect... + print "Checking " + src + p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools', + 'aapt'), + 'dump', 'badging', src], + stdout=subprocess.PIPE) + output = p.communicate()[0] + if thisbuild.get('novcheck', 'no') == "yes": + vercode = thisbuild['vercode'] + version = thisbuild['version'] + else: + vercode = None + version = None + for line in output.splitlines(): + if line.startswith("package:"): + pat = re.compile(".*versionCode='([0-9]*)'.*") + vercode = re.match(pat, line).group(1) + pat = re.compile(".*versionName='([^']*)'.*") + version = re.match(pat, line).group(1) + if version == None or vercode == None: + raise BuildException("Could not find version information in build in output") - # Some apps (e.g. Timeriffic) have had the bonkers idea of - # including the entire changelog in the version number. Remove - # it so we can compare. (TODO: might be better to remove it - # before we compile, in fact) - index = version.find(" //") - if index != -1: - version = version[:index] + # Some apps (e.g. Timeriffic) have had the bonkers idea of + # including the entire changelog in the version number. Remove + # it so we can compare. (TODO: might be better to remove it + # before we compile, in fact) + index = version.find(" //") + if index != -1: + version = version[:index] - if (version != thisbuild['version'] or - vercode != thisbuild['vercode']): - print "Unexpected version/version code in output" - print "APK:" + version + " / " + str(vercode) - print ("Expected: " + thisbuild['version'] + " / " + - str(thisbuild['vercode'])) - sys.exit(1) + if (version != thisbuild['version'] or + vercode != thisbuild['vercode']): + raise BuildException(("Unexpected version/version code in output" + "APK: %s / %s" + "Expected: %s / %s") + ) % (version, str(vercode), thisbuild['version'], str(thisbuild['vercode'])) - # Copy the unsigned apk to our temp directory for further - # processing... - shutil.copyfile(src, dest_unsigned) + # Copy the unsigned apk to our temp directory for further + # processing... + shutil.copyfile(src, dest_unsigned) - # Figure out the key alias name we'll use. Only the first 8 - # characters are significant, so we'll use the first 8 from - # the MD5 of the app's ID and hope there are no collisions. - # If a collision does occur later, we're going to have to - # come up with a new alogrithm, AND rename all existing keys - # in the keystore! - if keyaliases.has_key(app['id']): - # For this particular app, the key alias is overridden... - keyalias = keyaliases[app['id']] - else: - m = md5.new() - m.update(app['id']) - keyalias = m.hexdigest()[:8] - print "Key alias: " + keyalias + # Figure out the key alias name we'll use. Only the first 8 + # characters are significant, so we'll use the first 8 from + # the MD5 of the app's ID and hope there are no collisions. + # If a collision does occur later, we're going to have to + # come up with a new alogrithm, AND rename all existing keys + # in the keystore! + if keyaliases.has_key(app['id']): + # For this particular app, the key alias is overridden... + keyalias = keyaliases[app['id']] + else: + m = md5.new() + m.update(app['id']) + keyalias = m.hexdigest()[:8] + print "Key alias: " + keyalias - # See if we already have a key for this application, and - # if not generate one... - p = subprocess.Popen(['keytool', '-list', - '-alias', keyalias, '-keystore', keystore, - '-storepass', keystorepass], stdout=subprocess.PIPE) - output = p.communicate()[0] - if p.returncode !=0: - print "Key does not exist - generating..." - p = subprocess.Popen(['keytool', '-genkey', - '-keystore', keystore, '-alias', keyalias, - '-keyalg', 'RSA', '-keysize', '2048', - '-validity', '10000', + # See if we already have a key for this application, and + # if not generate one... + p = subprocess.Popen(['keytool', '-list', + '-alias', keyalias, '-keystore', keystore, + '-storepass', keystorepass], stdout=subprocess.PIPE) + output = p.communicate()[0] + if p.returncode !=0: + print "Key does not exist - generating..." + p = subprocess.Popen(['keytool', '-genkey', + '-keystore', keystore, '-alias', keyalias, + '-keyalg', 'RSA', '-keysize', '2048', + '-validity', '10000', + '-storepass', keystorepass, '-keypass', keypass, + '-dname', keydname], stdout=subprocess.PIPE) + output = p.communicate()[0] + print output + if p.returncode != 0: + raise BuildException("Failed to generate key") + + # Sign the application... + p = subprocess.Popen(['jarsigner', '-keystore', keystore, '-storepass', keystorepass, '-keypass', keypass, - '-dname', keydname], stdout=subprocess.PIPE) + dest_unsigned, keyalias], stdout=subprocess.PIPE) output = p.communicate()[0] print output if p.returncode != 0: - print "Failed to generate key" - sys.exit(1) + raise BuildException("Failed to sign application") - # Sign the application... - p = subprocess.Popen(['jarsigner', '-keystore', keystore, - '-storepass', keystorepass, '-keypass', keypass, - dest_unsigned, keyalias], stdout=subprocess.PIPE) - output = p.communicate()[0] - print output - if p.returncode != 0: - print "Failed to sign application" - sys.exit(1) + # Zipalign it... + p = subprocess.Popen([os.path.join(sdk_path,'tools','zipalign'), + '-v', '4', dest_unsigned, dest], + stdout=subprocess.PIPE) + output = p.communicate()[0] + print output + if p.returncode != 0: + raise BuildException("Failed to align application") + os.remove(dest_unsigned) + except BuildException as be: + print "Could not build app %s due to BuildException: %s" % (app['id'], be) + failed_apps[app['id']] = be + except VCSException as vcse: + print "VCS error while building app %s: %s" % (app['id'], vcse) + failed_apps[app['id']] = vcse + except Exception as e: + print "Could not build app %s due to unknown error: %s" % (app['id'], e) + failed_apps[app['id']] = e + build_succeeded.append(app) - # Zipalign it... - p = subprocess.Popen([os.path.join(sdk_path,'tools','zipalign'), - '-v', '4', dest_unsigned, dest], - stdout=subprocess.PIPE) - output = p.communicate()[0] - print output - if p.returncode != 0: - print "Failed to align application" - sys.exit(1) - os.remove(dest_unsigned) +for app in build_succeeded: + print "success: %s" % (app['id']) + +for fa in failed_apps: + print "Build for app %s failed: %s" % (fa, failed_apps[fa]) print "Finished." +if len(failed_apps) > 0: + print str(len(failed_apps)) + 'builds failed' diff --git a/checkmarket.py b/checkmarket.py index 47a212bf..f8e5c80c 100755 --- a/checkmarket.py +++ b/checkmarket.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python # -*- coding: utf-8 -*- # # checkmarket.py - part of the FDroid server tools diff --git a/checkmarket2.py b/checkmarket2.py new file mode 100755 index 00000000..193a63fa --- /dev/null +++ b/checkmarket2.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# checkmarket2.py - part of the FDroid server tools +# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import sys +import os +import shutil +import re +import urllib +import time +from optparse import OptionParser +import HTMLParser +import common + +#Read configuration... +execfile('config.py') + + +# Parse command line... +parser = OptionParser() +parser.add_option("-v", "--verbose", action="store_true", default=False, + help="Spew out even more information than normal") +(options, args) = parser.parse_args() + +# Get all apps... +apps = common.read_metadata(options.verbose) + +html_parser = HTMLParser.HTMLParser() + +for app in apps: + + print "Processing " + app['id'] + url = 'http://market.android.com/details?id=' + app['id'] + page = urllib.urlopen(url).read() + + version = None + vercode = None + + m = re.search('
([^>]+)
', page) + if m: + version = html_parser.unescape(m.group(1)) + + m = re.search('data-paramValue="(\d+)">
Latest Version<', page) + if m: + vercode = m.group(1) + + if not vercode: + print "...couldn't find version code" + elif not version: + print "...couldn't find version" + elif vercode == app['marketvercode'] and version == app['marketversion']: + print "...up to date" + else: + print '...updating to version:' + version + ' vercode:' + vercode + newdata = '' + metafile = os.path.join('metadata', app['id'] + '.txt') + mf = open(metafile, 'r') + for line in mf: + if line.startswith('Market Version:'): + newdata += 'Market Version:' + version + '\n' + elif line.startswith('Market Version Code:'): + newdata += 'Market Version Code:' + vercode + '\n' + else: + newdata += line + mf.close() + mf = open(metafile, 'w') + mf.write(newdata) + mf.close() + + time.sleep(5) + +print "Finished." + diff --git a/common.py b/common.py index 43630e1b..a7d4d613 100644 --- a/common.py +++ b/common.py @@ -29,8 +29,7 @@ def getvcs(vcstype, remote, local): return vcs_hg(remote,local) elif vcstype == 'bzr': return vcs_bzr(remote,local) - print "Invalid vcs type " + vcstype - sys.exit(1) + raise VCSException("Invalid vcs type " + vcstype) class vcs: def __init__(self, remote, local): @@ -44,8 +43,7 @@ class vcs: remote = remote[index+1:] index = self.username.find(':') if index == -1: - print "Password required with username" - sys.exit(1) + raise VCSException("Password required with username") self.password = self.username[index+1:] self.username = self.username[:index] else: @@ -86,41 +84,34 @@ class vcs_git(vcs): def clone(self): if subprocess.call(['git', 'clone', self.remote, self.local]) != 0: - print "Git clone failed" - sys.exit(1) + raise VCSException("Git clone failed") def reset(self, rev=None): if rev is None: rev = 'origin' if subprocess.call(['git', 'reset', '--hard', rev], cwd=self.local) != 0: - print "Git reset failed" - sys.exit(1) + raise VCSException("Git reset failed") if subprocess.call(['git', 'clean', '-dfx'], cwd=self.local) != 0: - print "Git clean failed" - sys.exit(1) + raise VCSException("Git clean failed") def pull(self): if subprocess.call(['git', 'pull', 'origin'], cwd=self.local) != 0: - print "Git pull failed" - sys.exit(1) + raise VCSException("Git pull failed") # Might need tags that aren't on a branch. if subprocess.call(['git', 'fetch', '--tags', 'origin'], cwd=self.local) != 0: - print "Git fetch failed" - sys.exit(1) + raise VCSException("Git fetch failed") def initsubmodules(self): if subprocess.call(['git', 'submodule', 'init'], cwd=self.local) != 0: - print "Git submodule init failed" - sys.exit(1) + raise VCSException("Git submodule init failed") if subprocess.call(['git', 'submodule', 'update'], cwd=self.local) != 0: - print "Git submodule update failed" - sys.exit(1) + raise VCSException("Git submodule update failed") @@ -128,7 +119,7 @@ class vcs_svn(vcs): def userargs(self): if self.username is None: - return [] + return ['--non-interactive'] return ['--username', self.username, '--password', self.password, '--non-interactive'] @@ -136,8 +127,7 @@ class vcs_svn(vcs): def clone(self): if subprocess.call(['svn', 'checkout', self.remote, self.local] + self.userargs()) != 0: - print "Svn checkout failed" - sys.exit(1) + raise VCSException("Svn checkout failed") def reset(self, rev=None): if rev is None: @@ -149,25 +139,21 @@ class vcs_svn(vcs): r"svn status | awk '/\?/ {print $2}' | xargs rm -rf"): if subprocess.call(svncommand, cwd=self.local, shell=True) != 0: - print "Svn reset failed" - sys.exit(1) + raise VCSException("Svn reset failed") if subprocess.call(['svn', 'update', '--force'] + revargs + self.userargs(), cwd=self.local) != 0: - print "Svn update failed" - sys.exit(1) + raise VCSException("Svn update failed") def pull(self): if subprocess.call(['svn', 'update'] + self.userargs(), cwd=self.local) != 0: - print "Svn update failed" - sys.exit(1) + raise VCSException("Svn update failed") class vcs_hg(vcs): def clone(self): if subprocess.call(['hg', 'clone', self.remote, self.local]) !=0: - print "Hg clone failed" - sys.exit(1) + raise VCSException("Hg clone failed") def reset(self, rev=None): if rev is None: @@ -176,25 +162,21 @@ class vcs_hg(vcs): revargs = [rev] if subprocess.call('hg status -u | xargs rm -rf', cwd=self.local, shell=True) != 0: - print "Hg clean failed" - sys.exit(1) + raise VCSException("Hg clean failed") if subprocess.call(['hg', 'checkout', '-C'] + revargs, cwd=self.local) != 0: - print "Hg checkout failed" - sys.exit(1) + raise VCSException("Hg checkout failed") def pull(self): if subprocess.call(['hg', 'pull'], cwd=self.local) != 0: - print "Hg pull failed" - sys.exit(1) + raise VCSException("Hg pull failed") class vcs_bzr(vcs): def clone(self): if subprocess.call(['bzr', 'branch', self.remote, self.local]) != 0: - print "Bzr branch failed" - sys.exit(1) + raise VCSException("Bzr branch failed") def reset(self, rev=None): if rev is None: @@ -203,18 +185,15 @@ class vcs_bzr(vcs): revargs = ['-r', rev] if subprocess.call(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local) != 0: - print "Bzr revert failed" - sys.exit(1) + raise VCSException("Bzr revert failed") if subprocess.call(['bzr', 'revert'] + revargs, cwd=self.local) != 0: - print "Bzr revert failed" - sys.exit(1) + raise VCSException("Bzr revert failed") def pull(self): if subprocess.call(['bzr', 'pull'], cwd=self.local) != 0: - print "Bzr update failed" - sys.exit(1) + raise VCSException("Bzr update failed") @@ -224,8 +203,7 @@ def parse_metadata(metafile, **kw): parts = [p.replace("\\,", ",") for p in re.split(r"(?ant_out.txt -Market Version:4.4 beta 39 -Market Version Code:31 +Market Version:4.4 +Market Version Code:33 diff --git a/metadata/org.helllabs.android.xmp.txt b/metadata/org.helllabs.android.xmp.txt index e9fc3295..464757a6 100644 --- a/metadata/org.helllabs.android.xmp.txt +++ b/metadata/org.helllabs.android.xmp.txt @@ -20,5 +20,5 @@ Build Version:2.1.0,15,639549fda2111cb800fabe468b4a64bf4ae27003,\ buildjni=yes,subdir=src/android/project,target=android-8,\ insertversion=2.1.0[^"]* -Market Version:2.3.0 -Market Version Code:19 +Market Version:2.4.0 +Market Version Code:20 diff --git a/metadata/org.jtb.alogcat.txt b/metadata/org.jtb.alogcat.txt index 44193029..da8dfccc 100644 --- a/metadata/org.jtb.alogcat.txt +++ b/metadata/org.jtb.alogcat.txt @@ -9,8 +9,6 @@ Description: Log viewer - an app equivalent of logcat from the Android shell, or adb logcat from the SDK tools. . -Market Version:2.3.2 -Market Version Code:38 Repo Type:svn Repo:http://alogcat.googlecode.com/svn/trunk @@ -19,3 +17,6 @@ Build Version:2.1.6,34,28 Build Version:2.3,36,30 Build Version:2.3.2,38,31,prebuild=find . -type f -name \*.java -print0 | xargs -0 sed -i "s/^import org\.jtb\.alogcat\.donate\.R;$/import org.jtb.alogcat.R;/g" && sed -i "s/org.jtb.alogcat.donate/org.jtb.alogcat/" AndroidManifest.xml +Market Version:2.4 +Market Version Code:39 + diff --git a/metadata/org.pocketworkstation.pckeyboard.txt b/metadata/org.pocketworkstation.pckeyboard.txt index 2caf1e55..4d1f84ef 100644 --- a/metadata/org.pocketworkstation.pckeyboard.txt +++ b/metadata/org.pocketworkstation.pckeyboard.txt @@ -19,5 +19,5 @@ Build Version:v1.18,1018,8298cd8728c2,subdir=java,target=android-11,buildjni=yes Build Version:v1.20,1020,e703fa82a4c3,subdir=java,target=android-11,buildjni=yes Build Version:v1.22,1022,154e21230e81,subdir=java,target=android-11,buildjni=yes -Market Version:v1.22 -Market Version Code:1022 +Market Version:v1.27 +Market Version Code:1027 diff --git a/metadata/org.scoutant.blokish.txt b/metadata/org.scoutant.blokish.txt index 9347b6bd..b8b88b1d 100644 --- a/metadata/org.scoutant.blokish.txt +++ b/metadata/org.scoutant.blokish.txt @@ -15,6 +15,7 @@ Repo:git://github.com/scoutant/blokish.git Build Version:0.5,1,ca2e9096c6d5140c14e7,target=android-9 Build Version:1.4,7,82ca58cdd26797f9ba9519e79c54031282d066af,target=android-9 Build Version:1.5,8,e6ab83e8832f03d78bf3f8deae33acb22a7ba520,target=android-9 +Build Version:1.6,9,ebd1b966b70e,target=android-9 -Market Version:1.5 -Market Version Code:8 +Market Version:1.6 +Market Version Code:9 diff --git a/metadata/org.sipdroid.sipua.txt b/metadata/org.sipdroid.sipua.txt index f8f65a9c..735464d2 100644 --- a/metadata/org.sipdroid.sipua.txt +++ b/metadata/org.sipdroid.sipua.txt @@ -7,5 +7,5 @@ Summary:A SIP (VOIP) client Description: A SIP (VOIP) client with video calling capabilities. . -Market Version:2.4 beta +Market Version:2.4 Market Version Code:87 diff --git a/metadata/org.sparkleshare.android.txt b/metadata/org.sparkleshare.android.txt new file mode 100644 index 00000000..735fa5fe --- /dev/null +++ b/metadata/org.sparkleshare.android.txt @@ -0,0 +1,18 @@ +License:GPLv3 +Category:System +Web Site:http://www.sparkleshare.org +Source Code:https://github.com/NewProggie/SparkleShare-Android +Issue Tracker:https://github.com/NewProggie/SparkleShare-Android/issues +Summary:distributed collaboration and sharing tool +Description: +SparkleShare is a collaboration and sharing tool that is designed to keep +things simple and to stay out of your way. + +Setup your own cloud with SparkleShare (http://sparkleshare.org/) and +browse your files right from your Android device. +. + +Repo Type:git +Repo:https://github.com/NewProggie/SparkleShare-Android.git + +Build Version:1.0,1,a9e23f0f9ae6161a786bf48cb48ab3dec20110c9,prebuild=rm -rf gen/ && rm -rf bin/,target=android-7 diff --git a/metadata/org.thoughtcrime.securesms.txt b/metadata/org.thoughtcrime.securesms.txt new file mode 100644 index 00000000..00ca9444 --- /dev/null +++ b/metadata/org.thoughtcrime.securesms.txt @@ -0,0 +1,20 @@ +License:GPLv3 +Category:Cryptography +Web Site:http://www.whispersys.com/ +Source Code:https://github.com/WhisperSystems/TextSecure +Issue Tracker: +Donate: +Summary:SMS Encryption +Description: +TextSecure is a drop-in replacement for the standard text messaging application, +allowing you to send and receive text messages as normal. All text messages +sent or received with TextSecure are stored in an encrypted database on your +phone, and text messages are encrypted during transmission when communicating +with someone else also using TextSecure. +. + +Repo Type:git +Repo:https://github.com/WhisperSystems/TextSecure.git + +Build Version:0.5.7,21,6a30867fd481474e8634b508d524863bc8a6b565,target=android-9 + diff --git a/metadata/org.ttrssreader.txt b/metadata/org.ttrssreader.txt index 8cdd79eb..859aaa57 100644 --- a/metadata/org.ttrssreader.txt +++ b/metadata/org.ttrssreader.txt @@ -4,6 +4,7 @@ Category:Internet Web Site:http://code.google.com/p/ttrss-reader-fork/ Source Code:http://code.google.com/p/ttrss-reader-fork/source/checkout Issue Tracker:http://code.google.com/p/ttrss-reader-fork/issues/list +Donate:http://code.google.com/p/ttrss-reader-fork/wiki/Donations Summary:An RSS reader for Tiny Tiny RSS Description: An offline reader that works with the Tiny Tiny RSS web-based feed reader/aggregator.. @@ -38,6 +39,7 @@ Build Version:1.07,1075,1.07 (4),subdir=ttrss-reader,target=android-8 #Build Version:1.08,1080,1.08,subdir=ttrss-reader,target=android-8 Build Version:1.21,1210,1.21,subdir=ttrss-reader,target=android-8 +Build Version:1.22,1220,1.22,subdir=ttrss-reader,target=android-8 -Market Version:1.21 -Market Version Code:1210 +Market Version:1.22 +Market Version Code:1220 diff --git a/metadata/org.wahtod.wififixer.txt b/metadata/org.wahtod.wififixer.txt index dc698d83..dcafbb58 100644 --- a/metadata/org.wahtod.wififixer.txt +++ b/metadata/org.wahtod.wififixer.txt @@ -19,6 +19,7 @@ Build Version:0.8.0.2,760,ff612a31874fc89cb124,subdir=wififixer,target=android-8 Build Version:0.8.0.3.1,770,59b8e66a0409a5d4c6a6,subdir=wififixer,target=android-8 Build Version:0.8.0.5,822,846b5c057e886f8436eb,subdir=wififixer,target=android-8 Build Version:0.8.0.6,838,81e9557f73d8529755cd,subdir=wififixer,target=android-8 +Build Version:0.9.0.1,915,!commit 7155ca938dc1 but can't compile for some reason -Market Version:0.8.0.6 -Market Version Code:838 +Market Version:0.9.0.1 +Market Version Code:915 diff --git a/metadata/org.wordpress.android.txt b/metadata/org.wordpress.android.txt index 860eb349..0c9540fd 100644 --- a/metadata/org.wordpress.android.txt +++ b/metadata/org.wordpress.android.txt @@ -21,5 +21,5 @@ Repo:http://android.svn.wordpress.org/trunk/ Build Version:1.3.9,31,202,prebuild=mkdir libs && mv *.jar libs && sed -i "s@checkStats(accounts.size());@// MY PRIVACY > YOUR STATS@" src/org/wordpress/android/wpAndroid.java,encoding=utf-8 Build Version:1.4.1,33,228,prebuild=mkdir libs && mv *.jar libs && sed -i "s@checkStats(accounts.size());@// MY PRIVACY > YOUR STATS@" src/org/wordpress/android/wpAndroid.java,encoding=utf-8 -Market Version:1.5.1 -Market Version Code:36 +Market Version:2.0.1 +Market Version Code:39 diff --git a/scanner.py b/scanner.py new file mode 100755 index 00000000..843c287f --- /dev/null +++ b/scanner.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- +# +# scanner.py - part of the FDroid server tools +# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import sys +import os +import shutil +import re +import urllib +import time +import subprocess +from optparse import OptionParser +import HTMLParser +import common + +#Read configuration... +execfile('config.py') + + +# Parse command line... +parser = OptionParser() +parser.add_option("-v", "--verbose", action="store_true", default=False, + help="Spew out even more information than normal") +(options, args) = parser.parse_args() + +# Get all apps... +apps = common.read_metadata(options.verbose) + +html_parser = HTMLParser.HTMLParser() + +problems = [] + +for app in apps: + + if app['disabled']: + print "Skipping %s: disabled" % app['id'] + elif not app['builds']: + print "Skipping %s: no builds specified" % app['id'] + + if (app['disabled'] is None and app['repo'] != '' + and app['repotype'] != '' and len(app['builds']) > 0): + + print "Processing " + app['id'] + + build_dir = 'build/' + app['id'] + + # Set up vcs interface and make sure we have the latest code... + vcs = common.getvcs(app['repotype'], app['repo'], build_dir) + + refreshed_source = False + + + for thisbuild in app['builds']: + + if thisbuild['commit'].startswith('!'): + print ("..skipping version " + thisbuild['version'] + " - " + + thisbuild['commit'][1:]) + else: + print "..scanning version " + thisbuild['version'] + + if not refreshed_source: + vcs.refreshlocal() + refreshed_source = True + + # Optionally, the actual app source can be in a subdirectory... + if thisbuild.has_key('subdir'): + root_dir = os.path.join(build_dir, thisbuild['subdir']) + else: + root_dir = build_dir + + # Get a working copy of the right revision... + if options.verbose: + print "Resetting repository to " + thisbuild['commit'] + vcs.reset(thisbuild['commit']) + + # Initialise submodules if requred... + if thisbuild.get('submodules', 'no') == 'yes': + vcs.initsubmodules() + + # Generate (or update) the ant build file, build.xml... + if (thisbuild.get('update', 'yes') == 'yes' and + not thisbuild.has_key('maven')): + parms = [os.path.join(sdk_path, 'tools', 'android'), + 'update', 'project', '-p', '.'] + parms.append('--subprojects') + if thisbuild.has_key('target'): + parms.append('-t') + parms.append(thisbuild['target']) + if subprocess.call(parms, cwd=root_dir) != 0: + print "Failed to update project" + sys.exit(1) + + # Delete unwanted file... + if thisbuild.has_key('rm'): + os.remove(os.path.join(build_dir, thisbuild['rm'])) + + # Run a pre-build command if one is required... + if thisbuild.has_key('prebuild'): + if subprocess.call(thisbuild['prebuild'], + cwd=root_dir, shell=True) != 0: + print "Error running pre-build command" + sys.exit(1) + + # Apply patches if any + if 'patch' in thisbuild: + for patch in thisbuild['patch'].split(';'): + print "Applying " + patch + patch_path = os.path.join('metadata', app['id'], patch) + if subprocess.call(['patch', '-p1', + '-i', os.path.abspath(patch_path)], cwd=build_dir) != 0: + print "Failed to apply patch %s" % patch_path + sys.exit(1) + + # Scan for common known non-free blobs: + usual_suspects = ['flurryagent.jar', 'paypal_mpl.jar'] + for r,d,f in os.walk(build_dir): + for curfile in f: + if curfile.lower() in usual_suspects: + msg = 'Found probable non-free blob ' + os.path.join(r,file) + msg += ' in ' + app['id'] + ' ' + thisbuild['version'] + problems.append(msg) + +print "Finished:" +for problem in problems: + print problem +print str(len(problems)) + ' problems.' + diff --git a/update.py b/update.py index dc228a2e..e6758d12 100755 --- a/update.py +++ b/update.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python # -*- coding: utf-8 -*- # # update.py - part of the FDroid server tools