diff --git a/.gitignore b/.gitignore index c8ac9607..8863e72d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,9 @@ config.py repo/ logs/ built/ -build/ +tmp/ build_*/ *~ *.pyc +buildserver.box +unsigned/ diff --git a/README b/README index e3824d51..140b6a0d 100644 --- a/README +++ b/README @@ -186,6 +186,8 @@ configuration to the build. These are: last character on a line to join that line with the next. It has no special meaning in other contexts; in particular, literal backslashes should not be escaped. + init=xxxx As for 'prebuild', but runs on the source code BEFORE any + other processing takes place. novcheck=yes Don't check that the version name and code in the resulting apk are correct by looking at the build output - assume the metadata is correct. This takes away a useful level of @@ -204,6 +206,16 @@ configuration to the build. These are: files within a directory below the metadata, with the same name as the metadata file but without the extension. Each of these patches is applied to the code in turn. + extlibs=a;b;c Specifies a list of external libraries (jar files) from the + build/extlib library, which will be placed in the libs + directory of the project. Separate items with semicolons. + srclibs=a@r;b@r1 Specifies a list of source libraries (kept up to date using + version control) from a predefined set. Separate items with + semicolons, and each item is of the form name@rev where name + is the predefined source library name and rev is the + revision in source control to use. You can then also use + $$name$$ in the prebuild command to substitute the relative + path to the library directory. Another example, using extra parameters: diff --git a/README.buildserver b/README.buildserver new file mode 100644 index 00000000..a28bab97 --- /dev/null +++ b/README.buildserver @@ -0,0 +1,17 @@ + +Setting up a build server: + +1. Install VirtualBox, vagrant and vagrant-snap +2. In the buildserver directory, run 'vagrant up'. Will take a long time. +3. Log in with 'vagrant ssh' +4. Check it all looks ok, then 'sudo shutdown -h now' +5. Back in the main directory, run 'VBoxManage listvms' look for + buildserver_xxxx +6. Run 'vagrant package --base buildserver_xxxx --output buildserver.box'. + Will take a while. +7. You should now have a new 'buildserver.box' + +You should now be able to use the --server option on build.py and builds will +take place in the clean, secure, isolated environment of a fresh virtual +machine for each app built. + diff --git a/build.py b/build.py index 0fcdcd67..5356aac6 100755 --- a/build.py +++ b/build.py @@ -24,8 +24,7 @@ import subprocess import re import zipfile import tarfile -import md5 -import shlex +import traceback from xml.dom.minidom import Document from optparse import OptionParser @@ -44,8 +43,20 @@ parser.add_option("-p", "--package", default=None, help="Build only the specified package") parser.add_option("-s", "--stop", action="store_true", default=False, help="Make the build stop on exceptions") +parser.add_option("-t", "--test", action="store_true", default=False, + help="Test mode - put output in the tmp directory only.") +parser.add_option("--server", action="store_true", default=False, + help="Use build server") +parser.add_option("--on-server", action="store_true", default=False, + help="Specify that we're running on the build server") +parser.add_option("-f", "--force", action="store_true", default=False, + help="Force build of disabled app. Only allowed in test mode.") (options, args) = parser.parse_args() +if options.force and not options.test: + print "Force is only allowed in test mode" + sys.exit(1) + # Get all apps... apps = common.read_metadata(options.verbose) @@ -57,32 +68,40 @@ if not os.path.isdir(log_dir): print "Creating log directory" os.makedirs(log_dir) -output_dir = 'repo' -if not os.path.isdir(output_dir): - print "Creating output directory" - os.makedirs(output_dir) - tmp_dir = 'tmp' if not os.path.isdir(tmp_dir): print "Creating temporary directory" os.makedirs(tmp_dir) +if options.test: + output_dir = tmp_dir +else: + output_dir = 'unsigned' + if not os.path.isdir(output_dir): + print "Creating output directory" + os.makedirs(output_dir) + +repo_dir = 'repo' + build_dir = 'build' if not os.path.isdir(build_dir): print "Creating build directory" os.makedirs(build_dir) +extlib_dir = os.path.join(build_dir, 'extlib') +# Build applications... for app in apps: if options.package and options.package != app['id']: # Silent skip... pass - elif app['Disabled']: - print "Skipping %s: disabled" % app['id'] + elif app['Disabled'] and not options.force: + if options.verbose: + print "Skipping %s: disabled" % app['id'] elif (not app['builds']) or app['Repo Type'] =='' or len(app['builds']) == 0: - print "Skipping %s: no builds specified" % app['id'] + if options.verbose: + print "Skipping %s: no builds specified" % app['id'] else: - print "Processing " + app['id'] build_dir = 'build/' + app['id'] @@ -96,180 +115,217 @@ for app in apps: try: dest = os.path.join(output_dir, app['id'] + '_' + thisbuild['vercode'] + '.apk') - dest_unsigned = os.path.join(tmp_dir, app['id'] + '_' + - thisbuild['vercode'] + '_unsigned.apk') + dest_repo = os.path.join(repo_dir, app['id'] + '_' + + thisbuild['vercode'] + '.apk') - if os.path.exists(dest): - print "..version " + thisbuild['version'] + " already exists" + if os.path.exists(dest) or (not options.test and os.path.exists(dest_repo)): + if options.verbose: + print "..version " + thisbuild['version'] + " already exists" elif thisbuild['commit'].startswith('!'): - print ("..skipping version " + thisbuild['version'] + " - " + + if options.verbose: + print ("..skipping version " + thisbuild['version'] + " - " + thisbuild['commit'][1:]) else: - print "..building version " + thisbuild['version'] + if options.verbose: + mstart = '.. building version ' + else: + mstart = 'Building version ' + print mstart + thisbuild['version'] + ' of ' + app['id'] - # Prepare the source code... - root_dir = common.prepare_source(vcs, app, thisbuild, - build_dir, sdk_path, ndk_path, javacc_path, - not refreshed_source) - refreshed_source = True + if options.server: - # Build the source tarball right before we build the release... - tarname = app['id'] + '_' + thisbuild['vercode'] + '_src' - tarball = tarfile.open(os.path.join(output_dir, - tarname + '.tar.gz'), "w:gz") - def tarexc(f): - if f in ['.svn', '.git', '.hg', '.bzr']: - return True - return False - tarball.add(build_dir, tarname, exclude=tarexc) - tarball.close() + import paramiko - # Build native stuff if required... - if thisbuild.get('buildjni', 'no') == 'yes': - ndkbuild = os.path.join(ndk_path, "ndk-build") - p = subprocess.Popen([ndkbuild], cwd=root_dir, - stdout=subprocess.PIPE) - output = p.communicate()[0] + # Start up the virtual maachine... + if subprocess.call(['vagrant', 'up'], cwd='builder') != 0: + # Not a very helpful message yet! + raise BuildException("Failed to set up build server") + # Get SSH configuration settings for us to connect... + subprocess.call('vagrant ssh-config >sshconfig', + cwd='builder', shell=True) + vagranthost = 'default' # Host in ssh config file + + # Load and parse the SSH config... + sshconfig = paramiko.SSHConfig() + sshconfig.parse('builder/sshconfig') + sshconfig = sshconfig.lookup(vagranthost) + + # Open SSH connection... + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AcceptPolicy) + ssh.connect(sshconfig['HostName'], username=sshconfig['Username'], + port=sshconfig['Port'], timeout=10, look_for_keys=False, + key_filename=sshconfig['IdentityFile']) + + # Get an SFTP connection... + ftp = ssh.open_sftp() + ftp.get_channel().settimeout(15) + + # Put all the necessary files in place... + ftp.chdir('/home/vagrant') + ftp.put('build.py', 'build.py') + ftp.put('common.py', 'common.py') + ftp.put('config.buildserver.py', 'config.py') + ftp.mkdir('build') + ftp.chdir('build') + ftp.mkdir('extlib') + def send_dir(path): + lastdir = path + for r, d, f in os.walk(base): + while lastdir != os.path.commonprefix([lastdir, root]): + ftp.chdir('..') + lastdir = os.path.split(lastdir)[0] + lastdir = r + for ff in f: + ftp.put(os.path.join(r, ff), ff) + ftp.send_dir(app['id']) + # TODO: send relevant extlib directories too + ftp.chdir('/home/vagrant') + + # Execute the build script... + ssh.exec_command('python build.py --on-server -p ' + + app['id']) + + # Retrieve the built files... + apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk' + tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz' + ftp.chdir('unsigned') + ftp.get(apkfile, os.path.join(output_dir, apkfile)) + ftp.get(tarball, os.path.join(output_dir, tarball)) + + # Get rid of the virtual machine... + if subprocess.call(['vagrant', 'destroy'], cwd='builder') != 0: + # Not a very helpful message yet! + raise BuildException("Failed to destroy") + + else: + + # Prepare the source code... + root_dir = common.prepare_source(vcs, app, thisbuild, + build_dir, extlib_dir, sdk_path, ndk_path, + javacc_path, not refreshed_source) + refreshed_source = True + + # Scan before building... + buildprobs = common.scan_source(build_dir, root_dir, thisbuild) + if len(buildprobs) > 0: + print 'Scanner found ' + str(len(buildprobs)) + ' problems:' + for problem in buildprobs: + print '...' + problem + raise BuildException("Can't build due to " + + str(len(buildprobs)) + " scanned problems") + + # Build the source tarball right before we build the release... + tarname = app['id'] + '_' + thisbuild['vercode'] + '_src' + tarball = tarfile.open(os.path.join(tmp_dir, + tarname + '.tar.gz'), "w:gz") + def tarexc(f): + if f in ['.svn', '.git', '.hg', '.bzr']: + return True + return False + tarball.add(build_dir, tarname, exclude=tarexc) + tarball.close() + + # Build native stuff if required... + if thisbuild.get('buildjni', 'no') == 'yes': + ndkbuild = os.path.join(ndk_path, "ndk-build") + p = subprocess.Popen([ndkbuild], cwd=root_dir, + stdout=subprocess.PIPE) + output = p.communicate()[0] + if p.returncode != 0: + print output + raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version'])) + elif options.verbose: + print output + + # Build the release... + if thisbuild.has_key('maven'): + p = subprocess.Popen(['mvn', 'clean', 'install', + '-Dandroid.sdk.path=' + sdk_path], + cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + else: + if thisbuild.has_key('antcommand'): + antcommand = thisbuild['antcommand'] + else: + antcommand = 'release' + p = subprocess.Popen(['ant', antcommand], cwd=root_dir, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error = p.communicate() if p.returncode != 0: - print output - raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version'])) + raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) elif options.verbose: print output + print "Build successful" - # Build the release... - if thisbuild.has_key('maven'): - p = subprocess.Popen(['mvn', 'clean', 'install', - '-Dandroid.sdk.path=' + sdk_path], - cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - else: - if thisbuild.has_key('antcommand'): - antcommand = thisbuild['antcommand'] + # Find the apk name in the output... + if thisbuild.has_key('bindir'): + bindir = os.path.join(build_dir, thisbuild['bindir']) else: - antcommand = 'release' - p = subprocess.Popen(['ant', antcommand], cwd=root_dir, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output, error = p.communicate() - if p.returncode != 0: - raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) - elif options.verbose: - print output - print "Build successful" - - # Find the apk name in the output... - if thisbuild.has_key('bindir'): - bindir = os.path.join(build_dir, thisbuild['bindir']) - else: - bindir = os.path.join(root_dir, 'bin') - if thisbuild.get('initfun', 'no') == "yes": - # Special case (again!) for funambol... - src = ("funambol-android-sync-client-" + - thisbuild['version'] + "-unsigned.apk") - src = os.path.join(bindir, src) - elif thisbuild.has_key('maven'): - src = re.match(r".*^\[INFO\] Installing /.*/([^/]*)\.apk", - output, re.S|re.M).group(1) - src = os.path.join(bindir, src) + '.apk' + bindir = os.path.join(root_dir, 'bin') + if thisbuild.get('initfun', 'no') == "yes": + # Special case (again!) for funambol... + src = ("funambol-android-sync-client-" + + thisbuild['version'] + "-unsigned.apk") + src = os.path.join(bindir, src) + elif thisbuild.has_key('maven'): + src = re.match(r".*^\[INFO\] Installing /.*/([^/]*)\.apk", + output, re.S|re.M).group(1) + src = os.path.join(bindir, src) + '.apk' #[INFO] Installing /home/ciaran/fdroidserver/tmp/mainline/application/target/callerid-1.0-SNAPSHOT.apk - else: - src = re.match(r".*^.*Creating (\S+) for release.*$.*", output, - re.S|re.M).group(1) - src = os.path.join(bindir, src) + else: + src = re.match(r".*^.*Creating (\S+) for release.*$.*", output, + re.S|re.M).group(1) + src = os.path.join(bindir, src) - # By way of a sanity check, make sure the version and version - # code in our new apk match what we expect... - print "Checking " + src - p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools', - 'aapt'), - 'dump', 'badging', src], - stdout=subprocess.PIPE) - output = p.communicate()[0] - if thisbuild.get('novcheck', 'no') == "yes": - vercode = thisbuild['vercode'] - version = thisbuild['version'] - else: - vercode = None - version = None - for line in output.splitlines(): - if line.startswith("package:"): - pat = re.compile(".*versionCode='([0-9]*)'.*") - vercode = re.match(pat, line).group(1) - pat = re.compile(".*versionName='([^']*)'.*") - version = re.match(pat, line).group(1) - if version == None or vercode == None: - raise BuildException("Could not find version information in build in output") - - # Some apps (e.g. Timeriffic) have had the bonkers idea of - # including the entire changelog in the version number. Remove - # it so we can compare. (TODO: might be better to remove it - # before we compile, in fact) - index = version.find(" //") - if index != -1: - version = version[:index] - - if (version != thisbuild['version'] or - vercode != thisbuild['vercode']): - raise BuildException(("Unexpected version/version code in output" - "APK: %s / %s" - "Expected: %s / %s") - % (version, str(vercode), thisbuild['version'], str(thisbuild['vercode'])) - ) - - # Copy the unsigned apk to our temp directory for further - # processing... - shutil.copyfile(src, dest_unsigned) - - # Figure out the key alias name we'll use. Only the first 8 - # characters are significant, so we'll use the first 8 from - # the MD5 of the app's ID and hope there are no collisions. - # If a collision does occur later, we're going to have to - # come up with a new alogrithm, AND rename all existing keys - # in the keystore! - if keyaliases.has_key(app['id']): - # For this particular app, the key alias is overridden... - keyalias = keyaliases[app['id']] - else: - m = md5.new() - m.update(app['id']) - keyalias = m.hexdigest()[:8] - print "Key alias: " + keyalias - - # See if we already have a key for this application, and - # if not generate one... - p = subprocess.Popen(['keytool', '-list', - '-alias', keyalias, '-keystore', keystore, - '-storepass', keystorepass], stdout=subprocess.PIPE) - output = p.communicate()[0] - if p.returncode !=0: - print "Key does not exist - generating..." - p = subprocess.Popen(['keytool', '-genkey', - '-keystore', keystore, '-alias', keyalias, - '-keyalg', 'RSA', '-keysize', '2048', - '-validity', '10000', - '-storepass', keystorepass, '-keypass', keypass, - '-dname', keydname], stdout=subprocess.PIPE) + # By way of a sanity check, make sure the version and version + # code in our new apk match what we expect... + print "Checking " + src + p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools', + 'aapt'), + 'dump', 'badging', src], + stdout=subprocess.PIPE) output = p.communicate()[0] - print output - if p.returncode != 0: - raise BuildException("Failed to generate key") + if thisbuild.get('novcheck', 'no') == "yes": + vercode = thisbuild['vercode'] + version = thisbuild['version'] + else: + vercode = None + version = None + for line in output.splitlines(): + if line.startswith("package:"): + pat = re.compile(".*versionCode='([0-9]*)'.*") + vercode = re.match(pat, line).group(1) + pat = re.compile(".*versionName='([^']*)'.*") + version = re.match(pat, line).group(1) + if version == None or vercode == None: + raise BuildException("Could not find version information in build in output") - # Sign the application... - p = subprocess.Popen(['jarsigner', '-keystore', keystore, - '-storepass', keystorepass, '-keypass', keypass, - dest_unsigned, keyalias], stdout=subprocess.PIPE) - output = p.communicate()[0] - print output - if p.returncode != 0: - raise BuildException("Failed to sign application") + # Some apps (e.g. Timeriffic) have had the bonkers idea of + # including the entire changelog in the version number. Remove + # it so we can compare. (TODO: might be better to remove it + # before we compile, in fact) + index = version.find(" //") + if index != -1: + version = version[:index] + + if (version != thisbuild['version'] or + vercode != thisbuild['vercode']): + raise BuildException(("Unexpected version/version code in output" + "APK: %s / %s" + "Expected: %s / %s") + % (version, str(vercode), thisbuild['version'], str(thisbuild['vercode'])) + ) + + # Copy the unsigned apk to our destination directory for further + # processing (by publish.py)... + shutil.copyfile(src, dest) + + # Move the source tarball into the output directory... + if output_dir != tmp_dir: + tarfilename = tarname + '.tar.gz' + shutil.move(os.path.join(tmp_dir, tarfilename), + os.path.join(output_dir, tarfilename)) - # Zipalign it... - p = subprocess.Popen([os.path.join(sdk_path,'tools','zipalign'), - '-v', '4', dest_unsigned, dest], - stdout=subprocess.PIPE) - output = p.communicate()[0] - print output - if p.returncode != 0: - raise BuildException("Failed to align application") - os.remove(dest_unsigned) build_succeeded.append(app) except BuildException as be: if options.stop: @@ -287,7 +343,7 @@ for app in apps: except Exception as e: if options.stop: raise - print "Could not build app %s due to unknown error: %s" % (app['id'], e) + print "Could not build app %s due to unknown error: %s" % (app['id'], traceback.format_exc()) failed_apps[app['id']] = e for app in build_succeeded: diff --git a/build/.gitignore b/build/.gitignore new file mode 100644 index 00000000..90b3dc52 --- /dev/null +++ b/build/.gitignore @@ -0,0 +1,2 @@ +*/ +!extlib/ diff --git a/build/extlib/.gitignore b/build/extlib/.gitignore new file mode 100644 index 00000000..0eedae0e --- /dev/null +++ b/build/extlib/.gitignore @@ -0,0 +1,6 @@ +!*/ +GreenDroid/ +ActionBarSherlock/ +FacebookSDK/ +OI/ +JOpenDocument/ diff --git a/build/extlib/HtmlSpanner/htmlspanner-0.1-SNAPSHOT.jar b/build/extlib/HtmlSpanner/htmlspanner-0.1-SNAPSHOT.jar new file mode 100644 index 00000000..790956e7 Binary files /dev/null and b/build/extlib/HtmlSpanner/htmlspanner-0.1-SNAPSHOT.jar differ diff --git a/build/extlib/HtmlSpanner/source.txt b/build/extlib/HtmlSpanner/source.txt new file mode 100644 index 00000000..5be1cec8 --- /dev/null +++ b/build/extlib/HtmlSpanner/source.txt @@ -0,0 +1 @@ +https://github.com/NightWhistler/HtmlSpanner.git diff --git a/build/extlib/acra/acra-4.2.3.jar b/build/extlib/acra/acra-4.2.3.jar new file mode 100644 index 00000000..c6b57a9d Binary files /dev/null and b/build/extlib/acra/acra-4.2.3.jar differ diff --git a/build/extlib/commons-io/LICENSE.txt b/build/extlib/commons-io/LICENSE.txt new file mode 100644 index 00000000..43e91eb0 --- /dev/null +++ b/build/extlib/commons-io/LICENSE.txt @@ -0,0 +1,203 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/build/extlib/commons-io/commons-io-2.0.1.jar b/build/extlib/commons-io/commons-io-2.0.1.jar new file mode 100644 index 00000000..5b64b7d6 Binary files /dev/null and b/build/extlib/commons-io/commons-io-2.0.1.jar differ diff --git a/build/extlib/commons-io/origin.txt b/build/extlib/commons-io/origin.txt new file mode 100644 index 00000000..791a769b --- /dev/null +++ b/build/extlib/commons-io/origin.txt @@ -0,0 +1 @@ +http://archive.apache.org/dist/commons/io/binaries/commons-io-2.0.1-bin.zip diff --git a/build/extlib/epublib/epublib-core-3.0-SNAPSHOT.jar b/build/extlib/epublib/epublib-core-3.0-SNAPSHOT.jar new file mode 100644 index 00000000..945682b9 Binary files /dev/null and b/build/extlib/epublib/epublib-core-3.0-SNAPSHOT.jar differ diff --git a/build/extlib/epublib/source.txt b/build/extlib/epublib/source.txt new file mode 100644 index 00000000..58417932 --- /dev/null +++ b/build/extlib/epublib/source.txt @@ -0,0 +1 @@ +https://github.com/psiegman/epublib diff --git a/build/extlib/guava-r08/COPYING b/build/extlib/guava-r08/COPYING new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/build/extlib/guava-r08/COPYING @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/build/extlib/guava-r08/guava-r08.jar b/build/extlib/guava-r08/guava-r08.jar new file mode 100644 index 00000000..d18d0bda Binary files /dev/null and b/build/extlib/guava-r08/guava-r08.jar differ diff --git a/build/extlib/guava-r08/source.txt b/build/extlib/guava-r08/source.txt new file mode 100644 index 00000000..c1f45086 --- /dev/null +++ b/build/extlib/guava-r08/source.txt @@ -0,0 +1 @@ +http://guava-libraries.googlecode.com/files/guava-r08.zip diff --git a/build/extlib/guice/guice-2.0-no_aop.jar b/build/extlib/guice/guice-2.0-no_aop.jar new file mode 100644 index 00000000..613aa6d6 Binary files /dev/null and b/build/extlib/guice/guice-2.0-no_aop.jar differ diff --git a/build/extlib/guice/source.txt b/build/extlib/guice/source.txt new file mode 100644 index 00000000..c2a43c05 --- /dev/null +++ b/build/extlib/guice/source.txt @@ -0,0 +1 @@ +http://code.google.com/p/google-guice/ diff --git a/build/extlib/htmlcleaner/htmlcleaner-2.2.jar b/build/extlib/htmlcleaner/htmlcleaner-2.2.jar new file mode 100644 index 00000000..5322a852 Binary files /dev/null and b/build/extlib/htmlcleaner/htmlcleaner-2.2.jar differ diff --git a/build/extlib/roboguice/roboguice-1.1.jar b/build/extlib/roboguice/roboguice-1.1.jar new file mode 100644 index 00000000..0262f153 Binary files /dev/null and b/build/extlib/roboguice/roboguice-1.1.jar differ diff --git a/build/extlib/sl4j/slf4j-android-1.6.1-RC1.jar b/build/extlib/sl4j/slf4j-android-1.6.1-RC1.jar new file mode 100644 index 00000000..480348f0 Binary files /dev/null and b/build/extlib/sl4j/slf4j-android-1.6.1-RC1.jar differ diff --git a/builder/.gitignore b/builder/.gitignore new file mode 100644 index 00000000..7fc03613 --- /dev/null +++ b/builder/.gitignore @@ -0,0 +1 @@ +sshconfig diff --git a/builder/Vagrantfile b/builder/Vagrantfile new file mode 100644 index 00000000..865a496a --- /dev/null +++ b/builder/Vagrantfile @@ -0,0 +1,8 @@ +Vagrant::Config.run do |config| + + config.vm.box = "buildserver" + config.vm.box_url = "../buildserver.box" + + config.vm.customize ["modifyvm", :id, "--memory", "768"] + +end diff --git a/buildserver/.gitignore b/buildserver/.gitignore new file mode 100644 index 00000000..8000dd9d --- /dev/null +++ b/buildserver/.gitignore @@ -0,0 +1 @@ +.vagrant diff --git a/buildserver/Vagrantfile b/buildserver/Vagrantfile new file mode 100644 index 00000000..d4f8d533 --- /dev/null +++ b/buildserver/Vagrantfile @@ -0,0 +1,26 @@ +Vagrant::Config.run do |config| + + config.vm.box = "debian6-32" + config.vm.box_url = "/shares/software/OS and Boot/debian6-32.box" + + config.vm.customize ["modifyvm", :id, "--memory", "1024"] + + config.vm.provision :shell, :path => "fixpaths.sh" + # Set apt proxy - remove, or adjust this, accordingly! + config.vm.provision :shell, :inline => 'sudo echo "Acquire::http { Proxy \"http://thurlow:3142\"; };" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update' + + config.vm.provision :chef_solo do |chef| + chef.cookbooks_path = "cookbooks" + chef.log_level = :debug + chef.json = { + :settings => { + :sdk_loc => "/home/vagrant/android-sdk", + :ndk_loc => "/home/vagrant/android-ndk", + :user => "vagrant" + } + } + chef.add_recipe "android-sdk" + chef.add_recipe "android-ndk" + chef.add_recipe "fdroidbuild-general" + end +end diff --git a/buildserver/cookbooks/android-ndk/recipes/default.rb b/buildserver/cookbooks/android-ndk/recipes/default.rb new file mode 100644 index 00000000..80e06346 --- /dev/null +++ b/buildserver/cookbooks/android-ndk/recipes/default.rb @@ -0,0 +1,17 @@ + +ndk_loc = node[:settings][:ndk_loc] + +script "setup-android-ndk" do + interpreter "bash" + user node[:settings][:user] + cwd "/tmp" + code " + wget http://dl.google.com/android/ndk/android-ndk-r7-linux-x86.tar.bz2 + tar jxvf android-ndk-r7-linux-x86.tar.bz2 + mv android-ndk-r7 #{ndk_loc} + " + not_if do + File.exists?("#{ndk_loc}") + end +end + diff --git a/buildserver/cookbooks/android-sdk/recipes/default.rb b/buildserver/cookbooks/android-sdk/recipes/default.rb new file mode 100644 index 00000000..91b5a084 --- /dev/null +++ b/buildserver/cookbooks/android-sdk/recipes/default.rb @@ -0,0 +1,34 @@ +%w{openjdk-6-jdk}.each do |pkg| + package pkg do + action :install + end +end + +sdk_loc = node[:settings][:sdk_loc] +user = node[:settings][:user] + +script "setup-android-sdk" do + interpreter "bash" + user user + cwd "/tmp" + code " + wget http://dl.google.com/android/android-sdk_r16-linux.tgz + tar zxvf android-sdk_r16-linux.tgz + mv android-sdk-linux #{sdk_loc} + rm android-sdk_r16-linux.tgz + #{sdk_loc}/tools/android update sdk --no-ui -t platform-tool + #{sdk_loc}/tools/android update sdk --no-ui -t platform + #{sdk_loc}/tools/android update sdk --no-ui -t tool,platform-tool + " + not_if do + File.exists?("#{sdk_loc}") + end +end + +execute "add-android-sdk-path" do + user user + path = "#{sdk_loc}/tools:#{sdk_loc}/platform-tools" + command "echo \"export PATH=\\$PATH:#{path}\" >> /home/#{user}/.bashrc" + not_if "grep #{sdk_loc} /home/#{user}/.bashrc" +end + diff --git a/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb b/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb new file mode 100644 index 00000000..81e4c949 --- /dev/null +++ b/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb @@ -0,0 +1,7 @@ + +%w{ant ant-contrib maven2 javacc python}.each do |pkg| + package pkg do + action :install + end +end + diff --git a/buildserver/fixpaths.sh b/buildserver/fixpaths.sh new file mode 100644 index 00000000..eb8a81fb --- /dev/null +++ b/buildserver/fixpaths.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +fixit() +{ + #Fix sudoers so the PATH gets passed through, otherwise chef + #provisioning doesn't work. + if [ -z "$1" ]; then + export EDITOR=$0 && sudo -E visudo + else + echo "Fix sudoers" + echo "Defaults exempt_group=admin" >> $1 + fi + #Stick the gems bin onto root's path as well. + sudo echo "PATH=$PATH:/var/lib/gems/1.8/bin" >>/root/.bashrc + # Restart sudo so it gets the changes straight away + sudo /etc/init.d/sudo restart +} + +sudo grep "exempt_group" /etc/sudoers -q +if [ "$?" -eq "1" ]; then + fixit +fi + diff --git a/checkupdates.py b/checkupdates.py index caa27c82..2c083044 100755 --- a/checkupdates.py +++ b/checkupdates.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# checkmarket2.py - part of the FDroid server tools +# checkupdates.py - part of the FDroid server tools # Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com # # This program is free software: you can redistribute it and/or modify @@ -35,9 +35,14 @@ execfile('config.py') # Returns (None, "a message") if this didn't work, or (version, vercode) for # the details of the current version. def check_market(app): - time.sleep(5) + time.sleep(10) url = 'http://market.android.com/details?id=' + app['id'] - page = urllib.urlopen(url).read() + req = urllib.urlopen(url) + if req.getcode() == 404: + return (None, 'Not in market') + elif req.getcode() != 200: + return (None, 'Return code ' + str(req.getcode())) + page = req.read() version = None vercode = None @@ -46,6 +51,9 @@ def check_market(app): if m: version = html_parser.unescape(m.group(1)) + if version == 'Varies with device': + return (None, 'Device-variable version, cannot use this method') + m = re.search('data-paramValue="(\d+)">