Make the server tools an installable package (with distutils) - wip

fdroidserver/build.py

@@ -0,0 +1,442 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# build.py - part of the FDroid server tools
+# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import os
+import shutil
+import subprocess
+import re
+import zipfile
+import tarfile
+import traceback
+from xml.dom.minidom import Document
+from optparse import OptionParser
+
+import common
+from common import BuildException
+from common import VCSException
+
+
+def build_server(app, thisbuild, build_dir, output_dir):
+    """Do a build on the build server."""
+
+    import paramiko
+
+    # Destroy the builder vm if it already exists...
+    # TODO: need to integrate the snapshot stuff so it doesn't have to
+    # keep wasting time doing this unnecessarily.
+    if os.path.exists(os.path.join('builder', '.vagrant')):
+        if subprocess.call(['vagrant', 'destroy'], cwd='builder') != 0:
+            raise BuildException("Failed to destroy build server")
+
+    # Start up the virtual maachine...
+    if subprocess.call(['vagrant', 'up'], cwd='builder') != 0:
+        # Not a very helpful message yet!
+        raise BuildException("Failed to set up build server")
+    # Get SSH configuration settings for us to connect...
+    subprocess.call('vagrant ssh-config >sshconfig',
+            cwd='builder', shell=True)
+    vagranthost = 'default' # Host in ssh config file
+
+    # Load and parse the SSH config...
+    sshconfig = paramiko.SSHConfig()
+    sshf = open('builder/sshconfig', 'r')
+    sshconfig.parse(sshf)
+    sshf.close()
+    sshconfig = sshconfig.lookup(vagranthost)
+
+    # Open SSH connection...
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    print sshconfig
+    ssh.connect(sshconfig['hostname'], username=sshconfig['user'],
+        port=int(sshconfig['port']), timeout=10, look_for_keys=False,
+        key_filename=sshconfig['identityfile'])
+
+    # Get an SFTP connection...
+    ftp = ssh.open_sftp()
+    ftp.get_channel().settimeout(15)
+
+    # Put all the necessary files in place...
+    ftp.chdir('/home/vagrant')
+    ftp.put('build.py', 'build.py')
+    ftp.put('common.py', 'common.py')
+    ftp.put('config.buildserver.py', 'config.py')
+    ftp.mkdir('metadata')
+    ftp.chdir('metadata')
+    ftp.put(os.path.join('metadata', app['id'] + '.txt'),
+            app['id'] + '.txt')
+    ftp.chdir('..')
+    ftp.mkdir('build')
+    ftp.chdir('build')
+    ftp.mkdir('extlib')
+    ftp.mkdir(app['id'])
+    ftp.chdir('..')
+    def send_dir(path):
+        lastdir = path
+        for r, d, f in os.walk(path):
+            ftp.chdir(r)
+            for dd in d:
+                ftp.mkdir(dd)
+            for ff in f:
+                ftp.put(os.path.join(r, ff), ff)
+            for i in range(len(r.split('/'))):
+                ftp.chdir('..')
+    send_dir(build_dir)
+    # TODO: send relevant extlib and srclib directories too
+
+    # Execute the build script...
+    ssh.exec_command('python build.py --on-server -p ' +
+            app['id'] + ' --vercode ' + thisbuild['vercode'])
+
+    # Retrieve the built files...
+    apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk'
+    tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz'
+    ftp.chdir('/home/vagrant/unsigned')
+    ftp.get(apkfile, os.path.join(output_dir, apkfile))
+    ftp.get(tarball, os.path.join(output_dir, tarball))
+
+    # Get rid of the virtual machine...
+    if subprocess.call(['vagrant', 'destroy'], cwd='builder') != 0:
+        # Not a very helpful message yet!
+        raise BuildException("Failed to destroy")
+
+
+def build_local(app, thisbuild, vcs, build_dir, output_dir, extlib_dir, tmp_dir, install, force):
+    """Do a build locally."""
+
+    # Prepare the source code...
+    root_dir = common.prepare_source(vcs, app, thisbuild,
+            build_dir, extlib_dir, sdk_path, ndk_path,
+            javacc_path)
+
+    # Scan before building...
+    buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
+    if len(buildprobs) > 0:
+        print 'Scanner found ' + str(len(buildprobs)) + ' problems:'
+        for problem in buildprobs:
+            print '...' + problem
+        if not force:
+            raise BuildException("Can't build due to " +
+                str(len(buildprobs)) + " scanned problems")
+
+    # Build the source tarball right before we build the release...
+    tarname = app['id'] + '_' + thisbuild['vercode'] + '_src'
+    tarball = tarfile.open(os.path.join(tmp_dir,
+        tarname + '.tar.gz'), "w:gz")
+    def tarexc(f):
+        for vcs_dir in ['.svn', '.git', '.hg', '.bzr']:
+            if f.endswith(vcs_dir):
+                return True
+        return False
+    tarball.add(build_dir, tarname, exclude=tarexc)
+    tarball.close()
+
+    # Build native stuff if required...
+    if thisbuild.get('buildjni') not in (None, 'no'):
+        jni_components = thisbuild.get('buildjni')
+        if jni_components == 'yes':
+            jni_components = ['']
+        else:
+            jni_components = jni_components.split(';')
+        ndkbuild = os.path.join(ndk_path, "ndk-build")
+        for d in jni_components:
+            if options.verbose:
+                print "Running ndk-build in " + root_dir + '/' + d
+            p = subprocess.Popen([ndkbuild], cwd=root_dir + '/' + d,
+                    stdout=subprocess.PIPE)
+        output = p.communicate()[0]
+        if p.returncode != 0:
+            print output
+            raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version']))
+
+    # Build the release...
+    if thisbuild.has_key('maven'):
+        p = subprocess.Popen(['mvn', 'clean', 'install',
+            '-Dandroid.sdk.path=' + sdk_path],
+            cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    else:
+        if install:
+            antcommands = ['debug','install']
+        elif thisbuild.has_key('antcommand'):
+            antcommands = [thisbuild['antcommand']]
+        else:
+            antcommands = ['release']
+        p = subprocess.Popen(['ant'] + antcommands, cwd=root_dir, 
+                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    output, error = p.communicate()
+    if p.returncode != 0:
+        raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip())
+    if install:
+        return
+    print "Build successful"
+
+    # Find the apk name in the output...
+    if thisbuild.has_key('bindir'):
+        bindir = os.path.join(build_dir, thisbuild['bindir'])
+    else:
+        bindir = os.path.join(root_dir, 'bin')
+    if thisbuild.get('initfun', 'no')  == "yes":
+        # Special case (again!) for funambol...
+        src = ("funambol-android-sync-client-" +
+                thisbuild['version'] + "-unsigned.apk")
+        src = os.path.join(bindir, src)
+    elif thisbuild.has_key('maven'):
+        src = re.match(r".*^\[INFO\] Installing /.*/([^/]*)\.apk",
+                output, re.S|re.M).group(1)
+        src = os.path.join(bindir, src) + '.apk'
+#[INFO] Installing /home/ciaran/fdroidserver/tmp/mainline/application/target/callerid-1.0-SNAPSHOT.apk
+    else:
+        src = re.match(r".*^.*Creating (\S+) for release.*$.*", output,
+            re.S|re.M).group(1)
+        src = os.path.join(bindir, src)
+
+    # By way of a sanity check, make sure the version and version
+    # code in our new apk match what we expect...
+    print "Checking " + src
+    p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools',
+                                    'aapt'),
+                        'dump', 'badging', src],
+                        stdout=subprocess.PIPE)
+    output = p.communicate()[0]
+    if thisbuild.get('novcheck', 'no') == "yes":
+        vercode = thisbuild['vercode']
+        version = thisbuild['version']
+    else:
+        vercode = None
+        version = None
+        for line in output.splitlines():
+            if line.startswith("package:"):
+                pat = re.compile(".*versionCode='([0-9]*)'.*")
+                vercode = re.match(pat, line).group(1)
+                pat = re.compile(".*versionName='([^']*)'.*")
+                version = re.match(pat, line).group(1)
+        if version == None or vercode == None:
+            raise BuildException("Could not find version information in build in output")
+
+    # Some apps (e.g. Timeriffic) have had the bonkers idea of
+    # including the entire changelog in the version number. Remove
+    # it so we can compare. (TODO: might be better to remove it
+    # before we compile, in fact)
+    index = version.find(" //")
+    if index != -1:
+        version = version[:index]
+
+    if (version != thisbuild['version'] or
+            vercode != thisbuild['vercode']):
+        raise BuildException(("Unexpected version/version code in output"
+                             "APK: %s / %s"
+                             "Expected: %s / %s")
+                             % (version, str(vercode), thisbuild['version'], str(thisbuild['vercode']))
+                            )
+
+    # Copy the unsigned apk to our destination directory for further
+    # processing (by publish.py)...
+    dest = os.path.join(output_dir, app['id'] + '_' +
+            thisbuild['vercode'] + '.apk')
+    shutil.copyfile(src, dest)
+
+    # Move the source tarball into the output directory...
+    if output_dir != tmp_dir:
+        tarfilename = tarname + '.tar.gz'
+        shutil.move(os.path.join(tmp_dir, tarfilename),
+            os.path.join(output_dir, tarfilename))
+
+
+def trybuild(app, thisbuild, build_dir, output_dir, extlib_dir, tmp_dir,
+        repo_dir, vcs, test, server, install, force):
+    """
+    Build a particular version of an application, if it needs building.
+
+    Returns True if the build was done, False if it wasn't necessary.
+    """
+
+    dest = os.path.join(output_dir, app['id'] + '_' +
+            thisbuild['vercode'] + '.apk')
+    dest_repo = os.path.join(repo_dir, app['id'] + '_' +
+            thisbuild['vercode'] + '.apk')
+
+    if os.path.exists(dest) or (not test and os.path.exists(dest_repo)):
+        return False
+
+    if thisbuild['commit'].startswith('!'):
+        return False
+
+    print "Building version " + thisbuild['version'] + ' of ' + app['id']
+
+    if server:
+        build_server(app, thisbuild, build_dir, output_dir)
+    else:
+        build_local(app, thisbuild, vcs, build_dir, output_dir, extlib_dir, tmp_dir, install, force)
+    return True
+
+
+def parse_commandline():
+    """Parse the command line. Returns options, args."""
+
+    parser = OptionParser()
+    parser.add_option("-v", "--verbose", action="store_true", default=False,
+                      help="Spew out even more information than normal")
+    parser.add_option("-p", "--package", default=None,
+                      help="Build only the specified package")
+    parser.add_option("-c", "--vercode", default=None,
+                      help="Build only the specified version code")
+    parser.add_option("-s", "--stop", action="store_true", default=False,
+                      help="Make the build stop on exceptions")
+    parser.add_option("-t", "--test", action="store_true", default=False,
+                      help="Test mode - put output in the tmp directory only.")
+    parser.add_option("--server", action="store_true", default=False,
+                      help="Use build server")
+    parser.add_option("--on-server", action="store_true", default=False,
+                      help="Specify that we're running on the build server")
+    parser.add_option("-f", "--force", action="store_true", default=False,
+                      help="Force build of disabled apps, and carries on regardless of scan problems. Only allowed in test mode.")
+    parser.add_option("--install", action="store_true", default=False,
+                      help="Use 'ant debug install' to build and install a " +
+                      "debug version on your device or emulator. " +
+                      "Implies --force and --test")
+    parser.add_option("--all", action="store_true", default=False,
+                      help="Use with --install, when not using --package"
+                      " to confirm you really want to build and install everything.")
+    options, args = parser.parse_args()
+
+    # The --install option implies --test and --force...
+    if options.install:
+        if options.server:
+            print "Can't install when building on a build server."
+            sys.exit(1)
+        if not options.package and not options.all:
+            print "This would build and install everything in the repo to the device."
+            print "You probably want to use --package and maybe also --vercode."
+            print "If you really want to install everything, use --all."
+            sys.exit(1)
+        options.force = True
+        options.test = True
+
+    if options.force and not options.test:
+        print "Force is only allowed in test mode"
+        sys.exit(1)
+
+    return options, args
+
+options = None
+
+def main():
+
+    global options
+    # Read configuration...
+    execfile('config.py', globals())
+    options, args = parse_commandline()
+
+    # Get all apps...
+    apps = common.read_metadata(options.verbose)
+
+    log_dir = 'logs'
+    if not os.path.isdir(log_dir):
+        print "Creating log directory"
+        os.makedirs(log_dir)
+
+    tmp_dir = 'tmp'
+    if not os.path.isdir(tmp_dir):
+        print "Creating temporary directory"
+        os.makedirs(tmp_dir)
+
+    if options.test:
+        output_dir = tmp_dir
+    else:
+        output_dir = 'unsigned'
+        if not os.path.isdir(output_dir):
+            print "Creating output directory"
+            os.makedirs(output_dir)
+
+    repo_dir = 'repo'
+
+    build_dir = 'build'
+    if not os.path.isdir(build_dir):
+        print "Creating build directory"
+        os.makedirs(build_dir)
+    extlib_dir = os.path.join(build_dir, 'extlib')
+
+    # Filter apps and build versions according to command-line options, etc...
+    if options.package:
+        apps = [app for app in apps if app['id'] == options.package]
+        if len(apps) == 0:
+            print "No such package"
+            sys.exit(1)
+    apps = [app for app in apps if (options.force or not app['Disabled']) and
+            app['builds'] and len(app['Repo Type']) > 0 and len(app['builds']) > 0]
+    if len(apps) == 0:
+        print "Nothing to do - all apps are disabled or have no builds defined."
+        sys.exit(1)
+    if options.vercode:
+        for app in apps:
+            app['builds'] = [b for b in app['builds']
+                    if str(b['vercode']) == options.vercode]
+
+    # Build applications...
+    failed_apps = {}
+    build_succeeded = []
+    for app in apps:
+
+        build_dir = 'build/' + app['id']
+
+        # Set up vcs interface and make sure we have the latest code...
+        vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
+
+        for thisbuild in app['builds']:
+            try:
+                if trybuild(app, thisbuild, build_dir, output_dir, extlib_dir,
+                        tmp_dir, repo_dir, vcs, options.test, options.server,
+                        options.install, options.force):
+                    build_succeeded.append(app)
+            except BuildException as be:
+                if options.stop:
+                    raise
+                print "Could not build app %s due to BuildException: %s" % (app['id'], be)
+                logfile = open(os.path.join(log_dir, app['id'] + '.log'), 'a+')
+                logfile.write(str(be))
+                logfile.close
+                failed_apps[app['id']] = be
+            except VCSException as vcse:
+                if options.stop:
+                    raise
+                print "VCS error while building app %s: %s" % (app['id'], vcse)
+                failed_apps[app['id']] = vcse
+            except Exception as e:
+                if options.stop:
+                    raise
+                print "Could not build app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
+                failed_apps[app['id']] = e
+
+    for app in build_succeeded:
+        print "success: %s" % (app['id'])
+
+    for fa in failed_apps:
+        print "Build for app %s failed:\n%s" % (fa, failed_apps[fa])
+
+    print "Finished."
+    if len(build_succeeded) > 0:
+        print str(len(build_succeeded)) + ' builds succeeded'
+    if len(failed_apps) > 0:
+        print str(len(failed_apps)) + ' builds failed'
+
+
+if __name__ == "__main__":
+    main()
+

fdroidserver/checkupdates.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# checkupdates.py - part of the FDroid server tools
+# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import os
+import shutil
+import re
+import urllib
+import time
+from optparse import OptionParser
+import HTMLParser
+import common
+
+
+# Check for a new version by looking at the AndroidManifest.xml at the HEAD
+# of the source repo. Whether this can be used reliably or not depends on
+# the development procedures used by the project's developers. Use it with
+# caution, because it's inappropriate for many projects.
+# Returns (None, "a message") if this didn't work, or (version, vercode) for
+# the details of the current version.
+def check_repomanifest(app):
+
+    try:
+
+        build_dir = 'build/' + app['id']
+
+        if app['Repo Type'] != 'git':
+            return (None, 'RepoManifest update mode only works for git repositories currently')
+
+        # Set up vcs interface and make sure we have the latest code...
+        vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
+        vcs.gotorevision('origin/master')
+
+        if len(app['builds']) == 0:
+            return (None, "Can't use RepoManifest with no builds defined")
+
+        manifest = build_dir
+        if app['builds'][-1].has_key('subdir'):
+            manifest = os.path.join(manifest, app['builds'][-1]['subdir'])
+        manifest = os.path.join(manifest, 'AndroidManifest.xml')
+
+        version, vercode, package = common.parse_androidmanifest(manifest)
+        if not package:
+            return (None, "Couldn't find ipackage ID")
+        if package != app['id']:
+            return (None, "Package ID mismatch")
+        if not version:
+            return (None,"Couldn't find latest version name")
+        if not vercode:
+            return (None,"Couldn't find latest version code")
+
+        return (version, vercode)
+
+    except BuildException as be:
+        msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
+        return (None, msg)
+    except VCSException as vcse:
+        msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
+        return (None, msg)
+    except Exception:
+        msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
+        return (None, msg)
+
+
+# Check for a new version by looking at the Google market.
+# Returns (None, "a message") if this didn't work, or (version, vercode) for
+# the details of the current version.
+def check_market(app):
+    time.sleep(10)
+    url = 'http://market.android.com/details?id=' + app['id']
+    req = urllib.urlopen(url)
+    if req.getcode() == 404:
+        return (None, 'Not in market')
+    elif req.getcode() != 200:
+        return (None, 'Return code ' + str(req.getcode()))
+    page = req.read()
+
+    version = None
+    vercode = None
+
+    m = re.search('<dd itemprop="softwareVersion">([^>]+)</dd>', page)
+    if m:
+        html_parser = HTMLParser.HTMLParser()
+        version = html_parser.unescape(m.group(1))
+
+    if version == 'Varies with device':
+        return (None, 'Device-variable version, cannot use this method')
+
+    m = re.search('data-paramValue="(\d+)"><div class="goog-menuitem-content">Latest Version<', page)
+    if m:
+        vercode = m.group(1)
+
+    if not vercode:
+        return (None, "Couldn't find version code")
+    if not version:
+        return (None, "Couldn't find version")
+    return (version, vercode)
+
+
+
+def main():
+
+    #Read configuration...
+    execfile('config.py', globals())
+
+    # Parse command line...
+    parser = OptionParser()
+    parser.add_option("-v", "--verbose", action="store_true", default=False,
+                      help="Spew out even more information than normal")
+    parser.add_option("-p", "--package", default=None,
+                      help="Build only the specified package")
+    (options, args) = parser.parse_args()
+
+    # Get all apps...
+    apps = common.read_metadata(options.verbose)
+
+    for app in apps:
+
+        if options.package and options.package != app['id']:
+            # Silent skip...
+            pass
+        else:
+            print "Processing " + app['id'] + '...'
+
+            mode = app['Update Check Mode']
+            if mode == 'Market':
+                (version, vercode) = check_market(app)
+            elif mode == 'RepoManifest':
+                (version, vercode) = check_repomanifest(app)
+            elif mode == 'None':
+                version = None
+                vercode = 'Checking disabled'
+            else:
+                version = None
+                vercode = 'Invalid update check method'
+
+            if not version:
+                print "..." + vercode
+            elif vercode == app['Current Version Code'] and version == app['Current Version']:
+                print "...up to date"
+            else:
+                print '...updating to version:' + version + ' vercode:' + vercode
+                app['Current Version'] = version
+                app['Current Version Code'] = vercode
+                metafile = os.path.join('metadata', app['id'] + '.txt')
+                common.write_metadata(metafile, app)
+
+    print "Finished."
+
+if __name__ == "__main__":
+    main()
+

fdroidserver/import.py

@@ -0,0 +1,241 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# import.py - part of the FDroid server tools
+# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import os
+import shutil
+import subprocess
+import re
+import urllib
+from optparse import OptionParser
+
+def main():
+
+    # Read configuration...
+    execfile('config.py', globals())
+
+    import common
+
+    # Parse command line...
+    parser = OptionParser()
+    parser.add_option("-u", "--url", default=None,
+                      help="Project URL to import from.")
+    parser.add_option("-s", "--subdir", default=None,
+                      help="Path to main android project subdirectory, if not in root.")
+    (options, args) = parser.parse_args()
+
+    if not options.url:
+        print "Specify project url."
+        sys.exit(1)
+    url = options.url
+
+    tmp_dir = 'tmp'
+    if not os.path.isdir(tmp_dir):
+        print "Creating temporary directory"
+        os.makedirs(tmp_dir)
+
+    # Get all apps...
+    apps = common.read_metadata()
+
+    # Figure out what kind of project it is...
+    projecttype = None
+    issuetracker = None
+    license = None
+    if url.startswith('https://github.com'):
+        projecttype = 'github'
+        repo = url + '.git'
+        repotype = 'git'
+        sourcecode = url
+    elif url.startswith('https://gitorious.org/'):
+        projecttype = 'gitorious'
+        repo = 'https://git.gitorious.org/' + url[22:] + '.git'
+        repotype = 'git'
+        sourcecode = url
+    elif url.startswith('https://bitbucket.org/'):
+        if url.endswith('/'):
+            url = url[:-1]
+        projecttype = 'bitbucket'
+        sourcecode = url + '/src'
+        issuetracker = url + '/issues'
+        repotype = 'hg'
+        repo = url
+    elif url.startswith('http://code.google.com/p/'):
+        if not url.endswith('/'):
+            url += '/';
+        projecttype = 'googlecode'
+        sourcecode = url + 'source/checkout'
+        issuetracker = url + 'issues/list'
+
+        # Figure out the repo type and adddress...
+        req = urllib.urlopen(sourcecode)
+        if req.getcode() != 200:
+            print 'Unable to find source at ' + sourcecode + ' - return code ' + str(req.getcode())
+            sys.exit(1)
+        page = req.read()
+        repotype = None
+        index = page.find('hg clone')
+        if index != -1:
+            repotype = 'hg'
+            repo = page[index + 9:]
+            index = repo.find('<')
+            if index == -1:
+                print "Error while getting repo address"
+                sys.exit(1)
+            repo = repo[:index]
+        if not repotype:
+            index=page.find('git clone')
+            if index != -1:
+                repotype = 'git'
+                repo = page[index + 10:]
+                index = repo.find('<')
+                if index == -1:
+                    print "Error while getting repo address"
+                    sys.exit(1)
+                repo = repo[:index]
+        if not repotype:
+            index=page.find('svn checkout')
+            if index != -1:
+                repotype = 'git-svn'
+                repo = page[index + 13:]
+                prefix = '<strong><em>http</em></strong>'
+                if not repo.startswith(prefix):
+                    print "Unexpected checkout instructions format"
+                    sys.exit(1)
+                repo = 'http' + repo[len(prefix):]
+                index = repo.find('<')
+                if index == -1:
+                    print "Error while getting repo address - no end tag? '" + repo + "'"
+                    sys.exit(1)
+                repo = repo[:index]
+                index = repo.find(' ')
+                if index == -1:
+                    print "Error while getting repo address - no space? '" + repo + "'"
+                    sys.exit(1)
+                repo = repo[:index]
+        if not repotype:
+            print "Unable to determine vcs type"
+            sys.exit(1)
+
+        # Figure out the license...
+        req = urllib.urlopen(url)
+        if req.getcode() != 200:
+            print 'Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode())
+            sys.exit(1)
+        page = req.read()
+        index = page.find('Code license')
+        if index == -1:
+            print "Couldn't find license data"
+            sys.exit(1)
+        ltext = page[index:]
+        lprefix = 'rel="nofollow">'
+        index = ltext.find(lprefix)
+        if index == -1:
+            print "Couldn't find license text"
+            sys.exit(1)
+        ltext = ltext[index + len(lprefix):]
+        index = ltext.find('<')
+        if index == -1:
+            print "License text not formatted as expected"
+            sys.exit(1)
+        ltext = ltext[:index]
+        if ltext == 'GNU GPL v3':
+            license = 'GPLv3'
+        elif ltext == 'GNU GPL v2':
+            license = 'GPLv2'
+        elif ltext == 'Apache License 2.0':
+            license = 'Apache2'
+        elif ltext == 'MIT License':
+            license = 'MIT'
+        else:
+            print "License " + ltext + " is not recognised"
+            sys.exit(1)
+
+    if not projecttype:
+        print "Unable to determine the project type."
+        sys.exit(1)
+
+    # Get a copy of the source so we can extract some info...
+    print 'Getting source from ' + repotype + ' repo at ' + repo
+    src_dir = os.path.join(tmp_dir, 'importer')
+    if os.path.exists(tmp_dir):
+        shutil.rmtree(tmp_dir)
+    vcs = common.getvcs(repotype, repo, src_dir)
+    vcs.gotorevision(None)
+    if options.subdir:
+        root_dir = os.path.join(src_dir, options.subdir)
+    else:
+        root_dir = src_dir
+
+    # Check AndroidManiifest.xml exists...
+    manifest = os.path.join(root_dir, 'AndroidManifest.xml')
+    if not os.path.exists(manifest):
+        print "AndroidManifest.xml did not exist in the expected location. Specify --subdir?"
+        sys.exit(1)
+
+    # Extract some information...
+    version, vercode, package = common.parse_androidmanifest(manifest)
+    if not package:
+        print "Couldn't find package ID"
+        sys.exit(1)
+    if not version:
+        print "Couldn't find latest version name"
+        sys.exit(1)
+    if not vercode:
+        print "Couldn't find latest version code"
+        sys.exit(1)
+
+    # Make sure it's actually new...
+    for app in apps:
+        if app['id'] == package:
+            print "Package " + package + " already exists"
+            sys.exit(1)
+
+    # Construct the metadata...
+    app = common.parse_metadata(None)
+    app['id'] = package
+    app['Web Site'] = url
+    app['Source Code'] = sourcecode
+    if issuetracker:
+        app['Issue Tracker'] = issuetracker
+    if license:
+        app['License'] = license
+    app['Repo Type'] = repotype
+    app['Repo'] = repo
+
+    # Create a build line...
+    build = {}
+    build['version'] = version
+    build['vercode'] = vercode
+    build['commit'] = '?'
+    if options.subdir:
+        build['subdir'] = options.subdir
+    if os.path.exists(os.path.join(root_dir, 'jni')):
+        build['buildjni'] = 'yes'
+    app['builds'].append(build)
+    app['comments'].append(('build:' + version,
+        "#Generated by import.py - check this is the right version, and find the right commit!"))
+
+    metafile = os.path.join('metadata', package + '.txt')
+    common.write_metadata(metafile, app)
+    print "Wrote " + metafile
+
+
+if __name__ == "__main__":
+    main()
+

fdroidserver/publish.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# publish.py - part of the FDroid server tools
+# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import os
+import shutil
+import subprocess
+import re
+import zipfile
+import tarfile
+import md5
+import glob
+from optparse import OptionParser
+
+import common
+from common import BuildException
+
+def main():
+
+    #Read configuration...
+    execfile('config.py', globals())
+
+    # Parse command line...
+    parser = OptionParser()
+    parser.add_option("-v", "--verbose", action="store_true", default=False,
+                      help="Spew out even more information than normal")
+    parser.add_option("-p", "--package", default=None,
+                      help="Publish only the specified package")
+    (options, args) = parser.parse_args()
+
+    log_dir = 'logs'
+    if not os.path.isdir(log_dir):
+        print "Creating log directory"
+        os.makedirs(log_dir)
+
+    tmp_dir = 'tmp'
+    if not os.path.isdir(tmp_dir):
+        print "Creating temporary directory"
+        os.makedirs(tmp_dir)
+
+    output_dir = 'repo'
+    if not os.path.isdir(output_dir):
+        print "Creating output directory"
+        os.makedirs(output_dir)
+
+    unsigned_dir = 'unsigned'
+    if not os.path.isdir(unsigned_dir):
+        print "No unsigned directory - nothing to do"
+        sys.exit(0)
+
+    for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))):
+
+        apkfilename = os.path.basename(apkfile)
+        i = apkfilename.rfind('_')
+        if i == -1:
+            raise BuildException("Invalid apk name")
+        appid = apkfilename[:i]
+        print "Processing " + appid
+
+        if not options.package or options.package == appid:
+
+            # Figure out the key alias name we'll use. Only the first 8
+            # characters are significant, so we'll use the first 8 from
+            # the MD5 of the app's ID and hope there are no collisions.
+            # If a collision does occur later, we're going to have to
+            # come up with a new alogrithm, AND rename all existing keys
+            # in the keystore!
+            if keyaliases.has_key(appid):
+                # For this particular app, the key alias is overridden...
+                keyalias = keyaliases[appid]
+            else:
+                m = md5.new()
+                m.update(appid)
+                keyalias = m.hexdigest()[:8]
+            print "Key alias: " + keyalias
+
+            # See if we already have a key for this application, and
+            # if not generate one...
+            p = subprocess.Popen(['keytool', '-list',
+                '-alias', keyalias, '-keystore', keystore,
+                '-storepass', keystorepass], stdout=subprocess.PIPE)
+            output = p.communicate()[0]
+            if p.returncode !=0:
+                print "Key does not exist - generating..."
+                p = subprocess.Popen(['keytool', '-genkey',
+                    '-keystore', keystore, '-alias', keyalias,
+                    '-keyalg', 'RSA', '-keysize', '2048',
+                    '-validity', '10000',
+                    '-storepass', keystorepass, '-keypass', keypass,
+                    '-dname', keydname], stdout=subprocess.PIPE)
+                output = p.communicate()[0]
+                print output
+                if p.returncode != 0:
+                    raise BuildException("Failed to generate key")
+
+            # Sign the application...
+            p = subprocess.Popen(['jarsigner', '-keystore', keystore,
+                '-storepass', keystorepass, '-keypass', keypass,
+                    apkfile, keyalias], stdout=subprocess.PIPE)
+            output = p.communicate()[0]
+            print output
+            if p.returncode != 0:
+                raise BuildException("Failed to sign application")
+
+            # Zipalign it...
+            p = subprocess.Popen([os.path.join(sdk_path,'tools','zipalign'),
+                                '-v', '4', apkfile,
+                                os.path.join(output_dir, apkfilename)],
+                                stdout=subprocess.PIPE)
+            output = p.communicate()[0]
+            print output
+            if p.returncode != 0:
+                raise BuildException("Failed to align application")
+            os.remove(apkfile)
+
+            # Move the source tarball into the output directory...
+            tarfilename = apkfilename[:-4] + '_src.tar.gz'
+            shutil.move(os.path.join(unsigned_dir, tarfilename),
+                    os.path.join(output_dir, tarfilename))
+
+            print 'Published ' + apkfilename
+
+
+if __name__ == "__main__":
+    main()
+

fdroidserver/rewritemeta.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# rewritemeta.py - part of the FDroid server tools
+# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import os
+import shutil
+import re
+import urllib
+import time
+from optparse import OptionParser
+import HTMLParser
+import common
+
+def main():
+
+    #Read configuration...
+    execfile('config.py', globals())
+
+    # Parse command line...
+    parser = OptionParser()
+    parser.add_option("-v", "--verbose", action="store_true", default=False,
+                      help="Spew out even more information than normal")
+    (options, args) = parser.parse_args()
+
+    # Get all apps...
+    apps = common.read_metadata(options.verbose)
+
+    for app in apps:
+        print "Writing " + app['id']
+        common.write_metadata(os.path.join('metadata', app['id']) + '.txt', app)
+
+    print "Finished."
+
+if __name__ == "__main__":
+    main()
+

fdroidserver/scanner.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# scanner.py - part of the FDroid server tools
+# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import os
+import shutil
+import re
+import urllib
+import time
+import subprocess
+import traceback
+from optparse import OptionParser
+import HTMLParser
+import common
+from common import BuildException
+from common import VCSException
+
+def main():
+
+    # Read configuration...
+    execfile('config.py', globals())
+
+
+    # Parse command line...
+    parser = OptionParser()
+    parser.add_option("-v", "--verbose", action="store_true", default=False,
+                      help="Spew out even more information than normal")
+    parser.add_option("-p", "--package", default=None,
+                      help="Scan only the specified package")
+    parser.add_option("--nosvn", action="store_true", default=False,
+                      help="Skip svn repositories - for test purposes, because they are too slow.")
+    (options, args) = parser.parse_args()
+
+    # Get all apps...
+    apps = common.read_metadata(options.verbose)
+
+    html_parser = HTMLParser.HTMLParser()
+
+    problems = []
+
+    extlib_dir = os.path.join('build', 'extlib')
+
+    for app in apps:
+
+        skip = False
+        if options.package and app['id'] != options.package:
+            skip = True
+        elif app['Disabled']:
+            print "Skipping %s: disabled" % app['id']
+            skip = True
+        elif not app['builds']:
+            print "Skipping %s: no builds specified" % app['id']
+            skip = True
+        elif options.nosvn and app['Repo Type'] == 'svn':
+            skip = True
+
+        if not skip:
+
+            print "Processing " + app['id']
+
+            try:
+
+                build_dir = 'build/' + app['id']
+
+                # Set up vcs interface and make sure we have the latest code...
+                vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
+
+                for thisbuild in app['builds']:
+
+                    if thisbuild['commit'].startswith('!'):
+                        print ("..skipping version " + thisbuild['version'] + " - " +
+                                thisbuild['commit'][1:])
+                    else:
+                        print "..scanning version " + thisbuild['version']
+
+                        # Prepare the source code...
+                        root_dir = common.prepare_source(vcs, app, thisbuild,
+                                build_dir, extlib_dir, sdk_path, ndk_path, javacc_path)
+
+                        # Do the scan...
+                        buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
+                        for problem in buildprobs:
+                            problems.append(problem + 
+                                ' in ' + app['id'] + ' ' + thisbuild['version'])
+
+            except BuildException as be:
+                msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
+                problems.append(msg)
+            except VCSException as vcse:
+                msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
+                problems.append(msg)
+            except Exception:
+                msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
+                problems.append(msg)
+
+    print "Finished:"
+    for problem in problems:
+        print problem
+    print str(len(problems)) + ' problems.'
+
+if __name__ == "__main__":
+    main()
+

fdroidserver/stats.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# stats.py - part of the FDroid server tools
+# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import os
+import shutil
+import re
+import urllib
+import time
+import traceback
+import glob
+from optparse import OptionParser
+import HTMLParser
+import paramiko
+import common
+
+def main():
+
+    # Read configuration...
+    execfile('config.py', globals())
+
+    # Parse command line...
+    parser = OptionParser()
+    parser.add_option("-v", "--verbose", action="store_true", default=False,
+                      help="Spew out even more information than normal")
+    parser.add_option("-d", "--download", action="store_true", default=False,
+                      help="Download logs we don't have")
+    (options, args) = parser.parse_args()
+
+
+    statsdir = 'stats'
+    logsdir = os.path.join(statsdir, 'logs')
+    logsarchivedir = os.path.join(logsdir, 'archive')
+    datadir = os.path.join(statsdir, 'data')
+    if not os.path.exists(statsdir):
+        os.mkdir(statsdir)
+    if not os.path.exists(logsdir):
+        os.mkdir(logsdir)
+    if not os.path.exists(datadir):
+        os.mkdir(datadir)
+
+    if options.download:
+        # Get any access logs we don't have...
+        ssh = None
+        ftp = None
+        try:
+            print 'Retrieving logs'
+            ssh = paramiko.SSHClient()
+            ssh.load_system_host_keys()
+            ssh.connect('f-droid.org', username='fdroid', timeout=10,
+                    key_filename=webserver_keyfile)
+            ftp = ssh.open_sftp()
+            ftp.get_channel().settimeout(15)
+            print "...connected"
+
+            ftp.chdir('logs')
+            files = ftp.listdir()
+            for f in files:
+                if f.startswith('access-') and f.endswith('.log'):
+
+                    destpath = os.path.join(logsdir, f)
+                    archivepath = os.path.join(logsarchivedir, f + '.gz')
+                    if os.path.exists(archivepath):
+                        if os.path.exists(destpath):
+                            # Just in case we have it archived but failed to remove
+                            # the original...
+                            os.remove(destpath)
+                    else:
+                        destsize = ftp.stat(f).st_size
+                        if (not os.path.exists(destpath) or
+                                os.path.getsize(destpath) != destsize):
+                            print "...retrieving " + f
+                            ftp.get(f, destpath)
+        except Exception as e:
+            traceback.print_exc()
+            sys.exit(1)
+        finally:
+            #Disconnect
+            if ftp != None:
+                ftp.close()
+            if ssh != None:
+                ssh.close()
+
+    # Process logs
+    logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] "GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) \d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
+    logsearch = re.compile(logexpr).search
+    apps = {}
+    unknownapks = []
+    knownapks = common.KnownApks()
+    for logfile in glob.glob(os.path.join(logsdir,'access-*.log')):
+        logdate = logfile[len(logsdir) + 1 + len('access-'):-4]
+        matches = (logsearch(line) for line in file(logfile))
+        for match in matches:
+            if match and match.group('statuscode') == '200':
+                uri = match.group('uri')
+                if uri.endswith('.apk'):
+                    _, apkname = os.path.split(uri)
+                    app = knownapks.getapp(apkname)
+                    if app:
+                        appid, _ = app
+                        if appid in apps:
+                            apps[appid] += 1
+                        else:
+                            apps[appid] = 1
+                    else:
+                        if not apkname in unknownapks:
+                            unknownapks.append(apkname)
+
+    # Calculate and write stats for total downloads...
+    f = open('stats/total_downloads_app.txt', 'w')
+    lst = []
+    alldownloads = 0
+    for app, count in apps.iteritems():
+        lst.append(app + " " + str(count))
+        alldownloads += count
+    lst.append("ALL " + str(alldownloads))
+    f.write('# Total downloads by application, since October 2011\n')
+    for line in sorted(lst):
+        f.write(line + '\n')
+    f.close()
+
+    # Write list of latest apps added to the repo...
+    latest = knownapks.getlatest(10)
+    f = open('stats/latestapps.txt', 'w')
+    for app in latest:
+        f.write(app + '\n')
+    f.close()
+
+    if len(unknownapks) > 0:
+        print '\nUnknown apks:'
+        for apk in unknownapks:
+            print apk
+
+    print "Finished."
+
+if __name__ == "__main__":
+    main()
+

fdroidserver/update.py

@@ -0,0 +1,513 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# update.py - part of the FDroid server tools
+# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import os
+import shutil
+import glob
+import subprocess
+import re
+import zipfile
+import hashlib
+from xml.dom.minidom import Document
+from optparse import OptionParser
+import time
+
+def main():
+
+    # Read configuration...
+    execfile('config.py', globals())
+
+    import common
+
+    # Parse command line...
+    parser = OptionParser()
+    parser.add_option("-c", "--createmeta", action="store_true", default=False,
+                      help="Create skeleton metadata files that are missing")
+    parser.add_option("-v", "--verbose", action="store_true", default=False,
+                      help="Spew out even more information than normal")
+    parser.add_option("-q", "--quiet", action="store_true", default=False,
+                      help="No output, except for warnings and errors")
+    parser.add_option("-b", "--buildreport", action="store_true", default=False,
+                      help="Report on build data status")
+    parser.add_option("-i", "--interactive", default=False, action="store_true",
+                      help="Interactively ask about things that need updating.")
+    parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
+                      help="Specify editor to use in interactive mode. Default "+
+                          "is /etc/alternatives/editor")
+    parser.add_option("", "--pretty", action="store_true", default=False,
+                      help="Produce human-readable index.xml")
+    (options, args) = parser.parse_args()
+
+
+    icon_dir=os.path.join('repo','icons')
+
+    # Delete and re-create the icon directory...
+    if os.path.exists(icon_dir):
+        shutil.rmtree(icon_dir)
+    os.mkdir(icon_dir)
+
+    warnings = 0
+
+    # Get all apps...
+    apps = common.read_metadata(verbose=options.verbose)
+
+    # Generate a list of categories...
+    categories = []
+    for app in apps:
+        if app['Category'] not in categories:
+            categories.append(app['Category'])
+
+    # Gather information about all the apk files in the repo directory...
+    apks = []
+    for apkfile in glob.glob(os.path.join('repo','*.apk')):
+
+        apkfilename = apkfile[5:]
+        if apkfilename.find(' ') != -1:
+            print "No spaces in APK filenames!"
+            sys.exit(1)
+        srcfilename = apkfilename[:-4] + "_src.tar.gz"
+
+        if not options.quiet:
+            print "Processing " + apkfilename
+        thisinfo = {}
+        thisinfo['apkname'] = apkfilename
+        if os.path.exists(os.path.join('repo', srcfilename)):
+            thisinfo['srcname'] = srcfilename
+        thisinfo['size'] = os.path.getsize(apkfile)
+        thisinfo['permissions'] = []
+        thisinfo['features'] = []
+        p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools', 'aapt'),
+                              'dump', 'badging', apkfile],
+                             stdout=subprocess.PIPE)
+        output = p.communicate()[0]
+        if options.verbose:
+            print output
+        if p.returncode != 0:
+            print "ERROR: Failed to get apk information"
+            sys.exit(1)
+        for line in output.splitlines():
+            if line.startswith("package:"):
+                pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
+                thisinfo['id'] = re.match(pat, line).group(1)
+                pat = re.compile(".*versionCode='([0-9]*)'.*")
+                thisinfo['versioncode'] = int(re.match(pat, line).group(1))
+                pat = re.compile(".*versionName='([^']*)'.*")
+                thisinfo['version'] = re.match(pat, line).group(1)
+            if line.startswith("application:"):
+                pat = re.compile(".*label='([^']*)'.*")
+                thisinfo['name'] = re.match(pat, line).group(1)
+                pat = re.compile(".*icon='([^']*)'.*")
+                thisinfo['iconsrc'] = re.match(pat, line).group(1)
+            if line.startswith("sdkVersion:"):
+                pat = re.compile(".*'([0-9]*)'.*")
+                thisinfo['sdkversion'] = re.match(pat, line).group(1)
+            if line.startswith("native-code:"):
+                pat = re.compile(".*'([^']*)'.*")
+                thisinfo['nativecode'] = re.match(pat, line).group(1)
+            if line.startswith("uses-permission:"):
+                pat = re.compile(".*'([^']*)'.*")
+                perm = re.match(pat, line).group(1)
+                if perm.startswith("android.permission."):
+                    perm = perm[19:]
+                thisinfo['permissions'].append(perm)
+            if line.startswith("uses-feature:"):
+                pat = re.compile(".*'([^']*)'.*")
+                perm = re.match(pat, line).group(1)
+                #Filter out this, it's only added with the latest SDK tools and
+                #causes problems for lots of apps.
+                if (perm != "android.hardware.screen.portrait" and
+                    perm != "android.hardware.screen.landscape"):
+                    if perm.startswith("android.feature."):
+                        perm = perm[16:]
+                    thisinfo['features'].append(perm)
+
+        if not thisinfo.has_key('sdkversion'):
+            print "  WARNING: no SDK version information found"
+            thisinfo['sdkversion'] = 0
+
+        # Calculate the md5 and sha256...
+        m = hashlib.md5()
+        sha = hashlib.sha256()
+        f = open(apkfile, 'rb')
+        while True:
+            t = f.read(1024)
+            if len(t) == 0:
+                break
+            m.update(t)
+            sha.update(t)
+        thisinfo['md5'] = m.hexdigest()
+        thisinfo['sha256'] = sha.hexdigest()
+        f.close()
+
+        # Get the signature (or md5 of, to be precise)...
+        p = subprocess.Popen(['java', 'getsig',
+                              os.path.join(os.getcwd(), apkfile)],
+                             cwd=os.path.join(sys.path[0], 'getsig'),
+                             stdout=subprocess.PIPE)
+        output = p.communicate()[0]
+        if options.verbose:
+            print output
+        if p.returncode != 0 or not output.startswith('Result:'):
+            print "ERROR: Failed to get apk signature"
+            sys.exit(1)
+        thisinfo['sig'] = output[7:].strip()
+
+        # Extract the icon file...
+        apk = zipfile.ZipFile(apkfile, 'r')
+        thisinfo['icon'] = (thisinfo['id'] + '.' +
+            str(thisinfo['versioncode']) + '.png')
+        iconfilename = os.path.join(icon_dir, thisinfo['icon'])
+        try:
+            iconfile = open(iconfilename, 'wb')
+            iconfile.write(apk.read(thisinfo['iconsrc']))
+            iconfile.close()
+        except:
+            print "WARNING: Error retrieving icon file"
+            warnings += 1
+        apk.close()
+
+        apks.append(thisinfo)
+
+    # Some information from the apks needs to be applied up to the application
+    # level. When doing this, we use the info from the most recent version's apk.
+    for app in apps:
+        bestver = 0 
+        for apk in apks:
+            if apk['id'] == app['id']:
+                if apk['versioncode'] > bestver:
+                    bestver = apk['versioncode']
+                    bestapk = apk
+
+        if bestver == 0:
+            if app['Name'] is None:
+                app['Name'] = app['id']
+            app['icon'] = ''
+            if app['Disabled'] is None:
+                print "WARNING: Application " + app['id'] + " has no packages"
+        else:
+            if app['Name'] is None:
+                app['Name'] = bestapk['name']
+            app['icon'] = bestapk['icon']
+
+    # Generate warnings for apk's with no metadata (or create skeleton
+    # metadata files, if requested on the command line)
+    for apk in apks:
+        found = False
+        for app in apps:
+            if app['id'] == apk['id']:
+                found = True
+                break
+        if not found:
+            if options.createmeta:
+                f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
+                f.write("License:Unknown\n")
+                f.write("Web Site:\n")
+                f.write("Source Code:\n")
+                f.write("Issue Tracker:\n")
+                f.write("Summary:" + apk['name'] + "\n")
+                f.write("Description:\n")
+                f.write(apk['name'] + "\n")
+                f.write(".\n")
+                f.close()
+                print "Generated skeleton metadata for " + apk['id']
+            else:
+                print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
+                print "       " + apk['name'] + " - " + apk['version']  
+
+    #Sort the app list by name, then the web site doesn't have to by default:
+    apps = sorted(apps, key=lambda app: app['Name'].upper())
+
+    # Create the index
+    doc = Document()
+
+    def addElement(name, value, doc, parent):
+        el = doc.createElement(name)
+        el.appendChild(doc.createTextNode(value))
+        parent.appendChild(el)
+
+    root = doc.createElement("fdroid")
+    doc.appendChild(root)
+
+    repoel = doc.createElement("repo")
+    repoel.setAttribute("name", repo_name)
+    repoel.setAttribute("icon", os.path.basename(repo_icon))
+    repoel.setAttribute("url", repo_url)
+
+    if repo_keyalias != None:
+
+        # Generate a certificate fingerprint the same way keytool does it
+        # (but with slightly different formatting)
+        def cert_fingerprint(data):
+            digest = hashlib.sha1(data).digest()
+            ret = []
+            for i in range(4):
+                ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
+            return " ".join(ret)
+
+        def extract_pubkey():
+            p = subprocess.Popen(['keytool', '-exportcert',
+                                  '-alias', repo_keyalias,
+                                  '-keystore', keystore,
+                                  '-storepass', keystorepass],
+                                 stdout=subprocess.PIPE)
+            cert = p.communicate()[0]
+            if p.returncode != 0:
+                print "ERROR: Failed to get repo pubkey"
+                sys.exit(1)
+            global repo_pubkey_fingerprint
+            repo_pubkey_fingerprint = cert_fingerprint(cert)
+            return "".join("%02x" % ord(b) for b in cert)
+
+        repoel.setAttribute("pubkey", extract_pubkey())
+
+    addElement('description', repo_description, doc, repoel)
+    root.appendChild(repoel)
+
+    apps_inrepo = 0
+    apps_disabled = 0
+    apps_nopkg = 0
+
+    for app in apps:
+
+        if app['Disabled'] is None:
+
+            # Get a list of the apks for this app...
+            gotcurrentver = False
+            apklist = []
+            for apk in apks:
+                if apk['id'] == app['id']:
+                    if str(apk['versioncode']) == app['Current Version Code']:
+                        gotcurrentver = True
+                    apklist.append(apk)
+
+            if len(apklist) == 0:
+                apps_nopkg += 1
+            else:
+                apps_inrepo += 1
+                apel = doc.createElement("application")
+                apel.setAttribute("id", app['id'])
+                root.appendChild(apel)
+
+                addElement('id', app['id'], doc, apel)
+                addElement('name', app['Name'], doc, apel)
+                addElement('summary', app['Summary'], doc, apel)
+                addElement('icon', app['icon'], doc, apel)
+                addElement('description',
+                        common.parse_description(app['Description']), doc, apel)
+                addElement('license', app['License'], doc, apel)
+                if 'Category' in app:
+                    addElement('category', app['Category'], doc, apel)
+                addElement('web', app['Web Site'], doc, apel)
+                addElement('source', app['Source Code'], doc, apel)
+                addElement('tracker', app['Issue Tracker'], doc, apel)
+                if app['Donate'] != None:
+                    addElement('donate', app['Donate'], doc, apel)
+
+                # These elements actually refer to the current version (i.e. which
+                # one is recommended. They are historically mis-named, and need
+                # changing, but stay like this for now to support existing clients.
+                addElement('marketversion', app['Current Version'], doc, apel)
+                addElement('marketvercode', app['Current Version Code'], doc, apel)
+
+                if not (app['AntiFeatures'] is None):
+                    addElement('antifeatures', app['AntiFeatures'], doc, apel)
+                if app['Requires Root']:
+                    addElement('requirements', 'root', doc, apel)
+
+                # Sort the apk list into version order, just so the web site
+                # doesn't have to do any work by default...
+                apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
+
+                # Check for duplicates - they will make the client unhappy...
+                for i in range(len(apklist) - 1):
+                    if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
+                        print "ERROR - duplicate versions"
+                        print apklist[i]['apkname']
+                        print apklist[i+1]['apkname']
+                        sys.exit(1)
+
+                for apk in apklist:
+                    apkel = doc.createElement("package")
+                    apel.appendChild(apkel)
+                    addElement('version', apk['version'], doc, apkel)
+                    addElement('versioncode', str(apk['versioncode']), doc, apkel)
+                    addElement('apkname', apk['apkname'], doc, apkel)
+                    if apk.has_key('srcname'):
+                        addElement('srcname', apk['srcname'], doc, apkel)
+                    for hash_type in ('sha256', 'md5'):
+                        if not hash_type in apk:
+                            continue
+                        hashel = doc.createElement("hash")
+                        hashel.setAttribute("type", hash_type)
+                        hashel.appendChild(doc.createTextNode(apk[hash_type]))
+                        apkel.appendChild(hashel)
+                    addElement('sig', apk['sig'], doc, apkel)
+                    addElement('size', str(apk['size']), doc, apkel)
+                    addElement('sdkver', str(apk['sdkversion']), doc, apkel)
+                    perms = ""
+                    for p in apk['permissions']:
+                        if len(perms) > 0:
+                            perms += ","
+                        perms += p
+                    if len(perms) > 0:
+                        addElement('permissions', perms, doc, apkel)
+                    features = ""
+                    for f in apk['features']:
+                        if len(features) > 0:
+                            features += ","
+                        features += f
+                    if len(features) > 0:
+                        addElement('features', features, doc, apkel)
+
+            if options.buildreport:
+                if len(app['builds']) == 0:
+                    print ("WARNING: No builds defined for " + app['id'] +
+                            " Source: " + app['Source Code'])
+                    warnings += 1
+                else:
+                    if app['Current Version Code'] != '0':
+                        gotbuild = False
+                        for build in app['builds']:
+                            if build['vercode'] == app['Current Version Code']:
+                                gotbuild = True
+                        if not gotbuild:
+                            print ("WARNING: No build data for current version of "
+                                    + app['id'] + " (" + app['Current Version']
+                                    + ") " + app['Source Code'])
+                            warnings += 1
+
+            # If we don't have the current version, check if there is a build
+            # with a commit ID starting with '!' - this means we can't build it
+            # for some reason, and don't want hassling about it...
+            if not gotcurrentver and app['Current Version Code'] != '0':
+                for build in app['builds']:
+                    if build['vercode'] == app['Current Version Code']:
+                        gotcurrentver = True
+
+            # Output a message of harassment if we don't have the current version:
+            if not gotcurrentver and app['Current Version Code'] != '0':
+                addr = app['Source Code']
+                print "WARNING: Don't have current version (" + app['Current Version'] + ") of " + app['Name']
+                print "         (" + app['id'] + ") " + addr
+                warnings += 1
+                if options.verbose:
+                    # A bit of extra debug info, basically for diagnosing
+                    # app developer mistakes:
+                    print "         Current vercode:" + app['Current Version Code']
+                    print "         Got:"
+                    for apk in apks:
+                        if apk['id'] == app['id']:
+                            print "           " + str(apk['versioncode']) + " - " + apk['version']
+                if options.interactive:
+                    print "Build data out of date for " + app['id']
+                    while True:
+                        answer = raw_input("[I]gnore, [E]dit or [Q]uit?").lower()
+                        if answer == 'i':
+                            break
+                        elif answer == 'e':
+                            subprocess.call([options.editor,
+                                os.path.join('metadata',
+                                app['id'] + '.txt')])
+                            break
+                        elif answer == 'q':
+                            sys.exit(0)
+        else:
+            apps_disabled += 1
+
+    of = open(os.path.join('repo','index.xml'), 'wb')
+    if options.pretty:
+        output = doc.toprettyxml()
+    else:
+        output = doc.toxml()
+    of.write(output)
+    of.close()
+
+    if repo_keyalias != None:
+
+        if not options.quiet:
+            print "Creating signed index."
+            print "Key fingerprint:", repo_pubkey_fingerprint
+        
+        #Create a jar of the index...
+        p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
+            cwd='repo', stdout=subprocess.PIPE)
+        output = p.communicate()[0]
+        if options.verbose:
+            print output
+        if p.returncode != 0:
+            print "ERROR: Failed to create jar file"
+            sys.exit(1)
+
+        # Sign the index...
+        p = subprocess.Popen(['jarsigner', '-keystore', keystore,
+            '-storepass', keystorepass, '-keypass', keypass,
+            os.path.join('repo', 'index.jar') , repo_keyalias], stdout=subprocess.PIPE)
+        output = p.communicate()[0]
+        if p.returncode != 0:
+            print "Failed to sign index"
+            print output
+            sys.exit(1)
+        if options.verbose:
+            print output
+
+    # Copy the repo icon into the repo directory...
+    iconfilename = os.path.join(icon_dir, os.path.basename(repo_icon))
+    shutil.copyfile(repo_icon, iconfilename)
+
+    # Write a category list in the repo to allow quick access...
+    catdata = ''
+    for cat in categories:
+        catdata += cat + '\n'
+    f = open('repo/categories.txt', 'w')
+    f.write(catdata)
+    f.close()
+
+    # Update known apks info...
+    knownapks = common.KnownApks()
+    for apk in apks:
+        knownapks.recordapk(apk['apkname'], apk['id'])
+    knownapks.writeifchanged()
+
+    # Generate latest apps data for widget
+    data = ''
+    for line in file(os.path.join('stats', 'latestapps.txt')):
+        appid = line.rstrip()
+        data += appid + "\t"
+        for app in apps:
+            if app['id'] == appid:
+                data += app['Name'] + "\t"
+                data += app['icon'] + "\t"
+                data += app['License'] + "\n"
+                break
+    f = open('repo/latestapps.dat', 'w')
+    f.write(data)
+    f.close()
+
+
+
+    print "Finished."
+    print str(apps_inrepo) + " apps in repo"
+    print str(apps_disabled) + " disabled"
+    print str(apps_nopkg) + " with no packages"
+    print str(warnings) + " warnings"
+
+if __name__ == "__main__":
+    main()
+