Moved the vcs-related code out of the build script and fixed a few other things

build.py

@@ -74,7 +74,10 @@ for app in apps:
 
         build_dir = 'build/' + app['id']
 
-        got_source = False
+        # Set up vcs interface and make sure we have the latest code...
+        vcs = common.getvcs(app['repotype'], app['repo'], build_dir)
+
+        refreshed_source = False
 
 
         for thisbuild in app['builds']:
@@ -91,129 +94,22 @@ for app in apps:
             else:
                 print "..building version " + thisbuild['version']
 
-                if not got_source:
+                if not refreshed_source:
-
+                    vcs.refreshlocal()
-                    got_source = True
+                    refreshed_source = True
-
-                    # Remove the build directory if it already exists...
-                    if os.path.exists(build_dir):
-                        shutil.rmtree(build_dir)
-
-                    # Strip username/password out of repo address if specified (relevant
-                    # only for SVN) and store for use later.
-                    repo = app['repo']
-                    index = repo.find('@')
-                    if index != -1:
-                        username = repo[:index]
-                        repo = repo[index+1:]
-                        index = username.find(':')
-                        if index == -1:
-                            print "Password required with username"
-                            sys.exit(1)
-                        password = username[index+1:]
-                        username = username[:index]
-                        repouserargs = ['--username', username, 
-                                '--password', password, '--non-interactive']
-                    else:
-                        repouserargs = []
-
-                    # Get the source code...
-                    if app['repotype'] == 'git':
-                        if subprocess.call(['git', 'clone', repo, build_dir]) != 0:
-                            print "Git clone failed"
-                            sys.exit(1)
-                    elif app['repotype'] == 'svn':
-                        if not repo.endswith("*"):
-                            if subprocess.call(['svn', 'checkout', repo, build_dir] +
-                                    repouserargs) != 0:
-                                print "Svn checkout failed"
-                                sys.exit(1)
-                    elif app['repotype'] == 'hg':
-                        if subprocess.call(['hg', 'clone', repo, build_dir]) !=0:
-                            print "Hg clone failed"
-                            sys.exit(1)
-                    elif app['repotype'] == 'bzr':
-                        if subprocess.call(['bzr', 'branch', repo, build_dir]) !=0:
-                            print "Bzr branch failed"
-                            sys.exit(1)
-                    else:
-                        print "Invalid repo type " + app['repotype'] + " in " + app['id']
-                        sys.exit(1)
 
                 # Optionally, the actual app source can be in a subdirectory...
-                doupdate = True
                 if thisbuild.has_key('subdir'):
-                    if app['repotype'] == 'svn' and repo.endswith("*"):
-                        root_dir = build_dir
-                        # Remove the build directory if it already exists...
-                        if os.path.exists(build_dir):
-                            shutil.rmtree(build_dir)
-                        if subprocess.call(['svn', 'checkout',
-                                repo[:-1] + thisbuild['subdir'],
-                                '-r', thisbuild['commit'],
-                                build_dir] + repouserargs) != 0:
-                            print "Svn checkout failed"
-                            sys.exit(1)
-                        # Because we're checking out for every version we build,
-                        # we've already checked out the repo at the correct revision
-                        # and don't need to update to it...
-                        doupdate = False
-                    else:
                     root_dir = os.path.join(build_dir, thisbuild['subdir'])
                 else:
                     root_dir = build_dir
 
-                if doupdate:
+                # Get a working copy of the right revision...
-                    if app['repotype'] == 'git':
+                vcs.reset(thisbuild['commit'])
-                        if subprocess.call(['git', 'reset', '--hard', thisbuild['commit']],
-                                cwd=build_dir) != 0:
-                            print "Git reset failed"
-                            sys.exit(1)
-                        if subprocess.call(['git', 'clean', '-dfx'],
-                                cwd=build_dir) != 0:
-                            print "Git clean failed"
-                            sys.exit(1)
-                    elif app['repotype'] == 'svn':
-                        for svncommand in (
-                        'svn revert -R .',
-                        r"svn status | awk '/\?/ {print $2}' | xargs rm -rf",
-                        'svn update --force -r '+thisbuild['commit'],):
-                            if subprocess.call(svncommand, cwd=build_dir,
-                                    shell=True) != 0:
-                                print "Svn update failed"
-                                sys.exit(1)
-                    elif app['repotype'] == 'hg':
-                        if subprocess.call(['hg', 'checkout', thisbuild['commit']],
-                                cwd=build_dir) != 0:
-                            print "Hg checkout failed"
-                            sys.exit(1)
-                        if subprocess.call('hg status -u -0 | xargs rm -rf',
-                                cwd=build_dir, shell=True) != 0:
-                            print "Hg clean failed"
-                            sys.exit(1)
-                    elif app['repotype'] == 'bzr':
-                        if subprocess.call(['bzr', 'revert', '-r', thisbuild['commit']],
-                                cwd=build_dir) != 0:
-                            print "Bzr revert failed"
-                            sys.exit(1)
-                        if subprocess.call(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'],
-                                cwd=build_dir) != 0:
-                            print "Bzr revert failed"
-                            sys.exit(1)
-                    else:
-                        print "Invalid repo type " + app['repotype']
-                        sys.exit(1)
 
                 # Initialise submodules if requred...
                 if thisbuild.get('submodules', 'no')  == 'yes':
-                        if subprocess.call(['git', 'submodule', 'init'],
+                    vcs.initsubmodules()
-                                cwd=build_dir) != 0:
-                            print "Git submodule init failed"
-                            sys.exit(1)
-                        if subprocess.call(['git', 'submodule', 'update'],
-                                cwd=build_dir) != 0:
-                            print "Git submodule update failed"
-                            sys.exit(1)
 
                 # Generate (or update) the ant build file, build.xml...
                 if (thisbuild.get('update', 'yes') == 'yes' and

common.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # common.py - part of the FDroid server tools
-# Copyright (C) 2010, Ciaran Gultnieks, ciaran@ciarang.com
+# Copyright (C) 2010-11, Ciaran Gultnieks, ciaran@ciarang.com
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,6 +17,201 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import glob, os, sys, re
+import subprocess
+
+
+def getvcs(vcstype, remote, local):
+    if vcstype == 'git':
+        return vcs_git(remote, local)
+    elif vcstype == 'svn':
+        return vcs_svn(remote, local)
+    elif vcstype == 'hg':
+        return vcs_hg(remote,local)
+    elif vcstype == 'bzr':
+        return vcs_bzr(remote,local)
+    print "Invalid vcs type " + vcstype
+    sys.exit(1)
+
+class vcs:
+    def __init__(self, remote, local):
+
+        # It's possible to sneak a username and password in with
+        # the remote address... (this really only applies to svn
+        # and we should probably be more specific!)
+        index = remote.find('@')
+        if index != -1:
+            self.username = remote[:index]
+            remote = remote[index+1:]
+            index = self.username.find(':')
+            if index == -1:
+                print "Password required with username"
+                sys.exit(1)
+            self.password = self.username[index+1:]
+            self.username = self.username[:index]
+        else:
+            self.username = None
+
+        self.remote = remote
+        self.local = local
+                    
+    # Refresh the local repository - i.e. get the latest code. This
+    # works either by updating if a local copy already exists, or by
+    # cloning from scratch if it doesn't.
+    def refreshlocal(self):
+        if not os.path.exists(self.local):
+            self.clone()
+        else:
+            self.reset()
+            self.pull()
+
+    # Clone the remote repository. It must not already exist locally.
+    def clone(self):
+        assert False    # Must be defined in child
+
+    # Reset the local repository. Remove changes, untracked files, etc.
+    # Put the working tree to either the given revision, or to the HEAD
+    # if not specified.
+    def reset(self, rev=None):
+        assert False    # Must be defined in child
+
+    # Get new commits from the remote repository. Local must be clean.
+    def pull(self):
+        assert False    # Must be defined in child
+
+    # Initialise and update submodules
+    def initsubmodules(self):
+        assert False    # Not supported unless overridden
+
+class vcs_git(vcs):
+
+    def clone(self):
+        if subprocess.call(['git', 'clone', self.remote, self.local]) != 0:
+            print "Git clone failed"
+            sys.exit(1)
+
+    def reset(self, rev=None):
+        if rev is None:
+            rev = 'HEAD'
+        if subprocess.call(['git', 'reset', '--hard', rev],
+                cwd=self.local) != 0:
+            print "Git reset failed"
+            sys.exit(1)
+        if subprocess.call(['git', 'clean', '-dfx'],
+                cwd=self.local) != 0:
+            print "Git clean failed"
+            sys.exit(1)
+
+    def pull(self):
+        if subprocess.call(['git', 'pull', 'origin'],
+                cwd=self.local) != 0:
+            print "Git pull failed"
+            sys.exit(1)
+
+    def initsubmodules(self):
+        if subprocess.call(['git', 'submodule', 'init'],
+                cwd=self.local) != 0:
+            print "Git submodule init failed"
+            sys.exit(1)
+        if subprocess.call(['git', 'submodule', 'update'],
+                cwd=self.local) != 0:
+            print "Git submodule update failed"
+            sys.exit(1)
+
+
+
+class vcs_svn(vcs):
+
+    def userargs(self):
+        if self.username is None:
+            return []
+        return ['--username', self.username, 
+                '--password', self.password,
+                '--non-interactive']
+
+    def clone(self):
+        if subprocess.call(['svn', 'checkout', self.remote, self.local] +
+                self.userargs()) != 0:
+            print "Svn checkout failed"
+            sys.exit(1)
+
+    def reset(self, rev=None):
+        if rev is None:
+            revargs = []
+        else:
+            revargs = [' -r ', rev]
+        for svncommand in (
+                'svn revert -R .',
+                r"svn status | awk '/\?/ {print $2}' | xargs rm -rf"):
+            if subprocess.call(svncommand, cwd=self.local,
+                    shell=True) != 0:
+                print "Svn reset failed"
+                sys.exit(1)
+        if subprocess.call(['svn', 'update', '--force'] + revargs +
+                self.userargs(), cwd=self.local) != 0:
+            print "Svn update failed"
+            sys.exit(1)
+
+    def pull(self):
+        if subprocess.call(['svn', 'update'] +
+                self.userargs(), cwd=self.local) != 0:
+            print "Svn update failed"
+            sys.exit(1)
+
+class vcs_hg(vcs):
+
+    def clone(self):
+        if subprocess.call(['hg', 'clone', self.remote, self.local]) !=0:
+            print "Hg clone failed"
+            sys.exit(1)
+
+    def reset(self, rev=None):
+        if rev is None:
+            revargs = []
+        else:
+            revargs = [rev]
+        if subprocess.call('hg status -u | xargs rm -rf',
+                cwd=self.local, shell=True) != 0:
+            print "Hg clean failed"
+            sys.exit(1)
+        if subprocess.call(['hg', 'checkout', '-C'] + revargs,
+                cwd=self.local) != 0:
+            print "Hg checkout failed"
+            sys.exit(1)
+
+    def pull(self):
+        if subprocess.call(['hg', 'pull'],
+                cwd=self.local) != 0:
+            print "Hg pull failed"
+            sys.exit(1)
+
+class vcs_bzr(vcs):
+
+    def clone(self):
+        if subprocess.call(['bzr', 'branch', self.remote, self.local]) !=0:
+            print "Bzr branch failed"
+            sys.exit(1)
+
+    def reset(self, rev=None):
+        if rev is None:
+            revargs = []
+        else:
+            revargs = ['-r', rev]
+        if subprocess.call(['bzr', 'clean-tree', '--force',
+                '--unknown', '--ignored'], cwd=self.local) != 0:
+            print "Bzr revert failed"
+            sys.exit(1)
+        if subprocess.call(['bzr', 'revert'] + revargs,
+                cwd=self.local) != 0:
+            print "Bzr revert failed"
+            sys.exit(1)
+
+    def pull(self):
+        if subprocess.call(['bzr', 'update'],
+                cwd=self.local) != 0:
+            print "Bzr update failed"
+            sys.exit(1)
+
+
 
 def parse_metadata(metafile, **kw):