Various fixes and improvements, and git-svn support

README

@@ -91,7 +91,11 @@ The type of repository - for automatic building from source. If this is not
 specified, automatic building is disabled for this application. Possible
 values are:
 
-  git, svn, hg, bzr
+  git, git-svn, svn, hg, bzr
+ 
+The git-svn option connects to an SVN repository, and you specify the URL in
+exactly the same way, but git is used as a back-end. This is preferable for
+performance reasons.
 
 ==Repo==
 
@@ -99,7 +103,7 @@ The repository location. Usually a git: or svn: URL.
 
 For a Subversion repo that requires authentication, you can precede the repo
 URL with username:password@ and those parameters will be passed as --username
-and --password to the SVN checkout command.
+and --password to the SVN checkout command. (Doesn't work for git-svn).
 
 ==Build Version==
 
@@ -125,9 +129,7 @@ configuration to the build. These are:
 
  subdir=<path>   - Specifies to build from a subdirectory of the checked out
                    source code. Normally this directory is changed to before
-                   building, but there is a special case for SVN repositories
+                   building.
-                   where the URL is specified with a * at the end. See the
-                   documentation for the Repo field for more information.
  bindir=<path>   - Normally the build output (apk) is expected to be in the
                    bin subdirectory below the ant build files. If the project
                    is configured to put it elsewhere, that can be specified

build.py

@@ -104,7 +104,7 @@ for app in apps:
 
                     # Prepare the source code...
                     root_dir = common.prepare_source(vcs, app, thisbuild,
-                            build_dir, sdk_path, ndk_path,
+                            build_dir, sdk_path, ndk_path, javacc_path,
                             not refreshed_source)
                     refreshed_source = True
 
@@ -112,7 +112,11 @@ for app in apps:
                     tarname = app['id'] + '_' + thisbuild['vercode'] + '_src'
                     tarball = tarfile.open(os.path.join(output_dir,
                         tarname + '.tar.gz'), "w:gz")
-                    tarball.add(build_dir, tarname)
+                    def tarexc(f):
+                        if f in ['.svn', '.git', '.hg', '.bzr']:
+                            return True
+                        return False
+                    tarball.add(build_dir, tarname, exclude=tarexc)
                     tarball.close()
 
                     # Build native stuff if required...

common.py

@@ -17,6 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import glob, os, sys, re
+import shutil
 import subprocess
 
 
@@ -25,6 +26,8 @@ def getvcs(vcstype, remote, local):
         return vcs_git(remote, local)
     elif vcstype == 'svn':
         return vcs_svn(remote, local)
+    elif vcstype == 'git-svn':
+        return vcs_gitsvn(remote, local)
     elif vcstype == 'hg':
         return vcs_hg(remote,local)
     elif vcstype == 'bzr':
@@ -114,6 +117,33 @@ class vcs_git(vcs):
             raise VCSException("Git submodule update failed")
 
 
+class vcs_gitsvn(vcs):
+
+    def clone(self):
+        if subprocess.call(['git', 'svn', 'clone', self.remote, self.local]) != 0:
+            raise VCSException("Git clone failed")
+
+    def reset(self, rev=None):
+        if rev is None:
+            rev = 'HEAD'
+        else:
+            p = subprocess.Popen(['git', 'svn', 'find-rev', 'r' + rev],
+                cwd=self.local, stdout=subprocess.PIPE)
+            rev = p.communicate()[0].rstrip()
+            if p.returncode != 0:
+                raise VCSException("Failed to get git treeish from svn rev")
+        if subprocess.call(['git', 'reset', '--hard', rev],
+                cwd=self.local) != 0:
+            raise VCSException("Git reset failed")
+        if subprocess.call(['git', 'clean', '-dfx'],
+                cwd=self.local) != 0:
+            raise VCSException("Git clean failed")
+
+    def pull(self):
+        if subprocess.call(['git', 'svn', 'rebase'],
+                cwd=self.local) != 0:
+            raise VCSException("Git svn rebase failed")
+
 
 class vcs_svn(vcs):
 
@@ -363,10 +393,11 @@ class MetaDataException(Exception):
 #  'build_dir'   - the path to the build directory
 #  'sdk_path'    - the path to the Android SDK
 #  'ndk_path'    - the path to the Android NDK
+#  'javacc_path' - the path to javacc
 #  'refresh'     - True to refresh from the remote repo
 # Returns the root directory, which may be the same as 'build_dir' or may
 # be a subdirectory of it.
-def prepare_source(vcs, app, build, build_dir, sdk_path, ndk_path, refresh):
+def prepare_source(vcs, app, build, build_dir, sdk_path, ndk_path, javacc_path, refresh):
 
     if refresh:
         vcs.refreshlocal()
@@ -374,11 +405,12 @@ def prepare_source(vcs, app, build, build_dir, sdk_path, ndk_path, refresh):
     # Optionally, the actual app source can be in a subdirectory...
     if build.has_key('subdir'):
         root_dir = os.path.join(build_dir, build['subdir'])
+        if not os.path.exists(root_dir):
+            raise BuildException('Missing subdir ' + root_dir)
     else:
         root_dir = build_dir
 
     # Get a working copy of the right revision...
-    if options.verbose:
     print "Resetting repository to " + build['commit']
     vcs.reset(build['commit'])
 

metadata/An.stop.txt

@@ -9,7 +9,7 @@ Description:A simple stopwatch, that also supports lap timing and a countdown
 timer.
 .
 
-Repo Type:svn
+Repo Type:git-svn
 Repo:http://anstop.googlecode.com/svn/trunk
 
 Build Version:1.4,9,34

metadata/android.androidVNC.txt

@@ -8,7 +8,7 @@ Summary:VNC viewer
 Description:
 A VNC ('remote desktop') client.
 .
-Repo Type:svn
+Repo Type:git-svn
 Repo:http://android-vnc-viewer.googlecode.com/svn/branches/antlersoft
 
 Build Version:0.5.0,13,197,subdir=androidVNC

metadata/com.agiro.scanner.android.txt

@@ -8,7 +8,7 @@ server. The server component is available separately:
 https://github.com/johannilsson/agiro-server
 .
 
-Repo Type:
+Repo Type:git
 Repo:https://github.com/pakerfeldt/aGiro.git
 
 Build Version:alpha 2,2,!repo moved and renamed 20bd0f021dd852afcc9aa9008ee713419ae8e05c

metadata/com.android.inputmethod.norwegian.txt

@@ -9,7 +9,7 @@ A modified version of the standard onscreen keyboard in Android
 with support for Norwegian, Swedish, Danish, Faroese, German,
 Icelandic and Northern Sámi keyboard layouts.
 .
-Repo Type:svn
+Repo Type:git-svn
 Repo:http://scandinavian-keyboard.googlecode.com/svn/trunk
 Build Version:1.4.4,13,15,target=android-4
 Build Version:1.4.6,15,17,target=android-4

metadata/com.funambol.androidsync.txt

@@ -13,7 +13,7 @@ Repo:guest:x@https://android-client.forge.funambol.org/svn/android-client/
 
 Build Version:8.7.3,8,1032,subdir=tags/8.7.3,update=no,initfun=yes
 Build Version:9.0.1,9,1437,subdir=tags/9.0.1,update=no,initfun=yes
-Build Version:9.0.3,10,1546,subdir=tags/9.0.3,update=no,initfun=yes
+Build Version:9.0.3,10,1547,subdir=tags/9.0.3,update=no,initfun=yes
 Build Version:10.0.4,14,2162,subdir=tags/10.0.4,update=no,initfun=yes
 Build Version:10.0.5,15,2211,subdir=tags/10.0.5,update=no,initfun=yes
 Build Version:10.0.6,16,2337,subdir=tags/10.0.6,update=no,initfun=yes

metadata/org.jessies.mathdroid.txt

@@ -10,10 +10,10 @@ Description:
 A calculator with full on-screen history and many functions.
 .
 
-Repo Type:svn
+Repo Type:git-svn
 Repo:http://enh.googlecode.com/svn/trunk
 
-Build Version:2.5,25,525,oldsdkloc=yes,target=android-9,subdir=mathdroid,prebuild=rm src/org/jessies/test && mkdir src/org/jessies/test && wget http://software.jessies.org/svn/salma-hayek/trunk/src/org/jessies/test/Assert.java -O src/org/jessies/test/Assert.java && wget http://software.jessies.org/svn/salma-hayek/trunk/src/org/jessies/test/Test.java -O src/org/jessies/test/Test.java && wget http://software.jessies.org/svn/salma-hayek/trunk/src/org/jessies/test/TestHelper.java -O src/org/jessies/test/TestHelper.java
+Build Version:2.5,25,525,oldsdkloc=yes,target=android-9,subdir=mathdroid,prebuild=rm -rf src/org/jessies/test && mkdir src/org/jessies/test && wget http://software.jessies.org/svn/salma-hayek/trunk/src/org/jessies/test/Assert.java -O src/org/jessies/test/Assert.java && wget http://software.jessies.org/svn/salma-hayek/trunk/src/org/jessies/test/Test.java -O src/org/jessies/test/Test.java && wget http://software.jessies.org/svn/salma-hayek/trunk/src/org/jessies/test/TestHelper.java -O src/org/jessies/test/TestHelper.java
 
 Market Version:2.5
 Market Version Code:25

scanner.py

@@ -24,6 +24,7 @@ import re
 import urllib
 import time
 import subprocess
+import traceback
 from optparse import OptionParser
 import HTMLParser
 import common
@@ -38,6 +39,8 @@ execfile('config.py')
 parser = OptionParser()
 parser.add_option("-v", "--verbose", action="store_true", default=False,
                   help="Spew out even more information than normal")
+parser.add_option("-p", "--package", default=None,
+                  help="Scan only the specified package")
 (options, args) = parser.parse_args()
 
 # Get all apps...
@@ -49,13 +52,17 @@ problems = []
 
 for app in apps:
 
-    if app['disabled']:
+    skip = False
+    if options.package and app['id'] != options.package:
+        skip = True
+    elif app['disabled']:
         print "Skipping %s: disabled" % app['id']
+        skip = True
     elif not app['builds']:
         print "Skipping %s: no builds specified" % app['id']
+        skip = True
 
-    if (app['disabled'] is None and app['repo'] != '' 
+    if not skip:
-            and app['repotype'] != '' and len(app['builds']) > 0):
 
         print "Processing " + app['id']
 
@@ -79,16 +86,18 @@ for app in apps:
 
                     # Prepare the source code...
                     root_dir = common.prepare_source(vcs, app, thisbuild,
-                            build_dir, sdk_path, ndk_path,
+                            build_dir, sdk_path, ndk_path, javacc_path,
                             not refreshed_source)
                     refreshed_source = True
 
                     # Scan for common known non-free blobs:
-                    usual_suspects = ['flurryagent.jar', 'paypal_mpl.jar']
+                    usual_suspects = ['flurryagent.jar',
+                                      'paypal_mpl.jar',
+                                      'admob-sdk-android.jar']
                     for r,d,f in os.walk(build_dir):
                         for curfile in f:
                             if curfile.lower() in usual_suspects:
-                                msg = 'Found probable non-free blob ' + os.path.join(r,file)
+                                msg = 'Found probable non-free blob ' + os.path.join(r, curfile)
                                 msg += ' in ' + app['id'] + ' ' + thisbuild['version']
                                 problems.append(msg)
 
@@ -98,8 +107,8 @@ for app in apps:
         except VCSException as vcse:
             msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
             problems.append(msg)
-        except Exception as e:
+        except Exception:
-            msg = "Could not scan app %s due to unknown error: %s" % (app['id'], e)
+            msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
             problems.append(msg)
 
 print "Finished:"