Merge branch 'master' into logging

Conflicts:
	fdroidserver/build.py

buildserver/cookbooks/fdroidbuild-general/recipes/default.rb

@@ -5,7 +5,8 @@ execute "apt-get-update" do
   command "apt-get update"
 end
 
-%w{ant ant-contrib autoconf autopoint bison cmake expect libtool libsaxonb-java libssl1.0.0 libssl-dev maven openjdk-7-jdk javacc python python-magic git-core mercurial subversion bzr git-svn make perlmagick pkg-config zip ruby rubygems librmagick-ruby yasm imagemagick gettext realpath transfig texinfo curl}.each do |pkg|
+%w{ant ant-contrib autoconf autopoint bison cmake expect libtool libsaxonb-java libssl1.0.0 libssl-dev maven openjdk-7-jdk javacc python python-magic git-core mercurial subversion bzr git-svn make perlmagick pkg-config zip ruby rubygems librmagick-ruby yasm imagemagick gettext realpath transfig texinfo curl
+librsvg2-bin xsltproc}.each do |pkg|
   package pkg do
     action :install
   end

docs/fdroid.texi

@@ -820,8 +820,10 @@ Specifies to build from a subdirectory of the checked out source code.
 Normally this directory is changed to before building,
 
 @item submodules=yes
-Use if the project (git only) has submodules - causes git submodule
+Use if the project (git only) has submodules - causes @code{git submodule
-init and update to be executed after the source is cloned.
+update --init --recursive} to be executed after the source is cloned.
+Submodules are reset and cleaned like the main app repository itself before
+each build.
 
 @item init=xxxx
 As for 'prebuild', but runs on the source code BEFORE any other processing
@@ -1013,6 +1015,11 @@ in a gradle project build.
 Specify an alternate ant command (target) instead of the default
 'release'. It can't be given any flags, such as the path to a build.xml.
 
+@item output=path/to/output.apk
+To be used when app is built with a tool other than the ones natively
+supported, like GNU Make. The given path will be where the build= set of
+commands should produce the final unsigned release apk.
+
 @item novcheck=yes
 Don't check that the version name and code in the resulting apk are
 correct by looking at the build output - assume the metadata is

fd-commit

@@ -62,6 +62,7 @@ while read line; do
 
 		onlybuild=true
 		newbuild=false
+		disable=false
 		while read l; do
 			if [[ "$l" == "-Build:"* ]]; then
 				onlybuild=false
@@ -74,7 +75,7 @@ while read line; do
 				version=${build%%,*}
 				build=${build#*,}
 				vercode=${build%%,*}
-			elif $newbuild && $onlybuild [[ "$l" == "+"*"disable="* ]]; then
+			elif $newbuild && $onlybuild && [[ "$l" == "+"*"disable="* ]]; then
 				disable=true
 			fi
 		done < <(git diff HEAD -- "$file")

fdroidserver/build.py

@@ -345,7 +345,8 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, force):
         tarball = common.getsrcname(app,thisbuild)
         try:
             ftp.get(apkfile, os.path.join(output_dir, apkfile))
-            ftp.get(tarball, os.path.join(output_dir, tarball))
+            if not options.notarball:
+                ftp.get(tarball, os.path.join(output_dir, tarball))
         except:
             raise BuildException("Build failed for %s:%s - missing output files" % (app['id'], thisbuild['version']), output)
         ftp.close()
@@ -428,14 +429,15 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
             raise BuildException("Can't build due to " +
                 str(len(buildprobs)) + " scanned problems")
 
-    # Build the source tarball right before we build the release...
+    if not options.notarball:
-    logging.info("Creating source tarball...")
+        # Build the source tarball right before we build the release...
-    tarname = common.getsrcname(app,thisbuild)
+        logging.info("Creating source tarball...")
-    tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz")
+        tarname = common.getsrcname(app,thisbuild)
-    def tarexc(f):
+        tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz")
-        return any(f.endswith(s) for s in ['.svn', '.git', '.hg', '.bzr'])
+        def tarexc(f):
-    tarball.add(build_dir, tarname, exclude=tarexc)
+            return any(f.endswith(s) for s in ['.svn', '.git', '.hg', '.bzr'])
-    tarball.close()
+        tarball.add(build_dir, tarname, exclude=tarexc)
+        tarball.close()
 
     # Run a build command if one is required...
     if 'build' in thisbuild:
@@ -588,7 +590,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
 
         p = FDroidPopen(commands, cwd=gradle_dir)
 
-    else:
+    elif thisbuild['type'] == 'ant':
         logging.info("Building Ant project...")
         cmd = ['ant']
         if 'antcommand' in thisbuild:
@@ -599,7 +601,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
 
         bindir = os.path.join(root_dir, 'bin')
 
-    if p.returncode != 0:
+    if p is not None and p.returncode != 0:
         raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout)
     logging.info("Successfully built version " + thisbuild['version'] + ' of ' + app['id'])
 
@@ -636,6 +638,9 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
         src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk,
             re.S|re.M).group(1)
         src = os.path.join(bindir, src)
+    elif thisbuild['type'] == 'raw':
+        src = os.path.join(root_dir, thisbuild['output'])
+        src = os.path.normpath(src)
 
     # Make sure it's not debuggable...
     if common.isApkDebuggable(src, config):
@@ -701,7 +706,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
     shutil.copyfile(src, dest)
 
     # Move the source tarball into the output directory...
-    if output_dir != tmp_dir:
+    if output_dir != tmp_dir and not options.notarball:
         shutil.move(os.path.join(tmp_dir, tarname),
             os.path.join(output_dir, tarname))
 
@@ -773,6 +778,8 @@ def parse_commandline():
                       help="Reset and create a brand new build server, even if the existing one appears to be ok.")
     parser.add_option("--on-server", dest="onserver", action="store_true", default=False,
                       help="Specify that we're running on the build server")
+    parser.add_option("--no-tarball", dest="notarball", action="store_true", default=False,
+                      help="Don't create a source tarball, useful when testing a build")
     parser.add_option("-f", "--force", action="store_true", default=False,
                       help="Force build of disabled apps, and carries on regardless of scan problems. Only allowed in test mode.")
     parser.add_option("-a", "--all", action="store_true", default=False,

fdroidserver/common.py

@@ -333,20 +333,18 @@ class vcs_git(vcs):
 
     def initsubmodules(self):
         self.checkrepo()
-        if subprocess.call(['git', 'submodule', 'init'],
+        if subprocess.call(['git', 'submodule', 'foreach', '--recursive',
-                cwd=self.local) != 0:
-            raise VCSException("Git submodule init failed")
-        if subprocess.call(['git', 'submodule', 'update'],
-                cwd=self.local) != 0:
-            raise VCSException("Git submodule update failed")
-        if subprocess.call(['git', 'submodule', 'foreach',
             'git', 'reset', '--hard'],
                 cwd=self.local) != 0:
             raise VCSException("Git submodule reset failed")
-        if subprocess.call(['git', 'submodule', 'foreach',
+        if subprocess.call(['git', 'submodule', 'foreach', '--recursive',
             'git', 'clean', '-dffx'],
                 cwd=self.local) != 0:
             raise VCSException("Git submodule clean failed")
+        if subprocess.call(['git', 'submodule', 'update',
+            '--init', '--force', '--recursive'],
+                cwd=self.local) != 0:
+            raise VCSException("Git submodule update failed")
 
     def gettags(self):
         self.checkrepo()

fdroidserver/metadata.py

@@ -471,6 +471,8 @@ def parse_metadata(metafile):
         for t in ['maven', 'gradle', 'kivy']:
             if build.get(t, 'no') != 'no':
                 return t
+        if 'output' in build:
+            return 'raw'
         return 'ant'
 
     thisinfo = {}
@@ -699,7 +701,7 @@ def write_metadata(dest, app):
         # This defines the preferred order for the build items - as in the
         # manual, they're roughly in order of application.
         keyorder = ['disable', 'commit', 'subdir', 'submodules', 'init',
-                    'gradle', 'maven', 'oldsdkloc', 'target',
+                    'gradle', 'maven', 'output', 'oldsdkloc', 'target',
                     'update', 'encoding', 'forceversion', 'forcevercode', 'rm',
                     'fixtrans', 'fixapos', 'extlibs', 'srclibs', 'patch',
                     'prebuild', 'scanignore', 'scandelete', 'build', 'buildjni',