From 61dca767f4c175a75a2991f14a90eb99a3b1e480 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 28 Jan 2014 16:59:27 +0100 Subject: [PATCH 1/8] Fix typo in fd-commit --- fd-commit | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fd-commit b/fd-commit index 478850b6..9a8b51f3 100755 --- a/fd-commit +++ b/fd-commit @@ -62,6 +62,7 @@ while read line; do onlybuild=true newbuild=false + disable=false while read l; do if [[ "$l" == "-Build:"* ]]; then onlybuild=false @@ -74,7 +75,7 @@ while read line; do version=${build%%,*} build=${build#*,} vercode=${build%%,*} - elif $newbuild && $onlybuild [[ "$l" == "+"*"disable="* ]]; then + elif $newbuild && $onlybuild && [[ "$l" == "+"*"disable="* ]]; then disable=true fi done < <(git diff HEAD -- "$file") From d42dd276a002ddd21a59641e4da16de65f75cdfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 28 Jan 2014 22:13:18 +0100 Subject: [PATCH 2/8] New build option: --no-tarball --- fdroidserver/build.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index c151c69c..33512710 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -359,7 +359,8 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, force): tarball = common.getsrcname(app,thisbuild) try: ftp.get(apkfile, os.path.join(output_dir, apkfile)) - ftp.get(tarball, os.path.join(output_dir, tarball)) + if not options.notarball: + ftp.get(tarball, os.path.join(output_dir, tarball)) except: raise BuildException("Build failed for %s:%s - missing output files" % (app['id'], thisbuild['version']), output) ftp.close() @@ -443,14 +444,15 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d raise BuildException("Can't build due to " + str(len(buildprobs)) + " scanned problems") - # Build the source tarball right before we build the release... - print "Creating source tarball..." - tarname = common.getsrcname(app,thisbuild) - tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz") - def tarexc(f): - return any(f.endswith(s) for s in ['.svn', '.git', '.hg', '.bzr']) - tarball.add(build_dir, tarname, exclude=tarexc) - tarball.close() + if not options.notarball: + # Build the source tarball right before we build the release... + print "Creating source tarball..." + tarname = common.getsrcname(app,thisbuild) + tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz") + def tarexc(f): + return any(f.endswith(s) for s in ['.svn', '.git', '.hg', '.bzr']) + tarball.add(build_dir, tarname, exclude=tarexc) + tarball.close() # Run a build command if one is required... if 'build' in thisbuild: @@ -720,7 +722,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d shutil.copyfile(src, dest) # Move the source tarball into the output directory... - if output_dir != tmp_dir: + if output_dir != tmp_dir and not options.notarball: shutil.move(os.path.join(tmp_dir, tarname), os.path.join(output_dir, tarname)) @@ -792,6 +794,8 @@ def parse_commandline(): help="Reset and create a brand new build server, even if the existing one appears to be ok.") parser.add_option("--on-server", dest="onserver", action="store_true", default=False, help="Specify that we're running on the build server") + parser.add_option("--no-tarball", dest="notarball", action="store_true", default=False, + help="Don't create a source tarball, useful when testing a build") parser.add_option("-f", "--force", action="store_true", default=False, help="Force build of disabled apps, and carries on regardless of scan problems. Only allowed in test mode.") parser.add_option("-a", "--all", action="store_true", default=False, From 8bbe38f172de51d3efaff5fead9ac1f2eadbf973 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 28 Jan 2014 22:14:28 +0100 Subject: [PATCH 3/8] Avoid default cases in build types --- fdroidserver/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 33512710..109425cc 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -607,7 +607,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d p = FDroidPopen(commands, cwd=gradle_dir) - else: + elif thisbuild['type'] == 'ant': print "Building Ant project..." cmd = ['ant'] if 'antcommand' in thisbuild: @@ -649,7 +649,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d else: name = '-'.join([os.path.basename(dd), '-'.join(flavours), 'release', 'unsigned']) src = os.path.join(dd, 'build', 'apk', name+'.apk') - else: + elif thisbuild['type'] == 'ant': stdout_apk = '\n'.join([ line for line in p.stdout.splitlines() if '.apk' in line]) src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk, From 13bcd38372017564247310fcffaaaac4201c92f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 28 Jan 2014 22:14:53 +0100 Subject: [PATCH 4/8] New build flag: output= for e.g. full make builds --- fdroidserver/build.py | 5 ++++- fdroidserver/metadata.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 109425cc..c60069d5 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -618,7 +618,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d bindir = os.path.join(root_dir, 'bin') - if p.returncode != 0: + if p is not None and p.returncode != 0: raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout) print "Successfully built version " + thisbuild['version'] + ' of ' + app['id'] @@ -655,6 +655,9 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk, re.S|re.M).group(1) src = os.path.join(bindir, src) + elif thisbuild['type'] == 'raw': + src = os.path.join(root_dir, thisbuild['output']) + src = os.path.normpath(src) # Make sure it's not debuggable... if common.isApkDebuggable(src, config): diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index c31a451a..4f1a60b0 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -470,6 +470,8 @@ def parse_metadata(metafile): for t in ['maven', 'gradle', 'kivy']: if build.get(t, 'no') != 'no': return t + if 'output' in build: + return 'raw' return 'ant' thisinfo = {} @@ -698,7 +700,7 @@ def write_metadata(dest, app): # This defines the preferred order for the build items - as in the # manual, they're roughly in order of application. keyorder = ['disable', 'commit', 'subdir', 'submodules', 'init', - 'gradle', 'maven', 'oldsdkloc', 'target', + 'gradle', 'maven', 'output', 'oldsdkloc', 'target', 'update', 'encoding', 'forceversion', 'forcevercode', 'rm', 'fixtrans', 'fixapos', 'extlibs', 'srclibs', 'patch', 'prebuild', 'scanignore', 'scandelete', 'build', 'buildjni', From f054cf49733e56826e524a9ead60a87d622ea598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 28 Jan 2014 22:26:20 +0100 Subject: [PATCH 5/8] Add output= into the docs --- docs/fdroid.texi | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/fdroid.texi b/docs/fdroid.texi index 6cac6d4e..cb5d17a2 100644 --- a/docs/fdroid.texi +++ b/docs/fdroid.texi @@ -1013,6 +1013,11 @@ in a gradle project build. Specify an alternate ant command (target) instead of the default 'release'. It can't be given any flags, such as the path to a build.xml. +@item output=path/to/output.apk +To be used when app is built with a tool other than the ones natively +supported, like GNU Make. The given path will be where the build= set of +commands should produce the final unsigned release apk. + @item novcheck=yes Don't check that the version name and code in the resulting apk are correct by looking at the build output - assume the metadata is From c06fafdb6f32e87b49dbc69a20fc723e821ed268 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 28 Jan 2014 23:49:36 +0100 Subject: [PATCH 6/8] Do submodules recursively, do init and update at the same time --- docs/fdroid.texi | 6 ++++-- fdroidserver/common.py | 6 ++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/fdroid.texi b/docs/fdroid.texi index cb5d17a2..765dd5e8 100644 --- a/docs/fdroid.texi +++ b/docs/fdroid.texi @@ -820,8 +820,10 @@ Specifies to build from a subdirectory of the checked out source code. Normally this directory is changed to before building, @item submodules=yes -Use if the project (git only) has submodules - causes git submodule -init and update to be executed after the source is cloned. +Use if the project (git only) has submodules - causes @code{git submodule +update --init --recursive} to be executed after the source is cloned. +Submodules are reset and cleaned like the main app repository itself before +each build. @item init=xxxx As for 'prebuild', but runs on the source code BEFORE any other processing diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 3bca96d3..6ccb30d0 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -336,10 +336,8 @@ class vcs_git(vcs): def initsubmodules(self): self.checkrepo() - if subprocess.call(['git', 'submodule', 'init'], - cwd=self.local) != 0: - raise VCSException("Git submodule init failed") - if subprocess.call(['git', 'submodule', 'update'], + if subprocess.call(['git', 'submodule', 'update', + '--init', '--recursive'], cwd=self.local) != 0: raise VCSException("Git submodule update failed") if subprocess.call(['git', 'submodule', 'foreach', From 9fa73faf2e478defd3877b0ba35ada15416c0b9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Wed, 29 Jan 2014 00:55:53 +0100 Subject: [PATCH 7/8] Add missing xcsoar deps --- buildserver/cookbooks/fdroidbuild-general/recipes/default.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb b/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb index 6837ee6c..7eef71e8 100644 --- a/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb +++ b/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb @@ -5,7 +5,8 @@ execute "apt-get-update" do command "apt-get update" end -%w{ant ant-contrib autoconf autopoint bison cmake expect libtool libsaxonb-java libssl1.0.0 libssl-dev maven openjdk-7-jdk javacc python python-magic git-core mercurial subversion bzr git-svn make perlmagick pkg-config zip ruby rubygems librmagick-ruby yasm imagemagick gettext realpath transfig texinfo curl}.each do |pkg| +%w{ant ant-contrib autoconf autopoint bison cmake expect libtool libsaxonb-java libssl1.0.0 libssl-dev maven openjdk-7-jdk javacc python python-magic git-core mercurial subversion bzr git-svn make perlmagick pkg-config zip ruby rubygems librmagick-ruby yasm imagemagick gettext realpath transfig texinfo curl +librsvg2-bin xsltproc}.each do |pkg| package pkg do action :install end From 63a0e859ff81cb03182af68e6c6bfff4d83386d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Wed, 29 Jan 2014 12:38:21 +0100 Subject: [PATCH 8/8] Run submodule reset and clean before update. Also do --recursive for these. --- fdroidserver/common.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 6ccb30d0..b348af1f 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -336,18 +336,18 @@ class vcs_git(vcs): def initsubmodules(self): self.checkrepo() - if subprocess.call(['git', 'submodule', 'update', - '--init', '--recursive'], - cwd=self.local) != 0: - raise VCSException("Git submodule update failed") - if subprocess.call(['git', 'submodule', 'foreach', + if subprocess.call(['git', 'submodule', 'foreach', '--recursive', 'git', 'reset', '--hard'], cwd=self.local) != 0: raise VCSException("Git submodule reset failed") - if subprocess.call(['git', 'submodule', 'foreach', + if subprocess.call(['git', 'submodule', 'foreach', '--recursive', 'git', 'clean', '-dffx'], cwd=self.local) != 0: raise VCSException("Git submodule clean failed") + if subprocess.call(['git', 'submodule', 'update', + '--init', '--force', '--recursive'], + cwd=self.local) != 0: + raise VCSException("Git submodule update failed") def gettags(self): self.checkrepo()