From ecab81d69d19f4513c1bf55161c1d4dd5840965e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 17 Jun 2016 14:29:41 +0200 Subject: [PATCH 1/5] buildserver: suppress unzip file list when unpacking SDK/NDK Just too much pointless info in the log. --- buildserver/provision-android-ndk | 4 ++-- buildserver/provision-android-sdk | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/buildserver/provision-android-ndk b/buildserver/provision-android-ndk index 3c9b0a6a..80411b34 100644 --- a/buildserver/provision-android-ndk +++ b/buildserver/provision-android-ndk @@ -17,8 +17,8 @@ if [ ! -e $ANDROID_NDK_HOME/r9b ]; then else SUFFIX='' fi - tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX.tar.bz2 - tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX-legacy-toolchains.tar.bz2 + tar xjf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX.tar.bz2 + tar xjf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX-legacy-toolchains.tar.bz2 mv android-ndk-r9b r9b fi diff --git a/buildserver/provision-android-sdk b/buildserver/provision-android-sdk index 5560ecb1..907b2489 100644 --- a/buildserver/provision-android-sdk +++ b/buildserver/provision-android-sdk @@ -43,7 +43,7 @@ for f in `ls -1 build-tools*.zip`; do done ${ANDROID_HOME}/tools/android --silent update sdk --no-ui --all \ - --filter platform-tools,extra-android-m2repository${cached} < /dev/null < Date: Mon, 20 Jun 2016 11:18:03 +0200 Subject: [PATCH 2/5] buildserver: run tests with verbose logs --- jenkins-build-makebuildserver | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-build-makebuildserver b/jenkins-build-makebuildserver index 81d88e9b..d32e1442 100755 --- a/jenkins-build-makebuildserver +++ b/jenkins-build-makebuildserver @@ -46,7 +46,7 @@ cd $WORKSPACE echo "debian_mirror = 'http://ftp.uk.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py -./makebuildserver +./makebuildserver --verbose # this can be handled in the jenkins job, or here: if [ -e fdroiddata ]; then From 226237c5f75e6ad9796295f04c7af45be90855cc Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 17 Jun 2016 11:29:00 +0200 Subject: [PATCH 3/5] buildserver: do not set NDK env vars, they are handled by `fdroid build` `fdroid build` handles setting the NDK env vars since the NDK version can change depending on the app being build. Unlike ANDROID_HOME, there is no single global NDK location. The NDK installs are all versioned. --- buildserver/provision-android-ndk | 17 +++++++---------- buildserver/setup-env-vars | 1 - makebuildserver | 9 ++++----- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/buildserver/provision-android-ndk b/buildserver/provision-android-ndk index 80411b34..9d4a54c0 100644 --- a/buildserver/provision-android-ndk +++ b/buildserver/provision-android-ndk @@ -3,15 +3,12 @@ set -e -if [ -z $ANDROID_NDK_HOME ]; then - echo "ANDROID_NDK_HOME env var must be set!" - exit 1 -fi +NDK_BASE=$1 -test -e $ANDROID_NDK_HOME || mkdir -p $ANDROID_NDK_HOME -cd $ANDROID_NDK_HOME +test -e $NDK_BASE || mkdir -p $NDK_BASE +cd $NDK_BASE -if [ ! -e $ANDROID_NDK_HOME/r9b ]; then +if [ ! -e $NDK_BASE/r9b ]; then if [ `uname -m` == 'x86_64' ] ; then SUFFIX='_64' else @@ -22,7 +19,7 @@ if [ ! -e $ANDROID_NDK_HOME/r9b ]; then mv android-ndk-r9b r9b fi -if [ ! -e $ANDROID_NDK_HOME/r10e ]; then +if [ ! -e $NDK_BASE/r10e ]; then if [ `uname -m` == 'x86_64' ] ; then SUFFIX='_64' else @@ -32,5 +29,5 @@ if [ ! -e $ANDROID_NDK_HOME/r10e ]; then mv android-ndk-r10e r10e fi -chmod -R a+rX $ANDROID_NDK_HOME/ -find $ANDROID_NDK_HOME/ -type f -executable -print0 | xargs -0 chmod a+x +chmod -R a+rX $NDK_BASE/ +find $NDK_BASE/ -type f -executable -print0 | xargs -0 chmod a+x diff --git a/buildserver/setup-env-vars b/buildserver/setup-env-vars index 83d433eb..00220a23 100644 --- a/buildserver/setup-env-vars +++ b/buildserver/setup-env-vars @@ -10,7 +10,6 @@ bsenv=/etc/profile.d/bsenv.sh echo "# generated on "`date` > $bsenv echo export ANDROID_HOME=$1 >> $bsenv -echo export ANDROID_NDK_HOME=$2 >> $bsenv echo 'export PATH=$PATH:${ANDROID_HOME}/tools:${ANDROID_HOME}/platform-tools:/opt/gradle/bin' >> $bsenv chmod 0644 $bsenv diff --git a/makebuildserver b/makebuildserver index 05616452..44c43256 100755 --- a/makebuildserver +++ b/makebuildserver @@ -377,10 +377,8 @@ if config['apt_package_cache']: vagrantfile += """ - config.vm.provision "shell" do |shell| - shell.path = "setup-env-vars" - shell.args = ["/home/vagrant/android-sdk", "/home/vagrant/android-ndk"] - end + config.vm.provision "shell", path: "setup-env-vars", + args: ["/home/vagrant/android-sdk"] config.vm.provision :chef_solo do |chef| chef.cookbooks_path = "cookbooks" @@ -397,7 +395,8 @@ vagrantfile += """ end config.vm.provision "shell", path: "provision-android-sdk" - config.vm.provision "shell", path: "provision-android-ndk" + config.vm.provision "shell", path: "provision-android-ndk", + args: ["/home/vagrant/android-ndk"] config.vm.provision "shell", path: "provision-gradle" config.vm.provision "file", source: "gradle", destination: "/opt/gradle/bin/gradle" From 400acdefabceab83d0eed96c4cf7d905a014751e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 23 Mar 2016 21:04:20 +0100 Subject: [PATCH 4/5] check whether metadata file is a duplicate before parsing it By checking first, this prevents a stacktrace when the duplicate metadata file is not valid. For example, in the tests, the duplicate is just a zero length file, which was causing a stacktrace. --- fdroidserver/metadata.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index ded6cd1a..9ba9b1fd 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -783,9 +783,10 @@ def read_metadata(xref=True): + glob.glob('.fdroid.json') + glob.glob('.fdroid.xml') + glob.glob('.fdroid.yml')): + packageName, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath)) + if packageName in apps: + raise MetaDataException("Found multiple metadata files for " + packageName) app = parse_metadata(metadatapath) - if app.id in apps: - raise MetaDataException("Found multiple metadata files for " + app.id) check_metadata(app) apps[app.id] = app From c35260576805a89c12f94f5fcec82b81a082280d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 20 Jun 2016 20:00:59 +0200 Subject: [PATCH 5/5] add force_build_tools config option This replaces the current default behavior of always forcing the build_tools version and allows the user to set build-tools forcing in config.py. closes #147 --- examples/config.py | 8 ++++++-- fdroidserver/build.py | 14 +++++++------- fdroidserver/common.py | 1 + tests/build.TestCase | 7 +++---- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/examples/config.py b/examples/config.py index 56ed7b86..1a3367e8 100644 --- a/examples/config.py +++ b/examples/config.py @@ -25,6 +25,10 @@ # Build tools version to be used # build_tools = "23.0.3" +# Force all build to use the above version of build -tools, good for testing +# builds without having all of the possible build-tools installed. +# force_build_tools = True + # Command or path to binary for running Ant # ant = "ant" @@ -232,10 +236,10 @@ The repository of older versions of applications from the main demo repository. # --server option on dedicated secure build server hosts. # build_server_always = True -# By default, fdroid will use YAML and the custom .txt metadata formats. It +# By default, fdroid will use YAML .yml and the custom .txt metadata formats. It # is also possible to have metadata in JSON and XML by adding 'json' and # 'xml'. -# accepted_formats = ['txt', 'yaml'] +# accepted_formats = ['txt', 'yml'] # Limit in number of characters that fields can take up # Only the fields listed here are supported, defaults shown diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 7709a6bf..bad025a6 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -431,8 +431,7 @@ def build_server(app, build, vcs, build_dir, output_dir, force): release_vm() -def adapt_gradle(build_dir): - filename = 'build.gradle' +def force_gradle_build_tools(build_dir, build_tools): for root, dirs, files in os.walk(build_dir): for filename in files: if not filename.endswith('.gradle'): @@ -440,9 +439,9 @@ def adapt_gradle(build_dir): path = os.path.join(root, filename) if not os.path.isfile(path): continue - logging.debug("Adapting %s at %s" % (filename, path)) + logging.debug("Forcing build-tools %s in %s" % (build_tools, path)) common.regsub_file(r"""(\s*)buildToolsVersion([\s=]+).*""", - r"""\1buildToolsVersion\2'%s'""" % config['build_tools'], + r"""\1buildToolsVersion\2'%s'""" % build_tools, path) @@ -512,9 +511,10 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, gradletasks += ['assemble' + flavours_cmd + 'Release'] - adapt_gradle(build_dir) - for name, number, libpath in srclibpaths: - adapt_gradle(libpath) + if config['force_build_tools']: + force_gradle_build_tools(build_dir, config['build_tools']) + for name, number, libpath in srclibpaths: + force_gradle_build_tools(libpath, config['build_tools']) cmd = [config['gradle']] if build.gradleprops: diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 17df02ae..f27177fe 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -59,6 +59,7 @@ default_config = { 'r10e': "$ANDROID_NDK", }, 'build_tools': "23.0.3", + 'force_build_tools': False, 'java_paths': None, 'ant': "ant", 'mvn3': "mvn", diff --git a/tests/build.TestCase b/tests/build.TestCase index 369c3836..9391a042 100755 --- a/tests/build.TestCase +++ b/tests/build.TestCase @@ -46,7 +46,7 @@ class BuildTest(unittest.TestCase): self.assertTrue(os.path.exists(path)) self.assertTrue(os.path.isfile(path)) - def test_adapt_gradle(self): + def test_force_gradle_build_tools(self): testsbase = os.path.join(os.path.dirname(__file__), '..', '.testfiles') if not os.path.exists(testsbase): os.makedirs(testsbase) @@ -54,9 +54,7 @@ class BuildTest(unittest.TestCase): shutil.copytree(os.path.join(os.path.dirname(__file__), 'source-files'), os.path.join(testsdir, 'source-files')) teststring = 'FAKE_VERSION_FOR_TESTING' - fdroidserver.build.config = {} - fdroidserver.build.config['build_tools'] = teststring - fdroidserver.build.adapt_gradle(testsdir) + fdroidserver.build.force_gradle_build_tools(testsdir, teststring) pattern = re.compile(bytes("buildToolsVersion[\s=]+'%s'\s+" % teststring, 'utf8')) for p in ('source-files/fdroid/fdroidclient/build.gradle', 'source-files/Zillode/syncthing-silk/build.gradle', @@ -67,6 +65,7 @@ class BuildTest(unittest.TestCase): filedata = f.read() self.assertIsNotNone(pattern.search(filedata)) + if __name__ == "__main__": parser = optparse.OptionParser() parser.add_option("-v", "--verbose", action="store_true", default=False,