From d01e814e366a0b22046b43c650b5ca6c8bb3eebc Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 24 Jul 2015 15:39:00 -0700 Subject: [PATCH 01/14] run-tests: run `fdroid readmeta` after each --create-metadata Just another basic check, this time for `fdroid readmeta`. --- tests/run-tests | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/run-tests b/tests/run-tests index 8f14dadc..8971bd04 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -124,6 +124,7 @@ REPOROOT=`create_test_dir` cd $REPOROOT $fdroid init $fdroid update --create-metadata +$fdroid readmeta $fdroid server update --local-copy-dir=/tmp/fdroid # now test the errors work @@ -160,6 +161,7 @@ cd $REPOROOT $fdroid init copy_apks_into_repo $REPOROOT $fdroid update --create-metadata +$fdroid readmeta grep -F ' Date: Mon, 27 Jul 2015 11:34:01 -0700 Subject: [PATCH 04/14] tests: short args for mktemp to support BSD *BSD and OSX do not have compatible long args --- tests/run-tests | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/run-tests b/tests/run-tests index df4d6355..b8f55763 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -35,12 +35,12 @@ create_fake_android_home() { create_test_dir() { test -e $WORKSPACE/.testfiles || mkdir $WORKSPACE/.testfiles - mktemp --directory --tmpdir=$WORKSPACE/.testfiles + TMPDIR=$WORKSPACE/.testfiles mktemp -d } create_test_file() { test -e $WORKSPACE/.testfiles || mkdir $WORKSPACE/.testfiles - mktemp --tmpdir=$WORKSPACE/.testfiles + TMPDIR=$WORKSPACE/.testfiles mktemp } #------------------------------------------------------------------------------# From 85febd40d1bfc5498b7079e8921ac9cbaf44d3e3 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 30 Jul 2015 22:13:12 +0200 Subject: [PATCH 05/14] use common.regsub_file() instead of Popen(sed) Python libraries work better in Python than running external commands, and it also makes the code much more portable. For example, the GNU and BSD sed commands have different, and sometimes conflicting, flags. This also reworks the regexp patterns to be more tightly focused, and not change the same variable name in comments or elsewhere. --- fdroidserver/build.py | 23 +++++++----------- fdroidserver/common.py | 54 +++++++++++++++++++++--------------------- 2 files changed, 36 insertions(+), 41 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 84af7f41..74004d29 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -436,10 +436,9 @@ def adapt_gradle(build_dir): if not os.path.isfile(path): continue logging.debug("Adapting %s at %s" % (filename, path)) - - FDroidPopen(['sed', '-i', - r's@buildToolsVersion\([ =]\+\).*@buildToolsVersion\1"' - + config['build_tools'] + '"@g', path]) + common.regsub_file(r"""(\s*)buildToolsVersion[\s'"=]+.*""", + r"""\1buildToolsVersion '%s'""" % config['build_tools'], + path) def capitalize_intact(string): @@ -631,17 +630,13 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d 'package'] if thisbuild['target']: target = thisbuild["target"].split('-')[1] - FDroidPopen(['sed', '-i', - 's@[0-9]*@' - + target + '@g', - 'pom.xml'], - cwd=root_dir) + common.regsub_file(r'[0-9]*', + r'%s' % target, + os.path.join(root_dir, 'pom.xml')) if '@' in thisbuild['maven']: - FDroidPopen(['sed', '-i', - 's@[0-9]*@' - + target + '@g', - 'pom.xml'], - cwd=maven_dir) + common.regsub_file(r'[0-9]*', + r'%s' % target, + os.path.join(maven_dir, 'pom.xml')) p = FDroidPopen(mvncmd, cwd=maven_dir) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 3c672728..e6306020 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -121,6 +121,14 @@ def fill_config_defaults(thisconfig): thisconfig[k][k2 + '_orig'] = v +def regsub_file(pattern, repl, path): + with open(path, 'r') as f: + text = f.read() + text = re.sub(pattern, repl, text) + with open(path, 'w') as f: + f.write(text) + + def read_config(opts, config_file='config.py'): """Read the repository config @@ -965,10 +973,9 @@ def remove_debuggable_flags(root_dir): logging.debug("Removing debuggable flags from %s" % root_dir) for root, dirs, files in os.walk(root_dir): if 'AndroidManifest.xml' in files: - path = os.path.join(root, 'AndroidManifest.xml') - p = FDroidPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path], output=False) - if p.returncode != 0: - raise BuildException("Failed to remove debuggable flags of %s" % path) + regsub_file(r'android:debuggable="[^"]*"', + '', + os.path.join(root, 'AndroidManifest.xml')) # Extract some information from the AndroidManifest.xml at the given path. @@ -1305,9 +1312,9 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= if build['target']: n = build["target"].split('-')[1] - FDroidPopen(['sed', '-i', - 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g', - 'build.gradle'], cwd=root_dir, output=False) + regsub_file(r'compileSdkVersion[ =]+[0-9]+', + r'compileSdkVersion %s' % n, + os.path.join(root_dir, 'build.gradle')) # Remove forced debuggable flags remove_debuggable_flags(root_dir) @@ -1319,34 +1326,27 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= if not os.path.isfile(path): continue if has_extension(path, 'xml'): - p = FDroidPopen(['sed', '-i', - 's/android:versionName="[^"]*"/android:versionName="' + build['version'] + '"/g', - path], output=False) - if p.returncode != 0: - raise BuildException("Failed to amend manifest") + regsub_file(r'android:versionName="[^"]*"', + r'android:versionName="%s"' % build['version'], + path) elif has_extension(path, 'gradle'): - p = FDroidPopen(['sed', '-i', - 's/versionName *=* *.*/versionName = "' + build['version'] + '"/g', - path], output=False) - if p.returncode != 0: - raise BuildException("Failed to amend build.gradle") + regsub_file(r"""(\s*)versionName[\s'"=]+.*""", + r"""\1versionName '%s'""" % build['version'], + path) + if build['forcevercode']: logging.info("Changing the version code") for path in manifest_paths(root_dir, flavours): if not os.path.isfile(path): continue if has_extension(path, 'xml'): - p = FDroidPopen(['sed', '-i', - 's/android:versionCode="[^"]*"/android:versionCode="' + build['vercode'] + '"/g', - path], output=False) - if p.returncode != 0: - raise BuildException("Failed to amend manifest") + regsub_file(r'android:versionCode="[^"]*"', + r'android:versionCode="%s"' % build['vercode'], + path) elif has_extension(path, 'gradle'): - p = FDroidPopen(['sed', '-i', - 's/versionCode *=* *[0-9]*/versionCode = ' + build['vercode'] + '/g', - path], output=False) - if p.returncode != 0: - raise BuildException("Failed to amend build.gradle") + regsub_file(r'versionCode[ =]+[0-9]+', + r'versionCode %s' % build['vercode'], + path) # Delete unwanted files if build['rm']: From 64a9c93ce74132a5a0a2daf2dc8f87267e62d158 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 30 Jul 2015 22:13:36 +0200 Subject: [PATCH 06/14] test new common.regsub_file() method that replaces sed calls --- .gitignore | 4 + tests/AndroidManifest.xml | 484 ++++++++++++++++++++++++++++++++++++++ tests/build.TestCase | 65 +++++ tests/build.gradle | 218 +++++++++++++++++ tests/common.TestCase | 44 ++++ 5 files changed, 815 insertions(+) create mode 100644 tests/AndroidManifest.xml create mode 100755 tests/build.TestCase create mode 100644 tests/build.gradle diff --git a/.gitignore b/.gitignore index 277ca280..4371053f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ *.pyc *.class *.box + # files generated by build build/ dist/ @@ -11,3 +12,6 @@ env/ fdroidserver.egg-info/ pylint.parseable /.testfiles/ + +# files generated by tests +tests/local.properties diff --git a/tests/AndroidManifest.xml b/tests/AndroidManifest.xml new file mode 100644 index 00000000..bd84256b --- /dev/null +++ b/tests/AndroidManifest.xml @@ -0,0 +1,484 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/build.TestCase b/tests/build.TestCase new file mode 100755 index 00000000..5eae2b41 --- /dev/null +++ b/tests/build.TestCase @@ -0,0 +1,65 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +# http://www.drdobbs.com/testing/unit-testing-with-python/240165163 + +import inspect +import optparse +import os +import re +import sys +import unittest + +localmodule = os.path.realpath( + os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) +print('localmodule: ' + localmodule) +if localmodule not in sys.path: + sys.path.insert(0, localmodule) + +import fdroidserver.build +import fdroidserver.common + + +class BuildTest(unittest.TestCase): + '''fdroidserver/build.py''' + + def _set_build_tools(self): + build_tools = os.path.join(fdroidserver.common.config['sdk_path'], 'build-tools') + if os.path.exists(build_tools): + fdroidserver.common.config['build_tools'] = '' + for f in sorted(os.listdir(build_tools), reverse=True): + versioned = os.path.join(build_tools, f) + if os.path.isdir(versioned) \ + and os.path.isfile(os.path.join(versioned, 'aapt')): + fdroidserver.common.config['build_tools'] = versioned + break + return True + else: + print 'no build-tools found: ' + build_tools + return False + + def _find_all(self): + for cmd in ('aapt', 'adb', 'android', 'zipalign'): + path = fdroidserver.common.find_sdk_tools_cmd(cmd) + if path is not None: + self.assertTrue(os.path.exists(path)) + self.assertTrue(os.path.isfile(path)) + + def test_adapt_gradle(self): + teststring = 'FAKE_VERSION_FOR_TESTING' + fdroidserver.build.config = {} + fdroidserver.build.config['build_tools'] = teststring + fdroidserver.build.adapt_gradle(os.path.dirname(__file__)) + filedata = open(os.path.join(os.path.dirname(__file__), 'build.gradle')).read() + self.assertIsNotNone(re.search("\s+buildToolsVersion '%s'\s+" % teststring, filedata)) + + +if __name__ == "__main__": + parser = optparse.OptionParser() + parser.add_option("-v", "--verbose", action="store_true", default=False, + help="Spew out even more information than normal") + (fdroidserver.common.options, args) = parser.parse_args(['--verbose']) + + newSuite = unittest.TestSuite() + newSuite.addTest(unittest.makeSuite(BuildTest)) + unittest.main() diff --git a/tests/build.gradle b/tests/build.gradle new file mode 100644 index 00000000..1d994dc8 --- /dev/null +++ b/tests/build.gradle @@ -0,0 +1,218 @@ +apply plugin: 'com.android.application' + +if ( !hasProperty( 'sourceDeps' ) ) { + + logger.info "Setting up *binary* dependencies for F-Droid (if you'd prefer to build from source, pass the -PsourceDeps argument to gradle while building)." + + repositories { + jcenter() + + // This is here until we sort out all dependencies from mavenCentral/jcenter. Once all of + // the dependencies below have been sorted out, this can be removed. + flatDir { + dirs 'libs/binaryDeps' + } + } + + dependencies { + + compile 'com.android.support:support-v4:22.1.0', + 'com.android.support:appcompat-v7:22.1.0', + 'com.android.support:support-annotations:22.1.0', + + 'org.thoughtcrime.ssl.pinning:AndroidPinning:1.0.0', + 'com.nostra13.universalimageloader:universal-image-loader:1.9.4', + 'com.google.zxing:core:3.2.0', + 'eu.chainfire:libsuperuser:1.0.0.201504231659', + + // We use a slightly modified spongycastle, see + // openkeychain/spongycastle with some changes on top of 1.51.0.0 + 'com.madgag.spongycastle:pkix:1.51.0.0', + 'com.madgag.spongycastle:prov:1.51.0.0', + 'com.madgag.spongycastle:core:1.51.0.0' + + // Upstream doesn't have a binary on mavenCentral/jcenter yet: + // https://github.com/kolavar/android-support-v4-preferencefragment/issues/13 + compile(name: 'support-v4-preferencefragment-release', ext: 'aar') + + // Fork for F-Droid, including support for https. Not merged into upstream + // yet (seems to be a little unsupported as of late), so not using mavenCentral/jcenter. + compile(name: 'nanohttpd-2.1.0') + + // Upstream doesn't have a binary on mavenCentral. + compile(name: 'zipsigner') + + // Project semi-abandoned, 3.4.1 is from 2011 and we use trunk from 2013 + compile(name: 'jmdns') + + androidTestCompile 'commons-io:commons-io:2.2' + } + +} else { + + logger.info "Setting up *source* dependencies for F-Droid (because you passed in the -PsourceDeps argument to gradle while building)." + + repositories { + jcenter() + } + + dependencies { + compile project(':extern:AndroidPinning') + compile project(':extern:UniversalImageLoader:library') + compile project(':extern:libsuperuser:libsuperuser') + compile project(':extern:nanohttpd:core') + compile project(':extern:jmdns') + compile project(':extern:zipsigner') + compile project(':extern:zxing-core') + compile( project(':extern:support-v4-preferencefragment') ) { + exclude module: 'support-v4' + } + + // Until the android team updates the gradle plugin version from 0.10.0 to + // a newer version, we can't build this from source with our gradle version + // of 1.0.0. They use API's which have been moved in the newer plugin. + // So yes, this is a little annoying that our "source dependencies" include + // a bunch of binaries from jcenter - but the ant build file (which is the + // one used to build F-Droid which is distributed on https://f-droid.org + // builds these from source - well - not support-v4). + // + // If the android team gets the build script working with the newer plugin, + // then you can find the relevant portions of the ../build.gradle file that + // include magic required to make it work at around about the v0.78 git tag. + // They have since been removed to clean up the build file. + compile 'com.android.support:support-v4:22.1.0', + 'com.android.support:appcompat-v7:22.1.0', + 'com.android.support:support-annotations:22.1.0' + + androidTestCompile 'commons-io:commons-io:2.2' + } + +} + +task cleanBinaryDeps(type: Delete) { + + enabled = project.hasProperty('sourceDeps') + description = "Removes all .jar and .aar files from F-Droid/libs/. Requires the sourceDeps property to be set (\"gradle -PsourceDeps cleanBinaryDeps\")" + + delete fileTree('libs/binaryDeps') { + include '*.aar' + include '*.jar' + } +} + +task binaryDeps(type: Copy, dependsOn: ':F-Droid:prepareReleaseDependencies') { + + enabled = project.hasProperty('sourceDeps') + description = "Copies .jar and .aar files from subproject dependencies in extern/ to F-Droid/libs. Requires the sourceDeps property to be set (\"gradle -PsourceDeps binaryDeps\")" + + from ('../extern/' ) { + include 'support-v4-preferencefragment/build/outputs/aar/support-v4-preferencefragment-release.aar', + 'nanohttpd/core/build/libs/nanohttpd-2.1.0.jar', + 'zipsigner/build/libs/zipsigner.jar', + 'jmdns/build/libs/jmdns.jar', + 'Support/v4/build/libs/support-v4.jar' + } + + into 'libs/binaryDeps' + + includeEmptyDirs false + + eachFile { FileCopyDetails details -> + // Don't copy to a sub folder such as libs/binaryDeps/Project/build/outputs/aar/project.aar, but + // rather libs/binaryDeps/project.aar. + details.path = details.name + } + +} + +android { + compileSdkVersion 21 + buildToolsVersion '22.0.1' + + sourceSets { + main { + manifest.srcFile 'AndroidManifest.xml' + java.srcDirs = ['src'] + resources.srcDirs = ['src'] + aidl.srcDirs = ['src'] + renderscript.srcDirs = ['src'] + res.srcDirs = ['res'] + assets.srcDirs = ['assets'] + } + + androidTest.setRoot('test') + androidTest { + manifest.srcFile 'test/AndroidManifest.xml' + java.srcDirs = ['test/src'] + resources.srcDirs = ['test/src'] + aidl.srcDirs = ['test/src'] + renderscript.srcDirs = ['test/src'] + res.srcDirs = ['test/res'] + assets.srcDirs = ['test/assets'] + } + } + + buildTypes { + release { + minifyEnabled false + } + buildTypes { + debug { + debuggable true + } + } + } + + compileOptions { + compileOptions.encoding = "UTF-8" + + // Use Java 1.7, requires minSdk 8 + sourceCompatibility JavaVersion.VERSION_1_7 + targetCompatibility JavaVersion.VERSION_1_7 + } + + lintOptions { + checkReleaseBuilds false + abortOnError false + } + + // Enable all Android lint warnings + gradle.projectsEvaluated { + tasks.withType(JavaCompile) { + options.compilerArgs << "-Xlint:all" + } + } + +} + +// This person took the example code below from another blogpost online, however +// I lost the reference to it: +// http://stackoverflow.com/questions/23297562/gradle-javadoc-and-android-documentation +android.applicationVariants.all { variant -> + + task("generate${variant.name}Javadoc", type: Javadoc) { + title = "$name $version API" + description "Generates Javadoc for F-Droid." + source = variant.javaCompile.source + + def sdkDir + Properties properties = new Properties() + File localProps = project.rootProject.file('local.properties') + if (localProps.exists()) { + properties.load(localProps.newDataInputStream()) + sdkDir = properties.getProperty('sdk.dir') + } else { + sdkDir = System.getenv('ANDROID_HOME') + } + if (!sdkDir) { + throw new ProjectConfigurationException("Cannot find android sdk. Make sure sdk.dir is defined in local.properties or the environment variable ANDROID_HOME is set.", null) + } + + ext.androidJar = "${sdkDir}/platforms/${android.compileSdkVersion}/android.jar" + classpath = files(variant.javaCompile.classpath.files) + files(ext.androidJar) + options.links("http://docs.oracle.com/javase/7/docs/api/"); + options.links("http://d.android.com/reference/"); + exclude '**/BuildConfig.java' + exclude '**/R.java' + } +} diff --git a/tests/common.TestCase b/tests/common.TestCase index 0dca4f4d..cbabc5c2 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -6,6 +6,7 @@ import inspect import optparse import os +import re import sys import unittest @@ -17,6 +18,7 @@ if localmodule not in sys.path: sys.path.insert(0,localmodule) import fdroidserver.common +import fdroidserver.metadata class CommonTest(unittest.TestCase): '''fdroidserver/common.py''' @@ -95,6 +97,48 @@ class CommonTest(unittest.TestCase): self.assertFalse(fdroidserver.common.is_valid_package_name(name), "{0} should not be a valid package name".format(name)) + def test_prepare_sources(self): + testint = 99999999 + teststr = 'FAKE_STR_FOR_TESTING' + testdir = os.path.dirname(__file__) + + config = dict() + config['sdk_path'] = os.getenv('ANDROID_HOME') + config['build_tools'] = 'FAKE_BUILD_TOOLS_VERSION' + fdroidserver.common.config = config + app = dict() + app['id'] = 'org.fdroid.froid' + build = dict(fdroidserver.metadata.flag_defaults) + build['commit'] = 'master' + build['forceversion'] = True + build['forcevercode'] = True + build['gradle'] = ['yes'] + build['ndk_path'] = os.getenv('ANDROID_NDK_HOME') + build['target'] = 'android-' + str(testint) + build['type'] = 'gradle' + build['version'] = teststr + build['vercode'] = testint + + class FakeVcs(): + # no need to change to the correct commit here + def gotorevision(self, rev, refresh=True): + pass + + # no srclib info needed, but it could be added... + def getsrclib(self): + return None + + fdroidserver.common.prepare_source(FakeVcs(), app, build, testdir, testdir, testdir) + + filedata = open(os.path.join(testdir, 'build.gradle')).read() + self.assertIsNotNone(re.search("\s+compileSdkVersion %s\s+" % testint, filedata)) + + filedata = open(os.path.join(testdir, 'AndroidManifest.xml')).read() + self.assertIsNone(re.search('android:debuggable', filedata)) + self.assertIsNotNone(re.search('android:versionName="%s"' % build['version'], filedata)) + self.assertIsNotNone(re.search('android:versionCode="%s"' % build['vercode'], filedata)) + + if __name__ == "__main__": parser = optparse.OptionParser() parser.add_option("-v", "--verbose", action="store_true", default=False, From 176029aefb0aa3823339ae9450273ba4c3fe5a0a Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 31 Jul 2015 12:27:57 +0200 Subject: [PATCH 07/14] remove non-existant files from the MANIFEST.in --- MANIFEST.in | 2 -- 1 file changed, 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 2cf1078c..6cf3bd52 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,7 +4,6 @@ include fd-commit include fdroid include jenkins-build include makebuildserver -include updateplugin include buildserver/config.buildserver.py include buildserver/fixpaths.sh include buildserver/cookbooks/android-ndk/recipes/default.rb @@ -27,7 +26,6 @@ include examples/opensc-fdroid.cfg include tests/getsig/run.sh include tests/getsig/make.sh include tests/getsig/getsig.java -include tests/getsig/getsig.class include tests/run-tests include tests/update.TestCase include tests/urzip.apk From 37694a615585cbd430f9c84fe124ad66b6e21bae Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 31 Jul 2015 14:47:48 +0200 Subject: [PATCH 08/14] fix installing via OSX's easy_install --- setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bfd5d106..3518d869 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,12 @@ from setuptools import setup import sys +# workaround issue with easy_install on OSX, where sys.prefix is not an installable location +if sys.platform == 'darwin' and sys.prefix.startswith('/System'): + data_prefix = '/Library/Python/2.7/site-packages' +else: + data_prefix = sys.prefix + setup(name='fdroidserver', version='0.3.0', description='F-Droid Server Tools', @@ -13,7 +19,7 @@ setup(name='fdroidserver', packages=['fdroidserver'], scripts=['fdroid', 'fd-commit'], data_files=[ - (sys.prefix + '/share/doc/fdroidserver/examples', + (data_prefix + '/share/doc/fdroidserver/examples', ['buildserver/config.buildserver.py', 'examples/config.py', 'examples/makebs.config.py', From 752b258ba72756a800e15f47703d07524171f156 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 31 Jul 2015 15:50:20 +0200 Subject: [PATCH 09/14] get MIME types without strictly requiring python-magic There are two forms of python-magic, one included in libmagic that is default on Debian, and another Python wrapper for libmagic that is called 'python-magic' on pypi. Those both rely on the compiled binary library libmagic. For platforms without good package management, fallback to using the built-in 'mimetypes' library if 'magic' is not available. This supports OSX, Windows, and BSD. --- fdroidserver/common.py | 47 +++++++++++++++++++++++++++++------------- setup.py | 3 +-- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index e6306020..4d6dff47 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -29,7 +29,6 @@ import time import operator import Queue import threading -import magic import logging import hashlib import socket @@ -1441,6 +1440,37 @@ def getpaths(build_dir, build, field): return paths +def get_mime_type(path): + ''' + There are two incompatible versions of the 'magic' module, one + that comes as part of libmagic, which is what Debian includes as + python-magic, then another called python-magic that is a separate + project that wraps libmagic. The second is 'magic' on pypi, so + both need to be supported. Then on platforms where libmagic is + not easily included, e.g. OSX and Windows, fallback to the + built-in 'mimetypes' module so this will work without + libmagic. Hence this function with the following hacks: + ''' + + try: + import magic + ms = None + try: + ms = magic.open(magic.MIME_TYPE) + ms.load() + return magic.from_file(path, mime=True) + except AttributeError: + return ms.file(path) + if ms is not None: + ms.close() + except UnicodeError: + logging.warn('Found malformed magic number at %s' % path) + except ImportError: + import mimetypes + mimetypes.init() + return mimetypes.guess_type(path, strict=False) + + # Scan the source code in the given directory (and all subdirectories) # and return the number of fatal problems encountered def scan_source(build_dir, root_dir, thisbuild): @@ -1472,12 +1502,6 @@ def scan_source(build_dir, root_dir, thisbuild): scanignore_worked = set() scandelete_worked = set() - try: - ms = magic.open(magic.MIME_TYPE) - ms.load() - except AttributeError: - ms = None - def toignore(fd): for p in scanignore: if fd.startswith(p): @@ -1526,10 +1550,7 @@ def scan_source(build_dir, root_dir, thisbuild): fp = os.path.join(r, curfile) fd = fp[len(build_dir) + 1:] - try: - mime = magic.from_file(fp, mime=True) if ms is None else ms.file(fp) - except UnicodeError: - warnproblem('malformed magic number', fd) + mime = get_mime_type(fp) if mime == 'application/x-sharedlib': count += handleproblem('shared library', fd, fp) @@ -1537,7 +1558,7 @@ def scan_source(build_dir, root_dir, thisbuild): elif mime == 'application/x-archive': count += handleproblem('static library', fd, fp) - elif mime == 'application/x-executable': + elif mime == 'application/x-executable' or mime == 'application/x-mach-binary': count += handleproblem('binary executable', fd, fp) elif mime == 'application/x-java-applet': @@ -1581,8 +1602,6 @@ def scan_source(build_dir, root_dir, thisbuild): if any(suspect.match(line) for suspect in usual_suspects): count += handleproblem('usual suspect at line %d' % i, fd, fp) break - if ms is not None: - ms.close() for p in scanignore: if p not in scanignore_worked: diff --git a/setup.py b/setup.py index 3518d869..12cf73d3 100644 --- a/setup.py +++ b/setup.py @@ -26,11 +26,10 @@ setup(name='fdroidserver', 'examples/opensc-fdroid.cfg', 'examples/fdroid-icon.png']), ], - install_requires=[ + install_requires=[ # should include 'python-magic' but its not strictly required 'mwclient', 'paramiko', 'Pillow', - 'python-magic', 'apache-libcloud >= 0.14.1', 'pyasn1', 'pyasn1-modules', From 1c1f481fccdcb893cf78d8bb35ca9a678436d35d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 31 Jul 2015 15:53:39 +0200 Subject: [PATCH 10/14] support installing as a Python .egg For platforms where easy_install is a good option, like OSX, support the Python .egg library format. --- fdroidserver/init.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 1e77e2fe..388c2aab 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -70,7 +70,9 @@ def main(): # find root install prefix tmp = os.path.dirname(sys.argv[0]) if os.path.basename(tmp) == 'bin': - prefix = os.path.dirname(tmp) + prefix = os.path.dirname(os.path.dirname(__file__)) # use .egg layout + if not prefix.endswith('.egg'): # use UNIX layout + prefix = os.path.dirname(tmp) examplesdir = prefix + '/share/doc/fdroidserver/examples' else: # we're running straight out of the git repo From f38619ef5fbcad293a63fce0c53991fda14d609e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 31 Jul 2015 15:54:50 +0200 Subject: [PATCH 11/14] chmod keystore after checking whether it was created On OSX, when Java is not installed, it'll fail to create the keystore, but then give an error from chmod failing. This changes things so that the missing Java is reported instead. --- fdroidserver/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 4d6dff47..32db17d3 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -2051,9 +2051,9 @@ def genkeystore(localconfig): '-keypass:file', config['keypassfile'], '-dname', localconfig['keydname']]) # TODO keypass should be sent via stdin - os.chmod(localconfig['keystore'], 0o0600) if p.returncode != 0: raise BuildException("Failed to generate key", p.output) + os.chmod(localconfig['keystore'], 0o0600) # now show the lovely key that was just generated p = FDroidPopen(['keytool', '-list', '-v', '-keystore', localconfig['keystore'], From 45ffaac3f29e4282ce91d5354378539549c4267f Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 31 Jul 2015 16:23:05 +0200 Subject: [PATCH 12/14] update install instructions in README --- README | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/README b/README index 50a29d30..83f6af26 100644 --- a/README +++ b/README @@ -18,16 +18,36 @@ Installing The easiest way to install the `fdroidserver` tools is on Debian, Ubuntu, Mint or other Debian based distributions, you can install using: + sudo apt-get install fdroidserver + +For older Ubuntu releases or to get the latest version, you can get +`fdroidserver` from the Guardian Project PPA (the signing key +fingerprint is `6B80 A842 07B3 0AC9 DEE2 35FE F50E ADDD 2234 F563`) + sudo add-apt-repository ppa:guardianproject/ppa sudo apt-get update sudo apt-get install fdroidserver -But you can also use `virtualenv` and `pip` python tools that also work on other -distributions. +On OSX, `fdroidserver` is available from third party package managers, +like Homebrew, MacPorts, and Fink: -First, make sure you have installed the python header files, virtualenv and pip. -They should be included in your OS's default package manager or you can install -them via other mechanisms like Brew/dnf/pacman/emerge/Fink/MacPorts. + sudo brew install fdroidserver + +For any platform where Python's `easy_install` is an option (e.g. OSX +or Cygwin, you can use it: + + sudo easy_install fdroidserver + +Python's `pip` also works: + + sudo pip install fdroidserver + +The combination of `virtualenv` and `pip` is great for testing out the +latest versions of `fdroidserver`. Using `pip`, `fdroidserver` can +even be installed straight from git. First, make sure you have +installed the python header files, virtualenv and pip. They should be +included in your OS's default package manager or you can install them +via other mechanisms like Brew/dnf/pacman/emerge/Fink/MacPorts. For Debian based distributions: From a020625462fe3176e1bce0d6023ee2be0dd8a2ac Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Sat, 1 Aug 2015 00:36:51 +0200 Subject: [PATCH 13/14] support egg-link format when installed from git repo If you run `python setup.py install` from the git repo, then it will be installed using the .egg-link format, which just points to the git repo. `fdroid init` needs to handle that when looking for example files to copy. --- fdroidserver/init.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 388c2aab..0ed66d6b 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -70,10 +70,16 @@ def main(): # find root install prefix tmp = os.path.dirname(sys.argv[0]) if os.path.basename(tmp) == 'bin': - prefix = os.path.dirname(os.path.dirname(__file__)) # use .egg layout - if not prefix.endswith('.egg'): # use UNIX layout - prefix = os.path.dirname(tmp) - examplesdir = prefix + '/share/doc/fdroidserver/examples' + prefix = None + egg_link = os.path.join(tmp, '..', 'local/lib/python2.7/site-packages/fdroidserver.egg-link') + if os.path.exists(egg_link): + # installed from local git repo + examplesdir = os.path.join(open(egg_link).readline().rstrip(), 'examples') + else: + prefix = os.path.dirname(os.path.dirname(__file__)) # use .egg layout + if not prefix.endswith('.egg'): # use UNIX layout + prefix = os.path.dirname(tmp) + examplesdir = prefix + '/share/doc/fdroidserver/examples' else: # we're running straight out of the git repo prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..')) From e2cbaff3773021e46207f4098ece2be380439385 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Sat, 1 Aug 2015 00:49:33 +0200 Subject: [PATCH 14/14] jenkins-build: run test suite from `pip install` using source tarball Since `python setup.py sdist` provides the actual tarball that will be installed via `easy_install`, `pip install`, etc. it should also be tested. The existing `pip install -e $WORKSPACE` tests the .egg-link install format --- jenkins-build | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/jenkins-build b/jenkins-build index cb94ebdf..7e54312d 100755 --- a/jenkins-build +++ b/jenkins-build @@ -50,13 +50,21 @@ cd $WORKSPACE/tests #------------------------------------------------------------------------------# -# test building the source tarball +# test building the source tarball, then installing it cd $WORKSPACE python2 setup.py sdist +rm -rf $WORKSPACE/env +virtualenv --python=python2 $WORKSPACE/env +. $WORKSPACE/env/bin/activate +pip install dist/fdroidserver-*.tar.gz + +# run tests in new pip+virtualenv install +fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource + #------------------------------------------------------------------------------# -# test install using site packages +# test install using install direct from git repo cd $WORKSPACE rm -rf $WORKSPACE/env virtualenv --python=python2 --system-site-packages $WORKSPACE/env @@ -65,7 +73,6 @@ pip install -e $WORKSPACE python2 setup.py install # run tests in new pip+virtualenv install -. $WORKSPACE/env/bin/activate fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource