From 186aec46baed1fb50af9ffb2e296abca088a7028 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 23 Apr 2014 12:44:37 -0400 Subject: [PATCH 1/7] init: split out defconfig and sdk test to run before config is loaded `fdroid init` runs before any config.py exists, but it still needs to have the default config and the SDK path tests. So split those two bits out of common.read_config() so that they can be run separately before config.py is in place. --- fdroidserver/common.py | 67 ++++++++++++++++++++++++------------------ fdroidserver/init.py | 2 ++ 2 files changed, 41 insertions(+), 28 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index f9db5506..0c1d7dd3 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -34,6 +34,28 @@ import metadata config = None options = None +def get_default_config(): + return { + 'sdk_path': os.getenv("ANDROID_HOME"), + 'ndk_path': "$ANDROID_NDK", + 'build_tools': "19.0.3", + 'ant': "ant", + 'mvn3': "mvn", + 'gradle': 'gradle', + 'archive_older': 0, + 'update_stats': False, + 'stats_to_carbon': False, + 'repo_maxage': 0, + 'build_server_always': False, + 'keystore': '$HOME/.local/share/fdroidserver/keystore.jks', + 'smartcardoptions': [], + 'char_limits': { + 'Summary' : 50, + 'Description' : 1500 + }, + 'keyaliases': { }, + } + def read_config(opts, config_file='config.py'): """Read the repository config @@ -65,26 +87,7 @@ def read_config(opts, config_file='config.py'): 'sun.security.pkcs11.SunPKCS11', '-providerArg', 'opensc-fdroid.cfg'] - defconfig = { - 'sdk_path': "$ANDROID_HOME", - 'ndk_path': "$ANDROID_NDK", - 'build_tools': "19.0.3", - 'ant': "ant", - 'mvn3': "mvn", - 'gradle': 'gradle', - 'archive_older': 0, - 'update_stats': False, - 'stats_to_carbon': False, - 'repo_maxage': 0, - 'build_server_always': False, - 'keystore': '$HOME/.local/share/fdroidserver/keystore.jks', - 'smartcardoptions': [], - 'char_limits': { - 'Summary' : 50, - 'Description' : 1500 - }, - 'keyaliases': { }, - } + defconfig = get_default_config() for k, v in defconfig.items(): if k not in config: config[k] = v @@ -96,14 +99,7 @@ def read_config(opts, config_file='config.py'): v = os.path.expanduser(v) config[k] = os.path.expandvars(v) - if not config['sdk_path']: - logging.critical("Neither $ANDROID_HOME nor sdk_path is set, no Android SDK found!") - sys.exit(3) - if not os.path.exists(config['sdk_path']): - logging.critical('Android SDK path "' + config['sdk_path'] + '" does not exist!') - sys.exit(3) - if not os.path.isdir(config['sdk_path']): - logging.critical('Android SDK path "' + config['sdk_path'] + '" is not a directory!') + if not test_sdk_exists(config): sys.exit(3) if any(k in config for k in ["keystore", "keystorepass", "keypass"]): @@ -124,6 +120,21 @@ def read_config(opts, config_file='config.py'): return config +def test_sdk_exists(c): + if c['sdk_path'] == None: + # c['sdk_path'] is set to the value of ANDROID_HOME by default + logging.critical("Neither ANDROID_HOME nor sdk_path is set, no Android SDK found!") + logging.info('Set ANDROID_HOME to the path to your SDK, i.e.:') + logging.info('\texport ANDROID_HOME=/opt/android-sdk') + return False + if not os.path.exists(c['sdk_path']): + logging.critical('Android SDK path "' + c['sdk_path'] + '" does not exist!') + return False + if not os.path.isdir(c['sdk_path']): + logging.critical('Android SDK path "' + c['sdk_path'] + '" is not a directory!') + return False + return True + def write_password_file(pwtype, password=None): ''' writes out passwords to a protected file instead of passing passwords as diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 1b11bcd4..1448ae79 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -105,6 +105,8 @@ def main(): help="Alias of the repo signing key in the keystore") (options, args) = parser.parse_args() + common.test_sdk_exists(common.get_default_config()) + # find root install prefix tmp = os.path.dirname(sys.argv[0]) if os.path.basename(tmp) == 'bin': From cc089b49b1c47905e881a5e7ce1d4f23182ae824 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 23 Apr 2014 12:46:24 -0400 Subject: [PATCH 2/7] init: only overwrites config.py, so run even if repo/ exists Previously, `fdroid init` would exit if a repo/ subdir existed. Since it only changes config.py, that test just caused confusion. Now, only exit if config.py exists, and if repo/ does not exist, create it. --- fdroidserver/init.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 1448ae79..a6f53c6b 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -119,9 +119,10 @@ def main(): fdroiddir = os.getcwd() - if not os.path.exists('config.py') and not os.path.exists('repo'): + if not os.path.exists('config.py'): # 'metadata' and 'tmp' are created in fdroid - os.mkdir('repo') + if not os.path.exists('repo'): + os.mkdir('repo') shutil.copy(os.path.join(examplesdir, 'fdroid-icon.png'), fdroiddir) shutil.copyfile(os.path.join(examplesdir, 'config.py'), 'config.py') os.chmod('config.py', 0o0600) From 66df02d5f8df84bc624e7a1b8b8f3e16cb099896 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 23 Apr 2014 19:21:22 -0400 Subject: [PATCH 3/7] init: --android-home for forcing the path to the Android SDK This allows the user to set the path to their Android SDK from the command line. This option is named after the standard env var ANDROID_HOME, as used in the build.xml generated by `android update project`. --android-home takes precendence over the ANDROID_HOME env var if it is set. --- .gitignore | 1 + fdroidserver/common.py | 7 ++- fdroidserver/init.py | 49 +++++++++---------- tests/run-tests | 107 ++++++++++++++++++++++++++++++++++++++--- 4 files changed, 131 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index 2a4e5024..277ca280 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ dist/ env/ fdroidserver.egg-info/ pylint.parseable +/.testfiles/ diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 0c1d7dd3..dc6ac697 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -123,8 +123,8 @@ def read_config(opts, config_file='config.py'): def test_sdk_exists(c): if c['sdk_path'] == None: # c['sdk_path'] is set to the value of ANDROID_HOME by default - logging.critical("Neither ANDROID_HOME nor sdk_path is set, no Android SDK found!") - logging.info('Set ANDROID_HOME to the path to your SDK, i.e.:') + logging.critical('No Android SDK found! ANDROID_HOME is not set and sdk_path is not in config.py!') + logging.info('You can use ANDROID_HOME to set the path to your SDK, i.e.:') logging.info('\texport ANDROID_HOME=/opt/android-sdk') return False if not os.path.exists(c['sdk_path']): @@ -133,6 +133,9 @@ def test_sdk_exists(c): if not os.path.isdir(c['sdk_path']): logging.critical('Android SDK path "' + c['sdk_path'] + '" is not a directory!') return False + if not os.path.isdir(os.path.join(c['sdk_path'], 'build-tools')): + logging.critical('Android SDK path "' + c['sdk_path'] + '" does not contain "build-tools/"!') + return False return True def write_password_file(pwtype, password=None): diff --git a/fdroidserver/init.py b/fdroidserver/init.py index a6f53c6b..390a7c7d 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -103,10 +103,10 @@ def main(): help="Path to the keystore for the repo signing key") parser.add_option("--repo-keyalias", default=None, help="Alias of the repo signing key in the keystore") + parser.add_option("--android-home", default=None, + help="Path to the Android SDK (sometimes set in ANDROID_HOME)") (options, args) = parser.parse_args() - common.test_sdk_exists(common.get_default_config()) - # find root install prefix tmp = os.path.dirname(sys.argv[0]) if os.path.basename(tmp) == 'bin': @@ -118,6 +118,25 @@ def main(): examplesdir = prefix + '/examples' fdroiddir = os.getcwd() + test_config = common.get_default_config() + + # track down where the Android SDK is, the default is to use the path set + # in ANDROID_HOME if that exists, otherwise None + if options.android_home != None: + test_config['sdk_path'] = options.android_home + elif not common.test_sdk_exists(test_config): + # if neither --android-home nor the default sdk_path exist, prompt the user + default_sdk_path = '/opt/android-sdk' + while True: + s = raw_input('Enter the path to the Android SDK (' + default_sdk_path + ') here:\n> ') + if re.match('^\s*$', s) != None: + test_config['sdk_path'] = default_sdk_path + else: + test_config['sdk_path'] = s + if common.test_sdk_exists(test_config): + break + if not common.test_sdk_exists(test_config): + sys.exit(3) if not os.path.exists('config.py'): # 'metadata' and 'tmp' are created in fdroid @@ -126,6 +145,7 @@ def main(): shutil.copy(os.path.join(examplesdir, 'fdroid-icon.png'), fdroiddir) shutil.copyfile(os.path.join(examplesdir, 'config.py'), 'config.py') os.chmod('config.py', 0o0600) + write_to_config('sdk_path', test_config['sdk_path']) else: logging.warn('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...') logging.info('Try running `fdroid init` in an empty directory.') @@ -134,29 +154,8 @@ def main(): # now that we have a local config.py, read configuration... config = common.read_config(options) - # track down where the Android SDK is - if os.path.isdir(config['sdk_path']): - logging.info('Using "' + config['sdk_path'] + '" for the Android SDK') - sdk_path = config['sdk_path'] - elif 'ANDROID_HOME' in os.environ.keys(): - sdk_path = os.environ['ANDROID_HOME'] - else: - default_sdk_path = '/opt/android-sdk' - while True: - s = raw_input('Enter the path to the Android SDK (' + default_sdk_path + '): ') - if re.match('^\s*$', s) != None: - sdk_path = default_sdk_path - else: - sdk_path = s - if os.path.isdir(os.path.join(sdk_path, 'build-tools')): - break - else: - logging.info('"' + s + '" does not contain the Android SDK! Try again...') - if os.path.isdir(sdk_path): - write_to_config('sdk_path', sdk_path) - # try to find a working aapt, in all the recent possible paths - build_tools = os.path.join(sdk_path, 'build-tools') + build_tools = os.path.join(config['sdk_path'], 'build-tools') aaptdirs = [] aaptdirs.append(os.path.join(build_tools, config['build_tools'])) aaptdirs.append(build_tools) @@ -255,7 +254,7 @@ def main(): logging.info('Built repo based in "' + fdroiddir + '"') logging.info('with this config:') - logging.info(' Android SDK:\t\t\t' + sdk_path) + logging.info(' Android SDK:\t\t\t' + config['sdk_path']) logging.info(' Android SDK Build Tools:\t' + os.path.dirname(aapt)) logging.info(' Android NDK (optional):\t' + ndk_path) logging.info(' Keystore for signing key:\t' + keystore) diff --git a/tests/run-tests b/tests/run-tests index 09e7c94b..b4ff47f6 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -13,6 +13,22 @@ copy_apks_into_repo() { done } +create_fake_android_home() { + mkdir $1/build-tools + mkdir $1/build-tools/19.0.1 + touch $1/build-tools/19.0.1/aapt +} + +create_test_dir() { + test -e $WORKSPACE/.testfiles || mkdir $WORKSPACE/.testfiles + mktemp --directory --tmpdir=$WORKSPACE/.testfiles +} + +create_test_file() { + test -e $WORKSPACE/.testfiles || mkdir $WORKSPACE/.testfiles + mktemp --tmpdir=$WORKSPACE/.testfiles +} + if [ -z $WORKSPACE ]; then WORKSPACE=`dirname $(pwd)` echo "Setting Workspace to $WORKSPACE" @@ -24,9 +40,9 @@ if [ -z $fdroid ]; then fi #------------------------------------------------------------------------------# -# setup a new repo from scratch +echo "setup a new repo from scratch using ANDROID_HOME" -REPOROOT=`mktemp --directory --tmpdir=$WORKSPACE` +REPOROOT=`create_test_dir` cd $REPOROOT $fdroid init copy_apks_into_repo $REPOROOT @@ -35,9 +51,85 @@ $fdroid update #------------------------------------------------------------------------------# -# setup a new repo from scratch and generate a keystore +# check that --android-home fails when dir does not exist or is not a dir -REPOROOT=`mktemp --directory --tmpdir=$WORKSPACE` +REPOROOT=`create_test_dir` +KEYSTORE=$REPOROOT/keystore.jks +cd $REPOROOT +set +e +$fdroid init --keystore $KEYSTORE --android-home /opt/fakeandroidhome +if [ $? -eq 0 ]; then + echo "This should have failed because /opt/fakeandroidhome does not exist!" + exit 1 +else + echo "testing android-home path checker passed" +fi +TESTFILE=`create_test_file` +$fdroid init --keystore $KEYSTORE --android-home $TESTFILE +if [ $? -eq 0 ]; then + echo "This should have failed because $TESTFILE is a file not a dir!" + exit 1 +else + echo "testing android-home not-dir checker passed" +fi +set -e + + +#------------------------------------------------------------------------------# +echo "check that --android-home overrides ANDROID_HOME" + +REPOROOT=`create_test_dir` +FAKE_ANDROID_HOME=`create_test_dir` +create_fake_android_home $FAKE_ANDROID_HOME +KEYSTORE=$REPOROOT/keystore.jks +cd $REPOROOT +$fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME +set +e +grep $FAKE_ANDROID_HOME $REPOROOT/config.py +if [ $? -ne 0 ]; then + echo "the value set in --android-home '$FAKE_ANDROID_HOME' should override ANDROID_HOME '$ANDROID_HOME'" + exit 1 +fi +set -e + + +#------------------------------------------------------------------------------# +echo "setup a new repo from scratch with keystore and android-home set on cmd line" + +REPOROOT=`create_test_dir` +KEYSTORE=$REPOROOT/keystore.jks +FAKE_ANDROID_HOME=`create_test_dir` +create_fake_android_home $FAKE_ANDROID_HOME +STORED_ANDROID_HOME=$ANDROID_HOME +unset ANDROID_HOME +echo "ANDROID_HOME: $ANDROID_HOME" +cd $REPOROOT +$fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME +test -e $KEYSTORE +copy_apks_into_repo $REPOROOT +$fdroid update -c +$fdroid update +test -e repo/index.xml +test -e repo/index.jar +export ANDROID_HOME=$STORED_ANDROID_HOME + + +#------------------------------------------------------------------------------# +echo "setup new repo from scratch using ANDROID_HOME, putting APKs in repo first" + +REPOROOT=`create_test_dir` +cd $REPOROOT +mkdir repo +copy_apks_into_repo $REPOROOT +$fdroid init +$fdroid update -c +$fdroid update + + +#------------------------------------------------------------------------------# +echo "setup a new repo from scratch and generate a keystore" + +REPOROOT=`create_test_dir` KEYSTORE=$REPOROOT/keystore.jks cd $REPOROOT $fdroid init --keystore $KEYSTORE @@ -50,10 +142,13 @@ test -e repo/index.jar #------------------------------------------------------------------------------# -# setup a new repo from scratch with a HSM/smartcard +echo "setup a new repo from scratch with a HSM/smartcard" -REPOROOT=`mktemp --directory --tmpdir=$WORKSPACE` +REPOROOT=`create_test_dir` cd $REPOROOT $fdroid init --keystore NONE test -e opensc-fdroid.cfg test ! -e NONE + + +echo SUCCESS From ef7c9d89d2c07b0dfe3e63df8fa8378483b34d84 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 23 Apr 2014 19:22:26 -0400 Subject: [PATCH 4/7] init: --no-prompt to skip sdk_path prompt For running the tests and in other scripted setups, the user prompt is an annoying. Using --no-prompt means the script can test for failure. --- fdroidserver/init.py | 4 +++- tests/run-tests | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 390a7c7d..bf4a05fa 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -105,6 +105,8 @@ def main(): help="Alias of the repo signing key in the keystore") parser.add_option("--android-home", default=None, help="Path to the Android SDK (sometimes set in ANDROID_HOME)") + parser.add_option("--no-prompt", action="store_true", default=False, + help="Do not prompt for Android SDK path, just fail") (options, args) = parser.parse_args() # find root install prefix @@ -127,7 +129,7 @@ def main(): elif not common.test_sdk_exists(test_config): # if neither --android-home nor the default sdk_path exist, prompt the user default_sdk_path = '/opt/android-sdk' - while True: + while not options.no_prompt: s = raw_input('Enter the path to the Android SDK (' + default_sdk_path + ') here:\n> ') if re.match('^\s*$', s) != None: test_config['sdk_path'] = default_sdk_path diff --git a/tests/run-tests b/tests/run-tests index b4ff47f6..2d0ad5cd 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -104,7 +104,7 @@ STORED_ANDROID_HOME=$ANDROID_HOME unset ANDROID_HOME echo "ANDROID_HOME: $ANDROID_HOME" cd $REPOROOT -$fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME +$fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME --no-prompt test -e $KEYSTORE copy_apks_into_repo $REPOROOT $fdroid update -c From 21769e9f0af1730205268c870119bf8818850b8a Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 23 Apr 2014 19:52:37 -0400 Subject: [PATCH 5/7] server: test using config.get() in case dict value is None If a key 'foo' is set to None, `if config.get('foo'):` will be false while `if 'foo' in config:` will be true. A None value is not useful here, so config.get() is the better check. Thanks to Adam Pritchard for the suggestion. --- fdroidserver/server.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 72c75945..fd772fa8 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -42,7 +42,7 @@ def update_awsbucket(repo_section): from libcloud.storage.types import Provider, ContainerDoesNotExistError from libcloud.storage.providers import get_driver - if 'awsaccesskeyid' not in config or 'awssecretkey' not in config: + if not config.get('awsaccesskeyid') or not config.get('awssecretkey'): logging.error('To use awsbucket, you must set awssecretkey and awsaccesskeyid in config.py!') sys.exit(1) awsbucket = config['awsbucket'] @@ -154,12 +154,12 @@ def main(): logging.critical("The only commands currently supported are 'init' and 'update'") sys.exit(1) - if 'nonstandardwebroot' in config and config['nonstandardwebroot'] == True: + if config.get('nonstandardwebroot') == True: standardwebroot = False else: standardwebroot = True - if 'serverwebroot' in config: + if config.get('serverwebroot'): serverwebroot = config['serverwebroot'] host, fdroiddir = serverwebroot.rstrip('/').split(':') serverrepobase = os.path.basename(fdroiddir) @@ -169,7 +169,7 @@ def main(): + serverwebroot.rstrip('/') + '/fdroid\n\t' + serverwebroot.rstrip('/').rstrip(serverrepobase) + 'fdroid') sys.exit(1) - elif 'awsbucket' not in config: + elif not config.get('awsbucket'): logging.warn('No serverwebroot or awsbucket set! Edit your config.py to set one or both.') sys.exit(1) @@ -178,7 +178,7 @@ def main(): repo_sections.append('archive') if args[0] == 'init': - if serverwebroot != None: + if config.get('serverwebroot'): sshargs = ['ssh'] if options.quiet: sshargs += ['-q'] @@ -192,9 +192,9 @@ def main(): sys.exit(1) elif args[0] == 'update': for repo_section in repo_sections: - if 'serverwebroot' in config: + if config.get('serverwebroot'): update_serverwebroot(repo_section) - if 'awsbucket' in config: + if config.get('awsbucket'): update_awsbucket(repo_section) sys.exit(0) From a66bf2037c9a8e0f4a2faefa02e3959d063627bb Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 23 Apr 2014 19:55:53 -0400 Subject: [PATCH 6/7] server: fix logging of file upload to awsbucket Thanks to Adam Pritchard for reporting this --- fdroidserver/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/server.py b/fdroidserver/server.py index fd772fa8..0875d8dc 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -97,7 +97,7 @@ def update_awsbucket(repo_section): elif file_to_upload.endswith('.asc'): extra['content_type'] = 'application/pgp-signature' logging.info(' uploading ' + os.path.relpath(file_to_upload) - + ' to s3://' + awsbucket + '/' + obj.name) + + ' to s3://' + awsbucket + '/' + object_name) obj = driver.upload_object(file_path=file_to_upload, container=container, object_name=object_name, From 5e93b6c80b7227936b85c70a1dc58b26db9941fd Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 25 Apr 2014 21:07:47 -0400 Subject: [PATCH 7/7] update: report signing key fingerprint in same format as client fdroidclient now uses SHA256 fingerprints internally, and they are shown in the repo details view. This changes the digest algorithm to SHA256 and changes the format to match what is shown in the repo details view. --- fdroidserver/update.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 53c999a2..d2b02b7b 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -632,10 +632,9 @@ def make_index(apps, apks, repodir, archive, categories): # Generate a certificate fingerprint the same way keytool does it # (but with slightly different formatting) def cert_fingerprint(data): - digest = hashlib.sha1(data).digest() + digest = hashlib.sha256(data).digest() ret = [] - for i in range(4): - ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5])) + ret.append(' '.join("%02X" % ord(b) for b in digest)) return " ".join(ret) def extract_pubkey(): @@ -789,8 +788,8 @@ def make_index(apps, apks, repodir, archive, categories): if 'repo_keyalias' in config: - logging.info("Creating signed index.") - logging.info("Key fingerprint: %s" % repo_pubkey_fingerprint) + logging.info("Creating signed index with this key:") + logging.info("SHA256: %s" % repo_pubkey_fingerprint) #Create a jar of the index... p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)