Merge branch 'master' of https://gitlab.com/eighthave/fdroidserver

.gitignore

@@ -10,3 +10,4 @@ dist/
 env/
 fdroidserver.egg-info/
 pylint.parseable
+/.testfiles/

fdroidserver/common.py

@@ -34,6 +34,28 @@ import metadata
 config = None
 options = None
 
+def get_default_config():
+    return {
+        'sdk_path': os.getenv("ANDROID_HOME"),
+        'ndk_path': "$ANDROID_NDK",
+        'build_tools': "19.0.3",
+        'ant': "ant",
+        'mvn3': "mvn",
+        'gradle': 'gradle',
+        'archive_older': 0,
+        'update_stats': False,
+        'stats_to_carbon': False,
+        'repo_maxage': 0,
+        'build_server_always': False,
+        'keystore': '$HOME/.local/share/fdroidserver/keystore.jks',
+        'smartcardoptions': [],
+        'char_limits': {
+            'Summary' : 50,
+            'Description' : 1500
+        },
+        'keyaliases': { },
+    }
+
 def read_config(opts, config_file='config.py'):
     """Read the repository config
 
@@ -65,26 +87,7 @@ def read_config(opts, config_file='config.py'):
                                       'sun.security.pkcs11.SunPKCS11',
                                       '-providerArg', 'opensc-fdroid.cfg']
 
-    defconfig = {
+    defconfig = get_default_config()
-        'sdk_path': "$ANDROID_HOME",
-        'ndk_path': "$ANDROID_NDK",
-        'build_tools': "19.0.3",
-        'ant': "ant",
-        'mvn3': "mvn",
-        'gradle': 'gradle',
-        'archive_older': 0,
-        'update_stats': False,
-        'stats_to_carbon': False,
-        'repo_maxage': 0,
-        'build_server_always': False,
-        'keystore': '$HOME/.local/share/fdroidserver/keystore.jks',
-        'smartcardoptions': [],
-        'char_limits': {
-            'Summary' : 50,
-            'Description' : 1500
-        },
-        'keyaliases': { },
-    }
     for k, v in defconfig.items():
         if k not in config:
             config[k] = v
@@ -96,14 +99,7 @@ def read_config(opts, config_file='config.py'):
         v = os.path.expanduser(v)
         config[k] = os.path.expandvars(v)
 
-    if not config['sdk_path']:
+    if not test_sdk_exists(config):
-        logging.critical("Neither $ANDROID_HOME nor sdk_path is set, no Android SDK found!")
-        sys.exit(3)
-    if not os.path.exists(config['sdk_path']):
-        logging.critical('Android SDK path "' + config['sdk_path'] + '" does not exist!')
-        sys.exit(3)
-    if not os.path.isdir(config['sdk_path']):
-        logging.critical('Android SDK path "' + config['sdk_path'] + '" is not a directory!')
         sys.exit(3)
 
     if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
@@ -124,6 +120,24 @@ def read_config(opts, config_file='config.py'):
 
     return config
 
+def test_sdk_exists(c):
+    if c['sdk_path'] == None:
+        # c['sdk_path'] is set to the value of ANDROID_HOME by default
+        logging.critical('No Android SDK found! ANDROID_HOME is not set and sdk_path is not in config.py!')
+        logging.info('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
+        logging.info('\texport ANDROID_HOME=/opt/android-sdk')
+        return False
+    if not os.path.exists(c['sdk_path']):
+        logging.critical('Android SDK path "' + c['sdk_path'] + '" does not exist!')
+        return False
+    if not os.path.isdir(c['sdk_path']):
+        logging.critical('Android SDK path "' + c['sdk_path'] + '" is not a directory!')
+        return False
+    if not os.path.isdir(os.path.join(c['sdk_path'], 'build-tools')):
+        logging.critical('Android SDK path "' + c['sdk_path'] + '" does not contain "build-tools/"!')
+        return False
+    return True
+
 def write_password_file(pwtype, password=None):
     '''
     writes out passwords to a protected file instead of passing passwords as

fdroidserver/init.py

@@ -103,6 +103,10 @@ def main():
                       help="Path to the keystore for the repo signing key")
     parser.add_option("--repo-keyalias", default=None,
                       help="Alias of the repo signing key in the keystore")
+    parser.add_option("--android-home", default=None,
+                      help="Path to the Android SDK (sometimes set in ANDROID_HOME)")
+    parser.add_option("--no-prompt", action="store_true", default=False,
+                      help="Do not prompt for Android SDK path, just fail")
     (options, args) = parser.parse_args()
 
     # find root install prefix
@@ -116,13 +120,34 @@ def main():
         examplesdir = prefix + '/examples'
 
     fdroiddir = os.getcwd()
+    test_config = common.get_default_config()
 
-    if not os.path.exists('config.py') and not os.path.exists('repo'):
+    # track down where the Android SDK is, the default is to use the path set
+    # in ANDROID_HOME if that exists, otherwise None
+    if options.android_home != None:
+        test_config['sdk_path'] = options.android_home
+    elif not common.test_sdk_exists(test_config):
+        # if neither --android-home nor the default sdk_path exist, prompt the user
+        default_sdk_path = '/opt/android-sdk'
+        while not options.no_prompt:
+            s = raw_input('Enter the path to the Android SDK (' + default_sdk_path + ') here:\n> ')
+            if re.match('^\s*$', s) != None:
+                test_config['sdk_path'] = default_sdk_path
+            else:
+                test_config['sdk_path'] = s
+            if common.test_sdk_exists(test_config):
+                break
+    if not common.test_sdk_exists(test_config):
+        sys.exit(3)
+
+    if not os.path.exists('config.py'):
         # 'metadata' and 'tmp' are created in fdroid
-        os.mkdir('repo')
+        if not os.path.exists('repo'):
+            os.mkdir('repo')
         shutil.copy(os.path.join(examplesdir, 'fdroid-icon.png'), fdroiddir)
         shutil.copyfile(os.path.join(examplesdir, 'config.py'), 'config.py')
         os.chmod('config.py', 0o0600)
+        write_to_config('sdk_path', test_config['sdk_path'])
     else:
         logging.warn('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...')
         logging.info('Try running `fdroid init` in an empty directory.')
@@ -131,29 +156,8 @@ def main():
     # now that we have a local config.py, read configuration...
     config = common.read_config(options)
 
-    # track down where the Android SDK is
-    if os.path.isdir(config['sdk_path']):
-        logging.info('Using "' + config['sdk_path'] + '" for the Android SDK')
-        sdk_path = config['sdk_path']
-    elif 'ANDROID_HOME' in os.environ.keys():
-        sdk_path = os.environ['ANDROID_HOME']
-    else:
-        default_sdk_path = '/opt/android-sdk'
-        while True:
-            s = raw_input('Enter the path to the Android SDK (' + default_sdk_path + '): ')
-            if re.match('^\s*$', s) != None:
-                sdk_path = default_sdk_path
-            else:
-                sdk_path = s
-            if os.path.isdir(os.path.join(sdk_path, 'build-tools')):
-                break
-            else:
-                logging.info('"' + s + '" does not contain the Android SDK! Try again...')
-    if os.path.isdir(sdk_path):
-        write_to_config('sdk_path', sdk_path)
-
     # try to find a working aapt, in all the recent possible paths
-    build_tools = os.path.join(sdk_path, 'build-tools')
+    build_tools = os.path.join(config['sdk_path'], 'build-tools')
     aaptdirs = []
     aaptdirs.append(os.path.join(build_tools, config['build_tools']))
     aaptdirs.append(build_tools)
@@ -252,7 +256,7 @@ def main():
 
     logging.info('Built repo based in "' + fdroiddir + '"')
     logging.info('with this config:')
-    logging.info('  Android SDK:\t\t\t' + sdk_path)
+    logging.info('  Android SDK:\t\t\t' + config['sdk_path'])
     logging.info('  Android SDK Build Tools:\t' + os.path.dirname(aapt))
     logging.info('  Android NDK (optional):\t' + ndk_path)
     logging.info('  Keystore for signing key:\t' + keystore)

fdroidserver/server.py

@@ -42,7 +42,7 @@ def update_awsbucket(repo_section):
     from libcloud.storage.types import Provider, ContainerDoesNotExistError
     from libcloud.storage.providers import get_driver
 
-    if 'awsaccesskeyid' not in config or 'awssecretkey' not in config:
+    if not config.get('awsaccesskeyid') or not config.get('awssecretkey'):
         logging.error('To use awsbucket, you must set awssecretkey and awsaccesskeyid in config.py!')
         sys.exit(1)
     awsbucket = config['awsbucket']
@@ -97,7 +97,7 @@ def update_awsbucket(repo_section):
                 elif file_to_upload.endswith('.asc'):
                     extra['content_type'] = 'application/pgp-signature'
                 logging.info(' uploading ' + os.path.relpath(file_to_upload)
-                             + ' to s3://' + awsbucket + '/' + obj.name)
+                             + ' to s3://' + awsbucket + '/' + object_name)
                 obj = driver.upload_object(file_path=file_to_upload,
                                            container=container,
                                            object_name=object_name,
@@ -154,12 +154,12 @@ def main():
         logging.critical("The only commands currently supported are 'init' and 'update'")
         sys.exit(1)
 
-    if 'nonstandardwebroot' in config and config['nonstandardwebroot'] == True:
+    if config.get('nonstandardwebroot') == True:
         standardwebroot = False
     else:
         standardwebroot = True
 
-    if 'serverwebroot' in config:
+    if config.get('serverwebroot'):
         serverwebroot = config['serverwebroot']
         host, fdroiddir = serverwebroot.rstrip('/').split(':')
         serverrepobase = os.path.basename(fdroiddir)
@@ -169,7 +169,7 @@ def main():
                           + serverwebroot.rstrip('/') + '/fdroid\n\t'
                           + serverwebroot.rstrip('/').rstrip(serverrepobase) + 'fdroid')
             sys.exit(1)
-    elif 'awsbucket' not in config:
+    elif not config.get('awsbucket'):
         logging.warn('No serverwebroot or awsbucket set! Edit your config.py to set one or both.')
         sys.exit(1)
 
@@ -178,7 +178,7 @@ def main():
         repo_sections.append('archive')
 
     if args[0] == 'init':
-        if serverwebroot != None:
+        if config.get('serverwebroot'):
             sshargs = ['ssh']
             if options.quiet:
                 sshargs += ['-q']
@@ -192,9 +192,9 @@ def main():
                     sys.exit(1)
     elif args[0] == 'update':
         for repo_section in repo_sections:
-            if 'serverwebroot' in config:
+            if config.get('serverwebroot'):
                 update_serverwebroot(repo_section)
-            if 'awsbucket' in config:
+            if config.get('awsbucket'):
                 update_awsbucket(repo_section)
 
     sys.exit(0)

fdroidserver/update.py

@@ -632,10 +632,9 @@ def make_index(apps, apks, repodir, archive, categories):
         # Generate a certificate fingerprint the same way keytool does it
         # (but with slightly different formatting)
         def cert_fingerprint(data):
-            digest = hashlib.sha1(data).digest()
+            digest = hashlib.sha256(data).digest()
             ret = []
-            for i in range(4):
+            ret.append(' '.join("%02X" % ord(b) for b in digest))
-                ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
             return " ".join(ret)
 
         def extract_pubkey():
@@ -789,8 +788,8 @@ def make_index(apps, apks, repodir, archive, categories):
 
     if 'repo_keyalias' in config:
 
-        logging.info("Creating signed index.")
+        logging.info("Creating signed index with this key:")
-        logging.info("Key fingerprint: %s" % repo_pubkey_fingerprint)
+        logging.info("SHA256: %s" % repo_pubkey_fingerprint)
 
         #Create a jar of the index...
         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)

tests/run-tests

@@ -13,6 +13,22 @@ copy_apks_into_repo() {
     done
 }
 
+create_fake_android_home() {
+    mkdir $1/build-tools
+    mkdir $1/build-tools/19.0.1
+    touch $1/build-tools/19.0.1/aapt
+}
+
+create_test_dir() {
+    test -e $WORKSPACE/.testfiles || mkdir $WORKSPACE/.testfiles
+    mktemp --directory --tmpdir=$WORKSPACE/.testfiles
+}
+
+create_test_file() {
+    test -e $WORKSPACE/.testfiles || mkdir $WORKSPACE/.testfiles
+    mktemp --tmpdir=$WORKSPACE/.testfiles
+}
+
 if [ -z $WORKSPACE ]; then
     WORKSPACE=`dirname $(pwd)`
     echo "Setting Workspace to $WORKSPACE"
@@ -24,9 +40,9 @@ if [ -z $fdroid ]; then
 fi
 
 #------------------------------------------------------------------------------#
-# setup a new repo from scratch
+echo "setup a new repo from scratch using ANDROID_HOME"
 
-REPOROOT=`mktemp --directory --tmpdir=$WORKSPACE`
+REPOROOT=`create_test_dir`
 cd $REPOROOT
 $fdroid init
 copy_apks_into_repo $REPOROOT
@@ -35,9 +51,85 @@ $fdroid update
 
 
 #------------------------------------------------------------------------------#
-# setup a new repo from scratch and generate a keystore
+# check that --android-home fails when dir does not exist or is not a dir
 
-REPOROOT=`mktemp --directory --tmpdir=$WORKSPACE`
+REPOROOT=`create_test_dir`
+KEYSTORE=$REPOROOT/keystore.jks
+cd $REPOROOT
+set +e
+$fdroid init --keystore $KEYSTORE --android-home /opt/fakeandroidhome
+if [ $? -eq 0 ]; then
+    echo "This should have failed because /opt/fakeandroidhome does not exist!"
+    exit 1
+else
+    echo "testing android-home path checker passed"
+fi
+TESTFILE=`create_test_file`
+$fdroid init --keystore $KEYSTORE --android-home $TESTFILE
+if [ $? -eq 0 ]; then
+    echo "This should have failed because $TESTFILE is a file not a dir!"
+    exit 1
+else
+    echo "testing android-home not-dir checker passed"
+fi
+set -e
+
+
+#------------------------------------------------------------------------------#
+echo "check that --android-home overrides ANDROID_HOME"
+
+REPOROOT=`create_test_dir`
+FAKE_ANDROID_HOME=`create_test_dir`
+create_fake_android_home $FAKE_ANDROID_HOME
+KEYSTORE=$REPOROOT/keystore.jks
+cd $REPOROOT
+$fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME
+set +e
+grep $FAKE_ANDROID_HOME $REPOROOT/config.py
+if [ $? -ne 0 ]; then
+    echo "the value set in --android-home '$FAKE_ANDROID_HOME' should override ANDROID_HOME '$ANDROID_HOME'"
+    exit 1
+fi
+set -e
+
+
+#------------------------------------------------------------------------------#
+echo "setup a new repo from scratch with keystore and android-home set on cmd line"
+
+REPOROOT=`create_test_dir`
+KEYSTORE=$REPOROOT/keystore.jks
+FAKE_ANDROID_HOME=`create_test_dir`
+create_fake_android_home $FAKE_ANDROID_HOME
+STORED_ANDROID_HOME=$ANDROID_HOME
+unset ANDROID_HOME
+echo "ANDROID_HOME: $ANDROID_HOME"
+cd $REPOROOT
+$fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME --no-prompt
+test -e $KEYSTORE
+copy_apks_into_repo $REPOROOT
+$fdroid update -c
+$fdroid update
+test -e repo/index.xml
+test -e repo/index.jar
+export ANDROID_HOME=$STORED_ANDROID_HOME
+
+
+#------------------------------------------------------------------------------#
+echo "setup new repo from scratch using ANDROID_HOME, putting APKs in repo first"
+
+REPOROOT=`create_test_dir`
+cd $REPOROOT
+mkdir repo
+copy_apks_into_repo $REPOROOT
+$fdroid init
+$fdroid update -c
+$fdroid update
+
+
+#------------------------------------------------------------------------------#
+echo "setup a new repo from scratch and generate a keystore"
+
+REPOROOT=`create_test_dir`
 KEYSTORE=$REPOROOT/keystore.jks
 cd $REPOROOT
 $fdroid init --keystore $KEYSTORE
@@ -50,10 +142,13 @@ test -e repo/index.jar
 
 
 #------------------------------------------------------------------------------#
-# setup a new repo from scratch with a HSM/smartcard
+echo "setup a new repo from scratch with a HSM/smartcard"
 
-REPOROOT=`mktemp --directory --tmpdir=$WORKSPACE`
+REPOROOT=`create_test_dir`
 cd $REPOROOT
 $fdroid init --keystore NONE
 test -e opensc-fdroid.cfg
 test ! -e NONE
+
+
+echo SUCCESS