Merge branch 'test-updates-and-related-bug-fixes' into 'master'

Test updates and related bug fixes

I just set up some big tests of generating repos based on feeding as many random APKs into `fdroid update` as possible.  On our jenkins server, the tests copy all of the APKs that the jenkins server has generated and builds a repo from them.  This process caught lots of little glitches in the whole process.  While these little glitches are usually caused by problematic APKs, `fdroid update` should handle them gracefully.  Hopefully this set of fixes accomplishes that.

fdroidserver/common.py

@@ -144,6 +144,23 @@ def test_sdk_exists(c):
     return True
 
 
+def test_build_tools_exists(c):
+    if not test_sdk_exists(c):
+        return False
+    build_tools = os.path.join(c['sdk_path'], 'build-tools')
+    versioned_build_tools = os.path.join(build_tools, c['build_tools'])
+    if not os.path.isdir(versioned_build_tools):
+        logging.critical('Android Build Tools path "'
+                         + versioned_build_tools + '" does not exist!')
+        return False
+    if not os.path.exists(os.path.join(c['sdk_path'], 'build-tools', c['build_tools'], 'aapt')):
+        logging.critical('Android Build Tools "'
+                         + versioned_build_tools
+                         + '" does not contain "aapt"!')
+        return False
+    return True
+
+
 def write_password_file(pwtype, password=None):
     '''
     writes out passwords to a protected file instead of passing passwords as
@@ -1514,7 +1531,7 @@ def SilentPopen(commands, cwd=None, shell=False):
     return FDroidPopen(commands, cwd=cwd, shell=shell, output=False)
 
 
-def FDroidPopen(commands, cwd=None, shell=False, output=True):
+def FDroidPopen(commands, cwd=None, shell=False, output=False):
     """
     Run a command and capture the possibly huge output.
 
@@ -1541,7 +1558,7 @@ def FDroidPopen(commands, cwd=None, shell=False, output=True):
     while not stdout_reader.eof():
         while not stdout_queue.empty():
             line = stdout_queue.get()
-            if output and options.verbose:
+            if output or options.verbose:
                 # Output directly to console
                 sys.stdout.write(line)
                 sys.stdout.flush()

fdroidserver/init.py

@@ -132,7 +132,12 @@ def main():
         # if neither --android-home nor the default sdk_path exist, prompt the user
         default_sdk_path = '/opt/android-sdk'
         while not options.no_prompt:
-            s = raw_input('Enter the path to the Android SDK (' + default_sdk_path + ') here:\n> ')
+            try:
+                s = raw_input('Enter the path to the Android SDK ('
+                              + default_sdk_path + ') here:\n> ')
+            except KeyboardInterrupt:
+                print('')
+                sys.exit(1)
             if re.match('^\s*$', s) is not None:
                 test_config['sdk_path'] = default_sdk_path
             else:
@@ -155,18 +160,15 @@ def main():
         logging.info('Try running `fdroid init` in an empty directory.')
         sys.exit()
 
-    # now that we have a local config.py, read configuration...
-    config = common.read_config(options)
-
     # try to find a working aapt, in all the recent possible paths
-    build_tools = os.path.join(config['sdk_path'], 'build-tools')
+    build_tools = os.path.join(test_config['sdk_path'], 'build-tools')
     aaptdirs = []
-    aaptdirs.append(os.path.join(build_tools, config['build_tools']))
+    aaptdirs.append(os.path.join(build_tools, test_config['build_tools']))
     aaptdirs.append(build_tools)
-    for f in sorted(os.listdir(build_tools), reverse=True):
+    for f in os.listdir(build_tools):
         if os.path.isdir(os.path.join(build_tools, f)):
             aaptdirs.append(os.path.join(build_tools, f))
-    for d in aaptdirs:
+    for d in sorted(aaptdirs, reverse=True):
         if os.path.isfile(os.path.join(d, 'aapt')):
             aapt = os.path.join(d, 'aapt')
             break
@@ -174,9 +176,15 @@ def main():
         dirname = os.path.basename(os.path.dirname(aapt))
         if dirname == 'build-tools':
             # this is the old layout, before versioned build-tools
-            write_to_config('build_tools', '')
+            test_config['build_tools'] = ''
         else:
-            write_to_config('build_tools', dirname)
+            test_config['build_tools'] = dirname
+        write_to_config('build_tools', test_config['build_tools'])
+    if not common.test_build_tools_exists(test_config):
+        sys.exit(3)
+
+    # now that we have a local config.py, read configuration...
+    config = common.read_config(options)
 
     # track down where the Android NDK is
     ndk_path = '/opt/android-ndk'

fdroidserver/update.py

@@ -301,9 +301,6 @@ def resize_icon(iconpath, density):
                 iconpath, oldsize, im.size))
             im.save(iconpath, "PNG")
 
-        else:
-            logging.info("%s is small enough: %s" % im.size)
-
     except Exception, e:
         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 
@@ -366,8 +363,8 @@ def scan_apks(apps, apkcache, repodir, knownapks):
             thisinfo = apkcache[apkfilename]
 
         else:
-
-            logging.info("Processing " + apkfilename)
+            if options.verbose:
+                logging.info("Processing " + apkfilename)
             thisinfo = {}
             thisinfo['apkname'] = apkfilename
             srcfilename = apkfilename[:-4] + "_src.tar.gz"
@@ -382,8 +379,15 @@ def scan_apks(apps, apkcache, repodir, knownapks):
                                           config['build_tools'], 'aapt'),
                              'dump', 'badging', apkfile])
             if p.returncode != 0:
-                logging.critical("Failed to get apk information")
-                sys.exit(1)
+                if options.delete_unknown:
+                    if os.path.exists(apkfile):
+                        logging.error("Failed to get apk information, deleting " + apkfile)
+                        os.remove(apkfile)
+                    else:
+                        logging.error("Could not find {0} to remove it".format(apkfile))
+                else:
+                    logging.error("Failed to get apk information, skipping " + apkfile)
+                continue
             for line in p.stdout.splitlines():
                 if line.startswith("package:"):
                     try:
@@ -415,7 +419,12 @@ def scan_apks(apps, apkcache, repodir, knownapks):
                         path = match.group(2)
                         thisinfo['icons_src'][density] = path
                 elif line.startswith("sdkVersion:"):
-                    thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
+                    m = re.match(sdkversion_pat, line)
+                    if m is None:
+                        logging.error(line.replace('sdkVersion:', '')
+                                      + ' is not a valid minSdkVersion!')
+                    else:
+                        thisinfo['sdkversion'] = m.group(1)
                 elif line.startswith("maxSdkVersion:"):
                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
                 elif line.startswith("native-code:"):
@@ -443,7 +452,7 @@ def scan_apks(apps, apkcache, repodir, knownapks):
 
             # Check for debuggable apks...
             if common.isApkDebuggable(apkfile, config):
-                logging.warn("{0} is debuggable... {1}".format(apkfile, line))
+                logging.warn('{0} is set to android:debuggable="true"!'.format(apkfile))
 
             # Calculate the sha256...
             sha = hashlib.sha256()
@@ -804,8 +813,8 @@ def make_index(apps, apks, repodir, archive, categories):
 
     if 'repo_keyalias' in config:
 
-        logging.info("Creating signed index with this key:")
-        logging.info("SHA256: %s" % repo_pubkey_fingerprint)
+        logging.info("Creating signed index with this key (SHA256):")
+        logging.info("%s" % repo_pubkey_fingerprint)
 
         # Create a jar of the index...
         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
@@ -882,7 +891,7 @@ def main():
 
     # Parse command line...
     parser = OptionParser()
-    parser.add_option("-c", "--createmeta", action="store_true", default=False,
+    parser.add_option("-c", "--create-metadata", action="store_true", default=False,
                       help="Create skeleton metadata files that are missing")
     parser.add_option("--delete-unknown", action="store_true", default=False,
                       help="Delete APKs without metadata from the repo")
@@ -957,7 +966,10 @@ def main():
                 found = True
                 break
         if not found:
-            if options.createmeta:
+            if options.create_metadata:
+                if 'name' not in apk:
+                    logging.error(apk['id'] + ' does not have a name! Skipping...')
+                    continue
                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
                 f.write("License:Unknown\n")
                 f.write("Web Site:\n")