Merge branch 'apk-extension-obb-support' into 'master'

support APK Extension OBB files

Google Play specifies OBB aka "APK Extension" files for apps that need more than 100 MBs, which is the Play APK size limit.  They also provide a mechanism to deliver large data blobs that do not need to be part of the APK.  For example, a game's assets do not need to change often, so they can be shipped as an OBB, then APK updates do not need to include all those assets for each update.
    
https://developer.android.com/google/play/expansion-files.html

See merge request !143

fdroidserver/update.py

@@ -419,6 +419,82 @@ def get_icon_bytes(apkzip, iconsrc):
         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 
 
+def sha256sum(filename):
+    '''Calculate the sha256 of the given file'''
+    sha = hashlib.sha256()
+    with open(filename, 'rb') as f:
+        while True:
+            t = f.read(16384)
+            if len(t) == 0:
+                break
+            sha.update(t)
+    return sha.hexdigest()
+
+
+def insert_obbs(repodir, apps, apks):
+    """Scans the .obb files in a given repo directory and adds them to the
+    relevant APK instances.  OBB files have versionCodes like APK
+    files, and they are loosely associated.  If there is an OBB file
+    present, then any APK with the same or higher versionCode will use
+    that OBB file.  There are two OBB types: main and patch, each APK
+    can only have only have one of each.
+
+    https://developer.android.com/google/play/expansion-files.html
+
+    :param repodir: repo directory to scan
+    :param apps: list of current, valid apps
+    :param apks: current information on all APKs
+
+    """
+
+    def obbWarnDelete(f, msg):
+        logging.warning(msg + f)
+        if options.delete_unknown:
+            logging.error("Deleting unknown file: " + f)
+            os.remove(f)
+
+    obbs = []
+    java_Integer_MIN_VALUE = -pow(2, 31)
+    for f in glob.glob(os.path.join(repodir, '*.obb')):
+        obbfile = os.path.basename(f)
+        # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
+        chunks = obbfile.split('.')
+        if chunks[0] != 'main' and chunks[0] != 'patch':
+            obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
+            continue
+        if not re.match(r'^-?[0-9]+$', chunks[1]):
+            obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
+            continue
+        versioncode = int(chunks[1])
+        packagename = ".".join(chunks[2:-1])
+
+        highestVersionCode = java_Integer_MIN_VALUE
+        if packagename not in apps.keys():
+            obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
+            continue
+        for apk in apks:
+            if packagename == apk['id'] and apk['versioncode'] > highestVersionCode:
+                highestVersionCode = apk['versioncode']
+        if versioncode > highestVersionCode:
+            obbWarnDelete(f, 'OBB file has newer versioncode(' + str(versioncode)
+                          + ') than any APK: ')
+            continue
+        obbsha256 = sha256sum(f)
+        obbs.append((packagename, versioncode, obbfile, obbsha256))
+
+    for apk in apks:
+        for (packagename, versioncode, obbfile, obbsha256) in sorted(obbs, reverse=True):
+            if versioncode <= apk['versioncode'] and packagename == apk['id']:
+                if obbfile.startswith('main.') and 'obbMainFile' not in apk:
+                    apk['obbMainFile'] = obbfile
+                    apk['obbMainFileSha256'] = obbsha256
+                elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
+                    apk['obbPatchFile'] = obbfile
+                    apk['obbPatchFileSha256'] = obbsha256
+            if 'obbMainFile' in apk and 'obbPatchFile' in apk:
+                break
+
+
 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
     """Scan the apks in the given repo directory.
 
@@ -460,15 +536,7 @@ def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
             logging.critical("Spaces in filenames are not allowed.")
             sys.exit(1)
 
-        # Calculate the sha256...
+        shasum = sha256sum(apkfile)
-        sha = hashlib.sha256()
-        with open(apkfile, 'rb') as f:
-            while True:
-                t = f.read(16384)
-                if len(t) == 0:
-                    break
-                sha.update(t)
-            shasum = sha.hexdigest()
 
         usecache = False
         if apkfilename in apkcache:
@@ -971,6 +1039,10 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
                 addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
             if 'maxSdkVersion' in apk:
                 addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
+            addElementNonEmpty('obbMainFile', apk.get('obbMainFile'), doc, apkel)
+            addElementNonEmpty('obbMainFileSha256', apk.get('obbMainFileSha256'), doc, apkel)
+            addElementNonEmpty('obbPatchFile', apk.get('obbPatchFile'), doc, apkel)
+            addElementNonEmpty('obbPatchFileSha256', apk.get('obbPatchFileSha256'), doc, apkel)
             if 'added' in apk:
                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
@@ -1156,7 +1228,7 @@ def main():
     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
                         help="Create skeleton metadata files that are missing")
     parser.add_argument("--delete-unknown", action="store_true", default=False,
-                        help="Delete APKs without metadata from the repo")
+                        help="Delete APKs and/or OBBs without metadata from the repo")
     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
                         help="Report on build data status")
     parser.add_argument("-i", "--interactive", default=False, action="store_true",
@@ -1292,6 +1364,8 @@ def main():
     if newmetadata:
         apps = metadata.read_metadata()
 
+    insert_obbs(repodirs[0], apps, apks)
+
     # Scan the archive repo for apks as well
     if len(repodirs) > 1:
         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)

tests/metadata/obb.main.oldversion.txt

@@ -0,0 +1,12 @@
+Categories:Development
+License:GPLv3
+Source Code:https://github.com/eighthave/urzip
+Bitcoin:1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk
+
+Auto Name:OBB Main Old Version
+
+Repo Type:git
+Repo:https://github.com/eighthave/urzip.git
+
+
+Current Version Code:99999999

tests/metadata/obb.main.twoversions.txt

@@ -0,0 +1,12 @@
+Categories:Development
+License:GPLv3
+Source Code:https://github.com/eighthave/urzip
+Bitcoin:1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk
+
+Auto Name:OBB Two Versions
+
+Repo Type:git
+Repo:https://github.com/eighthave/urzip.git
+
+
+Current Version Code:99999999

tests/metadata/obb.mainpatch.current.txt

@@ -0,0 +1,12 @@
+Categories:Development
+License:GPLv3
+Source Code:https://github.com/eighthave/urzip
+Bitcoin:1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk
+
+Auto Name:OBB Main+Patch Current Version
+
+Repo Type:git
+Repo:https://github.com/eighthave/urzip.git
+
+
+Current Version Code:99999999

tests/repo/main.1101613.obb.main.twoversions.obb

@@ -0,0 +1 @@
+dummy

tests/repo/main.1101615.obb.main.twoversions.obb

@@ -0,0 +1 @@
+dummy

tests/repo/main.1434483388.obb.main.oldversion.obb

@@ -0,0 +1 @@
+dummy

tests/repo/main.1619.obb.mainpatch.current.obb

@@ -0,0 +1 @@
+dummy

tests/repo/patch.1619.obb.mainpatch.current.obb

@@ -0,0 +1 @@
+dummy

tests/update.TestCase

@@ -83,7 +83,7 @@ class UpdateTest(unittest.TestCase):
         pysig = fdroidserver.update.getsig(apkfile)
         self.assertIsNone(pysig, "python sig should be None: " + str(sig))
 
-    def testScanApks(self):
+    def testScanApksAndObbs(self):
         os.chdir(os.path.dirname(__file__))
         if os.path.basename(os.getcwd()) != 'tests':
             raise Exception('This test must be run in the "tests/" subdir')
@@ -97,18 +97,39 @@ class UpdateTest(unittest.TestCase):
 
         fdroidserver.update.options = type('', (), {})()
         fdroidserver.update.options.clean = True
+        fdroidserver.update.options.delete_unknown = True
 
-        alltestapps = fdroidserver.metadata.read_metadata(xref=True)
+        apps = fdroidserver.metadata.read_metadata(xref=True)
-        apps = dict()
-        apps['info.guardianproject.urzip'] = alltestapps['info.guardianproject.urzip']
         knownapks = fdroidserver.common.KnownApks()
         apks, cachechanged = fdroidserver.update.scan_apks(apps, {}, 'repo', knownapks, False)
-        self.assertEqual(len(apks), 1)
+        self.assertEqual(len(apks), 6)
         apk = apks[0]
         self.assertEqual(apk['minSdkVersion'], '4')
         self.assertEqual(apk['targetSdkVersion'], '18')
         self.assertFalse('maxSdkVersion' in apk)
 
+        fdroidserver.update.insert_obbs('repo', apps, apks)
+        for apk in apks:
+            if apk['id'] == 'obb.mainpatch.current':
+                self.assertEqual(apk.get('obbMainFile'), 'main.1619.obb.mainpatch.current.obb')
+                self.assertEqual(apk.get('obbPatchFile'), 'patch.1619.obb.mainpatch.current.obb')
+            elif apk['id'] == 'obb.main.oldversion':
+                self.assertEqual(apk.get('obbMainFile'), 'main.1434483388.obb.main.oldversion.obb')
+                self.assertIsNone(apk.get('obbPatchFile'))
+            elif apk['id'] == 'obb.main.twoversions':
+                self.assertIsNone(apk.get('obbPatchFile'))
+                if apk['versioncode'] == 1101613:
+                    self.assertEqual(apk.get('obbMainFile'), 'main.1101613.obb.main.twoversions.obb')
+                elif apk['versioncode'] == 1101615:
+                    self.assertEqual(apk.get('obbMainFile'), 'main.1101615.obb.main.twoversions.obb')
+                elif apk['versioncode'] == 1101617:
+                    self.assertEqual(apk.get('obbMainFile'), 'main.1101615.obb.main.twoversions.obb')
+                else:
+                    self.assertTrue(False)
+            elif apk['id'] == 'info.guardianproject.urzip':
+                self.assertIsNone(apk.get('obbMainFile'))
+                self.assertIsNone(apk.get('obbPatchFile'))
+
 
 if __name__ == "__main__":
     parser = optparse.OptionParser()