diff --git a/fdroidserver/update.py b/fdroidserver/update.py index abff8b65..dba1a409 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -419,6 +419,82 @@ def get_icon_bytes(apkzip, iconsrc): return apkzip.read(iconsrc.encode('utf-8').decode('cp437')) +def sha256sum(filename): + '''Calculate the sha256 of the given file''' + sha = hashlib.sha256() + with open(filename, 'rb') as f: + while True: + t = f.read(16384) + if len(t) == 0: + break + sha.update(t) + return sha.hexdigest() + + +def insert_obbs(repodir, apps, apks): + """Scans the .obb files in a given repo directory and adds them to the + relevant APK instances. OBB files have versionCodes like APK + files, and they are loosely associated. If there is an OBB file + present, then any APK with the same or higher versionCode will use + that OBB file. There are two OBB types: main and patch, each APK + can only have only have one of each. + + https://developer.android.com/google/play/expansion-files.html + + :param repodir: repo directory to scan + :param apps: list of current, valid apps + :param apks: current information on all APKs + + """ + + def obbWarnDelete(f, msg): + logging.warning(msg + f) + if options.delete_unknown: + logging.error("Deleting unknown file: " + f) + os.remove(f) + + obbs = [] + java_Integer_MIN_VALUE = -pow(2, 31) + for f in glob.glob(os.path.join(repodir, '*.obb')): + obbfile = os.path.basename(f) + # obbfile looks like: [main|patch]...obb + chunks = obbfile.split('.') + if chunks[0] != 'main' and chunks[0] != 'patch': + obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ') + continue + if not re.match(r'^-?[0-9]+$', chunks[1]): + obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ') + continue + versioncode = int(chunks[1]) + packagename = ".".join(chunks[2:-1]) + + highestVersionCode = java_Integer_MIN_VALUE + if packagename not in apps.keys(): + obbWarnDelete(f, "OBB's packagename does not match a supported APK: ") + continue + for apk in apks: + if packagename == apk['id'] and apk['versioncode'] > highestVersionCode: + highestVersionCode = apk['versioncode'] + if versioncode > highestVersionCode: + obbWarnDelete(f, 'OBB file has newer versioncode(' + str(versioncode) + + ') than any APK: ') + continue + obbsha256 = sha256sum(f) + obbs.append((packagename, versioncode, obbfile, obbsha256)) + + for apk in apks: + for (packagename, versioncode, obbfile, obbsha256) in sorted(obbs, reverse=True): + if versioncode <= apk['versioncode'] and packagename == apk['id']: + if obbfile.startswith('main.') and 'obbMainFile' not in apk: + apk['obbMainFile'] = obbfile + apk['obbMainFileSha256'] = obbsha256 + elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk: + apk['obbPatchFile'] = obbfile + apk['obbPatchFileSha256'] = obbsha256 + if 'obbMainFile' in apk and 'obbPatchFile' in apk: + break + + def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False): """Scan the apks in the given repo directory. @@ -460,15 +536,7 @@ def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False): logging.critical("Spaces in filenames are not allowed.") sys.exit(1) - # Calculate the sha256... - sha = hashlib.sha256() - with open(apkfile, 'rb') as f: - while True: - t = f.read(16384) - if len(t) == 0: - break - sha.update(t) - shasum = sha.hexdigest() + shasum = sha256sum(apkfile) usecache = False if apkfilename in apkcache: @@ -971,6 +1039,10 @@ def make_index(apps, sortedids, apks, repodir, archive, categories): addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel) if 'maxSdkVersion' in apk: addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel) + addElementNonEmpty('obbMainFile', apk.get('obbMainFile'), doc, apkel) + addElementNonEmpty('obbMainFileSha256', apk.get('obbMainFileSha256'), doc, apkel) + addElementNonEmpty('obbPatchFile', apk.get('obbPatchFile'), doc, apkel) + addElementNonEmpty('obbPatchFileSha256', apk.get('obbPatchFileSha256'), doc, apkel) if 'added' in apk: addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel) addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel) @@ -1156,7 +1228,7 @@ def main(): parser.add_argument("-c", "--create-metadata", action="store_true", default=False, help="Create skeleton metadata files that are missing") parser.add_argument("--delete-unknown", action="store_true", default=False, - help="Delete APKs without metadata from the repo") + help="Delete APKs and/or OBBs without metadata from the repo") parser.add_argument("-b", "--buildreport", action="store_true", default=False, help="Report on build data status") parser.add_argument("-i", "--interactive", default=False, action="store_true", @@ -1292,6 +1364,8 @@ def main(): if newmetadata: apps = metadata.read_metadata() + insert_obbs(repodirs[0], apps, apks) + # Scan the archive repo for apks as well if len(repodirs) > 1: archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk) diff --git a/tests/metadata/obb.main.oldversion.txt b/tests/metadata/obb.main.oldversion.txt new file mode 100644 index 00000000..56c4a9f5 --- /dev/null +++ b/tests/metadata/obb.main.oldversion.txt @@ -0,0 +1,12 @@ +Categories:Development +License:GPLv3 +Source Code:https://github.com/eighthave/urzip +Bitcoin:1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk + +Auto Name:OBB Main Old Version + +Repo Type:git +Repo:https://github.com/eighthave/urzip.git + + +Current Version Code:99999999 diff --git a/tests/metadata/obb.main.twoversions.txt b/tests/metadata/obb.main.twoversions.txt new file mode 100644 index 00000000..d06afa36 --- /dev/null +++ b/tests/metadata/obb.main.twoversions.txt @@ -0,0 +1,12 @@ +Categories:Development +License:GPLv3 +Source Code:https://github.com/eighthave/urzip +Bitcoin:1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk + +Auto Name:OBB Two Versions + +Repo Type:git +Repo:https://github.com/eighthave/urzip.git + + +Current Version Code:99999999 diff --git a/tests/metadata/obb.mainpatch.current.txt b/tests/metadata/obb.mainpatch.current.txt new file mode 100644 index 00000000..2f7571f5 --- /dev/null +++ b/tests/metadata/obb.mainpatch.current.txt @@ -0,0 +1,12 @@ +Categories:Development +License:GPLv3 +Source Code:https://github.com/eighthave/urzip +Bitcoin:1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk + +Auto Name:OBB Main+Patch Current Version + +Repo Type:git +Repo:https://github.com/eighthave/urzip.git + + +Current Version Code:99999999 diff --git a/tests/repo/main.1101613.obb.main.twoversions.obb b/tests/repo/main.1101613.obb.main.twoversions.obb new file mode 100644 index 00000000..421376db --- /dev/null +++ b/tests/repo/main.1101613.obb.main.twoversions.obb @@ -0,0 +1 @@ +dummy diff --git a/tests/repo/main.1101615.obb.main.twoversions.obb b/tests/repo/main.1101615.obb.main.twoversions.obb new file mode 100644 index 00000000..421376db --- /dev/null +++ b/tests/repo/main.1101615.obb.main.twoversions.obb @@ -0,0 +1 @@ +dummy diff --git a/tests/repo/main.1434483388.obb.main.oldversion.obb b/tests/repo/main.1434483388.obb.main.oldversion.obb new file mode 100644 index 00000000..421376db --- /dev/null +++ b/tests/repo/main.1434483388.obb.main.oldversion.obb @@ -0,0 +1 @@ +dummy diff --git a/tests/repo/main.1619.obb.mainpatch.current.obb b/tests/repo/main.1619.obb.mainpatch.current.obb new file mode 100644 index 00000000..421376db --- /dev/null +++ b/tests/repo/main.1619.obb.mainpatch.current.obb @@ -0,0 +1 @@ +dummy diff --git a/tests/repo/obb.main.oldversion_1444412523.apk b/tests/repo/obb.main.oldversion_1444412523.apk new file mode 100644 index 00000000..94ed9503 Binary files /dev/null and b/tests/repo/obb.main.oldversion_1444412523.apk differ diff --git a/tests/repo/obb.main.twoversions_1101613.apk b/tests/repo/obb.main.twoversions_1101613.apk new file mode 100644 index 00000000..259d0903 Binary files /dev/null and b/tests/repo/obb.main.twoversions_1101613.apk differ diff --git a/tests/repo/obb.main.twoversions_1101615.apk b/tests/repo/obb.main.twoversions_1101615.apk new file mode 100644 index 00000000..0d82052e Binary files /dev/null and b/tests/repo/obb.main.twoversions_1101615.apk differ diff --git a/tests/repo/obb.main.twoversions_1101617.apk b/tests/repo/obb.main.twoversions_1101617.apk new file mode 100644 index 00000000..202d6a03 Binary files /dev/null and b/tests/repo/obb.main.twoversions_1101617.apk differ diff --git a/tests/repo/obb.mainpatch.current_1619.apk b/tests/repo/obb.mainpatch.current_1619.apk new file mode 100644 index 00000000..23cf8232 Binary files /dev/null and b/tests/repo/obb.mainpatch.current_1619.apk differ diff --git a/tests/repo/patch.1619.obb.mainpatch.current.obb b/tests/repo/patch.1619.obb.mainpatch.current.obb new file mode 100644 index 00000000..421376db --- /dev/null +++ b/tests/repo/patch.1619.obb.mainpatch.current.obb @@ -0,0 +1 @@ +dummy diff --git a/tests/update.TestCase b/tests/update.TestCase index 1005a158..e2927932 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -83,7 +83,7 @@ class UpdateTest(unittest.TestCase): pysig = fdroidserver.update.getsig(apkfile) self.assertIsNone(pysig, "python sig should be None: " + str(sig)) - def testScanApks(self): + def testScanApksAndObbs(self): os.chdir(os.path.dirname(__file__)) if os.path.basename(os.getcwd()) != 'tests': raise Exception('This test must be run in the "tests/" subdir') @@ -97,18 +97,39 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.options = type('', (), {})() fdroidserver.update.options.clean = True + fdroidserver.update.options.delete_unknown = True - alltestapps = fdroidserver.metadata.read_metadata(xref=True) - apps = dict() - apps['info.guardianproject.urzip'] = alltestapps['info.guardianproject.urzip'] + apps = fdroidserver.metadata.read_metadata(xref=True) knownapks = fdroidserver.common.KnownApks() apks, cachechanged = fdroidserver.update.scan_apks(apps, {}, 'repo', knownapks, False) - self.assertEqual(len(apks), 1) + self.assertEqual(len(apks), 6) apk = apks[0] self.assertEqual(apk['minSdkVersion'], '4') self.assertEqual(apk['targetSdkVersion'], '18') self.assertFalse('maxSdkVersion' in apk) + fdroidserver.update.insert_obbs('repo', apps, apks) + for apk in apks: + if apk['id'] == 'obb.mainpatch.current': + self.assertEqual(apk.get('obbMainFile'), 'main.1619.obb.mainpatch.current.obb') + self.assertEqual(apk.get('obbPatchFile'), 'patch.1619.obb.mainpatch.current.obb') + elif apk['id'] == 'obb.main.oldversion': + self.assertEqual(apk.get('obbMainFile'), 'main.1434483388.obb.main.oldversion.obb') + self.assertIsNone(apk.get('obbPatchFile')) + elif apk['id'] == 'obb.main.twoversions': + self.assertIsNone(apk.get('obbPatchFile')) + if apk['versioncode'] == 1101613: + self.assertEqual(apk.get('obbMainFile'), 'main.1101613.obb.main.twoversions.obb') + elif apk['versioncode'] == 1101615: + self.assertEqual(apk.get('obbMainFile'), 'main.1101615.obb.main.twoversions.obb') + elif apk['versioncode'] == 1101617: + self.assertEqual(apk.get('obbMainFile'), 'main.1101615.obb.main.twoversions.obb') + else: + self.assertTrue(False) + elif apk['id'] == 'info.guardianproject.urzip': + self.assertIsNone(apk.get('obbMainFile')) + self.assertIsNone(apk.get('obbPatchFile')) + if __name__ == "__main__": parser = optparse.OptionParser()