mirrors/fdroidserver
1
0
Fork 0
mirror of https://github.com/f-droid/fdroidserver.git synced 2025-11-14 11:10:30 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

convert apkcache from pickle to JSON

Browse source 
pickle can serialize executable code, while JSON is only ever pure data.
The APK cache is only ever pure data, so no need for the security risks of
pickle.  For example, if some malicious thing gets write access on the
`fdroid update` machine, it can write out a custom tmp/apkcache which would
then be executed.  That is not possible with JSON.

This does just ignore any existing cache and rebuilds from scratch. That is
so we don't need to maintain pickle anywhere, and to ensure there are no
glitches from a conversion from pickle to JSON.

closes #163
This commit is contained in:
Hans-Christoph Steiner 2018-09-03 18:07:40 +02:00
parent 74776e026f
commit 3011953d0e
6 changed files with 97 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

2
tests/repo/index-v1.json
View file

@ -1,7 +1,7 @@
{
"repo": {
"timestamp": 1502845383782,
"version": 19,
"version": 20,
"name": "My First F-Droid Repo Demo",
"icon": "fdroid-icon.png",
"address": "https://MyFirstFDroidRepo.org/fdroid/repo",

2
tests/repo/index.xml
View file

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<fdroid>
<repo icon="fdroid-icon.png" name="My First F-Droid Repo Demo" pubkey="308204e1308202c9a003020102020434597643300d06092a864886f70d01010b050030213110300e060355040b1307462d44726f6964310d300b06035504031304736f7661301e170d3136303931333230313930395a170d3434303133303230313930395a30213110300e060355040b1307462d44726f6964310d300b06035504031304736f766130820222300d06092a864886f70d01010105000382020f003082020a028202010086ef94b5aacf2ba4f38c875f4194b44f5644392e3715575d7c92828577e692c352b567172823851c8c72347fbc9d99684cd7ca3e1db3e4cca126382c53f2a5869fb4c19bdec989b2930501af3e758ff40588915fe96b10076ce3346a193a0277d79e83e30fd8657c20e35260dd085aa32eac7c4b85786ffefbf1555cafe2bc928443430cdbba48cfbe701e12ae86e676477932730d4fc7c00af820aef85038a5b4df084cf6470d110dc4c49ea1b749b80b34709d199b3db516b223625c5de4501e861f7d261b3838f8f616aa78831d618d41d25872dc810c9b2087b5a9e146ca95be740316dcdbcb77314e23ab87d4487913b800b1113c0603ea2294188b71d3e49875df097b56f9151211fc6832f9790c5c83d17481f14ad37915fd164f4fd713f6732a15f4245714b84cd665bdbd085660ea33ad7d7095dcc414f09e3903604a40facc2314a115c0045bb50e9df38efb57e1b8e7cc105f340a26eeb46aba0fa6672953eee7f1f92dcb408e561909bbd4bdf4a4948c4d57c467d21aa238c34ba43be050398be963191fa2b49828bc1e4eeed224b40dbe9dc3e570890a71a974a2f4527edb1b07105071755105edcb2af2f269facfb89180903a572a99b46456e80d4a01685a80b233278805f2c876678e731f4ec4f52075aeef6b2b023efbb8a3637ef507c4c37c27e428152ec1817fcba640ad601cb09f72f0fbe2d274a2410203010001a321301f301d0603551d0e04160414c28bf33dd5a9a17338e5b1d1a6edd8c7d141ed0b300d06092a864886f70d01010b0500038202010084e20458b2aafd7fc27146b0986f9324f4260f244920417a77c9bf15e2e2d22d2725bdd8093ec261c3779c3ca03312516506f9410075b90595b41345956d8eb2786fb5994f195611382c2b99dba13381b0100a30bc9e6e47248bf4325e2f6eec9d789216dc7536e753bf1f4be603d9fa2e6f5e192b4eb988b8cdb0bb1e8668a9225426f7d4636479f73ed24ad1d2657c31e63c93d9679b9080171b3bd1bf10a3b92b80bd790fbf62d3644900cd08eae8b9bf9c2567be98dc8cdd2ae19a8d57a3e3e2de899f81f1279f578989e6af906f80c8c2b67651730ee7e568c1af5bcb845b6d685dc55332a9984aeceaea3b7e883447edf1c76b155d95253e39b9710eaa22efa6c81468829702b5dce7126538f3ca70c2f0ad9a5795435fdb1f715f20d60359ef9a9926c7050116e802df651727447848827815f70bd82af3cedd08783156102d2d8ce995c4c43b8e47e91a3e6927f3505a5d395e6bebb84542c570903eeab4382a1c2151f1471c7a06a34dc4d268d8fa72e93bdcd2dccc4302ecac47b9e7e3d8bc9b46d21cd097874a24d529548018dc190ff568c6aa428f0a5eedff1a347730931c74f19277538e49647a4ad7254f4c1ec7d4da12cce9e1fad9607534e66ab40a56b473d9d7e3d563fd03cad2052bad365c5a29f8ae54f09b60dbca3ea768d7767cbe1c133ca08ce725c1c1370f4aab8e5b6e286f52dc0be8d0982b5a" timestamp="1480431575" url="https://MyFirstFDroidRepo.org/fdroid/repo" version="19">
<repo icon="fdroid-icon.png" name="My First F-Droid Repo Demo" pubkey="308204e1308202c9a003020102020434597643300d06092a864886f70d01010b050030213110300e060355040b1307462d44726f6964310d300b06035504031304736f7661301e170d3136303931333230313930395a170d3434303133303230313930395a30213110300e060355040b1307462d44726f6964310d300b06035504031304736f766130820222300d06092a864886f70d01010105000382020f003082020a028202010086ef94b5aacf2ba4f38c875f4194b44f5644392e3715575d7c92828577e692c352b567172823851c8c72347fbc9d99684cd7ca3e1db3e4cca126382c53f2a5869fb4c19bdec989b2930501af3e758ff40588915fe96b10076ce3346a193a0277d79e83e30fd8657c20e35260dd085aa32eac7c4b85786ffefbf1555cafe2bc928443430cdbba48cfbe701e12ae86e676477932730d4fc7c00af820aef85038a5b4df084cf6470d110dc4c49ea1b749b80b34709d199b3db516b223625c5de4501e861f7d261b3838f8f616aa78831d618d41d25872dc810c9b2087b5a9e146ca95be740316dcdbcb77314e23ab87d4487913b800b1113c0603ea2294188b71d3e49875df097b56f9151211fc6832f9790c5c83d17481f14ad37915fd164f4fd713f6732a15f4245714b84cd665bdbd085660ea33ad7d7095dcc414f09e3903604a40facc2314a115c0045bb50e9df38efb57e1b8e7cc105f340a26eeb46aba0fa6672953eee7f1f92dcb408e561909bbd4bdf4a4948c4d57c467d21aa238c34ba43be050398be963191fa2b49828bc1e4eeed224b40dbe9dc3e570890a71a974a2f4527edb1b07105071755105edcb2af2f269facfb89180903a572a99b46456e80d4a01685a80b233278805f2c876678e731f4ec4f52075aeef6b2b023efbb8a3637ef507c4c37c27e428152ec1817fcba640ad601cb09f72f0fbe2d274a2410203010001a321301f301d0603551d0e04160414c28bf33dd5a9a17338e5b1d1a6edd8c7d141ed0b300d06092a864886f70d01010b0500038202010084e20458b2aafd7fc27146b0986f9324f4260f244920417a77c9bf15e2e2d22d2725bdd8093ec261c3779c3ca03312516506f9410075b90595b41345956d8eb2786fb5994f195611382c2b99dba13381b0100a30bc9e6e47248bf4325e2f6eec9d789216dc7536e753bf1f4be603d9fa2e6f5e192b4eb988b8cdb0bb1e8668a9225426f7d4636479f73ed24ad1d2657c31e63c93d9679b9080171b3bd1bf10a3b92b80bd790fbf62d3644900cd08eae8b9bf9c2567be98dc8cdd2ae19a8d57a3e3e2de899f81f1279f578989e6af906f80c8c2b67651730ee7e568c1af5bcb845b6d685dc55332a9984aeceaea3b7e883447edf1c76b155d95253e39b9710eaa22efa6c81468829702b5dce7126538f3ca70c2f0ad9a5795435fdb1f715f20d60359ef9a9926c7050116e802df651727447848827815f70bd82af3cedd08783156102d2d8ce995c4c43b8e47e91a3e6927f3505a5d395e6bebb84542c570903eeab4382a1c2151f1471c7a06a34dc4d268d8fa72e93bdcd2dccc4302ecac47b9e7e3d8bc9b46d21cd097874a24d529548018dc190ff568c6aa428f0a5eedff1a347730931c74f19277538e49647a4ad7254f4c1ec7d4da12cce9e1fad9607534e66ab40a56b473d9d7e3d563fd03cad2052bad365c5a29f8ae54f09b60dbca3ea768d7767cbe1c133ca08ce725c1c1370f4aab8e5b6e286f52dc0be8d0982b5a" timestamp="1480431575" url="https://MyFirstFDroidRepo.org/fdroid/repo" version="20">
<description>This is a repository of apps to be used with F-Droid. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitlab.com/u/fdroid. </description>
<mirror>http://foobarfoobarfoobar.onion/fdroid/repo</mirror>
<mirror>https://foo.bar/fdroid/repo</mirror>

24
tests/run-tests
View file

@ -177,8 +177,8 @@ if which zipalign || ls -1 $ANDROID_HOME/build-tools/*/zipalign; then
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
test -e tmp/apkcache
! test -z tmp/apkcache
test -e tmp/apkcache.json
! test -z tmp/apkcache.json
test -L urzip.apk
grep -F '<application id=' repo/index.xml > /dev/null
fi
@ -808,8 +808,8 @@ else
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
test -e tmp/apkcache
! test -z tmp/apkcache
test -e tmp/apkcache.json
! test -z tmp/apkcache.json
export ANDROID_HOME=$STORED_ANDROID_HOME
fi
@ -860,8 +860,8 @@ $fdroid readmeta
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
test -e tmp/apkcache
! test -z tmp/apkcache
test -e tmp/apkcache.json
! test -z tmp/apkcache.json
grep -F '<application id=' repo/index.xml > /dev/null
@ -890,8 +890,8 @@ $fdroid readmeta
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
test -e tmp/apkcache
! test -z tmp/apkcache
test -e tmp/apkcache.json
! test -z tmp/apkcache.json
grep -F '<application id=' repo/index.xml > /dev/null
@ -917,8 +917,8 @@ $fdroid readmeta
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
test -e tmp/apkcache
! test -z tmp/apkcache
test -e tmp/apkcache.json
! test -z tmp/apkcache.json
grep -F '<application id=' repo/index.xml > /dev/null
@ -1009,8 +1009,8 @@ $fdroid readmeta
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
test -e tmp/apkcache
! test -z tmp/apkcache
test -e tmp/apkcache.json
! test -z tmp/apkcache.json
grep -F '<application id=' repo/index.xml > /dev/null
# now set fake repo_keyalias

40
tests/update.TestCase
View file

@ -3,6 +3,7 @@
# http://www.drdobbs.com/testing/unit-testing-with-python/240165163
import git
import glob
import inspect
import logging
import optparse
@ -289,6 +290,45 @@ class UpdateTest(unittest.TestCase):
self.assertIsNone(apk.get('obbMainFile'))
self.assertIsNone(apk.get('obbPatchFile'))
def test_apkcache_json(self):
"""test the migration from pickle to json"""
os.chdir(os.path.join(localmodule, 'tests'))
if os.path.basename(os.getcwd()) != 'tests':
raise Exception('This test must be run in the "tests/" subdir')
config = dict()
fdroidserver.common.fill_config_defaults(config)
config['ndk_paths'] = dict()
config['accepted_formats'] = ['json', 'txt', 'yml']
fdroidserver.common.config = config
fdroidserver.update.config = config
fdroidserver.update.options = type('', (), {})()
fdroidserver.update.options.clean = True
fdroidserver.update.options.delete_unknown = True
fdroidserver.update.options.rename_apks = False
fdroidserver.update.options.allow_disabled_algorithms = False
fdroidserver.metadata.read_metadata(xref=True)
knownapks = fdroidserver.common.KnownApks()
apkcache = fdroidserver.update.get_cache()
self.assertEqual(2, len(apkcache))
self.assertEqual(fdroidserver.update.METADATA_VERSION, apkcache["METADATA_VERSION"])
self.assertEqual(fdroidserver.update.options.allow_disabled_algorithms,
apkcache['allow_disabled_algorithms'])
apks, cachechanged = fdroidserver.update.process_apks(apkcache, 'repo', knownapks, False)
fdroidserver.update.write_cache(apkcache)
fdroidserver.update.options.clean = False
read_from_json = fdroidserver.update.get_cache()
self.assertEqual(16, len(read_from_json))
for f in glob.glob('repo/*.apk'):
self.assertTrue(os.path.basename(f) in read_from_json)
fdroidserver.update.options.clean = True
reset = fdroidserver.update.get_cache()
self.assertEqual(2, len(reset))
def test_scan_apk(self):
config = dict()
fdroidserver.common.fill_config_defaults(config)