mirror of
https://github.com/f-droid/fdroidserver.git
synced 2025-10-08 10:21:05 +03:00
Cache apk info to speed up updates
This commit is contained in:
Ciaran Gultnieks
parent
a7a966a837
commit
2f5417e589
1 changed files with 121 additions and 99 deletions
|
|
@ -25,6 +25,7 @@ import subprocess
|
||||||
import re
|
import re
|
||||||
import zipfile
|
import zipfile
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import pickle
|
||||||
from xml.dom.minidom import Document
|
from xml.dom.minidom import Document
|
||||||
from optparse import OptionParser
|
from optparse import OptionParser
|
||||||
import time
|
import time
|
||||||
|
|
@ -217,7 +218,15 @@ def main():
|
||||||
# Read known apks data (will be updated and written back when we've finished)
|
# Read known apks data (will be updated and written back when we've finished)
|
||||||
knownapks = common.KnownApks()
|
knownapks = common.KnownApks()
|
||||||
|
|
||||||
# Gather information about all the apk files in the repo directory...
|
# Gather information about all the apk files in the repo directory, using
|
||||||
|
# cached data if possible.
|
||||||
|
apkcachefile = os.path.join('tmp', 'apkcache')
|
||||||
|
if os.path.exists(apkcachefile):
|
||||||
|
with open(apkcachefile, 'rb') as cf:
|
||||||
|
apkcache = pickle.load(cf)
|
||||||
|
else:
|
||||||
|
apkcache = {}
|
||||||
|
cachechanged = False
|
||||||
apks = []
|
apks = []
|
||||||
for apkfile in glob.glob(os.path.join('repo','*.apk')):
|
for apkfile in glob.glob(os.path.join('repo','*.apk')):
|
||||||
|
|
||||||
|
|
@ -227,112 +236,125 @@ def main():
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
srcfilename = apkfilename[:-4] + "_src.tar.gz"
|
srcfilename = apkfilename[:-4] + "_src.tar.gz"
|
||||||
|
|
||||||
if not options.quiet:
|
if apkcache.has_key(apkfilename):
|
||||||
print "Processing " + apkfilename
|
if options.verbose:
|
||||||
thisinfo = {}
|
print "Reading " + apkfilename + " from cache"
|
||||||
thisinfo['apkname'] = apkfilename
|
thisinfo = apkcache[apkfilename]
|
||||||
if os.path.exists(os.path.join('repo', srcfilename)):
|
|
||||||
thisinfo['srcname'] = srcfilename
|
|
||||||
thisinfo['size'] = os.path.getsize(apkfile)
|
|
||||||
thisinfo['permissions'] = []
|
|
||||||
thisinfo['features'] = []
|
|
||||||
p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools', 'aapt'),
|
|
||||||
'dump', 'badging', apkfile],
|
|
||||||
stdout=subprocess.PIPE)
|
|
||||||
output = p.communicate()[0]
|
|
||||||
if options.verbose:
|
|
||||||
print output
|
|
||||||
if p.returncode != 0:
|
|
||||||
print "ERROR: Failed to get apk information"
|
|
||||||
sys.exit(1)
|
|
||||||
for line in output.splitlines():
|
|
||||||
if line.startswith("package:"):
|
|
||||||
pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
|
|
||||||
thisinfo['id'] = re.match(pat, line).group(1)
|
|
||||||
pat = re.compile(".*versionCode='([0-9]*)'.*")
|
|
||||||
thisinfo['versioncode'] = int(re.match(pat, line).group(1))
|
|
||||||
pat = re.compile(".*versionName='([^']*)'.*")
|
|
||||||
thisinfo['version'] = re.match(pat, line).group(1)
|
|
||||||
if line.startswith("application:"):
|
|
||||||
pat = re.compile(".*label='([^']*)'.*")
|
|
||||||
thisinfo['name'] = re.match(pat, line).group(1)
|
|
||||||
pat = re.compile(".*icon='([^']*)'.*")
|
|
||||||
thisinfo['iconsrc'] = re.match(pat, line).group(1)
|
|
||||||
if line.startswith("sdkVersion:"):
|
|
||||||
pat = re.compile(".*'([0-9]*)'.*")
|
|
||||||
thisinfo['sdkversion'] = re.match(pat, line).group(1)
|
|
||||||
if line.startswith("native-code:"):
|
|
||||||
pat = re.compile(".*'([^']*)'.*")
|
|
||||||
thisinfo['nativecode'] = re.match(pat, line).group(1)
|
|
||||||
if line.startswith("uses-permission:"):
|
|
||||||
pat = re.compile(".*'([^']*)'.*")
|
|
||||||
perm = re.match(pat, line).group(1)
|
|
||||||
if perm.startswith("android.permission."):
|
|
||||||
perm = perm[19:]
|
|
||||||
thisinfo['permissions'].append(perm)
|
|
||||||
if line.startswith("uses-feature:"):
|
|
||||||
pat = re.compile(".*'([^']*)'.*")
|
|
||||||
perm = re.match(pat, line).group(1)
|
|
||||||
#Filter out this, it's only added with the latest SDK tools and
|
|
||||||
#causes problems for lots of apps.
|
|
||||||
if (perm != "android.hardware.screen.portrait" and
|
|
||||||
perm != "android.hardware.screen.landscape"):
|
|
||||||
if perm.startswith("android.feature."):
|
|
||||||
perm = perm[16:]
|
|
||||||
thisinfo['features'].append(perm)
|
|
||||||
|
|
||||||
if not thisinfo.has_key('sdkversion'):
|
else:
|
||||||
print " WARNING: no SDK version information found"
|
|
||||||
thisinfo['sdkversion'] = 0
|
|
||||||
|
|
||||||
# Calculate the md5 and sha256...
|
if not options.quiet:
|
||||||
m = hashlib.md5()
|
print "Processing " + apkfilename
|
||||||
sha = hashlib.sha256()
|
thisinfo = {}
|
||||||
f = open(apkfile, 'rb')
|
thisinfo['apkname'] = apkfilename
|
||||||
while True:
|
if os.path.exists(os.path.join('repo', srcfilename)):
|
||||||
t = f.read(1024)
|
thisinfo['srcname'] = srcfilename
|
||||||
if len(t) == 0:
|
thisinfo['size'] = os.path.getsize(apkfile)
|
||||||
break
|
thisinfo['permissions'] = []
|
||||||
m.update(t)
|
thisinfo['features'] = []
|
||||||
sha.update(t)
|
p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools', 'aapt'),
|
||||||
thisinfo['md5'] = m.hexdigest()
|
'dump', 'badging', apkfile],
|
||||||
thisinfo['sha256'] = sha.hexdigest()
|
stdout=subprocess.PIPE)
|
||||||
f.close()
|
output = p.communicate()[0]
|
||||||
|
if options.verbose:
|
||||||
|
print output
|
||||||
|
if p.returncode != 0:
|
||||||
|
print "ERROR: Failed to get apk information"
|
||||||
|
sys.exit(1)
|
||||||
|
for line in output.splitlines():
|
||||||
|
if line.startswith("package:"):
|
||||||
|
pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
|
||||||
|
thisinfo['id'] = re.match(pat, line).group(1)
|
||||||
|
pat = re.compile(".*versionCode='([0-9]*)'.*")
|
||||||
|
thisinfo['versioncode'] = int(re.match(pat, line).group(1))
|
||||||
|
pat = re.compile(".*versionName='([^']*)'.*")
|
||||||
|
thisinfo['version'] = re.match(pat, line).group(1)
|
||||||
|
if line.startswith("application:"):
|
||||||
|
pat = re.compile(".*label='([^']*)'.*")
|
||||||
|
thisinfo['name'] = re.match(pat, line).group(1)
|
||||||
|
pat = re.compile(".*icon='([^']*)'.*")
|
||||||
|
thisinfo['iconsrc'] = re.match(pat, line).group(1)
|
||||||
|
if line.startswith("sdkVersion:"):
|
||||||
|
pat = re.compile(".*'([0-9]*)'.*")
|
||||||
|
thisinfo['sdkversion'] = re.match(pat, line).group(1)
|
||||||
|
if line.startswith("native-code:"):
|
||||||
|
pat = re.compile(".*'([^']*)'.*")
|
||||||
|
thisinfo['nativecode'] = re.match(pat, line).group(1)
|
||||||
|
if line.startswith("uses-permission:"):
|
||||||
|
pat = re.compile(".*'([^']*)'.*")
|
||||||
|
perm = re.match(pat, line).group(1)
|
||||||
|
if perm.startswith("android.permission."):
|
||||||
|
perm = perm[19:]
|
||||||
|
thisinfo['permissions'].append(perm)
|
||||||
|
if line.startswith("uses-feature:"):
|
||||||
|
pat = re.compile(".*'([^']*)'.*")
|
||||||
|
perm = re.match(pat, line).group(1)
|
||||||
|
#Filter out this, it's only added with the latest SDK tools and
|
||||||
|
#causes problems for lots of apps.
|
||||||
|
if (perm != "android.hardware.screen.portrait" and
|
||||||
|
perm != "android.hardware.screen.landscape"):
|
||||||
|
if perm.startswith("android.feature."):
|
||||||
|
perm = perm[16:]
|
||||||
|
thisinfo['features'].append(perm)
|
||||||
|
|
||||||
# Get the signature (or md5 of, to be precise)...
|
if not thisinfo.has_key('sdkversion'):
|
||||||
p = subprocess.Popen(['java', 'getsig',
|
print " WARNING: no SDK version information found"
|
||||||
os.path.join(os.getcwd(), apkfile)],
|
thisinfo['sdkversion'] = 0
|
||||||
cwd=os.path.join(os.path.dirname(__file__), 'getsig'),
|
|
||||||
stdout=subprocess.PIPE)
|
|
||||||
output = p.communicate()[0]
|
|
||||||
if options.verbose:
|
|
||||||
print output
|
|
||||||
if p.returncode != 0 or not output.startswith('Result:'):
|
|
||||||
print "ERROR: Failed to get apk signature"
|
|
||||||
sys.exit(1)
|
|
||||||
thisinfo['sig'] = output[7:].strip()
|
|
||||||
|
|
||||||
# Extract the icon file...
|
# Calculate the md5 and sha256...
|
||||||
apk = zipfile.ZipFile(apkfile, 'r')
|
m = hashlib.md5()
|
||||||
thisinfo['icon'] = (thisinfo['id'] + '.' +
|
sha = hashlib.sha256()
|
||||||
str(thisinfo['versioncode']) + '.png')
|
with open(apkfile, 'rb') as f:
|
||||||
iconfilename = os.path.join(icon_dir, thisinfo['icon'])
|
while True:
|
||||||
try:
|
t = f.read(1024)
|
||||||
iconfile = open(iconfilename, 'wb')
|
if len(t) == 0:
|
||||||
iconfile.write(apk.read(thisinfo['iconsrc']))
|
break
|
||||||
iconfile.close()
|
m.update(t)
|
||||||
except:
|
sha.update(t)
|
||||||
print "WARNING: Error retrieving icon file"
|
thisinfo['md5'] = m.hexdigest()
|
||||||
warnings += 1
|
thisinfo['sha256'] = sha.hexdigest()
|
||||||
apk.close()
|
|
||||||
|
|
||||||
# Record in known apks, getting the added date at the same time..
|
# Get the signature (or md5 of, to be precise)...
|
||||||
added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
|
p = subprocess.Popen(['java', 'getsig',
|
||||||
if added:
|
os.path.join(os.getcwd(), apkfile)],
|
||||||
thisinfo['added'] = added
|
cwd=os.path.join(os.path.dirname(__file__), 'getsig'),
|
||||||
|
stdout=subprocess.PIPE)
|
||||||
|
output = p.communicate()[0]
|
||||||
|
if options.verbose:
|
||||||
|
print output
|
||||||
|
if p.returncode != 0 or not output.startswith('Result:'):
|
||||||
|
print "ERROR: Failed to get apk signature"
|
||||||
|
sys.exit(1)
|
||||||
|
thisinfo['sig'] = output[7:].strip()
|
||||||
|
|
||||||
|
# Extract the icon file...
|
||||||
|
apk = zipfile.ZipFile(apkfile, 'r')
|
||||||
|
thisinfo['icon'] = (thisinfo['id'] + '.' +
|
||||||
|
str(thisinfo['versioncode']) + '.png')
|
||||||
|
iconfilename = os.path.join(icon_dir, thisinfo['icon'])
|
||||||
|
try:
|
||||||
|
iconfile = open(iconfilename, 'wb')
|
||||||
|
iconfile.write(apk.read(thisinfo['iconsrc']))
|
||||||
|
iconfile.close()
|
||||||
|
except:
|
||||||
|
print "WARNING: Error retrieving icon file"
|
||||||
|
warnings += 1
|
||||||
|
apk.close()
|
||||||
|
|
||||||
|
# Record in known apks, getting the added date at the same time..
|
||||||
|
added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
|
||||||
|
if added:
|
||||||
|
thisinfo['added'] = added
|
||||||
|
|
||||||
|
apkcache[apkfilename] = thisinfo
|
||||||
|
cachechanged = True
|
||||||
|
|
||||||
apks.append(thisinfo)
|
apks.append(thisinfo)
|
||||||
|
|
||||||
|
if cachechanged:
|
||||||
|
with open(apkcachefile, 'wb') as cf:
|
||||||
|
pickle.dump(apkcache, cf)
|
||||||
|
|
||||||
# Some information from the apks needs to be applied up to the application
|
# Some information from the apks needs to be applied up to the application
|
||||||
# level. When doing this, we use the info from the most recent version's apk.
|
# level. When doing this, we use the info from the most recent version's apk.
|
||||||
# We deal with figuring out when the app was added and last updated at the
|
# We deal with figuring out when the app was added and last updated at the
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue