mirrors/fdroidserver
1
0
Fork 0
mirror of https://github.com/f-droid/fdroidserver.git synced 2025-11-10 09:10:30 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Rework build into a class

Browse source 
This simplifies usage, goes from

    build['flag']
to
    build.flag

Also makes static analyzers able to detect invalid attributes as the set
is now limited in the class definition.

As a bonus, setting of the default field values is now done in the
constructor, not separately and manually.

While at it, unify "build", "thisbuild", "info", "thisinfo", etc into
just "build".
This commit is contained in:
Daniel Martí 2015-11-28 17:55:27 +01:00
parent 2c12485aeb
commit bf8518ee8f
15 changed files with 6997 additions and 8636 deletions
Download patch file Download diff file Expand all files Collapse all files

182
fdroidserver/build.py
View file

@ -245,7 +245,7 @@ def release_vm():
# Note that 'force' here also implies test mode.
def build_server(app, thisbuild, vcs, build_dir, output_dir, force):
def build_server(app, build, vcs, build_dir, output_dir, force):
"""Do a build on the build server."""
try:
@ -333,9 +333,9 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, force):
ftp.mkdir('extlib')
ftp.mkdir('srclib')
# Copy any extlibs that are required...
if thisbuild['extlibs']:
if build.extlibs:
ftp.chdir(homedir + '/build/extlib')
for lib in thisbuild['extlibs']:
for lib in build.extlibs:
lib = lib.strip()
libsrc = os.path.join('build/extlib', lib)
if not os.path.exists(libsrc):
@ -350,8 +350,8 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, force):
ftp.chdir('..')
# Copy any srclibs that are required...
srclibpaths = []
if thisbuild['srclibs']:
for lib in thisbuild['srclibs']:
if build.srclibs:
for lib in build.srclibs:
srclibpaths.append(
common.getsrclib(lib, 'build/srclib', basepath=True, prepare=False))
@ -389,7 +389,7 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, force):
cmdline += ' --force --test'
if options.verbose:
cmdline += ' --verbose'
cmdline += " %s:%s" % (app.id, thisbuild['vercode'])
cmdline += " %s:%s" % (app.id, build.vercode)
chan.exec_command('bash -c ". ~/.bsenv && ' + cmdline + '"')
output = ''
while not chan.exit_status_ready():
@ -406,7 +406,7 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, force):
if returncode != 0:
raise BuildException(
"Build.py failed on server for {0}:{1}".format(
app.id, thisbuild['version']), output)
app.id, build.version), output)
# Retrieve the built files...
logging.info("Retrieving build output...")
@ -414,8 +414,8 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, force):
ftp.chdir(homedir + '/tmp')
else:
ftp.chdir(homedir + '/unsigned')
apkfile = common.getapkname(app, thisbuild)
tarball = common.getsrcname(app, thisbuild)
apkfile = common.getapkname(app, build)
tarball = common.getsrcname(app, build)
try:
ftp.get(apkfile, os.path.join(output_dir, apkfile))
if not options.notarball:
@ -423,7 +423,7 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, force):
except:
raise BuildException(
"Build failed for %s:%s - missing output files".format(
app.id, thisbuild['version']), output)
app.id, build.version), output)
ftp.close()
finally:
@ -457,32 +457,33 @@ def capitalize_intact(string):
return string[0].upper() + string[1:]
def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh):
def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh):
"""Do a build locally."""
if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
if not thisbuild['ndk_path']:
logging.critical("Android NDK version '%s' could not be found!" % thisbuild['ndk'])
ndk_path = build.ndk_path()
if build.buildjni and build.buildjni != ['no']:
if not ndk_path:
logging.critical("Android NDK version '%s' could not be found!" % build.ndk or 'r10e')
logging.critical("Configured versions:")
for k, v in config['ndk_paths'].iteritems():
if k.endswith("_orig"):
continue
logging.critical(" %s: %s" % (k, v))
sys.exit(3)
elif not os.path.isdir(thisbuild['ndk_path']):
logging.critical("Android NDK '%s' is not a directory!" % thisbuild['ndk_path'])
elif not os.path.isdir(ndk_path):
logging.critical("Android NDK '%s' is not a directory!" % ndk_path)
sys.exit(3)
# Set up environment vars that depend on each build
for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
common.env[n] = thisbuild['ndk_path']
common.env[n] = ndk_path
common.reset_env_path()
# Set up the current NDK to the PATH
common.add_to_env_path(thisbuild['ndk_path'])
common.add_to_env_path(ndk_path)
# Prepare the source code...
root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild,
root_dir, srclibpaths = common.prepare_source(vcs, app, build,
build_dir, srclib_dir,
extlib_dir, onserver, refresh)
@ -490,26 +491,27 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
# different from the default ones
p = None
gradletasks = []
if thisbuild['type'] == 'maven':
method = build.method()
if method == 'maven':
logging.info("Cleaning Maven project...")
cmd = [config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']]
if '@' in thisbuild['maven']:
maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@', 1)[1])
if '@' in build.maven:
maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1])
maven_dir = os.path.normpath(maven_dir)
else:
maven_dir = root_dir
p = FDroidPopen(cmd, cwd=maven_dir)
elif thisbuild['type'] == 'gradle':
elif method == 'gradle':
logging.info("Cleaning Gradle project...")
if thisbuild['preassemble']:
gradletasks += thisbuild['preassemble']
if build.preassemble:
gradletasks += build.preassemble
flavours = thisbuild['gradle']
flavours = build.gradle
if flavours == ['yes']:
flavours = []
@ -522,8 +524,8 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
adapt_gradle(libpath)
cmd = [config['gradle']]
if thisbuild['gradleprops']:
cmd += ['-P'+kv for kv in thisbuild['gradleprops']]
if build.gradleprops:
cmd += ['-P'+kv for kv in build.gradleprops]
for task in gradletasks:
parts = task.split(':')
@ -534,16 +536,16 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
p = FDroidPopen(cmd, cwd=root_dir)
elif thisbuild['type'] == 'kivy':
elif method == 'kivy':
pass
elif thisbuild['type'] == 'ant':
elif method == 'ant':
logging.info("Cleaning Ant project...")
p = FDroidPopen(['ant', 'clean'], cwd=root_dir)
if p is not None and p.returncode != 0:
raise BuildException("Error cleaning %s:%s" %
(app.id, thisbuild['version']), p.output)
(app.id, build.version), p.output)
for root, dirs, files in os.walk(build_dir):
@ -575,12 +577,12 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
del_dirs(['obj'])
if options.skipscan:
if thisbuild['scandelete']:
if build.scandelete:
raise BuildException("Refusing to skip source scan since scandelete is present")
else:
# Scan before building...
logging.info("Scanning source for common problems...")
count = scanner.scan_source(build_dir, root_dir, thisbuild)
count = scanner.scan_source(build_dir, root_dir, build)
if count > 0:
if force:
logging.warn('Scanner found %d problems' % count)
@ -590,7 +592,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
if not options.notarball:
# Build the source tarball right before we build the release...
logging.info("Creating source tarball...")
tarname = common.getsrcname(app, thisbuild)
tarname = common.getsrcname(app, build)
tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz")
def tarexc(f):
@ -599,9 +601,9 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
tarball.close()
# Run a build command if one is required...
if thisbuild['build']:
if build.build:
logging.info("Running 'build' commands in %s" % root_dir)
cmd = common.replace_config_vars(thisbuild['build'], thisbuild)
cmd = common.replace_config_vars(build.build, build)
# Substitute source library paths into commands...
for name, number, libpath in srclibpaths:
@ -612,16 +614,16 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
if p.returncode != 0:
raise BuildException("Error running build command for %s:%s" %
(app.id, thisbuild['version']), p.output)
(app.id, build.version), p.output)
# Build native stuff if required...
if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
if build.buildjni and build.buildjni != ['no']:
logging.info("Building the native code")
jni_components = thisbuild['buildjni']
jni_components = build.buildjni
if jni_components == ['yes']:
jni_components = ['']
cmd = [os.path.join(thisbuild['ndk_path'], "ndk-build"), "-j1"]
cmd = [os.path.join(ndk_path, "ndk-build"), "-j1"]
for d in jni_components:
if d:
logging.info("Building native code in '%s'" % d)
@ -640,15 +642,15 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
del manifest_text
p = FDroidPopen(cmd, cwd=os.path.join(root_dir, d))
if p.returncode != 0:
raise BuildException("NDK build failed for %s:%s" % (app.id, thisbuild['version']), p.output)
raise BuildException("NDK build failed for %s:%s" % (app.id, build.version), p.output)
p = None
# Build the release...
if thisbuild['type'] == 'maven':
if method == 'maven':
logging.info("Building Maven project...")
if '@' in thisbuild['maven']:
maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@', 1)[1])
if '@' in build.maven:
maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1])
else:
maven_dir = root_dir
@ -656,12 +658,12 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
'-Dmaven.jar.sign.skip=true', '-Dmaven.test.skip=true',
'-Dandroid.sign.debug=false', '-Dandroid.release=true',
'package']
if thisbuild['target']:
target = thisbuild["target"].split('-')[1]
if build.target:
target = build.target.split('-')[1]
common.regsub_file(r'<platform>[0-9]*</platform>',
r'<platform>%s</platform>' % target,
os.path.join(root_dir, 'pom.xml'))
if '@' in thisbuild['maven']:
if '@' in build.maven:
common.regsub_file(r'<platform>[0-9]*</platform>',
r'<platform>%s</platform>' % target,
os.path.join(maven_dir, 'pom.xml'))
@ -670,7 +672,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
bindir = os.path.join(root_dir, 'target')
elif thisbuild['type'] == 'kivy':
elif method == 'kivy':
logging.info("Building Kivy project...")
spec = os.path.join(root_dir, 'buildozer.spec')
@ -690,8 +692,8 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
modules = bconfig.get('app', 'requirements').split(',')
cmd = 'ANDROIDSDK=' + config['sdk_path']
cmd += ' ANDROIDNDK=' + thisbuild['ndk_path']
cmd += ' ANDROIDNDKVER=' + thisbuild['ndk']
cmd += ' ANDROIDNDK=' + ndk_path
cmd += ' ANDROIDNDKVER=' + build.ndk
cmd += ' ANDROIDAPI=' + str(bconfig.get('app', 'android.api'))
cmd += ' VIRTUALENV=virtualenv'
cmd += ' ./distribute.sh'
@ -731,27 +733,27 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
cmd.append('release')
p = FDroidPopen(cmd, cwd=distdir)
elif thisbuild['type'] == 'gradle':
elif method == 'gradle':
logging.info("Building Gradle project...")
# Avoid having to use lintOptions.abortOnError false
if thisbuild['gradlepluginver'] >= LooseVersion('0.7'):
if build.gradlepluginver >= LooseVersion('0.7'):
with open(os.path.join(root_dir, 'build.gradle'), "a") as f:
f.write("\nandroid { lintOptions { checkReleaseBuilds false } }\n")
cmd = [config['gradle']]
if thisbuild['gradleprops']:
cmd += ['-P'+kv for kv in thisbuild['gradleprops']]
if build.gradleprops:
cmd += ['-P'+kv for kv in build.gradleprops]
cmd += gradletasks
p = FDroidPopen(cmd, cwd=root_dir)
elif thisbuild['type'] == 'ant':
elif method == 'ant':
logging.info("Building Ant project...")
cmd = ['ant']
if thisbuild['antcommands']:
cmd += thisbuild['antcommands']
if build.antcommands:
cmd += build.antcommands
else:
cmd += ['release']
p = FDroidPopen(cmd, cwd=root_dir)
@ -759,10 +761,10 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
bindir = os.path.join(root_dir, 'bin')
if p is not None and p.returncode != 0:
raise BuildException("Build failed for %s:%s" % (app.id, thisbuild['version']), p.output)
logging.info("Successfully built version " + thisbuild['version'] + ' of ' + app.id)
raise BuildException("Build failed for %s:%s" % (app.id, build.version), p.output)
logging.info("Successfully built version " + build.version + ' of ' + app.id)
if thisbuild['type'] == 'maven':
if method == 'maven':
stdout_apk = '\n'.join([
line for line in p.output.splitlines() if any(
a in line for a in ('.apk', '.ap_', '.jar'))])
@ -782,14 +784,14 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
raise BuildException('Failed to find output')
src = m.group(1)
src = os.path.join(bindir, src) + '.apk'
elif thisbuild['type'] == 'kivy':
elif method == 'kivy':
src = os.path.join('python-for-android', 'dist', 'default', 'bin',
'{0}-{1}-release.apk'.format(
bconfig.get('app', 'title'),
bconfig.get('app', 'version')))
elif thisbuild['type'] == 'gradle':
elif method == 'gradle':
if thisbuild['gradlepluginver'] >= LooseVersion('0.11'):
if build.gradlepluginver >= LooseVersion('0.11'):
apks_dir = os.path.join(root_dir, 'build', 'outputs', 'apk')
else:
apks_dir = os.path.join(root_dir, 'build', 'apk')
@ -801,14 +803,14 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
if len(apks) < 1:
raise BuildException('Failed to find gradle output in %s' % apks_dir)
src = apks[0]
elif thisbuild['type'] == 'ant':
elif method == 'ant':
stdout_apk = '\n'.join([
line for line in p.output.splitlines() if '.apk' in line])
src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk,
re.S | re.M).group(1)
src = os.path.join(bindir, src)
elif thisbuild['type'] == 'raw':
src = os.path.join(root_dir, thisbuild['output'])
elif method == 'raw':
src = os.path.join(root_dir, build.output)
src = os.path.normpath(src)
# Make sure it's not debuggable...
@ -850,12 +852,12 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
nativecode = nativecode.strip()
nativecode = None if not nativecode else nativecode
if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
if build.buildjni and build.buildjni != ['no']:
if nativecode is None:
raise BuildException("Native code should have been built but none was packaged")
if thisbuild['novcheck']:
vercode = thisbuild['vercode']
version = thisbuild['version']
if build.novcheck:
vercode = build.vercode
version = build.version
if not version or not vercode:
raise BuildException("Could not find version information in build in output")
if not foundid:
@ -871,13 +873,13 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
if index != -1:
version = version[:index]
if (version != thisbuild['version'] or
vercode != thisbuild['vercode']):
if (version != build.version or
vercode != build.vercode):
raise BuildException(("Unexpected version/version code in output;"
" APK: '%s' / '%s', "
" Expected: '%s' / '%s'")
% (version, str(vercode), thisbuild['version'],
str(thisbuild['vercode']))
% (version, str(vercode), build.version,
str(build.vercode))
)
# Add information for 'fdroid verify' to be able to reproduce the build
@ -895,7 +897,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
# Copy the unsigned apk to our destination directory for further
# processing (by publish.py)...
dest = os.path.join(output_dir, common.getapkname(app, thisbuild))
dest = os.path.join(output_dir, common.getapkname(app, build))
shutil.copyfile(src, dest)
# Move the source tarball into the output directory...
@ -904,7 +906,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
os.path.join(output_dir, tarname))
def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir,
def trybuild(app, build, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir,
tmp_dir, repo_dir, vcs, test, server, force, onserver, refresh):
"""
Build a particular version of an application, if it needs building.
@ -923,7 +925,7 @@ def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir,
:returns: True if the build was done, False if it wasn't necessary.
"""
dest_apk = common.getapkname(app, thisbuild)
dest_apk = common.getapkname(app, build)
dest = os.path.join(output_dir, dest_apk)
dest_repo = os.path.join(repo_dir, dest_apk)
@ -937,20 +939,20 @@ def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir,
if os.path.exists(dest_also):
return False
if thisbuild['disable'] and not options.force:
if build.disable and not options.force:
return False
logging.info("Building version %s (%s) of %s" % (
thisbuild['version'], thisbuild['vercode'], app.id))
build.version, build.vercode, app.id))
if server:
# When using server mode, still keep a local cache of the repo, by
# grabbing the source now.
vcs.gotorevision(thisbuild['commit'])
vcs.gotorevision(build.commit)
build_server(app, thisbuild, vcs, build_dir, output_dir, force)
build_server(app, build, vcs, build_dir, output_dir, force)
else:
build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh)
build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh)
return True
@ -1060,7 +1062,7 @@ def main():
if options.latest:
for app in apps.itervalues():
for build in reversed(app.builds):
if build['disable'] and not options.force:
if build.disable and not options.force:
continue
app.builds = [build]
break
@ -1078,7 +1080,7 @@ def main():
first = True
for thisbuild in app.builds:
for build in app.builds:
wikilog = None
try:
@ -1098,8 +1100,8 @@ def main():
first = False
logging.debug("Checking " + thisbuild['version'])
if trybuild(app, thisbuild, build_dir, output_dir,
logging.debug("Checking " + build.version)
if trybuild(app, build, build_dir, output_dir,
also_check_dir, srclib_dir, extlib_dir,
tmp_dir, repo_dir, vcs, options.test,
options.server, options.force,
@ -1112,10 +1114,10 @@ def main():
# alongside our built one in the 'unsigend'
# directory.
url = app.Binaries
url = url.replace('%v', thisbuild['version'])
url = url.replace('%c', str(thisbuild['vercode']))
url = url.replace('%v', build.version)
url = url.replace('%c', str(build.vercode))
logging.info("...retrieving " + url)
of = "{0}_{1}.apk.binary".format(app.id, thisbuild['vercode'])
of = "{0}_{1}.apk.binary".format(app.id, build.vercode)
of = os.path.join(output_dir, of)
net.download_file(url, local_filename=of)
@ -1148,7 +1150,7 @@ def main():
if options.wiki and wikilog:
try:
# Write a page with the last build log for this version code
lastbuildpage = appid + '/lastbuild_' + thisbuild['vercode']
lastbuildpage = appid + '/lastbuild_' + build.vercode
newpage = site.Pages[lastbuildpage]
txt = "Build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + wikilog
newpage.save(txt, summary='Build log')

59
fdroidserver/checkupdates.py
View file

@ -108,10 +108,12 @@ def check_tags(app, pattern):
vcs.gotorevision(None)
flavours = []
last_build = metadata.Build()
if len(app.builds) > 0:
if app.builds[-1]['gradle']:
flavours = app.builds[-1]['gradle']
last_build = app.builds[-1]
if last_build.submodules:
vcs.initsubmodules()
hpak = None
htag = None
@ -143,7 +145,7 @@ def check_tags(app, pattern):
root_dir = build_dir
else:
root_dir = os.path.join(build_dir, subdir)
paths = common.manifest_paths(root_dir, flavours)
paths = common.manifest_paths(root_dir, last_build.gradle)
version, vercode, package = common.parse_androidmanifests(paths, app)
if vercode:
logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
@ -199,10 +201,12 @@ def check_repomanifest(app, branch=None):
elif repotype == 'bzr':
vcs.gotorevision(None)
flavours = []
last_build = metadata.Build()
if len(app.builds) > 0:
if app.builds[-1]['gradle']:
flavours = app.builds[-1]['gradle']
last_build = app.builds[-1]
if last_build.submodules:
vcs.initsubmodules()
hpak = None
hver = None
@ -212,7 +216,7 @@ def check_repomanifest(app, branch=None):
root_dir = build_dir
else:
root_dir = os.path.join(build_dir, subdir)
paths = common.manifest_paths(root_dir, flavours)
paths = common.manifest_paths(root_dir, last_build.gradle)
version, vercode, package = common.parse_androidmanifests(paths, app)
if vercode:
logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
@ -313,14 +317,12 @@ def possible_subdirs(app):
else:
build_dir = os.path.join('build', app.id)
flavours = []
last_build = metadata.Build()
if len(app.builds) > 0:
build = app.builds[-1]
if build['gradle']:
flavours = build['gradle']
last_build = app.builds[-1]
for d in dirs_with_manifest(build_dir):
m_paths = common.manifest_paths(d, flavours)
m_paths = common.manifest_paths(d, last_build.gradle)
package = common.parse_androidmanifests(m_paths, app)[2]
if package is not None:
subdir = os.path.relpath(d, build_dir)
@ -344,10 +346,9 @@ def fetch_autoname(app, tag):
except VCSException:
return None
flavours = []
last_build = metadata.Build()
if len(app.builds) > 0:
if app.builds[-1]['gradle']:
flavours = app.builds[-1]['gradle']
last_build = app.builds[-1]
logging.debug("...fetch auto name from " + build_dir)
new_name = None
@ -356,7 +357,7 @@ def fetch_autoname(app, tag):
root_dir = build_dir
else:
root_dir = os.path.join(build_dir, subdir)
new_name = common.fetch_real_name(root_dir, flavours)
new_name = common.fetch_real_name(root_dir, last_build.gradle)
if new_name is not None:
break
commitmsg = None
@ -458,25 +459,25 @@ def checkupdates_app(app, first=True):
gotcur = False
latest = None
for build in app.builds:
if int(build['vercode']) >= int(app.CurrentVersionCode):
if int(build.vercode) >= int(app.CurrentVersionCode):
gotcur = True
if not latest or int(build['vercode']) > int(latest['vercode']):
if not latest or int(build.vercode) > int(latest.vercode):
latest = build
if int(latest['vercode']) > int(app.CurrentVersionCode):
if int(latest.vercode) > int(app.CurrentVersionCode):
logging.info("Refusing to auto update, since the latest build is newer")
if not gotcur:
newbuild = latest.copy()
if 'origlines' in newbuild:
del newbuild['origlines']
newbuild['disable'] = False
newbuild['vercode'] = app.CurrentVersionCode
newbuild['version'] = app.CurrentVersion + suffix
logging.info("...auto-generating build for " + newbuild['version'])
commit = pattern.replace('%v', newbuild['version'])
commit = commit.replace('%c', newbuild['vercode'])
newbuild['commit'] = commit
if newbuild.origlines:
del newbuild.origlines[:]
newbuild.disable = False
newbuild.vercode = app.CurrentVersionCode
newbuild.version = app.CurrentVersion + suffix
logging.info("...auto-generating build for " + newbuild.version)
commit = pattern.replace('%v', newbuild.version)
commit = commit.replace('%c', newbuild.vercode)
newbuild.commit = commit
app.builds.append(newbuild)
name = common.getappname(app)
ver = common.getcvname(app)

116
fdroidserver/common.py
View file

@ -221,15 +221,6 @@ def read_config(opts, config_file='config.py'):
return config
def get_ndk_path(version):
if version is None:
version = 'r10e' # falls back to latest
paths = config['ndk_paths']
if version not in paths:
return ''
return paths[version] or ''
def find_sdk_tools_cmd(cmd):
'''find a working path to a tool from the Android SDK'''
@ -363,10 +354,10 @@ def read_app_args(args, allapps, allow_vercodes=False):
vc = vercodes[appid]
if not vc:
continue
app.builds = [b for b in app.builds if b['vercode'] in vc]
app.builds = [b for b in app.builds if b.vercode in vc]
if len(app.builds) != len(vercodes[appid]):
error = True
allvcs = [b['vercode'] for b in app.builds]
allvcs = [b.vercode for b in app.builds]
for v in vercodes[appid]:
if v not in allvcs:
logging.critical("No such vercode %s for app %s" % (v, appid))
@ -419,11 +410,11 @@ def apknameinfo(filename):
def getapkname(app, build):
return "%s_%s.apk" % (app.id, build['vercode'])
return "%s_%s.apk" % (app.id, build.vercode)
def getsrcname(app, build):
return "%s_%s_src.tar.gz" % (app.id, build['vercode'])
return "%s_%s_src.tar.gz" % (app.id, build.vercode)
def getappname(app):
@ -1250,17 +1241,17 @@ gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\
def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
# Optionally, the actual app source can be in a subdirectory
if build['subdir']:
root_dir = os.path.join(build_dir, build['subdir'])
if build.subdir:
root_dir = os.path.join(build_dir, build.subdir)
else:
root_dir = build_dir
# Get a working copy of the right revision
logging.info("Getting source for revision " + build['commit'])
vcs.gotorevision(build['commit'], refresh)
logging.info("Getting source for revision " + build.commit)
vcs.gotorevision(build.commit, refresh)
# Initialise submodules if required
if build['submodules']:
if build.submodules:
logging.info("Initialising submodules")
vcs.initsubmodules()
@ -1270,19 +1261,19 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
raise BuildException('Missing subdir ' + root_dir)
# Run an init command if one is required
if build['init']:
cmd = replace_config_vars(build['init'], build)
if build.init:
cmd = replace_config_vars(build.init, build)
logging.info("Running 'init' commands in %s" % root_dir)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
raise BuildException("Error running init command for %s:%s" %
(app.id, build['version']), p.output)
(app.id, build.version), p.output)
# Apply patches if any
if build['patch']:
if build.patch:
logging.info("Applying patches")
for patch in build['patch']:
for patch in build.patch:
patch = patch.strip()
logging.info("Applying " + patch)
patch_path = os.path.join('metadata', app.id, patch)
@ -1292,9 +1283,9 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
# Get required source libraries
srclibpaths = []
if build['srclibs']:
if build.srclibs:
logging.info("Collecting source libraries")
for lib in build['srclibs']:
for lib in build.srclibs:
srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
for name, number, libpath in srclibpaths:
@ -1307,8 +1298,8 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
# Update the local.properties file
localprops = [os.path.join(build_dir, 'local.properties')]
if build['subdir']:
parts = build['subdir'].split(os.sep)
if build.subdir:
parts = build.subdir.split(os.sep)
cur = build_dir
for d in parts:
cur = os.path.join(cur, d)
@ -1324,26 +1315,27 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
logging.info("Creating local.properties file at %s" % path)
# Fix old-fashioned 'sdk-location' by copying
# from sdk.dir, if necessary
if build['oldsdkloc']:
if build.oldsdkloc:
sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
re.S | re.M).group(1)
props += "sdk-location=%s\n" % sdkloc
else:
props += "sdk.dir=%s\n" % config['sdk_path']
props += "sdk-location=%s\n" % config['sdk_path']
if build['ndk_path']:
ndk_path = build.ndk_path()
if ndk_path:
# Add ndk location
props += "ndk.dir=%s\n" % build['ndk_path']
props += "ndk-location=%s\n" % build['ndk_path']
props += "ndk.dir=%s\n" % ndk_path
props += "ndk-location=%s\n" % ndk_path
# Add java.encoding if necessary
if build['encoding']:
props += "java.encoding=%s\n" % build['encoding']
if build.encoding:
props += "java.encoding=%s\n" % build.encoding
with open(path, 'w') as f:
f.write(props)
flavours = []
if build['type'] == 'gradle':
flavours = build['gradle']
if build.method() == 'gradle':
flavours = build.gradle
gradlepluginver = None
@ -1372,13 +1364,13 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
break
if gradlepluginver:
build['gradlepluginver'] = LooseVersion(gradlepluginver)
build.gradlepluginver = LooseVersion(gradlepluginver)
else:
logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
build['gradlepluginver'] = LooseVersion('0.11')
build.gradlepluginver = LooseVersion('0.11')
if build['target']:
n = build["target"].split('-')[1]
if build.target:
n = build.target.split('-')[1]
regsub_file(r'compileSdkVersion[ =]+[0-9]+',
r'compileSdkVersion %s' % n,
os.path.join(root_dir, 'build.gradle'))
@ -1387,38 +1379,38 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
remove_debuggable_flags(root_dir)
# Insert version code and number into the manifest if necessary
if build['forceversion']:
if build.forceversion:
logging.info("Changing the version name")
for path in manifest_paths(root_dir, flavours):
if not os.path.isfile(path):
continue
if has_extension(path, 'xml'):
regsub_file(r'android:versionName="[^"]*"',
r'android:versionName="%s"' % build['version'],
r'android:versionName="%s"' % build.version,
path)
elif has_extension(path, 'gradle'):
regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
r"""\1versionName '%s'""" % build['version'],
r"""\1versionName '%s'""" % build.version,
path)
if build['forcevercode']:
if build.forcevercode:
logging.info("Changing the version code")
for path in manifest_paths(root_dir, flavours):
if not os.path.isfile(path):
continue
if has_extension(path, 'xml'):
regsub_file(r'android:versionCode="[^"]*"',
r'android:versionCode="%s"' % build['vercode'],
r'android:versionCode="%s"' % build.vercode,
path)
elif has_extension(path, 'gradle'):
regsub_file(r'versionCode[ =]+[0-9]+',
r'versionCode %s' % build['vercode'],
r'versionCode %s' % build.vercode,
path)
# Delete unwanted files
if build['rm']:
if build.rm:
logging.info("Removing specified files")
for part in getpaths(build_dir, build['rm']):
for part in getpaths(build_dir, build.rm):
dest = os.path.join(build_dir, part)
logging.info("Removing {0}".format(part))
if os.path.lexists(dest):
@ -1432,12 +1424,12 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
remove_signing_keys(build_dir)
# Add required external libraries
if build['extlibs']:
if build.extlibs:
logging.info("Collecting prebuilt libraries")
libsdir = os.path.join(root_dir, 'libs')
if not os.path.exists(libsdir):
os.mkdir(libsdir)
for lib in build['extlibs']:
for lib in build.extlibs:
lib = lib.strip()
logging.info("...installing extlib {0}".format(lib))
libf = os.path.basename(lib)
@ -1447,10 +1439,10 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
shutil.copyfile(libsrc, os.path.join(libsdir, libf))
# Run a pre-build command if one is required
if build['prebuild']:
if build.prebuild:
logging.info("Running 'prebuild' commands in %s" % root_dir)
cmd = replace_config_vars(build['prebuild'], build)
cmd = replace_config_vars(build.prebuild, build)
# Substitute source library paths into prebuild commands
for name, number, libpath in srclibpaths:
@ -1460,20 +1452,20 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
raise BuildException("Error running prebuild command for %s:%s" %
(app.id, build['version']), p.output)
(app.id, build.version), p.output)
# Generate (or update) the ant build file, build.xml...
if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
if build.update and build.update != ['no'] and build.method() == 'ant':
parms = ['android', 'update', 'lib-project']
lparms = ['android', 'update', 'project']
if build['target']:
parms += ['-t', build['target']]
lparms += ['-t', build['target']]
if build['update'] == ['auto']:
update_dirs = ant_subprojects(root_dir) + ['.']
if build.target:
parms += ['-t', build.target]
lparms += ['-t', build.target]
if build.update:
update_dirs = build.update
else:
update_dirs = build['update']
update_dirs = ant_subprojects(root_dir) + ['.']
for d in update_dirs:
subdir = os.path.join(root_dir, d)
@ -1770,9 +1762,9 @@ def replace_config_vars(cmd, build):
cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
cmd = cmd.replace('$$MVN3$$', config['mvn3'])
if build is not None:
cmd = cmd.replace('$$COMMIT$$', build['commit'])
cmd = cmd.replace('$$VERSION$$', build['version'])
cmd = cmd.replace('$$VERCODE$$', build['vercode'])
cmd = cmd.replace('$$COMMIT$$', build.commit)
cmd = cmd.replace('$$VERSION$$', build.version)
cmd = cmd.replace('$$VERCODE$$', build.vercode)
return cmd

19
fdroidserver/import.py
View file

@ -223,20 +223,15 @@ def main():
sys.exit(1)
# Create a build line...
build = {}
build['version'] = version or '?'
build['vercode'] = vercode or '?'
build['commit'] = '?'
build['disable'] = 'Generated by import.py - check/set version fields and commit id'
build = metadata.Build()
build.version = version or '?'
build.vercode = vercode or '?'
build.commit = '?'
build.disable = 'Generated by import.py - check/set version fields and commit id'
if options.subdir:
build['subdir'] = options.subdir
build.subdir = options.subdir
if os.path.exists(os.path.join(root_dir, 'jni')):
build['buildjni'] = ['yes']
for flag, value in metadata.flag_defaults.iteritems():
if flag in build:
continue
build[flag] = value
build.buildjni = ['yes']
app.builds.append(build)

26
fdroidserver/lint.py
View file

@ -122,11 +122,11 @@ def get_lastbuild(builds):
lowest_vercode = -1
lastbuild = None
for build in builds:
if not build['disable']:
vercode = int(build['vercode'])
if not build.disable:
vercode = int(build.vercode)
if lowest_vercode == -1 or vercode < lowest_vercode:
lowest_vercode = vercode
if not lastbuild or int(build['vercode']) > int(lastbuild['vercode']):
if not lastbuild or int(build.vercode) > int(lastbuild.vercode):
lastbuild = build
return lastbuild
@ -134,14 +134,14 @@ def get_lastbuild(builds):
def check_ucm_tags(app):
lastbuild = get_lastbuild(app.builds)
if (lastbuild is not None
and lastbuild['commit']
and lastbuild.commit
and app.UpdateCheckMode == 'RepoManifest'
and not lastbuild['commit'].startswith('unknown')
and lastbuild['vercode'] == app.CurrentVersionCode
and not lastbuild['forcevercode']
and any(s in lastbuild['commit'] for s in '.,_-/')):
and not lastbuild.commit.startswith('unknown')
and lastbuild.vercode == app.CurrentVersionCode
and not lastbuild.forcevercode
and any(s in lastbuild.commit for s in '.,_-/')):
yield "Last used commit '%s' looks like a tag, but Update Check Mode is '%s'" % (
lastbuild['commit'], app.UpdateCheckMode)
lastbuild.commit, app.UpdateCheckMode)
def check_char_limits(app):
@ -285,12 +285,12 @@ def check_bulleted_lists(app):
def check_builds(app):
for build in app.builds:
if build['disable']:
if build.disable:
continue
for s in ['master', 'origin', 'HEAD', 'default', 'trunk']:
if build['commit'] and build['commit'].startswith(s):
yield "Branch '%s' used as commit in build '%s'" % (s, build['version'])
for srclib in build['srclibs']:
if build.commit and build.commit.startswith(s):
yield "Branch '%s' used as commit in build '%s'" % (s, build.version)
for srclib in build.srclibs:
ref = srclib.split('@')[1].split('/')[0]
if ref.startswith(s):
yield "Branch '%s' used as commit in srclib '%s'" % (s, srclib)

585
fdroidserver/metadata.py
View file

@ -38,8 +38,6 @@ except ImportError:
# use the C implementation when available
import xml.etree.cElementTree as ElementTree
from collections import OrderedDict
import common
srclibs = None
@ -155,7 +153,16 @@ class App():
# Constructs an old-fashioned dict with the human-readable field
# names. Should only be used for tests.
def field_dict(self):
return {App.attr_to_field(k): v for k, v in self.__dict__.iteritems()}
d = {}
for k, v in self.__dict__.iteritems():
if k == 'builds':
d['builds'] = []
for build in v:
d['builds'].append(build.__dict__)
else:
k = App.attr_to_field(k)
d[k] = v
return d
# Gets the value associated to a field name, e.g. 'Auto Name'
def get_field(self, f):
@ -184,44 +191,156 @@ class App():
# Like dict.update(), but using human-readable field names
def update_fields(self, d):
for f, v in d.iteritems():
self.set_field(f, v)
if f == 'builds':
for b in v:
build = Build()
build.update_flags(b)
self.builds.append(build)
else:
self.set_field(f, v)
def metafieldtype(name):
if name in ['Description', 'Maintainer Notes']:
return 'multiline'
if name in ['Categories', 'AntiFeatures']:
return 'list'
if name == 'Build Version':
return 'build'
if name == 'Build':
return 'buildv2'
if name == 'Use Built':
return 'obsolete'
if name not in app_fields:
return 'unknown'
return 'string'
# In the order in which they are laid out on files
# Sorted by their action and their place in the build timeline
# These variables can have varying datatypes. For example, anything with
# flagtype(v) == 'list' is inited as False, then set as a list of strings.
flag_defaults = OrderedDict([
('disable', False),
('commit', None),
('subdir', None),
('submodules', False),
('init', ''),
('patch', []),
('gradle', False),
('maven', False),
('kivy', False),
('output', None),
('srclibs', []),
('oldsdkloc', False),
('encoding', None),
('forceversion', False),
('forcevercode', False),
('rm', []),
('extlibs', []),
('prebuild', ''),
('update', ['auto']),
('target', None),
('scanignore', []),
('scandelete', []),
('build', ''),
('buildjni', []),
('ndk', 'r10e'), # defaults to latest
('preassemble', []),
('gradleprops', []),
('antcommands', None),
('novcheck', False),
])
build_flags_order = [
'disable',
'commit',
'subdir',
'submodules',
'init',
'patch',
'gradle',
'maven',
'kivy',
'output',
'srclibs',
'oldsdkloc',
'encoding',
'forceversion',
'forcevercode',
'rm',
'extlibs',
'prebuild',
'update',
'target',
'scanignore',
'scandelete',
'build',
'buildjni',
'ndk',
'preassemble',
'gradleprops',
'antcommands',
'novcheck',
]
build_flags = set(build_flags_order + ['version', 'vercode'])
class Build():
def __init__(self):
self.disable = False
self.commit = None
self.subdir = None
self.submodules = False
self.init = ''
self.patch = []
self.gradle = False
self.maven = False
self.kivy = False
self.output = None
self.srclibs = []
self.oldsdkloc = False
self.encoding = None
self.forceversion = False
self.forcevercode = False
self.rm = []
self.extlibs = []
self.prebuild = ''
self.update = None
self.target = None
self.scanignore = []
self.scandelete = []
self.build = ''
self.buildjni = []
self.ndk = None
self.preassemble = []
self.gradleprops = []
self.antcommands = None
self.novcheck = False
def get_flag(self, f):
if f not in build_flags:
raise MetaDataException('Unrecognised build flag: ' + f)
return getattr(self, f)
def set_flag(self, f, v):
if f == 'versionName':
f = 'version'
if f == 'versionCode':
f = 'vercode'
if f not in build_flags:
raise MetaDataException('Unrecognised build flag: ' + f)
setattr(self, f, v)
def append_flag(self, f, v):
if f not in build_flags:
raise MetaDataException('Unrecognised build flag: ' + f)
if f not in self.__dict__:
self.__dict__[f] = [v]
else:
self.__dict__[f].append(v)
def method(self):
for f in ['maven', 'gradle', 'kivy']:
if self.get_flag(f):
return f
if build.output:
return 'raw'
return 'ant'
def ndk_path(self):
version = self.ndk
if not version:
version = 'r10e' # falls back to latest
paths = common.config['ndk_paths']
if version not in paths:
return ''
return paths[version]
def update_flags(self, d):
for f, v in d.iteritems():
self.set_flag(f, v)
def flagtype(name):
if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni', 'preassemble',
'update', 'scanignore', 'scandelete', 'gradle', 'antcommands',
'gradleprops']:
return 'list'
if name in ['init', 'prebuild', 'build']:
return 'script'
if name in ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
'novcheck']:
return 'bool'
return 'string'
# Designates a metadata field type and checks that it matches
@ -230,18 +349,18 @@ flag_defaults = OrderedDict([
# 'matching' - List of possible values or regex expression
# 'sep' - Separator to use if value may be a list
# 'fields' - Metadata fields (Field:Value) of this type
# 'attrs' - Build attributes (attr=value) of this type
# 'flags' - Build flags (flag=value) of this type
#
class FieldValidator():
def __init__(self, name, matching, sep, fields, attrs):
def __init__(self, name, matching, sep, fields, flags):
self.name = name
self.matching = matching
if type(matching) is str:
self.compiled = re.compile(matching)
self.sep = sep
self.fields = fields
self.attrs = attrs
self.flags = flags
def _assert_regex(self, values, appid):
for v in values:
@ -257,13 +376,13 @@ class FieldValidator():
% (v, self.name, appid) +
"Possible values: %s" % (", ".join(self.matching)))
def check(self, value, appid):
if type(value) is not str or not value:
def check(self, v, appid):
if type(v) is not str or not v:
return
if self.sep is not None:
values = value.split(self.sep)
values = v.split(self.sep)
else:
values = [value]
values = [v]
if type(self.matching) is list:
self._assert_list(values, appid)
else:
@ -337,11 +456,11 @@ valuetypes = {
# Check an app's metadata information for integrity errors
def check_metadata(app):
for v in valuetypes:
for field in v.fields:
v.check(app.get_field(field), app.id)
for f in v.fields:
v.check(app.get_field(f), app.id)
for build in app.builds:
for attr in v.attrs:
v.check(build[attr], app.id)
for f in v.flags:
v.check(build.get_flag(f), app.id)
# Formatter for descriptions. Create an instance, and call parseline() with
@ -554,14 +673,14 @@ def parse_srclib(metadatapath):
continue
try:
field, value = line.split(':', 1)
f, v = line.split(':', 1)
except ValueError:
raise MetaDataException("Invalid metadata in %s:%d" % (line, n))
if field == "Subdir":
thisinfo[field] = value.split(',')
if f == "Subdir":
thisinfo[f] = v.split(',')
else:
thisinfo[field] = value
thisinfo[f] = v
return thisinfo
@ -640,54 +759,6 @@ def read_metadata(xref=True):
return apps
# Get the type expected for a given metadata field.
def metafieldtype(name):
if name in ['Description', 'Maintainer Notes']:
return 'multiline'
if name in ['Categories', 'AntiFeatures']:
return 'list'
if name == 'Build Version':
return 'build'
if name == 'Build':
return 'buildv2'
if name == 'Use Built':
return 'obsolete'
if name not in app_fields:
return 'unknown'
return 'string'
def flagtype(name):
if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni', 'preassemble',
'update', 'scanignore', 'scandelete', 'gradle', 'antcommands',
'gradleprops']:
return 'list'
if name in ['init', 'prebuild', 'build']:
return 'script'
if name in ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
'novcheck']:
return 'bool'
return 'string'
def fill_build_defaults(build):
def get_build_type():
for t in ['maven', 'gradle', 'kivy']:
if build[t]:
return t
if build['output']:
return 'raw'
return 'ant'
for flag, value in flag_defaults.iteritems():
if flag in build:
continue
build[flag] = value
build['type'] = get_build_type()
build['ndk_path'] = common.get_ndk_path(build['ndk'])
def split_list_values(s):
# Port legacy ';' separators
l = [v.strip() for v in s.replace(';', ',').split(',')]
@ -709,7 +780,7 @@ def get_default_app_info(metadatapath=None):
def sorted_builds(builds):
return sorted(builds, key=lambda build: int(build['vercode']))
return sorted(builds, key=lambda build: int(build.vercode))
def post_metadata_parse(app):
@ -726,60 +797,30 @@ def post_metadata_parse(app):
text = v.rstrip().lstrip()
app.set_field(f, text.split('\n'))
supported_flags = (flag_defaults.keys()
+ ['vercode', 'version', 'versionCode', 'versionName',
'type', 'ndk_path'])
esc_newlines = re.compile('\\\\( |\\n)')
for build in app.builds:
for k, v in build.items():
if k not in supported_flags:
raise MetaDataException("Unrecognised build flag: {0}={1}"
.format(k, v))
for k in build_flags:
v = build.get_flag(k)
if k == 'versionCode':
build['vercode'] = str(v)
del build['versionCode']
elif k == 'versionName':
build['version'] = str(v)
del build['versionName']
elif type(v) in (float, int):
build[k] = str(v)
if type(v) in (float, int):
build.set_flag(k, v)
else:
keyflagtype = flagtype(k)
if keyflagtype == 'list':
# these can be bools, strings or lists, but ultimately are lists
if isinstance(v, basestring):
build[k] = [v]
elif isinstance(v, bool):
build[k] = ['yes' if v else 'no']
elif isinstance(v, list):
build[k] = []
for e in v:
if isinstance(e, bool):
build[k].append('yes' if v else 'no')
else:
build[k].append(e)
elif keyflagtype == 'script':
build[k] = re.sub(esc_newlines, '', v).lstrip().rstrip()
if keyflagtype == 'script':
build.set_flag(k, re.sub(esc_newlines, '', v).lstrip().rstrip())
elif keyflagtype == 'bool':
# TODO handle this using <xsd:element type="xsd:boolean> in a schema
if isinstance(v, basestring):
if v == 'true':
build[k] = True
else:
build[k] = False
if isinstance(v, basestring) and v == 'true':
build.set_flag(k, 'true')
elif keyflagtype == 'string':
if isinstance(v, bool):
build[k] = 'yes' if v else 'no'
if isinstance(v, bool) and v:
build.set_flag(k, 'yes')
if not app.Description:
app.Description = ['No description available']
for build in app.builds:
fill_build_defaults(build)
app.builds = sorted_builds(app.builds)
@ -826,16 +867,16 @@ def _decode_list(data):
def _decode_dict(data):
'''convert items in a dict from unicode to basestring'''
rv = {}
for key, value in data.iteritems():
if isinstance(key, unicode):
key = key.encode('utf-8')
if isinstance(value, unicode):
value = value.encode('utf-8')
elif isinstance(value, list):
value = _decode_list(value)
elif isinstance(value, dict):
value = _decode_dict(value)
rv[key] = value
for k, v in data.iteritems():
if isinstance(k, unicode):
k = k.encode('utf-8')
if isinstance(v, unicode):
v = v.encode('utf-8')
elif isinstance(v, list):
v = _decode_list(v)
elif isinstance(v, dict):
v = _decode_dict(v)
rv[k] = v
return rv
@ -897,16 +938,14 @@ def parse_xml_metadata(metadatapath):
if child.tag == 'string':
app.set_field(name, child.text)
elif child.tag == 'string-array':
items = []
for item in child:
items.append(item.text)
app.set_field(name, items)
app.append_field(name, item.text)
elif child.tag == 'builds':
for build in child:
builddict = dict()
for key in build:
builddict[key.tag] = key.text
app.builds.append(builddict)
for b in child:
build = Build()
for key in b:
build.set_flag(key.tag, key.text)
app.builds.append(build)
# TODO handle this using <xsd:element type="xsd:boolean> in a schema
if not isinstance(app.RequiresRoot, bool):
@ -935,7 +974,7 @@ def parse_txt_metadata(metadatapath):
linedesc = None
def add_buildflag(p, thisbuild):
def add_buildflag(p, build):
if not p.strip():
raise MetaDataException("Empty build flag at {1}"
.format(buildlines[0], linedesc))
@ -943,13 +982,10 @@ def parse_txt_metadata(metadatapath):
if len(bv) != 2:
raise MetaDataException("Invalid build flag at {0} in {1}"
.format(buildlines[0], linedesc))
pk, pv = bv
if pk in thisbuild:
raise MetaDataException("Duplicate definition on {0} in version {1} of {2}"
.format(pk, thisbuild['version'], linedesc))
pk, pv = bv
pk = pk.lstrip()
if pk not in flag_defaults:
if pk not in build_flags:
raise MetaDataException("Unrecognised build flag at {0} in {1}"
.format(p, linedesc))
t = flagtype(pk)
@ -958,45 +994,45 @@ def parse_txt_metadata(metadatapath):
if pk == 'gradle':
if len(pv) == 1 and pv[0] in ['main', 'yes']:
pv = ['yes']
thisbuild[pk] = pv
build.set_flag(pk, pv)
elif t == 'string' or t == 'script':
thisbuild[pk] = pv
build.set_flag(pk, pv)
elif t == 'bool':
value = pv == 'yes'
if value:
thisbuild[pk] = True
v = pv == 'yes'
if v:
build.set_flag(pk, True)
else:
raise MetaDataException("Unrecognised build flag type '%s' at %s in %s"
% (t, p, linedesc))
def parse_buildline(lines):
value = "".join(lines)
v = "".join(lines)
parts = [p.replace("\\,", ",")
for p in re.split(r"(?<!\\),", value)]
for p in re.split(r"(?<!\\),", v)]
if len(parts) < 3:
raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
thisbuild = {}
thisbuild['origlines'] = lines
thisbuild['version'] = parts[0]
thisbuild['vercode'] = parts[1]
raise MetaDataException("Invalid build format: " + v + " in " + metafile.name)
build = Build()
build.origlines = lines
build.version = parts[0]
build.vercode = parts[1]
if parts[2].startswith('!'):
# For backwards compatibility, handle old-style disabling,
# including attempting to extract the commit from the message
thisbuild['disable'] = parts[2][1:]
build.disable = parts[2][1:]
commit = 'unknown - see disabled'
index = parts[2].rfind('at ')
if index != -1:
commit = parts[2][index + 3:]
if commit.endswith(')'):
commit = commit[:-1]
thisbuild['commit'] = commit
build.commit = commit
else:
thisbuild['commit'] = parts[2]
build.commit = parts[2]
for p in parts[3:]:
add_buildflag(p, thisbuild)
add_buildflag(p, build)
return thisbuild
return build
def add_comments(key):
if not curcomments:
@ -1010,7 +1046,7 @@ def parse_txt_metadata(metadatapath):
mode = 0
buildlines = []
curcomments = []
curbuild = None
build = None
vc_seen = {}
c = 0
@ -1020,13 +1056,12 @@ def parse_txt_metadata(metadatapath):
line = line.rstrip('\r\n')
if mode == 3:
if not any(line.startswith(s) for s in (' ', '\t')):
commit = curbuild['commit'] if 'commit' in curbuild else None
if not commit and 'disable' not in curbuild:
if not build.commit and not build.disable:
raise MetaDataException("No commit specified for {0} in {1}"
.format(curbuild['version'], linedesc))
.format(build.version, linedesc))
app.builds.append(curbuild)
add_comments('build:' + curbuild['vercode'])
app.builds.append(build)
add_comments('build:' + build.vercode)
mode = 0
else:
if line.endswith('\\'):
@ -1034,7 +1069,7 @@ def parse_txt_metadata(metadatapath):
else:
buildlines.append(line.lstrip())
bl = ''.join(buildlines)
add_buildflag(bl, curbuild)
add_buildflag(bl, build)
buildlines = []
if mode == 0:
@ -1044,74 +1079,74 @@ def parse_txt_metadata(metadatapath):
curcomments.append(line[1:].strip())
continue
try:
field, value = line.split(':', 1)
f, v = line.split(':', 1)
except ValueError:
raise MetaDataException("Invalid metadata in " + linedesc)
if field != field.strip() or value != value.strip():
if f != f.strip() or v != v.strip():
raise MetaDataException("Extra spacing found in " + linedesc)
# Translate obsolete fields...
if field == 'Market Version':
field = 'Current Version'
if field == 'Market Version Code':
field = 'Current Version Code'
if f == 'Market Version':
f = 'Current Version'
if f == 'Market Version Code':
f = 'Current Version Code'
fieldtype = metafieldtype(field)
fieldtype = metafieldtype(f)
if fieldtype not in ['build', 'buildv2']:
add_comments(field)
add_comments(f)
if fieldtype == 'multiline':
mode = 1
if value:
raise MetaDataException("Unexpected text on same line as " + field + " in " + linedesc)
if v:
raise MetaDataException("Unexpected text on same line as " + f + " in " + linedesc)
elif fieldtype == 'string':
app.set_field(field, value)
app.set_field(f, v)
elif fieldtype == 'list':
app.set_field(field, split_list_values(value))
app.set_field(f, split_list_values(v))
elif fieldtype == 'build':
if value.endswith("\\"):
if v.endswith("\\"):
mode = 2
buildlines = [value[:-1]]
buildlines = [v[:-1]]
else:
curbuild = parse_buildline([value])
app.builds.append(curbuild)
add_comments('build:' + app.builds[-1]['vercode'])
build = parse_buildline([v])
app.builds.append(build)
add_comments('build:' + app.builds[-1].vercode)
elif fieldtype == 'buildv2':
curbuild = {}
vv = value.split(',')
build = Build()
vv = v.split(',')
if len(vv) != 2:
raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'
.format(value, linedesc))
curbuild['version'] = vv[0]
curbuild['vercode'] = vv[1]
if curbuild['vercode'] in vc_seen:
.format(v, linedesc))
build.version = vv[0]
build.vercode = vv[1]
if build.vercode in vc_seen:
raise MetaDataException('Duplicate build recipe found for vercode %s in %s' % (
curbuild['vercode'], linedesc))
vc_seen[curbuild['vercode']] = True
build.vercode, linedesc))
vc_seen[build.vercode] = True
buildlines = []
mode = 3
elif fieldtype == 'obsolete':
pass # Just throw it away!
else:
raise MetaDataException("Unrecognised field type for " + field + " in " + linedesc)
raise MetaDataException("Unrecognised field type for " + f + " in " + linedesc)
elif mode == 1: # Multiline field
if line == '.':
mode = 0
else:
app.append_field(field, line)
app.append_field(f, line)
elif mode == 2: # Line continuation mode in Build Version
if line.endswith("\\"):
buildlines.append(line[:-1])
else:
buildlines.append(line)
curbuild = parse_buildline(buildlines)
app.builds.append(curbuild)
add_comments('build:' + app.builds[-1]['vercode'])
build = parse_buildline(buildlines)
app.builds.append(build)
add_comments('build:' + app.builds[-1].vercode)
mode = 0
add_comments(None)
# Mode at end of file should always be 0...
if mode == 1:
raise MetaDataException(field + " not terminated in " + metafile.name)
raise MetaDataException(f + " not terminated in " + metafile.name)
elif mode == 2:
raise MetaDataException("Unterminated continuation in " + metafile.name)
elif mode == 3:
@ -1130,18 +1165,18 @@ def write_plaintext_metadata(mf, app, w_comment, w_field, w_build):
for line in app.comments[key]:
w_comment(line)
def w_field_always(field, value=None):
if value is None:
value = app.get_field(field)
w_comments(field)
w_field(field, value)
def w_field_always(f, v=None):
if v is None:
v = app.get_field(f)
w_comments(f)
w_field(f, v)
def w_field_nonempty(field, value=None):
if value is None:
value = app.get_field(field)
w_comments(field)
if value:
w_field(field, value)
def w_field_nonempty(f, v=None):
if v is None:
v = app.get_field(f)
w_comments(f)
if v:
w_field(f, v)
w_field_nonempty('Disabled')
if app.AntiFeatures:
@ -1175,10 +1210,10 @@ def write_plaintext_metadata(mf, app, w_comment, w_field, w_build):
for build in sorted_builds(app.builds):
if build['version'] == "Ignore":
if build.version == "Ignore":
continue
w_comments('build:' + build['vercode'])
w_comments('build:' + build.vercode)
w_build(build)
mf.write('\n')
@ -1211,39 +1246,37 @@ def write_txt_metadata(mf, app):
def w_comment(line):
mf.write("# %s\n" % line)
def w_field(field, value):
t = metafieldtype(field)
def w_field(f, v):
t = metafieldtype(f)
if t == 'list':
value = ','.join(value)
v = ','.join(v)
elif t == 'multiline':
if type(value) == list:
value = '\n' + '\n'.join(value) + '\n.'
if type(v) == list:
v = '\n' + '\n'.join(v) + '\n.'
else:
value = '\n' + value + '\n.'
mf.write("%s:%s\n" % (field, value))
v = '\n' + v + '\n.'
mf.write("%s:%s\n" % (f, v))
def w_build(build):
mf.write("Build:%s,%s\n" % (build['version'], build['vercode']))
mf.write("Build:%s,%s\n" % (build.version, build.vercode))
for key in flag_defaults:
value = build[key]
if not value:
continue
if value == flag_defaults[key]:
for f in build_flags_order:
v = build.get_flag(f)
if not v:
continue
t = flagtype(key)
v = ' %s=' % key
t = flagtype(f)
out = ' %s=' % f
if t == 'string':
v += value
out += v
elif t == 'bool':
v += 'yes'
out += 'yes'
elif t == 'script':
v += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
out += '&& \\\n '.join([s.lstrip() for s in v.split('&& ')])
elif t == 'list':
v += ','.join(value) if type(value) == list else value
out += ','.join(v) if type(v) == list else v
mf.write(v)
mf.write(out)
mf.write('\n')
write_plaintext_metadata(mf, app, w_comment, w_field, w_build)
@ -1254,26 +1287,26 @@ def write_yaml_metadata(mf, app):
def w_comment(line):
mf.write("# %s\n" % line)
def escape(value):
if not value:
def escape(v):
if not v:
return ''
if any(c in value for c in [': ', '%', '@', '*']):
return "'" + value.replace("'", "''") + "'"
return value
if any(c in v for c in [': ', '%', '@', '*']):
return "'" + v.replace("'", "''") + "'"
return v
def w_field(field, value, prefix='', t=None):
def w_field(f, v, prefix='', t=None):
if t is None:
t = metafieldtype(field)
t = metafieldtype(f)
v = ''
if t == 'list':
v = '\n'
for e in value:
for e in v:
v += prefix + ' - ' + escape(e) + '\n'
elif t == 'multiline':
v = ' |\n'
lines = value
if type(value) == str:
lines = value.splitlines()
lines = v
if type(v) == str:
lines = v.splitlines()
for l in lines:
if l:
v += prefix + ' ' + l + '\n'
@ -1282,16 +1315,16 @@ def write_yaml_metadata(mf, app):
elif t == 'bool':
v = ' yes\n'
elif t == 'script':
cmds = [s + '&& \\' for s in value.split('&& ')]
cmds = [s + '&& \\' for s in v.split('&& ')]
if len(cmds) > 0:
cmds[-1] = cmds[-1][:-len('&& \\')]
w_field(field, cmds, prefix, 'multiline')
w_field(f, cmds, prefix, 'multiline')
return
else:
v = ' ' + escape(value) + '\n'
v = ' ' + escape(v) + '\n'
mf.write(prefix)
mf.write(field)
mf.write(f)
mf.write(":")
mf.write(v)
@ -1304,16 +1337,14 @@ def write_yaml_metadata(mf, app):
mf.write("builds:\n")
first_build = False
w_field('versionName', build['version'], ' - ', 'string')
w_field('versionCode', build['vercode'], ' ', 'strsng')
for key in flag_defaults:
value = build[key]
if not value:
continue
if value == flag_defaults[key]:
w_field('versionName', build.version, ' - ', 'string')
w_field('versionCode', build.vercode, ' ', 'strsng')
for f in build_flags_order:
v = build.get_flag(f)
if not v:
continue
w_field(key, value, ' ', flagtype(key))
w_field(f, v, ' ', flagtype(f))
write_plaintext_metadata(mf, app, w_comment, w_field, w_build)

32
fdroidserver/scanner.py
View file

@ -31,18 +31,18 @@ config = None
options = None
def get_gradle_compile_commands(thisbuild):
def get_gradle_compile_commands(build):
compileCommands = ['compile', 'releaseCompile']
if thisbuild['gradle'] and thisbuild['gradle'] != ['yes']:
compileCommands += [flavor + 'Compile' for flavor in thisbuild['gradle']]
compileCommands += [flavor + 'ReleaseCompile' for flavor in thisbuild['gradle']]
if build.gradle and build.gradle != ['yes']:
compileCommands += [flavor + 'Compile' for flavor in build.gradle]
compileCommands += [flavor + 'ReleaseCompile' for flavor in build.gradle]
return [re.compile(r'\s*' + c, re.IGNORECASE) for c in compileCommands]
# Scan the source code in the given directory (and all subdirectories)
# and return the number of fatal problems encountered
def scan_source(build_dir, root_dir, thisbuild):
def scan_source(build_dir, root_dir, build):
count = 0
@ -85,8 +85,8 @@ def scan_source(build_dir, root_dir, thisbuild):
]
]
scanignore = common.getpaths_map(build_dir, thisbuild['scanignore'])
scandelete = common.getpaths_map(build_dir, thisbuild['scandelete'])
scanignore = common.getpaths_map(build_dir, build.scanignore)
scandelete = common.getpaths_map(build_dir, build.scandelete)
scanignore_worked = set()
scandelete_worked = set()
@ -153,7 +153,7 @@ def scan_source(build_dir, root_dir, thisbuild):
return True
return False
gradle_compile_commands = get_gradle_compile_commands(thisbuild)
gradle_compile_commands = get_gradle_compile_commands(build)
def is_used_by_gradle(line):
return any(command.match(line) for command in gradle_compile_commands)
@ -240,7 +240,7 @@ def scan_source(build_dir, root_dir, thisbuild):
# indicate a problem (if it's not a problem, explicitly use
# buildjni=no to bypass this check)
if (os.path.exists(os.path.join(root_dir, 'jni')) and
not thisbuild['buildjni']):
not build.buildjni):
logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
count += 1
@ -293,24 +293,24 @@ def main():
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
for thisbuild in app.builds:
for build in app.builds:
if thisbuild['disable']:
if build.disable:
logging.info("...skipping version %s - %s" % (
thisbuild['version'], thisbuild.get('disable', thisbuild['commit'][1:])))
build.version, build.get('disable', build.commit[1:])))
else:
logging.info("...scanning version " + thisbuild['version'])
logging.info("...scanning version " + build.version)
# Prepare the source code...
root_dir, _ = common.prepare_source(vcs, app, thisbuild,
root_dir, _ = common.prepare_source(vcs, app, build,
build_dir, srclib_dir,
extlib_dir, False)
# Do the scan...
count = scan_source(build_dir, root_dir, thisbuild)
count = scan_source(build_dir, root_dir, build)
if count > 0:
logging.warn('Scanner found %d problems in %s (%s)' % (
count, appid, thisbuild['vercode']))
count, appid, build.vercode))
probcount += count
except BuildException as be:

136
fdroidserver/update.py
View file

@ -144,26 +144,26 @@ def update_wiki(apps, sortedids, apks):
gotcurrentver = True
apklist.append(apk)
# Include ones we can't build, as a special case...
for thisbuild in app.builds:
if thisbuild['disable']:
if thisbuild['vercode'] == app.CurrentVersionCode:
for build in app.builds:
if build.disable:
if build.vercode == app.CurrentVersionCode:
cantupdate = True
# TODO: Nasty: vercode is a string in the build, and an int elsewhere
apklist.append({'versioncode': int(thisbuild['vercode']),
'version': thisbuild['version'],
'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(thisbuild['disable']),
apklist.append({'versioncode': int(build.vercode),
'version': build.version,
'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
})
else:
builtit = False
for apk in apklist:
if apk['versioncode'] == int(thisbuild['vercode']):
if apk['versioncode'] == int(build.vercode):
builtit = True
break
if not builtit:
buildfails = True
apklist.append({'versioncode': int(thisbuild['vercode']),
'version': thisbuild['version'],
'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, thisbuild['vercode']),
apklist.append({'versioncode': int(build.vercode),
'version': build.version,
'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
})
if app.CurrentVersionCode == '0':
cantupdate = True
@ -291,12 +291,12 @@ def delete_disabled_builds(apps, apkcache, repodirs):
"""
for appid, app in apps.iteritems():
for build in app.builds:
if not build['disable']:
if not build.disable:
continue
apkfilename = appid + '_' + str(build['vercode']) + '.apk'
apkfilename = appid + '_' + str(build.vercode) + '.apk'
iconfilename = "%s.%s.png" % (
appid,
build['vercode'])
build.vercode)
for repodir in repodirs:
files = [
os.path.join(repodir, apkfilename),
@ -453,8 +453,8 @@ def scan_apks(apps, apkcache, repodir, knownapks):
usecache = False
if apkfilename in apkcache:
thisinfo = apkcache[apkfilename]
if thisinfo['sha256'] == shasum:
apk = apkcache[apkfilename]
if apk['sha256'] == shasum:
logging.debug("Reading " + apkfilename + " from cache")
usecache = True
else:
@ -462,17 +462,17 @@ def scan_apks(apps, apkcache, repodir, knownapks):
if not usecache:
logging.debug("Processing " + apkfilename)
thisinfo = {}
thisinfo['apkname'] = apkfilename
thisinfo['sha256'] = shasum
apk = {}
apk['apkname'] = apkfilename
apk['sha256'] = shasum
srcfilename = apkfilename[:-4] + "_src.tar.gz"
if os.path.exists(os.path.join(repodir, srcfilename)):
thisinfo['srcname'] = srcfilename
thisinfo['size'] = os.path.getsize(apkfile)
thisinfo['permissions'] = set()
thisinfo['features'] = set()
thisinfo['icons_src'] = {}
thisinfo['icons'] = {}
apk['srcname'] = srcfilename
apk['size'] = os.path.getsize(apkfile)
apk['permissions'] = set()
apk['features'] = set()
apk['icons_src'] = {}
apk['icons'] = {}
p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
if p.returncode != 0:
if options.delete_unknown:
@ -487,51 +487,51 @@ def scan_apks(apps, apkcache, repodir, knownapks):
for line in p.output.splitlines():
if line.startswith("package:"):
try:
thisinfo['id'] = re.match(name_pat, line).group(1)
thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
thisinfo['version'] = re.match(vername_pat, line).group(1)
apk['id'] = re.match(name_pat, line).group(1)
apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
apk['version'] = re.match(vername_pat, line).group(1)
except Exception, e:
logging.error("Package matching failed: " + str(e))
logging.info("Line was: " + line)
sys.exit(1)
elif line.startswith("application:"):
thisinfo['name'] = re.match(label_pat, line).group(1)
apk['name'] = re.match(label_pat, line).group(1)
# Keep path to non-dpi icon in case we need it
match = re.match(icon_pat_nodpi, line)
if match:
thisinfo['icons_src']['-1'] = match.group(1)
apk['icons_src']['-1'] = match.group(1)
elif line.startswith("launchable-activity:"):
# Only use launchable-activity as fallback to application
if not thisinfo['name']:
thisinfo['name'] = re.match(label_pat, line).group(1)
if '-1' not in thisinfo['icons_src']:
if not apk['name']:
apk['name'] = re.match(label_pat, line).group(1)
if '-1' not in apk['icons_src']:
match = re.match(icon_pat_nodpi, line)
if match:
thisinfo['icons_src']['-1'] = match.group(1)
apk['icons_src']['-1'] = match.group(1)
elif line.startswith("application-icon-"):
match = re.match(icon_pat, line)
if match:
density = match.group(1)
path = match.group(2)
thisinfo['icons_src'][density] = path
apk['icons_src'][density] = path
elif line.startswith("sdkVersion:"):
m = re.match(sdkversion_pat, line)
if m is None:
logging.error(line.replace('sdkVersion:', '')
+ ' is not a valid minSdkVersion!')
else:
thisinfo['sdkversion'] = m.group(1)
apk['sdkversion'] = m.group(1)
elif line.startswith("maxSdkVersion:"):
thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
apk['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
elif line.startswith("native-code:"):
thisinfo['nativecode'] = []
apk['nativecode'] = []
for arch in line[13:].split(' '):
thisinfo['nativecode'].append(arch[1:-1])
apk['nativecode'].append(arch[1:-1])
elif line.startswith("uses-permission:"):
perm = re.match(string_pat, line).group(1)
if perm.startswith("android.permission."):
perm = perm[19:]
thisinfo['permissions'].add(perm)
apk['permissions'].add(perm)
elif line.startswith("uses-feature:"):
perm = re.match(string_pat, line).group(1)
# Filter out this, it's only added with the latest SDK tools and
@ -540,11 +540,11 @@ def scan_apks(apps, apkcache, repodir, knownapks):
and perm != "android.hardware.screen.landscape":
if perm.startswith("android.feature."):
perm = perm[16:]
thisinfo['features'].add(perm)
apk['features'].add(perm)
if 'sdkversion' not in thisinfo:
if 'sdkversion' not in apk:
logging.warn("No SDK version information found in {0}".format(apkfile))
thisinfo['sdkversion'] = 0
apk['sdkversion'] = 0
# Check for debuggable apks...
if common.isApkDebuggable(apkfile, config):
@ -552,20 +552,20 @@ def scan_apks(apps, apkcache, repodir, knownapks):
# Get the signature (or md5 of, to be precise)...
logging.debug('Getting signature of {0}'.format(apkfile))
thisinfo['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
if not thisinfo['sig']:
apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
if not apk['sig']:
logging.critical("Failed to get apk signature")
sys.exit(1)
apk = zipfile.ZipFile(apkfile, 'r')
apkzip = zipfile.ZipFile(apkfile, 'r')
# if an APK has files newer than the system time, suggest updating
# the system clock. This is useful for offline systems, used for
# signing, which do not have another source of clock sync info. It
# has to be more than 24 hours newer because ZIP/APK files do not
# store timezone info
info = apk.getinfo('AndroidManifest.xml')
dt_obj = datetime(*info.date_time)
manifest = apkzip.getinfo('AndroidManifest.xml')
dt_obj = datetime(*manifest.date_time)
checkdt = dt_obj - timedelta(1)
if datetime.today() < checkdt:
logging.warn('System clock is older than manifest in: '
@ -573,44 +573,44 @@ def scan_apks(apps, apkcache, repodir, knownapks):
+ 'sudo date -s "' + str(dt_obj) + '"')
iconfilename = "%s.%s.png" % (
thisinfo['id'],
thisinfo['versioncode'])
apk['id'],
apk['versioncode'])
# Extract the icon file...
empty_densities = []
for density in screen_densities:
if density not in thisinfo['icons_src']:
if density not in apk['icons_src']:
empty_densities.append(density)
continue
iconsrc = thisinfo['icons_src'][density]
iconsrc = apk['icons_src'][density]
icon_dir = get_icon_dir(repodir, density)
icondest = os.path.join(icon_dir, iconfilename)
try:
with open(icondest, 'wb') as f:
f.write(apk.read(iconsrc))
thisinfo['icons'][density] = iconfilename
f.write(apkzip.read(iconsrc))
apk['icons'][density] = iconfilename
except:
logging.warn("Error retrieving icon file")
del thisinfo['icons'][density]
del thisinfo['icons_src'][density]
del apk['icons'][density]
del apk['icons_src'][density]
empty_densities.append(density)
if '-1' in thisinfo['icons_src']:
iconsrc = thisinfo['icons_src']['-1']
if '-1' in apk['icons_src']:
iconsrc = apk['icons_src']['-1']
iconpath = os.path.join(
get_icon_dir(repodir, '0'), iconfilename)
with open(iconpath, 'wb') as f:
f.write(apk.read(iconsrc))
f.write(apkzip.read(iconsrc))
try:
im = Image.open(iconpath)
dpi = px_to_dpi(im.size[0])
for density in screen_densities:
if density in thisinfo['icons']:
if density in apk['icons']:
break
if density == screen_densities[-1] or dpi >= int(density):
thisinfo['icons'][density] = iconfilename
apk['icons'][density] = iconfilename
shutil.move(iconpath,
os.path.join(get_icon_dir(repodir, density), iconfilename))
empty_densities.remove(density)
@ -618,10 +618,10 @@ def scan_apks(apps, apkcache, repodir, knownapks):
except Exception, e:
logging.warn("Failed reading {0} - {1}".format(iconpath, e))
if thisinfo['icons']:
thisinfo['icon'] = iconfilename
if apk['icons']:
apk['icon'] = iconfilename
apk.close()
apkzip.close()
# First try resizing down to not lose quality
last_density = None
@ -675,19 +675,19 @@ def scan_apks(apps, apkcache, repodir, knownapks):
# Copy from icons-mdpi to icons since mdpi is the baseline density
baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
if os.path.isfile(baseline):
thisinfo['icons']['0'] = iconfilename
apk['icons']['0'] = iconfilename
shutil.copyfile(baseline,
os.path.join(get_icon_dir(repodir, '0'), iconfilename))
# Record in known apks, getting the added date at the same time..
added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
added = knownapks.recordapk(apk['apkname'], apk['id'])
if added:
thisinfo['added'] = added
apk['added'] = added
apkcache[apkfilename] = thisinfo
apkcache[apkfilename] = apk
cachechanged = True
apks.append(thisinfo)
apks.append(apk)
return apks, cachechanged