mirrors/fdroidserver
1
0
Fork 0
mirror of https://github.com/f-droid/fdroidserver.git synced 2025-11-05 06:50:29 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

More logging

Browse source
This commit is contained in:
Daniel Martí 2014-01-27 17:04:22 +01:00
parent e20fa9d7f4
commit 1b743e8b74
4 changed files with 92 additions and 102 deletions
Download patch file Download diff file Expand all files Collapse all files

51
fdroidserver/build.py
View file

@ -462,8 +462,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
jni_components = [c.strip() for c in jni_components.split(';')] jni_components = [c.strip() for c in jni_components.split(';')]
ndkbuild = os.path.join(config['ndk_path'], "ndk-build") ndkbuild = os.path.join(config['ndk_path'], "ndk-build")
for d in jni_components: for d in jni_components:
if options.verbose: logging.info("Building native code in '%s'" % d)
print "Running ndk-build in " + root_dir + '/' + d
manifest = root_dir + '/' + d + '/AndroidManifest.xml' manifest = root_dir + '/' + d + '/AndroidManifest.xml'
if os.path.exists(manifest): if os.path.exists(manifest):
# Read and write the whole AM.xml to fix newlines and avoid # Read and write the whole AM.xml to fix newlines and avoid
@ -482,7 +481,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
p = None p = None
# Build the release... # Build the release...
if thisbuild['type'] == 'maven': if thisbuild['type'] == 'maven':
print "Building Maven project..." logging.info("Building Maven project...")
if '@' in thisbuild['maven']: if '@' in thisbuild['maven']:
maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@',1)[1]) maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@',1)[1])
@ -510,7 +509,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
bindir = os.path.join(root_dir, 'target') bindir = os.path.join(root_dir, 'target')
elif thisbuild['type'] == 'kivy': elif thisbuild['type'] == 'kivy':
print "Building Kivy project..." logging.info("Building Kivy project...")
spec = os.path.join(root_dir, 'buildozer.spec') spec = os.path.join(root_dir, 'buildozer.spec')
if not os.path.exists(spec): if not os.path.exists(spec):
@ -570,7 +569,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
p = FDroidPopen(cmd, cwd=distdir) p = FDroidPopen(cmd, cwd=distdir)
elif thisbuild['type'] == 'gradle': elif thisbuild['type'] == 'gradle':
print "Building Gradle project..." logging.info("Building Gradle project...")
if '@' in thisbuild['gradle']: if '@' in thisbuild['gradle']:
flavours = thisbuild['gradle'].split('@')[0].split(',') flavours = thisbuild['gradle'].split('@')[0].split(',')
gradle_dir = thisbuild['gradle'].split('@')[1] gradle_dir = thisbuild['gradle'].split('@')[1]
@ -601,7 +600,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
p = FDroidPopen(commands, cwd=gradle_dir) p = FDroidPopen(commands, cwd=gradle_dir)
else: else:
print "Building Ant project..." logging.info("Building Ant project...")
cmd = ['ant'] cmd = ['ant']
if 'antcommand' in thisbuild: if 'antcommand' in thisbuild:
cmd += [thisbuild['antcommand']] cmd += [thisbuild['antcommand']]
@ -613,7 +612,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
if p.returncode != 0: if p.returncode != 0:
raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout) raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout)
print "Successfully built version " + thisbuild['version'] + ' of ' + app['id'] logging.info("Successfully built version " + thisbuild['version'] + ' of ' + app['id'])
# Find the apk name in the output... # Find the apk name in the output...
if 'bindir' in thisbuild: if 'bindir' in thisbuild:
@ -659,7 +658,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
# By way of a sanity check, make sure the version and version # By way of a sanity check, make sure the version and version
# code in our new apk match what we expect... # code in our new apk match what we expect...
print "Checking " + src logging.info("Checking " + src)
if not os.path.exists(src): if not os.path.exists(src):
raise BuildException("Unsigned apk is not at expected location of " + src) raise BuildException("Unsigned apk is not at expected location of " + src)
@ -758,7 +757,7 @@ def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir,
if 'disable' in thisbuild: if 'disable' in thisbuild:
return False return False
print "Building version " + thisbuild['version'] + ' of ' + app['id'] logging.info("Building version " + thisbuild['version'] + ' of ' + app['id'])
if server: if server:
# When using server mode, still keep a local cache of the repo, by # When using server mode, still keep a local cache of the repo, by
@ -826,12 +825,12 @@ def main():
log_dir = 'logs' log_dir = 'logs'
if not os.path.isdir(log_dir): if not os.path.isdir(log_dir):
print "Creating log directory" logging.info("Creating log directory")
os.makedirs(log_dir) os.makedirs(log_dir)
tmp_dir = 'tmp' tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir): if not os.path.isdir(tmp_dir):
print "Creating temporary directory" logging.info("Creating temporary directory")
os.makedirs(tmp_dir) os.makedirs(tmp_dir)
if options.test: if options.test:
@ -839,7 +838,7 @@ def main():
else: else:
output_dir = 'unsigned' output_dir = 'unsigned'
if not os.path.isdir(output_dir): if not os.path.isdir(output_dir):
print "Creating output directory" logging.info("Creating output directory")
os.makedirs(output_dir) os.makedirs(output_dir)
if config['archive_older'] != 0: if config['archive_older'] != 0:
@ -851,7 +850,7 @@ def main():
build_dir = 'build' build_dir = 'build'
if not os.path.isdir(build_dir): if not os.path.isdir(build_dir):
print "Creating build directory" logging.info("Creating build directory")
os.makedirs(build_dir) os.makedirs(build_dir)
srclib_dir = os.path.join(build_dir, 'srclib') srclib_dir = os.path.join(build_dir, 'srclib')
extlib_dir = os.path.join(build_dir, 'extlib') extlib_dir = os.path.join(build_dir, 'extlib')
@ -901,15 +900,13 @@ def main():
build_dir = os.path.join('build', app['id']) build_dir = os.path.join('build', app['id'])
# Set up vcs interface and make sure we have the latest code... # Set up vcs interface and make sure we have the latest code...
if options.verbose: logging.info("Getting {0} vcs interface for {1}".format(
print "Getting {0} vcs interface for {1}".format( app['Repo Type'], app['Repo']))
app['Repo Type'], app['Repo'])
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
first = False first = False
if options.verbose: logging.info("Checking " + thisbuild['version'])
print "Checking " + thisbuild['version']
if trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, if trybuild(app, thisbuild, build_dir, output_dir, also_check_dir,
srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, options.test, srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, options.test,
options.server, options.force, options.onserver): options.server, options.force, options.onserver):
@ -919,19 +916,19 @@ def main():
logfile = open(os.path.join(log_dir, app['id'] + '.log'), 'a+') logfile = open(os.path.join(log_dir, app['id'] + '.log'), 'a+')
logfile.write(str(be)) logfile.write(str(be))
logfile.close() logfile.close()
print "Could not build app %s due to BuildException: %s" % (app['id'], be) logging.info("Could not build app %s due to BuildException: %s" % (app['id'], be))
if options.stop: if options.stop:
sys.exit(1) sys.exit(1)
failed_apps[app['id']] = be failed_apps[app['id']] = be
wikilog = be.get_wikitext() wikilog = be.get_wikitext()
except VCSException as vcse: except VCSException as vcse:
print "VCS error while building app %s: %s" % (app['id'], vcse) logging.info("VCS error while building app %s: %s" % (app['id'], vcse))
if options.stop: if options.stop:
sys.exit(1) sys.exit(1)
failed_apps[app['id']] = vcse failed_apps[app['id']] = vcse
wikilog = str(vcse) wikilog = str(vcse)
except Exception as e: except Exception as e:
print "Could not build app %s due to unknown error: %s" % (app['id'], traceback.format_exc()) logging.info("Could not build app %s due to unknown error: %s" % (app['id'], traceback.format_exc()))
if options.stop: if options.stop:
sys.exit(1) sys.exit(1)
failed_apps[app['id']] = e failed_apps[app['id']] = e
@ -946,20 +943,20 @@ def main():
txt = "Build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + txt txt = "Build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + txt
newpage.save(txt, summary='Build log') newpage.save(txt, summary='Build log')
except: except:
print "Error while attempting to publish build log" logging.info("Error while attempting to publish build log")
for app in build_succeeded: for app in build_succeeded:
print "success: %s" % (app['id']) logging.info("success: %s" % (app['id']))
if not options.verbose: if not options.verbose:
for fa in failed_apps: for fa in failed_apps:
print "Build for app %s failed:\n%s" % (fa, failed_apps[fa]) logging.info("Build for app %s failed:\n%s" % (fa, failed_apps[fa]))
print "Finished." logging.info("Finished.")
if len(build_succeeded) > 0: if len(build_succeeded) > 0:
print str(len(build_succeeded)) + ' builds succeeded' logging.info(str(len(build_succeeded)) + ' builds succeeded')
if len(failed_apps) > 0: if len(failed_apps) > 0:
print str(len(failed_apps)) + ' builds failed' logging.info(str(len(failed_apps)) + ' builds failed')
sys.exit(0) sys.exit(0)

63
fdroidserver/checkupdates.py
View file

@ -28,6 +28,8 @@ from optparse import OptionParser
import traceback import traceback
import HTMLParser import HTMLParser
from distutils.version import LooseVersion from distutils.version import LooseVersion
import logging
import common, metadata import common, metadata
from common import BuildException from common import BuildException
from common import VCSException from common import VCSException
@ -48,7 +50,7 @@ def check_http(app):
vercode = "99999999" vercode = "99999999"
if len(urlcode) > 0: if len(urlcode) > 0:
print "...requesting {0}".format(urlcode) logging.info("...requesting {0}".format(urlcode))
req = urllib2.Request(urlcode, None) req = urllib2.Request(urlcode, None)
resp = urllib2.urlopen(req, None, 20) resp = urllib2.urlopen(req, None, 20)
page = resp.read() page = resp.read()
@ -61,7 +63,7 @@ def check_http(app):
version = "??" version = "??"
if len(urlver) > 0: if len(urlver) > 0:
if urlver != '.': if urlver != '.':
print "...requesting {0}".format(urlver) logging.info("...requesting {0}".format(urlver))
req = urllib2.Request(urlver, None) req = urllib2.Request(urlver, None)
resp = urllib2.urlopen(req, None, 20) resp = urllib2.urlopen(req, None, 20)
page = resp.read() page = resp.read()
@ -114,16 +116,15 @@ def check_tags(app):
hcode = "0" hcode = "0"
for tag in vcs.gettags(): for tag in vcs.gettags():
if options.verbose: logging.info("Check tag: '{0}'".format(tag))
print "Check tag: '{0}'".format(tag)
vcs.gotorevision(tag) vcs.gotorevision(tag)
# Only process tags where the manifest exists... # Only process tags where the manifest exists...
paths = common.manifest_paths(build_dir, flavour) paths = common.manifest_paths(build_dir, flavour)
version, vercode, package = common.parse_androidmanifests(paths) version, vercode, package = common.parse_androidmanifests(paths)
if package and package == app['id'] and version and vercode: if package and package == app['id'] and version and vercode:
print "Manifest exists. Found version %s (%s)" % ( logging.info("Manifest exists. Found version %s (%s)" % (
version, vercode) version, vercode))
if int(vercode) > int(hcode): if int(vercode) > int(hcode):
htag = tag htag = tag
hcode = str(int(vercode)) hcode = str(int(vercode))
@ -201,7 +202,7 @@ def check_repomanifest(app, branch=None):
vercode = str(int(vercode)) vercode = str(int(vercode))
print "Manifest exists. Found version %s (%s)" % (version, vercode) logging.info("Manifest exists. Found version %s (%s)" % (version, vercode))
return (version, vercode) return (version, vercode)
@ -307,38 +308,36 @@ def main():
if options.gplay: if options.gplay:
for app in apps: for app in apps:
version, reason = check_gplay(app) version, reason = check_gplay(app)
if version is None and options.verbose: if version is None:
if reason == '404': if reason == '404':
print "%s is not in the Play Store" % common.getappname(app) logging.info("%s is not in the Play Store" % common.getappname(app))
else: else:
print "%s encountered a problem: %s" % (common.getappname(app), reason) logging.info("%s encountered a problem: %s" % (common.getappname(app), reason))
if version is not None: if version is not None:
stored = app['Current Version'] stored = app['Current Version']
if not stored: if not stored:
if options.verbose: logging.info("%s has no Current Version but has version %s on the Play Store" % (
print "%s has no Current Version but has version %s on the Play Store" % ( common.getappname(app), version))
common.getappname(app), version)
elif LooseVersion(stored) < LooseVersion(version): elif LooseVersion(stored) < LooseVersion(version):
print "%s has version %s on the Play Store, which is bigger than %s" % ( logging.info("%s has version %s on the Play Store, which is bigger than %s" % (
common.getappname(app), version, stored) common.getappname(app), version, stored))
elif options.verbose: else:
if stored != version: if stored != version:
print "%s has version %s on the Play Store, which differs from %s" % ( logging.info("%s has version %s on the Play Store, which differs from %s" % (
common.getappname(app), version, stored) common.getappname(app), version, stored))
else: else:
print "%s has the same version %s on the Play Store" % ( logging.info("%s has the same version %s on the Play Store" % (
common.getappname(app), version) common.getappname(app), version))
return return
for app in apps: for app in apps:
if options.autoonly and app['Auto Update Mode'] == 'None': if options.autoonly and app['Auto Update Mode'] == 'None':
if options.verbose: logging.info("Nothing to do for %s..." % app['id'])
print "Nothing to do for %s..." % app['id']
continue continue
print "Processing " + app['id'] + '...' logging.info("Processing " + app['id'] + '...')
writeit = False writeit = False
logmsg = None logmsg = None
@ -374,9 +373,9 @@ def main():
updating = False updating = False
if not version: if not version:
print "...%s" % msg logging.info("...%s" % msg)
elif vercode == app['Current Version Code']: elif vercode == app['Current Version Code']:
print "...up to date" logging.info("...up to date")
else: else:
app['Current Version'] = version app['Current Version'] = version
app['Current Version Code'] = str(int(vercode)) app['Current Version Code'] = str(int(vercode))
@ -413,12 +412,12 @@ def main():
app['Current Version'] = cv app['Current Version'] = cv
writeit = True writeit = True
except Exception: except Exception:
print "ERROR: Auto Name or Current Version failed for %s due to exception: %s" % (app['id'], traceback.format_exc()) logging.info("ERROR: Auto Name or Current Version failed for %s due to exception: %s" % (app['id'], traceback.format_exc()))
if updating: if updating:
name = common.getappname(app) name = common.getappname(app)
ver = common.getcvname(app) ver = common.getcvname(app)
print '...updating to version %s' % ver logging.info('...updating to version %s' % ver)
logmsg = 'Update CV of %s to %s' % (name, ver) logmsg = 'Update CV of %s to %s' % (name, ver)
if options.auto: if options.auto:
@ -447,7 +446,7 @@ def main():
del newbuild['origlines'] del newbuild['origlines']
newbuild['vercode'] = app['Current Version Code'] newbuild['vercode'] = app['Current Version Code']
newbuild['version'] = app['Current Version'] + suffix newbuild['version'] = app['Current Version'] + suffix
print "...auto-generating build for " + newbuild['version'] logging.info("...auto-generating build for " + newbuild['version'])
commit = pattern.replace('%v', newbuild['version']) commit = pattern.replace('%v', newbuild['version'])
commit = commit.replace('%c', newbuild['vercode']) commit = commit.replace('%c', newbuild['vercode'])
newbuild['commit'] = commit newbuild['commit'] = commit
@ -457,23 +456,23 @@ def main():
ver = common.getcvname(app) ver = common.getcvname(app)
logmsg = "Update %s to %s" % (name, ver) logmsg = "Update %s to %s" % (name, ver)
else: else:
print 'Invalid auto update mode "' + mode + '"' logging.info('Invalid auto update mode "' + mode + '"')
if writeit: if writeit:
metafile = os.path.join('metadata', app['id'] + '.txt') metafile = os.path.join('metadata', app['id'] + '.txt')
metadata.write_metadata(metafile, app) metadata.write_metadata(metafile, app)
if options.commit and logmsg: if options.commit and logmsg:
print "Commiting update for " + metafile logging.info("Commiting update for " + metafile)
gitcmd = ["git", "commit", "-m", gitcmd = ["git", "commit", "-m",
logmsg] logmsg]
if 'auto_author' in config: if 'auto_author' in config:
gitcmd.extend(['--author', config['auto_author']]) gitcmd.extend(['--author', config['auto_author']])
gitcmd.extend(["--", metafile]) gitcmd.extend(["--", metafile])
if subprocess.call(gitcmd) != 0: if subprocess.call(gitcmd) != 0:
print "Git commit failed" logging.info("Git commit failed")
sys.exit(1) sys.exit(1)
print "Finished." logging.info("Finished.")
if __name__ == "__main__": if __name__ == "__main__":
main() main()

41
fdroidserver/import.py
View file

@ -24,6 +24,7 @@ import shutil
import urllib import urllib
from optparse import OptionParser from optparse import OptionParser
from ConfigParser import ConfigParser from ConfigParser import ConfigParser
import logging
import common, metadata import common, metadata
# Get the repo type and address from the given web page. The page is scanned # Get the repo type and address from the given web page. The page is scanned
@ -106,13 +107,13 @@ def main():
config = common.read_config(options) config = common.read_config(options)
if not options.url: if not options.url:
print "Specify project url." logging.info("Specify project url.")
sys.exit(1) sys.exit(1)
url = options.url url = options.url
tmp_dir = 'tmp' tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir): if not os.path.isdir(tmp_dir):
print "Creating temporary directory" logging.info("Creating temporary directory")
os.makedirs(tmp_dir) os.makedirs(tmp_dir)
# Get all apps... # Get all apps...
@ -133,7 +134,7 @@ def main():
if url.endswith('/'): if url.endswith('/'):
url = url[:-1] url = url[:-1]
if url.endswith('.git'): if url.endswith('.git'):
print "A github URL should point to the project, not the git repo" logging.info("A github URL should point to the project, not the git repo")
sys.exit(1) sys.exit(1)
projecttype = 'github' projecttype = 'github'
repo = url + '.git' repo = url + '.git'
@ -154,7 +155,7 @@ def main():
# Figure out the repo type and adddress... # Figure out the repo type and adddress...
repotype, repo = getrepofrompage(sourcecode) repotype, repo = getrepofrompage(sourcecode)
if not repotype: if not repotype:
print "Unable to determine vcs type. " + repo logging.info("Unable to determine vcs type. " + repo)
sys.exit(1) sys.exit(1)
elif url.startswith('http://code.google.com/p/'): elif url.startswith('http://code.google.com/p/'):
if not url.endswith('/'): if not url.endswith('/'):
@ -168,29 +169,29 @@ def main():
# Figure out the repo type and adddress... # Figure out the repo type and adddress...
repotype, repo = getrepofrompage(sourcecode) repotype, repo = getrepofrompage(sourcecode)
if not repotype: if not repotype:
print "Unable to determine vcs type. " + repo logging.info("Unable to determine vcs type. " + repo)
sys.exit(1) sys.exit(1)
# Figure out the license... # Figure out the license...
req = urllib.urlopen(url) req = urllib.urlopen(url)
if req.getcode() != 200: if req.getcode() != 200:
print 'Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode()) logging.info('Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode()))
sys.exit(1) sys.exit(1)
page = req.read() page = req.read()
index = page.find('Code license') index = page.find('Code license')
if index == -1: if index == -1:
print "Couldn't find license data" logging.info("Couldn't find license data")
sys.exit(1) sys.exit(1)
ltext = page[index:] ltext = page[index:]
lprefix = 'rel="nofollow">' lprefix = 'rel="nofollow">'
index = ltext.find(lprefix) index = ltext.find(lprefix)
if index == -1: if index == -1:
print "Couldn't find license text" logging.info("Couldn't find license text")
sys.exit(1) sys.exit(1)
ltext = ltext[index + len(lprefix):] ltext = ltext[index + len(lprefix):]
index = ltext.find('<') index = ltext.find('<')
if index == -1: if index == -1:
print "License text not formatted as expected" logging.info("License text not formatted as expected")
sys.exit(1) sys.exit(1)
ltext = ltext[:index] ltext = ltext[:index]
if ltext == 'GNU GPL v3': if ltext == 'GNU GPL v3':
@ -208,17 +209,17 @@ def main():
elif ltext == 'New BSD License': elif ltext == 'New BSD License':
license = 'NewBSD' license = 'NewBSD'
else: else:
print "License " + ltext + " is not recognised" logging.info("License " + ltext + " is not recognised")
sys.exit(1) sys.exit(1)
if not projecttype: if not projecttype:
print "Unable to determine the project type." logging.info("Unable to determine the project type.")
print "The URL you supplied was not in one of the supported formats. Please consult" logging.info("The URL you supplied was not in one of the supported formats. Please consult")
print "the manual for a list of supported formats, and supply one of those." logging.info("the manual for a list of supported formats, and supply one of those.")
sys.exit(1) sys.exit(1)
# Get a copy of the source so we can extract some info... # Get a copy of the source so we can extract some info...
print 'Getting source from ' + repotype + ' repo at ' + repo logging.info('Getting source from ' + repotype + ' repo at ' + repo)
src_dir = os.path.join(tmp_dir, 'importer') src_dir = os.path.join(tmp_dir, 'importer')
if os.path.exists(src_dir): if os.path.exists(src_dir):
shutil.rmtree(src_dir) shutil.rmtree(src_dir)
@ -235,12 +236,12 @@ def main():
version, vercode, package = common.parse_androidmanifests(paths) version, vercode, package = common.parse_androidmanifests(paths)
if not package: if not package:
print "Couldn't find package ID" logging.info("Couldn't find package ID")
sys.exit(1) sys.exit(1)
if not version: if not version:
print "WARNING: Couldn't find latest version name" logging.info("WARNING: Couldn't find latest version name")
if not vercode: if not vercode:
print "WARNING: Couldn't find latest version code" logging.info("WARNING: Couldn't find latest version code")
else: else:
spec = os.path.join(root_dir, 'buildozer.spec') spec = os.path.join(root_dir, 'buildozer.spec')
if os.path.exists(spec): if os.path.exists(spec):
@ -252,13 +253,13 @@ def main():
version = bconfig.get('app', 'version') version = bconfig.get('app', 'version')
vercode = None vercode = None
else: else:
print "No android or kivy project could be found. Specify --subdir?" logging.info("No android or kivy project could be found. Specify --subdir?")
sys.exit(1) sys.exit(1)
# Make sure it's actually new... # Make sure it's actually new...
for app in apps: for app in apps:
if app['id'] == package: if app['id'] == package:
print "Package " + package + " already exists" logging.info("Package " + package + " already exists")
sys.exit(1) sys.exit(1)
# Construct the metadata... # Construct the metadata...
@ -295,7 +296,7 @@ def main():
metafile = os.path.join('metadata', package + '.txt') metafile = os.path.join('metadata', package + '.txt')
metadata.write_metadata(metafile, app) metadata.write_metadata(metafile, app)
print "Wrote " + metafile logging.info("Wrote " + metafile)
if __name__ == "__main__": if __name__ == "__main__":

39
fdroidserver/stats.py
View file

@ -26,6 +26,7 @@ import glob
from optparse import OptionParser from optparse import OptionParser
import paramiko import paramiko
import socket import socket
import logging
import common, metadata import common, metadata
from common import FDroidPopen from common import FDroidPopen
@ -57,7 +58,7 @@ def main():
config = common.read_config(options) config = common.read_config(options)
if not config['update_stats']: if not config['update_stats']:
print "Stats are disabled - check your configuration" logging.info("Stats are disabled - check your configuration")
sys.exit(1) sys.exit(1)
# Get all metadata-defined apps... # Get all metadata-defined apps...
@ -78,14 +79,14 @@ def main():
ssh = None ssh = None
ftp = None ftp = None
try: try:
print 'Retrieving logs' logging.info('Retrieving logs')
ssh = paramiko.SSHClient() ssh = paramiko.SSHClient()
ssh.load_system_host_keys() ssh.load_system_host_keys()
ssh.connect('f-droid.org', username='fdroid', timeout=10, ssh.connect('f-droid.org', username='fdroid', timeout=10,
key_filename=config['webserver_keyfile']) key_filename=config['webserver_keyfile'])
ftp = ssh.open_sftp() ftp = ssh.open_sftp()
ftp.get_channel().settimeout(60) ftp.get_channel().settimeout(60)
print "...connected" logging.info("...connected")
ftp.chdir('logs') ftp.chdir('logs')
files = ftp.listdir() files = ftp.listdir()
@ -96,7 +97,7 @@ def main():
destsize = ftp.stat(f).st_size destsize = ftp.stat(f).st_size
if (not os.path.exists(destpath) or if (not os.path.exists(destpath) or
os.path.getsize(destpath) != destsize): os.path.getsize(destpath) != destsize):
print "...retrieving " + f logging.info("...retrieving " + f)
ftp.get(f, destpath) ftp.get(f, destpath)
except Exception: except Exception:
traceback.print_exc() traceback.print_exc()
@ -113,15 +114,13 @@ def main():
if not options.nologs: if not options.nologs:
# Process logs # Process logs
if options.verbose: logging.info('Processing logs...')
print 'Processing logs...'
apps = {} apps = {}
appsVer = {} appsVer = {}
logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] "GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) \d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"' logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] "GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) \d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
logsearch = re.compile(logexpr).search logsearch = re.compile(logexpr).search
for logfile in glob.glob(os.path.join(logsdir,'access-*.log.gz')): for logfile in glob.glob(os.path.join(logsdir,'access-*.log.gz')):
if options.verbose: logging.info('...' + logfile)
print '...' + logfile
p = FDroidPopen(["zcat", logfile]) p = FDroidPopen(["zcat", logfile])
matches = (logsearch(line) for line in p.stdout) matches = (logsearch(line) for line in p.stdout)
for match in matches: for match in matches:
@ -172,8 +171,7 @@ def main():
f.close() f.close()
# Calculate and write stats for repo types... # Calculate and write stats for repo types...
if options.verbose: logging.info("Processing repo types...")
print "Processing repo types..."
repotypes = {} repotypes = {}
for app in metaapps: for app in metaapps:
if len(app['Repo Type']) == 0: if len(app['Repo Type']) == 0:
@ -193,8 +191,7 @@ def main():
f.close() f.close()
# Calculate and write stats for update check modes... # Calculate and write stats for update check modes...
if options.verbose: logging.info("Processing update check modes...")
print "Processing update check modes..."
ucms = {} ucms = {}
for app in metaapps: for app in metaapps:
checkmode = app['Update Check Mode'].split('/')[0] checkmode = app['Update Check Mode'].split('/')[0]
@ -207,8 +204,7 @@ def main():
f.write(checkmode + ' ' + str(count) + '\n') f.write(checkmode + ' ' + str(count) + '\n')
f.close() f.close()
if options.verbose: logging.info("Processing categories...")
print "Processing categories..."
ctgs = {} ctgs = {}
for app in metaapps: for app in metaapps:
if app['Categories'] is None: if app['Categories'] is None:
@ -224,8 +220,7 @@ def main():
f.write(category + ' ' + str(count) + '\n') f.write(category + ' ' + str(count) + '\n')
f.close() f.close()
if options.verbose: logging.info("Processing antifeatures...")
print "Processing antifeatures..."
afs = {} afs = {}
for app in metaapps: for app in metaapps:
if app['AntiFeatures'] is None: if app['AntiFeatures'] is None:
@ -242,8 +237,7 @@ def main():
f.close() f.close()
# Calculate and write stats for licenses... # Calculate and write stats for licenses...
if options.verbose: logging.info("Processing licenses...")
print "Processing licenses..."
licenses = {} licenses = {}
for app in metaapps: for app in metaapps:
license = app['License'] license = app['License']
@ -257,8 +251,7 @@ def main():
f.close() f.close()
# Write list of latest apps added to the repo... # Write list of latest apps added to the repo...
if options.verbose: logging.info("Processing latest apps...")
print "Processing latest apps..."
latest = knownapks.getlatest(10) latest = knownapks.getlatest(10)
f = open('stats/latestapps.txt', 'w') f = open('stats/latestapps.txt', 'w')
for app in latest: for app in latest:
@ -266,11 +259,11 @@ def main():
f.close() f.close()
if unknownapks: if unknownapks:
print '\nUnknown apks:' logging.info('\nUnknown apks:')
for apk in unknownapks: for apk in unknownapks:
print apk logging.info(apk)
print "Finished." logging.info("Finished.")
if __name__ == "__main__": if __name__ == "__main__":
main() main()