mirrors/fdroidserver
1
0
Fork 0
mirror of https://github.com/f-droid/fdroidserver.git synced 2025-09-16 16:02:33 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

All callable scripts now implement main()

Browse source
This commit is contained in:
Ciaran Gultnieks 2012-02-26 14:18:58 +00:00
parent 4e5b4fa77c
commit 00abc9527d
7 changed files with 970 additions and 936 deletions
Download patch file Download diff file Expand all files Collapse all files

30
checkupdates.py
View file

@ -27,9 +27,6 @@ from optparse import OptionParser
import HTMLParser import HTMLParser
import common import common
#Read configuration...
execfile('config.py')
# Check for a new version by looking at the Google market. # Check for a new version by looking at the Google market.
# Returns (None, "a message") if this didn't work, or (version, vercode) for # Returns (None, "a message") if this didn't work, or (version, vercode) for
@ -66,21 +63,25 @@ def check_market(app):
def main():
# Parse command line... #Read configuration...
parser = OptionParser() execfile('config.py')
parser.add_option("-v", "--verbose", action="store_true", default=False,
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal") help="Spew out even more information than normal")
parser.add_option("-p", "--package", default=None, parser.add_option("-p", "--package", default=None,
help="Build only the specified package") help="Build only the specified package")
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
# Get all apps... # Get all apps...
apps = common.read_metadata(options.verbose) apps = common.read_metadata(options.verbose)
html_parser = HTMLParser.HTMLParser() html_parser = HTMLParser.HTMLParser()
for app in apps: for app in apps:
if options.package and options.package != app['id']: if options.package and options.package != app['id']:
# Silent skip... # Silent skip...
@ -109,5 +110,8 @@ for app in apps:
metafile = os.path.join('metadata', app['id'] + '.txt') metafile = os.path.join('metadata', app['id'] + '.txt')
common.write_metadata(metafile, app) common.write_metadata(metafile, app)
print "Finished." print "Finished."
if __name__ == "__main__":
main()

146
import.py
View file

@ -25,46 +25,48 @@ import re
import urllib import urllib
from optparse import OptionParser from optparse import OptionParser
#Read configuration... def main():
repo_name = None
repo_description = None
repo_icon = None
repo_url = None
execfile('config.py')
import common # Read configuration...
repo_name = None
repo_description = None
repo_icon = None
repo_url = None
execfile('config.py')
# Parse command line... import common
parser = OptionParser()
parser.add_option("-u", "--url", default=None, # Parse command line...
parser = OptionParser()
parser.add_option("-u", "--url", default=None,
help="Project URL to import from.") help="Project URL to import from.")
parser.add_option("-s", "--subdir", default=None, parser.add_option("-s", "--subdir", default=None,
help="Path to main android project subdirectory, if not in root.") help="Path to main android project subdirectory, if not in root.")
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
if not options.url: if not options.url:
print "Specify project url." print "Specify project url."
sys.exit(1) sys.exit(1)
url = options.url url = options.url
tmp_dir = 'tmp' tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir): if not os.path.isdir(tmp_dir):
print "Creating temporary directory" print "Creating temporary directory"
os.makedirs(tmp_dir) os.makedirs(tmp_dir)
# Get all apps... # Get all apps...
apps = common.read_metadata() apps = common.read_metadata()
# Figure out what kind of project it is... # Figure out what kind of project it is...
projecttype = None projecttype = None
issuetracker = None issuetracker = None
license = None license = None
if url.startswith('https://github.com'): if url.startswith('https://github.com'):
projecttype = 'github' projecttype = 'github'
repo = url + '.git' repo = url + '.git'
repotype = 'git' repotype = 'git'
sourcecode = url sourcecode = url
elif url.startswith('http://code.google.com/p/'): elif url.startswith('http://code.google.com/p/'):
if not url.endswith('/'): if not url.endswith('/'):
print "Expected format for googlecode url is http://code.google.com/p/PROJECT/" print "Expected format for googlecode url is http://code.google.com/p/PROJECT/"
sys.exit(1) sys.exit(1)
@ -154,36 +156,36 @@ elif url.startswith('http://code.google.com/p/'):
print "License " + ltext + " is not recognised" print "License " + ltext + " is not recognised"
sys.exit(1) sys.exit(1)
if not projecttype: if not projecttype:
print "Unable to determine the project type." print "Unable to determine the project type."
sys.exit(1) sys.exit(1)
# Get a copy of the source so we can extract some info... # Get a copy of the source so we can extract some info...
print 'Getting source from ' + repotype + ' repo at ' + repo print 'Getting source from ' + repotype + ' repo at ' + repo
src_dir = os.path.join(tmp_dir, 'importer') src_dir = os.path.join(tmp_dir, 'importer')
if os.path.exists(tmp_dir): if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir) shutil.rmtree(tmp_dir)
vcs = common.getvcs(repotype, repo, src_dir) vcs = common.getvcs(repotype, repo, src_dir)
vcs.gotorevision(None) vcs.gotorevision(None)
if options.subdir: if options.subdir:
root_dir = os.path.join(src_dir, options.subdir) root_dir = os.path.join(src_dir, options.subdir)
else: else:
root_dir = src_dir root_dir = src_dir
# Check AndroidManiifest.xml exists... # Check AndroidManiifest.xml exists...
manifest = os.path.join(root_dir, 'AndroidManifest.xml') manifest = os.path.join(root_dir, 'AndroidManifest.xml')
if not os.path.exists(manifest): if not os.path.exists(manifest):
print "AndroidManifest.xml did not exist in the expected location. Specify --subdir?" print "AndroidManifest.xml did not exist in the expected location. Specify --subdir?"
sys.exit(1) sys.exit(1)
# Extract some information... # Extract some information...
vcsearch = re.compile(r'.*android:versionCode="([^"]+)".*').search vcsearch = re.compile(r'.*android:versionCode="([^"]+)".*').search
vnsearch = re.compile(r'.*android:versionName="([^"]+)".*').search vnsearch = re.compile(r'.*android:versionName="([^"]+)".*').search
psearch = re.compile(r'.*package="([^"]+)".*').search psearch = re.compile(r'.*package="([^"]+)".*').search
version = None version = None
vercode = None vercode = None
package = None package = None
for line in file(manifest): for line in file(manifest):
if not package: if not package:
matches = psearch(line) matches = psearch(line)
if matches: if matches:
@ -196,48 +198,52 @@ for line in file(manifest):
matches = vcsearch(line) matches = vcsearch(line)
if matches: if matches:
vercode = matches.group(1) vercode = matches.group(1)
if not package: if not package:
print "Couldn't find package ID" print "Couldn't find package ID"
sys.exit(1) sys.exit(1)
if not version: if not version:
print "Couldn't find latest version name" print "Couldn't find latest version name"
sys.exit(1) sys.exit(1)
if not vercode: if not vercode:
print "Couldn't find latest version code" print "Couldn't find latest version code"
sys.exit(1) sys.exit(1)
# Make sure it's actually new... # Make sure it's actually new...
for app in apps: for app in apps:
if app['id'] == package: if app['id'] == package:
print "Package " + package + " already exists" print "Package " + package + " already exists"
sys.exit(1) sys.exit(1)
# Construct the metadata... # Construct the metadata...
app = common.parse_metadata(None) app = common.parse_metadata(None)
app['id'] = package app['id'] = package
app['Web Site'] = url app['Web Site'] = url
app['Source Code'] = sourcecode app['Source Code'] = sourcecode
if issuetracker: if issuetracker:
app['Issue Tracker'] = issuetracker app['Issue Tracker'] = issuetracker
if license: if license:
app['License'] = license app['License'] = license
app['Repo Type'] = repotype app['Repo Type'] = repotype
app['Repo'] = repo app['Repo'] = repo
# Create a build line... # Create a build line...
build = {} build = {}
build['version'] = version build['version'] = version
build['vercode'] = vercode build['vercode'] = vercode
build['commit'] = '?' build['commit'] = '?'
if options.subdir: if options.subdir:
build['subdir'] = options.subdir build['subdir'] = options.subdir
if os.path.exists(os.path.join(root_dir, 'jni')): if os.path.exists(os.path.join(root_dir, 'jni')):
build['buildjni'] = 'yes' build['buildjni'] = 'yes'
app['builds'].append(build) app['builds'].append(build)
app['comments'].append(('build:' + version, app['comments'].append(('build:' + version,
"#Generated by import.py - check this is the right version, and find the right commit!")) "#Generated by import.py - check this is the right version, and find the right commit!"))
metafile = os.path.join('metadata', package + '.txt') metafile = os.path.join('metadata', package + '.txt')
common.write_metadata(metafile, app) common.write_metadata(metafile, app)
print "Wrote " + metafile print "Wrote " + metafile
if __name__ == "__main__":
main()

38
publish.py
View file

@ -31,38 +31,40 @@ from optparse import OptionParser
import common import common
from common import BuildException from common import BuildException
#Read configuration... def main():
execfile('config.py')
# Parse command line... #Read configuration...
parser = OptionParser() execfile('config.py')
parser.add_option("-v", "--verbose", action="store_true", default=False,
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal") help="Spew out even more information than normal")
parser.add_option("-p", "--package", default=None, parser.add_option("-p", "--package", default=None,
help="Publish only the specified package") help="Publish only the specified package")
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
log_dir = 'logs' log_dir = 'logs'
if not os.path.isdir(log_dir): if not os.path.isdir(log_dir):
print "Creating log directory" print "Creating log directory"
os.makedirs(log_dir) os.makedirs(log_dir)
tmp_dir = 'tmp' tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir): if not os.path.isdir(tmp_dir):
print "Creating temporary directory" print "Creating temporary directory"
os.makedirs(tmp_dir) os.makedirs(tmp_dir)
output_dir = 'repo' output_dir = 'repo'
if not os.path.isdir(output_dir): if not os.path.isdir(output_dir):
print "Creating output directory" print "Creating output directory"
os.makedirs(output_dir) os.makedirs(output_dir)
unsigned_dir = 'unsigned' unsigned_dir = 'unsigned'
if not os.path.isdir(unsigned_dir): if not os.path.isdir(unsigned_dir):
print "No unsigned directory - nothing to do" print "No unsigned directory - nothing to do"
sys.exit(0) sys.exit(0)
for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))): for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))):
apkfilename = os.path.basename(apkfile) apkfilename = os.path.basename(apkfile)
i = apkfilename.rfind('_') i = apkfilename.rfind('_')
@ -134,3 +136,7 @@ for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))):
print 'Published ' + apkfilename print 'Published ' + apkfilename
if __name__ == "__main__":
main()

24
rewritemeta.py
View file

@ -27,22 +27,26 @@ from optparse import OptionParser
import HTMLParser import HTMLParser
import common import common
#Read configuration... def main():
execfile('config.py')
#Read configuration...
execfile('config.py')
# Parse command line... # Parse command line...
parser = OptionParser() parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False, parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal") help="Spew out even more information than normal")
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
# Get all apps... # Get all apps...
apps = common.read_metadata(options.verbose) apps = common.read_metadata(options.verbose)
for app in apps: for app in apps:
print "Writing " + app['id'] print "Writing " + app['id']
common.write_metadata(os.path.join('metadata', app['id']) + '.txt', app) common.write_metadata(os.path.join('metadata', app['id']) + '.txt', app)
print "Finished." print "Finished."
if __name__ == "__main__":
main()

37
scanner.py
View file

@ -31,28 +31,30 @@ import common
from common import BuildException from common import BuildException
from common import VCSException from common import VCSException
#Read configuration... def main():
execfile('config.py')
# Read configuration...
execfile('config.py')
# Parse command line... # Parse command line...
parser = OptionParser() parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False, parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal") help="Spew out even more information than normal")
parser.add_option("-p", "--package", default=None, parser.add_option("-p", "--package", default=None,
help="Scan only the specified package") help="Scan only the specified package")
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
# Get all apps... # Get all apps...
apps = common.read_metadata(options.verbose) apps = common.read_metadata(options.verbose)
html_parser = HTMLParser.HTMLParser() html_parser = HTMLParser.HTMLParser()
problems = [] problems = []
extlib_dir = os.path.join('build', 'extlib') extlib_dir = os.path.join('build', 'extlib')
for app in apps: for app in apps:
skip = False skip = False
if options.package and app['id'] != options.package: if options.package and app['id'] != options.package:
@ -103,8 +105,11 @@ for app in apps:
msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc()) msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
problems.append(msg) problems.append(msg)
print "Finished:" print "Finished:"
for problem in problems: for problem in problems:
print problem print problem
print str(len(problems)) + ' problems.' print str(len(problems)) + ' problems.'
if __name__ == "__main__":
main()

177
update.py
View file

@ -29,63 +29,65 @@ from xml.dom.minidom import Document
from optparse import OptionParser from optparse import OptionParser
import time import time
#Read configuration... def main():
repo_name = None
repo_description = None
repo_icon = None
repo_url = None
execfile('config.py')
import common # Read configuration...
repo_name = None
repo_description = None
repo_icon = None
repo_url = None
execfile('config.py')
# Parse command line... import common
parser = OptionParser()
parser.add_option("-c", "--createmeta", action="store_true", default=False, # Parse command line...
parser = OptionParser()
parser.add_option("-c", "--createmeta", action="store_true", default=False,
help="Create skeleton metadata files that are missing") help="Create skeleton metadata files that are missing")
parser.add_option("-v", "--verbose", action="store_true", default=False, parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal") help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False, parser.add_option("-q", "--quiet", action="store_true", default=False,
help="No output, except for warnings and errors") help="No output, except for warnings and errors")
parser.add_option("-b", "--buildreport", action="store_true", default=False, parser.add_option("-b", "--buildreport", action="store_true", default=False,
help="Report on build data status") help="Report on build data status")
parser.add_option("-i", "--interactive", default=False, action="store_true", parser.add_option("-i", "--interactive", default=False, action="store_true",
help="Interactively ask about things that need updating.") help="Interactively ask about things that need updating.")
parser.add_option("-e", "--editor", default="/etc/alternatives/editor", parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
help="Specify editor to use in interactive mode. Default "+ help="Specify editor to use in interactive mode. Default "+
"is /etc/alternatives/editor") "is /etc/alternatives/editor")
parser.add_option("", "--pretty", action="store_true", default=False, parser.add_option("", "--pretty", action="store_true", default=False,
help="Produce human-readable index.xml") help="Produce human-readable index.xml")
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
icon_dir=os.path.join('repo','icons') icon_dir=os.path.join('repo','icons')
# Delete and re-create the icon directory... # Delete and re-create the icon directory...
if os.path.exists(icon_dir): if os.path.exists(icon_dir):
shutil.rmtree(icon_dir) shutil.rmtree(icon_dir)
os.mkdir(icon_dir) os.mkdir(icon_dir)
warnings = 0 warnings = 0
#Make sure we have the repository description... # Make sure we have the repository description...
if (repo_url is None or repo_name is None or if (repo_url is None or repo_name is None or
repo_icon is None or repo_description is None): repo_icon is None or repo_description is None):
print "Repository description fields are required in config.py" print "Repository description fields are required in config.py"
print "See config.sample.py for details" print "See config.sample.py for details"
sys.exit(1) sys.exit(1)
# Get all apps... # Get all apps...
apps = common.read_metadata(verbose=options.verbose) apps = common.read_metadata(verbose=options.verbose)
# Generate a list of categories... # Generate a list of categories...
categories = [] categories = []
for app in apps: for app in apps:
if app['Category'] not in categories: if app['Category'] not in categories:
categories.append(app['Category']) categories.append(app['Category'])
# Gather information about all the apk files in the repo directory... # Gather information about all the apk files in the repo directory...
apks = [] apks = []
for apkfile in glob.glob(os.path.join('repo','*.apk')): for apkfile in glob.glob(os.path.join('repo','*.apk')):
apkfilename = apkfile[5:] apkfilename = apkfile[5:]
if apkfilename.find(' ') != -1: if apkfilename.find(' ') != -1:
@ -194,9 +196,9 @@ for apkfile in glob.glob(os.path.join('repo','*.apk')):
apks.append(thisinfo) apks.append(thisinfo)
# Some information from the apks needs to be applied up to the application # Some information from the apks needs to be applied up to the application
# level. When doing this, we use the info from the most recent version's apk. # level. When doing this, we use the info from the most recent version's apk.
for app in apps: for app in apps:
bestver = 0 bestver = 0
for apk in apks: for apk in apks:
if apk['id'] == app['id']: if apk['id'] == app['id']:
@ -215,9 +217,9 @@ for app in apps:
app['Name'] = bestapk['name'] app['Name'] = bestapk['name']
app['icon'] = bestapk['icon'] app['icon'] = bestapk['icon']
# Generate warnings for apk's with no metadata (or create skeleton # Generate warnings for apk's with no metadata (or create skeleton
# metadata files, if requested on the command line) # metadata files, if requested on the command line)
for apk in apks: for apk in apks:
found = False found = False
for app in apps: for app in apps:
if app['id'] == apk['id']: if app['id'] == apk['id']:
@ -240,26 +242,26 @@ for apk in apks:
print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata" print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
print " " + apk['name'] + " - " + apk['version'] print " " + apk['name'] + " - " + apk['version']
#Sort the app list by name, then the web site doesn't have to by default: #Sort the app list by name, then the web site doesn't have to by default:
apps = sorted(apps, key=lambda app: app['Name'].upper()) apps = sorted(apps, key=lambda app: app['Name'].upper())
# Create the index # Create the index
doc = Document() doc = Document()
def addElement(name, value, doc, parent): def addElement(name, value, doc, parent):
el = doc.createElement(name) el = doc.createElement(name)
el.appendChild(doc.createTextNode(value)) el.appendChild(doc.createTextNode(value))
parent.appendChild(el) parent.appendChild(el)
root = doc.createElement("fdroid") root = doc.createElement("fdroid")
doc.appendChild(root) doc.appendChild(root)
repoel = doc.createElement("repo") repoel = doc.createElement("repo")
repoel.setAttribute("name", repo_name) repoel.setAttribute("name", repo_name)
repoel.setAttribute("icon", os.path.basename(repo_icon)) repoel.setAttribute("icon", os.path.basename(repo_icon))
repoel.setAttribute("url", repo_url) repoel.setAttribute("url", repo_url)
if repo_keyalias != None: if repo_keyalias != None:
# Generate a certificate fingerprint the same way keytool does it # Generate a certificate fingerprint the same way keytool does it
# (but with slightly different formatting) # (but with slightly different formatting)
@ -286,14 +288,14 @@ if repo_keyalias != None:
repoel.setAttribute("pubkey", extract_pubkey()) repoel.setAttribute("pubkey", extract_pubkey())
addElement('description', repo_description, doc, repoel) addElement('description', repo_description, doc, repoel)
root.appendChild(repoel) root.appendChild(repoel)
apps_inrepo = 0 apps_inrepo = 0
apps_disabled = 0 apps_disabled = 0
apps_nopkg = 0 apps_nopkg = 0
for app in apps: for app in apps:
if app['Disabled'] is None: if app['Disabled'] is None:
@ -440,15 +442,15 @@ for app in apps:
else: else:
apps_disabled += 1 apps_disabled += 1
of = open(os.path.join('repo','index.xml'), 'wb') of = open(os.path.join('repo','index.xml'), 'wb')
if options.pretty: if options.pretty:
output = doc.toprettyxml() output = doc.toprettyxml()
else: else:
output = doc.toxml() output = doc.toxml()
of.write(output) of.write(output)
of.close() of.close()
if repo_keyalias != None: if repo_keyalias != None:
if not options.quiet: if not options.quiet:
print "Creating signed index." print "Creating signed index."
@ -476,27 +478,27 @@ if repo_keyalias != None:
if options.verbose: if options.verbose:
print output print output
# Copy the repo icon into the repo directory... # Copy the repo icon into the repo directory...
iconfilename = os.path.join(icon_dir, os.path.basename(repo_icon)) iconfilename = os.path.join(icon_dir, os.path.basename(repo_icon))
shutil.copyfile(repo_icon, iconfilename) shutil.copyfile(repo_icon, iconfilename)
# Write a category list in the repo to allow quick access... # Write a category list in the repo to allow quick access...
catdata = '' catdata = ''
for cat in categories: for cat in categories:
catdata += cat + '\n' catdata += cat + '\n'
f = open('repo/categories.txt', 'w') f = open('repo/categories.txt', 'w')
f.write(catdata) f.write(catdata)
f.close() f.close()
# Update known apks info... # Update known apks info...
knownapks = common.KnownApks() knownapks = common.KnownApks()
for apk in apks: for apk in apks:
knownapks.recordapk(apk['apkname'], apk['id']) knownapks.recordapk(apk['apkname'], apk['id'])
knownapks.writeifchanged() knownapks.writeifchanged()
# Generate latest apps data for widget # Generate latest apps data for widget
data = '' data = ''
for line in file(os.path.join('stats', 'latestapps.txt')): for line in file(os.path.join('stats', 'latestapps.txt')):
appid = line.rstrip() appid = line.rstrip()
data += appid + "\t" data += appid + "\t"
for app in apps: for app in apps:
@ -505,15 +507,18 @@ for line in file(os.path.join('stats', 'latestapps.txt')):
data += app['icon'] + "\t" data += app['icon'] + "\t"
data += app['License'] + "\n" data += app['License'] + "\n"
break break
f = open('repo/latestapps.dat', 'w') f = open('repo/latestapps.dat', 'w')
f.write(data) f.write(data)
f.close() f.close()
print "Finished." print "Finished."
print str(apps_inrepo) + " apps in repo" print str(apps_inrepo) + " apps in repo"
print str(apps_disabled) + " disabled" print str(apps_disabled) + " disabled"
print str(apps_nopkg) + " with no packages" print str(apps_nopkg) + " with no packages"
print str(warnings) + " warnings" print str(warnings) + " warnings"
if __name__ == "__main__":
main()

80
updatestats.py
View file

@ -30,31 +30,32 @@ import HTMLParser
import paramiko import paramiko
import common import common
#Read configuration... def main():
execfile('config.py')
# Read configuration...
execfile('config.py')
# Parse command line... # Parse command line...
parser = OptionParser() parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False, parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal") help="Spew out even more information than normal")
parser.add_option("-d", "--download", action="store_true", default=False, parser.add_option("-d", "--download", action="store_true", default=False,
help="Download logs we don't have") help="Download logs we don't have")
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
statsdir = 'stats' statsdir = 'stats'
logsdir = os.path.join(statsdir, 'logs') logsdir = os.path.join(statsdir, 'logs')
logsarchivedir = os.path.join(logsdir, 'archive') logsarchivedir = os.path.join(logsdir, 'archive')
datadir = os.path.join(statsdir, 'data') datadir = os.path.join(statsdir, 'data')
if not os.path.exists(statsdir): if not os.path.exists(statsdir):
os.mkdir(statsdir) os.mkdir(statsdir)
if not os.path.exists(logsdir): if not os.path.exists(logsdir):
os.mkdir(logsdir) os.mkdir(logsdir)
if not os.path.exists(datadir): if not os.path.exists(datadir):
os.mkdir(datadir) os.mkdir(datadir)
if options.download: if options.download:
# Get any access logs we don't have... # Get any access logs we don't have...
ssh = None ssh = None
ftp = None ftp = None
@ -96,13 +97,13 @@ if options.download:
if ssh != None: if ssh != None:
ssh.close() ssh.close()
# Process logs # Process logs
logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] "GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) \d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"' logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] "GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) \d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
logsearch = re.compile(logexpr).search logsearch = re.compile(logexpr).search
apps = {} apps = {}
unknownapks = [] unknownapks = []
knownapks = common.KnownApks() knownapks = common.KnownApks()
for logfile in glob.glob(os.path.join(logsdir,'access-*.log')): for logfile in glob.glob(os.path.join(logsdir,'access-*.log')):
logdate = logfile[len(logsdir) + 1 + len('access-'):-4] logdate = logfile[len(logsdir) + 1 + len('access-'):-4]
matches = (logsearch(line) for line in file(logfile)) matches = (logsearch(line) for line in file(logfile))
for match in matches: for match in matches:
@ -121,30 +122,33 @@ for logfile in glob.glob(os.path.join(logsdir,'access-*.log')):
if not apkname in unknownapks: if not apkname in unknownapks:
unknownapks.append(apkname) unknownapks.append(apkname)
# Calculate and write stats for total downloads... # Calculate and write stats for total downloads...
f = open('stats/total_downloads_app.txt', 'w') f = open('stats/total_downloads_app.txt', 'w')
lst = [] lst = []
alldownloads = 0 alldownloads = 0
for app, count in apps.iteritems(): for app, count in apps.iteritems():
lst.append(app + " " + str(count)) lst.append(app + " " + str(count))
alldownloads += count alldownloads += count
lst.append("ALL " + str(alldownloads)) lst.append("ALL " + str(alldownloads))
f.write('# Total downloads by application, since October 2011\n') f.write('# Total downloads by application, since October 2011\n')
for line in sorted(lst): for line in sorted(lst):
f.write(line + '\n') f.write(line + '\n')
f.close() f.close()
# Write list of latest apps added to the repo... # Write list of latest apps added to the repo...
latest = knownapks.getlatest(10) latest = knownapks.getlatest(10)
f = open('stats/latestapps.txt', 'w') f = open('stats/latestapps.txt', 'w')
for app in latest: for app in latest:
f.write(app + '\n') f.write(app + '\n')
f.close() f.close()
if len(unknownapks) > 0: if len(unknownapks) > 0:
print '\nUnknown apks:' print '\nUnknown apks:'
for apk in unknownapks: for apk in unknownapks:
print apk print apk
print "Finished." print "Finished."
if __name__ == "__main__":
main()