mirrors/fdroidserver
1
0
Fork 0
mirror of https://github.com/f-droid/fdroidserver.git synced 2025-11-05 15:00:30 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

server: store virustotal JSON locally as reusable data cache

Browse source 
This helps with uploading faster when hitting the public API rating
limiting since it will not hit the API if there is already a matching
response on the local filesystem.  This also provides a source of data for
things like repomaker to diplay to users.
This commit is contained in:
Hans-Christoph Steiner 2017-04-12 15:01:21 +02:00
parent d74d6160dd
commit fcb0634eaa
1 changed files with 23 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

27
fdroidserver/server.py
View file

@ -409,13 +409,21 @@ def upload_to_virustotal(repo_section, vt_apikey):
logging.getLogger("requests").setLevel(logging.WARNING) logging.getLogger("requests").setLevel(logging.WARNING)
if repo_section == 'repo': if repo_section == 'repo':
if not os.path.exists('virustotal'):
os.mkdir('virustotal')
with open(os.path.join(repo_section, 'index-v1.json')) as fp: with open(os.path.join(repo_section, 'index-v1.json')) as fp:
index = json.load(fp) index = json.load(fp)
for packageName, packages in index['packages'].items(): for packageName, packages in index['packages'].items():
for package in packages: for package in packages:
outputfilename = os.path.join('virustotal',
packageName + '_' + str(package.get('versionCode'))
+ '_' + package['hash'] + '.json')
if os.path.exists(outputfilename):
logging.debug(package['apkName'] + ' results are in ' + outputfilename)
continue
filename = package['apkName'] filename = package['apkName']
repofilename = os.path.join(repo_section, filename) repofilename = os.path.join(repo_section, filename)
logging.info('Uploading ' + repofilename + ' to virustotal.com') logging.info('Checking if ' + repofilename + ' is on virustotal')
headers = { headers = {
"User-Agent": "F-Droid" "User-Agent": "F-Droid"
@ -424,15 +432,23 @@ def upload_to_virustotal(repo_section, vt_apikey):
'apikey': vt_apikey, 'apikey': vt_apikey,
'resource': package['hash'], 'resource': package['hash'],
} }
download = False needs_file_upload = False
while True: while True:
r = requests.post('https://www.virustotal.com/vtapi/v2/file/report', r = requests.post('https://www.virustotal.com/vtapi/v2/file/report',
params=params, headers=headers) params=params, headers=headers)
if r.status_code == 200: if r.status_code == 200:
response = r.json() response = r.json()
if response['response_code'] == 0: if response['response_code'] == 0:
download = True needs_file_upload = True
elif response['positives'] > 0: else:
response['filename'] = filename
response['packageName'] = packageName
response['versionCode'] = package.get('versionCode')
response['versionName'] = package.get('versionName')
with open(outputfilename, 'w') as fp:
json.dump(response, fp, indent=2, sort_keys=True)
if response.get('positives') > 0:
logging.warning(repofilename + ' has been flagged by virustotal ' logging.warning(repofilename + ' has been flagged by virustotal '
+ str(response['positives']) + ' times:' + str(response['positives']) + ' times:'
+ '\n\t' + response['permalink']) + '\n\t' + response['permalink'])
@ -440,7 +456,8 @@ def upload_to_virustotal(repo_section, vt_apikey):
elif r.status_code == 204: elif r.status_code == 204:
time.sleep(10) # wait for public API rate limiting time.sleep(10) # wait for public API rate limiting
if download: if needs_file_upload:
logging.info('Uploading ' + repofilename + ' to virustotal')
files = { files = {
'file': (filename, open(repofilename, 'rb')) 'file': (filename, open(repofilename, 'rb'))
} }