diff --git a/.gitignore b/.gitignore
index 25aab3b3..57da9fb8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -57,6 +57,7 @@ makebuildserver.config.py
/tests/repo/obb.mainpatch.current/en-US/icon_WI0pkO3LsklrsTAnRr-OQSxkkoMY41lYe2-fAvXLiLg=.png
/tests/repo/org.videolan.vlc/en-US/icon_yAfSvPRJukZzMMfUzvbYqwaD1XmHXNtiPBtuPVHW-6s=.png
/tests/urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk
+/tests/virustotal/
/unsigned/
# generated by gettext
diff --git a/fdroidserver/common.py b/fdroidserver/common.py
index 7b256476..b1b4f1c1 100644
--- a/fdroidserver/common.py
+++ b/fdroidserver/common.py
@@ -37,6 +37,8 @@ import logging
import hashlib
import socket
import base64
+import urllib.parse
+import urllib.request
import zipfile
import tempfile
import json
@@ -84,6 +86,9 @@ VALID_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-z_]+(?:\d*[a-zA-Z_]*)*)(?:\.[
re.IGNORECASE)
ANDROID_PLUGIN_REGEX = re.compile(r'''\s*(:?apply plugin:|id)\(?\s*['"](android|com\.android\.application)['"]\s*\)?''')
+SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?')
+GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:([^'"]+)['"]''')
+
MAX_VERSION_CODE = 0x7fffffff # Java's Integer.MAX_VALUE (2147483647)
XMLNS_ANDROID = '{http://schemas.android.com/apk/res/android}'
@@ -1653,6 +1658,152 @@ def is_strict_application_id(name):
and '.' in name
+def get_all_gradle_and_manifests(build_dir):
+ paths = []
+ for root, dirs, files in os.walk(build_dir):
+ for f in sorted(files):
+ if f == 'AndroidManifest.xml' \
+ or f.endswith('.gradle') or f.endswith('.gradle.kts'):
+ full = os.path.join(root, f)
+ paths.append(full)
+ return paths
+
+
+def get_gradle_subdir(build_dir, paths):
+ """get the subdir where the gradle build is based"""
+ first_gradle_dir = None
+ for path in paths:
+ if not first_gradle_dir:
+ first_gradle_dir = os.path.relpath(os.path.dirname(path), build_dir)
+ if os.path.exists(path) and SETTINGS_GRADLE_REGEX.match(os.path.basename(path)):
+ with open(path) as fp:
+ for m in GRADLE_SUBPROJECT_REGEX.finditer(fp.read()):
+ for f in glob.glob(os.path.join(os.path.dirname(path), m.group(1), 'build.gradle*')):
+ with open(f) as fp:
+ while True:
+ line = fp.readline()
+ if not line:
+ break
+ if ANDROID_PLUGIN_REGEX.match(line):
+ return os.path.relpath(os.path.dirname(f), build_dir)
+ if first_gradle_dir and first_gradle_dir != '.':
+ return first_gradle_dir
+
+ return ''
+
+
+def getrepofrompage(url):
+ """Get the repo type and address from the given web page.
+
+ The page is scanned in a rather naive manner for 'git clone xxxx',
+ 'hg clone xxxx', etc, and when one of these is found it's assumed
+ that's the information we want. Returns repotype, address, or
+ None, reason
+
+ """
+ if not url.startswith('http'):
+ return (None, _('{url} does not start with "http"!'.format(url=url)))
+ req = urllib.request.urlopen(url) # nosec B310 non-http URLs are filtered out
+ if req.getcode() != 200:
+ return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode()))
+ page = req.read().decode(req.headers.get_content_charset())
+
+ # Works for BitBucket
+ m = re.search('data-fetch-url="(.*)"', page)
+ if m is not None:
+ repo = m.group(1)
+
+ if repo.endswith('.git'):
+ return ('git', repo)
+
+ return ('hg', repo)
+
+ # Works for BitBucket (obsolete)
+ index = page.find('hg clone')
+ if index != -1:
+ repotype = 'hg'
+ repo = page[index + 9:]
+ index = repo.find('<')
+ if index == -1:
+ return (None, _("Error while getting repo address"))
+ repo = repo[:index]
+ repo = repo.split('"')[0]
+ return (repotype, repo)
+
+ # Works for BitBucket (obsolete)
+ index = page.find('git clone')
+ if index != -1:
+ repotype = 'git'
+ repo = page[index + 10:]
+ index = repo.find('<')
+ if index == -1:
+ return (None, _("Error while getting repo address"))
+ repo = repo[:index]
+ repo = repo.split('"')[0]
+ return (repotype, repo)
+
+ return (None, _("No information found.") + page)
+
+
+def get_app_from_url(url):
+ """Guess basic app metadata from the URL.
+
+ The URL must include a network hostname, unless it is an lp:,
+ file:, or git/ssh URL. This throws ValueError on bad URLs to
+ match urlparse().
+
+ """
+
+ parsed = urllib.parse.urlparse(url)
+ invalid_url = False
+ if not parsed.scheme or not parsed.path:
+ invalid_url = True
+
+ app = fdroidserver.metadata.App()
+ app.Repo = url
+ if url.startswith('git://') or url.startswith('git@'):
+ app.RepoType = 'git'
+ elif parsed.netloc == 'github.com':
+ app.RepoType = 'git'
+ app.SourceCode = url
+ app.IssueTracker = url + '/issues'
+ elif parsed.netloc == 'gitlab.com':
+ # git can be fussy with gitlab URLs unless they end in .git
+ if url.endswith('.git'):
+ url = url[:-4]
+ app.Repo = url + '.git'
+ app.RepoType = 'git'
+ app.SourceCode = url
+ app.IssueTracker = url + '/issues'
+ elif parsed.netloc == 'notabug.org':
+ if url.endswith('.git'):
+ url = url[:-4]
+ app.Repo = url + '.git'
+ app.RepoType = 'git'
+ app.SourceCode = url
+ app.IssueTracker = url + '/issues'
+ elif parsed.netloc == 'bitbucket.org':
+ if url.endswith('/'):
+ url = url[:-1]
+ app.SourceCode = url + '/src'
+ app.IssueTracker = url + '/issues'
+ # Figure out the repo type and adddress...
+ app.RepoType, app.Repo = getrepofrompage(url)
+ elif url.startswith('https://') and url.endswith('.git'):
+ app.RepoType = 'git'
+
+ if not parsed.netloc and parsed.scheme in ('git', 'http', 'https', 'ssh'):
+ invalid_url = True
+
+ if invalid_url:
+ raise ValueError(_('"{url}" is not a valid URL!'.format(url=url)))
+
+ if not app.RepoType:
+ raise FDroidException("Unable to determine vcs type. " + app.Repo)
+
+ return app
+
+
def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
raw=False, prepare=True, preponly=False, refresh=True,
build=None):
diff --git a/fdroidserver/import.py b/fdroidserver/import.py
index 42ba55e3..23d06d57 100644
--- a/fdroidserver/import.py
+++ b/fdroidserver/import.py
@@ -18,14 +18,10 @@
# along with this program. If not, see .
import git
-import glob
import json
import os
-import re
import shutil
import sys
-import urllib.parse
-import urllib.request
import yaml
from argparse import ArgumentParser
import logging
@@ -40,121 +36,12 @@ from . import common
from . import metadata
from .exception import FDroidException
-SETTINGS_GRADLE = re.compile(r'settings\.gradle(?:\.kts)?')
-GRADLE_SUBPROJECT = re.compile(r'''['"]:([^'"]+)['"]''')
-
-
-# Get the repo type and address from the given web page. The page is scanned
-# in a rather naive manner for 'git clone xxxx', 'hg clone xxxx', etc, and
-# when one of these is found it's assumed that's the information we want.
-# Returns repotype, address, or None, reason
-def getrepofrompage(url):
- if not url.startswith('http'):
- return (None, _('{url} does not start with "http"!'.format(url=url)))
- req = urllib.request.urlopen(url) # nosec B310 non-http URLs are filtered out
- if req.getcode() != 200:
- return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode()))
- page = req.read().decode(req.headers.get_content_charset())
-
- # Works for BitBucket
- m = re.search('data-fetch-url="(.*)"', page)
- if m is not None:
- repo = m.group(1)
-
- if repo.endswith('.git'):
- return ('git', repo)
-
- return ('hg', repo)
-
- # Works for BitBucket (obsolete)
- index = page.find('hg clone')
- if index != -1:
- repotype = 'hg'
- repo = page[index + 9:]
- index = repo.find('<')
- if index == -1:
- return (None, _("Error while getting repo address"))
- repo = repo[:index]
- repo = repo.split('"')[0]
- return (repotype, repo)
-
- # Works for BitBucket (obsolete)
- index = page.find('git clone')
- if index != -1:
- repotype = 'git'
- repo = page[index + 10:]
- index = repo.find('<')
- if index == -1:
- return (None, _("Error while getting repo address"))
- repo = repo[:index]
- repo = repo.split('"')[0]
- return (repotype, repo)
-
- return (None, _("No information found.") + page)
-
config = None
options = None
-def get_app_from_url(url):
- """Guess basic app metadata from the URL.
-
- The URL must include a network hostname, unless it is an lp:,
- file:, or git/ssh URL. This throws ValueError on bad URLs to
- match urlparse().
-
- """
-
- parsed = urllib.parse.urlparse(url)
- invalid_url = False
- if not parsed.scheme or not parsed.path:
- invalid_url = True
-
- app = metadata.App()
- app.Repo = url
- if url.startswith('git://') or url.startswith('git@'):
- app.RepoType = 'git'
- elif parsed.netloc == 'github.com':
- app.RepoType = 'git'
- app.SourceCode = url
- app.IssueTracker = url + '/issues'
- elif parsed.netloc == 'gitlab.com':
- # git can be fussy with gitlab URLs unless they end in .git
- if url.endswith('.git'):
- url = url[:-4]
- app.Repo = url + '.git'
- app.RepoType = 'git'
- app.SourceCode = url
- app.IssueTracker = url + '/issues'
- elif parsed.netloc == 'notabug.org':
- if url.endswith('.git'):
- url = url[:-4]
- app.Repo = url + '.git'
- app.RepoType = 'git'
- app.SourceCode = url
- app.IssueTracker = url + '/issues'
- elif parsed.netloc == 'bitbucket.org':
- if url.endswith('/'):
- url = url[:-1]
- app.SourceCode = url + '/src'
- app.IssueTracker = url + '/issues'
- # Figure out the repo type and adddress...
- app.RepoType, app.Repo = getrepofrompage(url)
- elif url.startswith('https://') and url.endswith('.git'):
- app.RepoType = 'git'
-
- if not parsed.netloc and parsed.scheme in ('git', 'http', 'https', 'ssh'):
- invalid_url = True
-
- if invalid_url:
- raise ValueError(_('"{url}" is not a valid URL!'.format(url=url)))
-
- if not app.RepoType:
- raise FDroidException("Unable to determine vcs type. " + app.Repo)
-
- return app
-
+# WARNING! This cannot be imported as a Python module, so reuseable functions need to go into common.py!
def clone_to_tmp_dir(app):
tmp_dir = 'tmp'
@@ -171,40 +58,6 @@ def clone_to_tmp_dir(app):
return tmp_dir
-def get_all_gradle_and_manifests(build_dir):
- paths = []
- for root, dirs, files in os.walk(build_dir):
- for f in sorted(files):
- if f == 'AndroidManifest.xml' \
- or f.endswith('.gradle') or f.endswith('.gradle.kts'):
- full = os.path.join(root, f)
- paths.append(full)
- return paths
-
-
-def get_gradle_subdir(build_dir, paths):
- """get the subdir where the gradle build is based"""
- first_gradle_dir = None
- for path in paths:
- if not first_gradle_dir:
- first_gradle_dir = os.path.relpath(os.path.dirname(path), build_dir)
- if os.path.exists(path) and SETTINGS_GRADLE.match(os.path.basename(path)):
- with open(path) as fp:
- for m in GRADLE_SUBPROJECT.finditer(fp.read()):
- for f in glob.glob(os.path.join(os.path.dirname(path), m.group(1), 'build.gradle*')):
- with open(f) as fp:
- while True:
- line = fp.readline()
- if not line:
- break
- if common.ANDROID_PLUGIN_REGEX.match(line):
- return os.path.relpath(os.path.dirname(f), build_dir)
- if first_gradle_dir and first_gradle_dir != '.':
- return first_gradle_dir
-
- return ''
-
-
def main():
global config, options
@@ -220,6 +73,8 @@ def main():
help=_("Comma separated list of categories."))
parser.add_argument("-l", "--license", default=None,
help=_("Overall license of the project."))
+ parser.add_argument("--omit-disable", default=False,
+ help=_("Do not add 'disable:' to the generated build entries"))
parser.add_argument("--rev", default=None,
help=_("Allows a different revision (or git branch) to be specified for the initial import"))
metadata.add_metadata_arguments(parser)
@@ -256,10 +111,11 @@ def main():
break
write_local_file = True
elif options.url:
- app = get_app_from_url(options.url)
+ app = common.get_app_from_url(options.url)
tmp_importer_dir = clone_to_tmp_dir(app)
git_repo = git.repo.Repo(tmp_importer_dir)
- build.disable = 'Generated by import.py - check/set version fields and commit id'
+ if not options.omit_disable:
+ build.disable = 'Generated by import.py - check/set version fields and commit id'
write_local_file = False
else:
raise FDroidException("Specify project url.")
@@ -268,8 +124,8 @@ def main():
build.commit = common.get_head_commit_id(git_repo)
# Extract some information...
- paths = get_all_gradle_and_manifests(tmp_importer_dir)
- subdir = get_gradle_subdir(tmp_importer_dir, paths)
+ paths = common.get_all_gradle_and_manifests(tmp_importer_dir)
+ subdir = common.get_gradle_subdir(tmp_importer_dir, paths)
if paths:
versionName, versionCode, package = common.parse_androidmanifests(paths, app)
if not package:
@@ -290,8 +146,10 @@ def main():
build.versionCode = versionCode or '0' # TODO heinous but this is still a str
if options.subdir:
build.subdir = options.subdir
+ build.gradle = ['yes']
elif subdir:
build.subdir = subdir
+ build.gradle = ['yes']
if options.license:
app.License = options.license
diff --git a/fdroidserver/net.py b/fdroidserver/net.py
index b9ddf72b..3f497999 100644
--- a/fdroidserver/net.py
+++ b/fdroidserver/net.py
@@ -17,16 +17,17 @@
# along with this program. If not, see .
import os
-
import requests
+HEADERS = {'User-Agent': 'F-Droid'}
+
def download_file(url, local_filename=None, dldir='tmp'):
filename = url.split('/')[-1]
if local_filename is None:
local_filename = os.path.join(dldir, filename)
# the stream=True parameter keeps memory usage low
- r = requests.get(url, stream=True, allow_redirects=True)
+ r = requests.get(url, stream=True, allow_redirects=True, headers=HEADERS)
r.raise_for_status()
with open(local_filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
@@ -48,16 +49,15 @@ def http_get(url, etag=None, timeout=600):
- The raw content that was downloaded or None if it did not change
- The new eTag as returned by the HTTP request
"""
- headers = {'User-Agent': 'F-Droid'}
# TODO disable TLS Session IDs and TLS Session Tickets
# (plain text cookie visible to anyone who can see the network traffic)
if etag:
- r = requests.head(url, headers=headers, timeout=timeout)
+ r = requests.head(url, headers=HEADERS, timeout=timeout)
r.raise_for_status()
if 'ETag' in r.headers and etag == r.headers['ETag']:
return None, etag
- r = requests.get(url, headers=headers, timeout=timeout)
+ r = requests.get(url, headers=HEADERS, timeout=timeout)
r.raise_for_status()
new_etag = None
diff --git a/fdroidserver/server.py b/fdroidserver/server.py
index c01c1f26..eb1d8236 100644
--- a/fdroidserver/server.py
+++ b/fdroidserver/server.py
@@ -19,6 +19,7 @@
import sys
import glob
import hashlib
+import json
import os
import paramiko
import pwd
@@ -447,9 +448,8 @@ def update_servergitmirrors(servergitmirrors, repo_section):
def upload_to_android_observatory(repo_section):
- # depend on requests and lxml only if users enable AO
import requests
- from lxml.html import fromstring
+ requests # stop unused import warning
if options.verbose:
logging.getLogger("requests").setLevel(logging.INFO)
@@ -460,44 +460,53 @@ def upload_to_android_observatory(repo_section):
if repo_section == 'repo':
for f in sorted(glob.glob(os.path.join(repo_section, '*.apk'))):
- fpath = f
- fname = os.path.basename(f)
- r = requests.post('https://androidobservatory.org/',
- data={'q': update.sha256sum(f), 'searchby': 'hash'})
- if r.status_code == 200:
- # from now on XPath will be used to retrieve the message in the HTML
- # androidobservatory doesn't have a nice API to talk with
- # so we must scrape the page content
- tree = fromstring(r.text)
+ upload_apk_to_android_observatory(f)
- href = None
- for element in tree.xpath("//html/body/div/div/table/tbody/tr/td/a"):
- a = element.attrib.get('href')
- if a:
- m = re.match(r'^/app/[0-9A-F]{40}$', a)
- if m:
- href = m.group()
- page = 'https://androidobservatory.org'
- message = ''
- if href:
- message = (_('Found {apkfilename} at {url}')
- .format(apkfilename=fname, url=(page + href)))
- if message:
- logging.debug(message)
- continue
+def upload_apk_to_android_observatory(path):
+ # depend on requests and lxml only if users enable AO
+ import requests
+ from . import net
+ from lxml.html import fromstring
- # upload the file with a post request
- logging.info(_('Uploading {apkfilename} to androidobservatory.org')
- .format(apkfilename=fname))
- r = requests.post('https://androidobservatory.org/upload',
- files={'apk': (fname, open(fpath, 'rb'))},
- allow_redirects=False)
+ apkfilename = os.path.basename(path)
+ r = requests.post('https://androidobservatory.org/',
+ data={'q': update.sha256sum(path), 'searchby': 'hash'},
+ headers=net.HEADERS)
+ if r.status_code == 200:
+ # from now on XPath will be used to retrieve the message in the HTML
+ # androidobservatory doesn't have a nice API to talk with
+ # so we must scrape the page content
+ tree = fromstring(r.text)
+
+ href = None
+ for element in tree.xpath("//html/body/div/div/table/tbody/tr/td/a"):
+ a = element.attrib.get('href')
+ if a:
+ m = re.match(r'^/app/[0-9A-F]{40}$', a)
+ if m:
+ href = m.group()
+
+ page = 'https://androidobservatory.org'
+ message = ''
+ if href:
+ message = (_('Found {apkfilename} at {url}')
+ .format(apkfilename=apkfilename, url=(page + href)))
+ if message:
+ logging.debug(message)
+
+ # upload the file with a post request
+ logging.info(_('Uploading {apkfilename} to androidobservatory.org')
+ .format(apkfilename=apkfilename))
+ r = requests.post('https://androidobservatory.org/upload',
+ files={'apk': (apkfilename, open(path, 'rb'))},
+ headers=net.HEADERS,
+ allow_redirects=False)
def upload_to_virustotal(repo_section, virustotal_apikey):
- import json
import requests
+ requests # stop unused import warning
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("requests").setLevel(logging.WARNING)
@@ -514,82 +523,96 @@ def upload_to_virustotal(repo_section, virustotal_apikey):
for packageName, packages in data['packages'].items():
for package in packages:
- outputfilename = os.path.join('virustotal',
- packageName + '_' + str(package.get('versionCode'))
- + '_' + package['hash'] + '.json')
- if os.path.exists(outputfilename):
- logging.debug(package['apkName'] + ' results are in ' + outputfilename)
- continue
- filename = package['apkName']
- repofilename = os.path.join(repo_section, filename)
- logging.info('Checking if ' + repofilename + ' is on virustotal')
+ upload_apk_to_virustotal(virustotal_apikey, **package)
- headers = {
- "User-Agent": "F-Droid"
- }
- data = {
- 'apikey': virustotal_apikey,
- 'resource': package['hash'],
- }
- needs_file_upload = False
- while True:
- r = requests.get('https://www.virustotal.com/vtapi/v2/file/report?'
- + urllib.parse.urlencode(data), headers=headers)
- if r.status_code == 200:
- response = r.json()
- if response['response_code'] == 0:
- needs_file_upload = True
- else:
- response['filename'] = filename
- response['packageName'] = packageName
- response['versionCode'] = package.get('versionCode')
- response['versionName'] = package.get('versionName')
- with open(outputfilename, 'w') as fp:
- json.dump(response, fp, indent=2, sort_keys=True)
- if response.get('positives', 0) > 0:
- logging.warning(repofilename + ' has been flagged by virustotal '
- + str(response['positives']) + ' times:'
- + '\n\t' + response['permalink'])
- break
- elif r.status_code == 204:
- time.sleep(10) # wait for public API rate limiting
+def upload_apk_to_virustotal(virustotal_apikey, packageName, apkName, hash,
+ versionCode, **kwargs):
+ import requests
- upload_url = None
- if needs_file_upload:
- manual_url = 'https://www.virustotal.com/'
- size = os.path.getsize(repofilename)
- if size > 200000000:
- # VirusTotal API 200MB hard limit
- logging.error(_('{path} more than 200MB, manually upload: {url}')
- .format(path=repofilename, url=manual_url))
- elif size > 32000000:
- # VirusTotal API requires fetching a URL to upload bigger files
- r = requests.get('https://www.virustotal.com/vtapi/v2/file/scan/upload_url?'
- + urllib.parse.urlencode(data), headers=headers)
- if r.status_code == 200:
- upload_url = r.json().get('upload_url')
- elif r.status_code == 403:
- logging.error(_('VirusTotal API key cannot upload files larger than 32MB, '
- + 'use {url} to upload {path}.')
- .format(path=repofilename, url=manual_url))
- else:
- r.raise_for_status()
- else:
- upload_url = 'https://www.virustotal.com/vtapi/v2/file/scan'
+ outputfilename = os.path.join('virustotal',
+ packageName + '_' + str(versionCode)
+ + '_' + hash + '.json')
+ if os.path.exists(outputfilename):
+ logging.debug(apkName + ' results are in ' + outputfilename)
+ return outputfilename
+ repofilename = os.path.join('repo', apkName)
+ logging.info('Checking if ' + repofilename + ' is on virustotal')
- if upload_url:
- logging.info(_('Uploading {apkfilename} to virustotal')
- .format(apkfilename=repofilename))
- files = {
- 'file': (filename, open(repofilename, 'rb'))
- }
- r = requests.post(upload_url, data=data, headers=headers, files=files)
- logging.debug(_('If this upload fails, try manually uploading to {url}')
- .format(url=manual_url))
- r.raise_for_status()
- response = r.json()
- logging.info(response['verbose_msg'] + " " + response['permalink'])
+ headers = {
+ "User-Agent": "F-Droid"
+ }
+ if 'headers' in kwargs:
+ for k, v in kwargs['headers'].items():
+ headers[k] = v
+
+ data = {
+ 'apikey': virustotal_apikey,
+ 'resource': hash,
+ }
+ needs_file_upload = False
+ while True:
+ r = requests.get('https://www.virustotal.com/vtapi/v2/file/report?'
+ + urllib.parse.urlencode(data), headers=headers)
+ if r.status_code == 200:
+ response = r.json()
+ if response['response_code'] == 0:
+ needs_file_upload = True
+ else:
+ response['filename'] = apkName
+ response['packageName'] = packageName
+ response['versionCode'] = versionCode
+ if kwargs.get('versionName'):
+ response['versionName'] = kwargs.get('versionName')
+ with open(outputfilename, 'w') as fp:
+ json.dump(response, fp, indent=2, sort_keys=True)
+
+ if response.get('positives', 0) > 0:
+ logging.warning(repofilename + ' has been flagged by virustotal '
+ + str(response['positives']) + ' times:'
+ + '\n\t' + response['permalink'])
+ break
+ elif r.status_code == 204:
+ logging.warning(_('virustotal.com is rate limiting, waiting to retry...'))
+ time.sleep(30) # wait for public API rate limiting
+
+ upload_url = None
+ if needs_file_upload:
+ manual_url = 'https://www.virustotal.com/'
+ size = os.path.getsize(repofilename)
+ if size > 200000000:
+ # VirusTotal API 200MB hard limit
+ logging.error(_('{path} more than 200MB, manually upload: {url}')
+ .format(path=repofilename, url=manual_url))
+ elif size > 32000000:
+ # VirusTotal API requires fetching a URL to upload bigger files
+ r = requests.get('https://www.virustotal.com/vtapi/v2/file/scan/upload_url?'
+ + urllib.parse.urlencode(data), headers=headers)
+ if r.status_code == 200:
+ upload_url = r.json().get('upload_url')
+ elif r.status_code == 403:
+ logging.error(_('VirusTotal API key cannot upload files larger than 32MB, '
+ + 'use {url} to upload {path}.')
+ .format(path=repofilename, url=manual_url))
+ else:
+ r.raise_for_status()
+ else:
+ upload_url = 'https://www.virustotal.com/vtapi/v2/file/scan'
+
+ if upload_url:
+ logging.info(_('Uploading {apkfilename} to virustotal')
+ .format(apkfilename=repofilename))
+ files = {
+ 'file': (apkName, open(repofilename, 'rb'))
+ }
+ r = requests.post(upload_url, data=data, headers=headers, files=files)
+ logging.debug(_('If this upload fails, try manually uploading to {url}')
+ .format(url=manual_url))
+ r.raise_for_status()
+ response = r.json()
+ logging.info(response['verbose_msg'] + " " + response['permalink'])
+
+ return outputfilename
def push_binary_transparency(git_repo_path, git_remote):
diff --git a/tests/common.TestCase b/tests/common.TestCase
index 856b71e7..268d5dd4 100755
--- a/tests/common.TestCase
+++ b/tests/common.TestCase
@@ -983,6 +983,48 @@ class CommonTest(unittest.TestCase):
self.assertEqual(('1.0-free', '1', 'com.kunzisoft.fdroidtest.applicationidsuffix'),
fdroidserver.common.parse_androidmanifests(paths, app))
+ def test_get_all_gradle_and_manifests(self):
+ a = fdroidserver.common.get_all_gradle_and_manifests(os.path.join('source-files', 'cn.wildfirechat.chat'))
+ paths = [
+ os.path.join('source-files', 'cn.wildfirechat.chat', 'avenginekit', 'build.gradle'),
+ os.path.join('source-files', 'cn.wildfirechat.chat', 'build.gradle'),
+ os.path.join('source-files', 'cn.wildfirechat.chat', 'chat', 'build.gradle'),
+ os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'build.gradle'),
+ os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'src', 'main', 'AndroidManifest.xml'),
+ os.path.join('source-files', 'cn.wildfirechat.chat', 'emojilibrary', 'build.gradle'),
+ os.path.join('source-files', 'cn.wildfirechat.chat', 'gradle', 'build_libraries.gradle'),
+ os.path.join('source-files', 'cn.wildfirechat.chat', 'imagepicker', 'build.gradle'),
+ os.path.join('source-files', 'cn.wildfirechat.chat', 'mars-core-release', 'build.gradle'),
+ os.path.join('source-files', 'cn.wildfirechat.chat', 'push', 'build.gradle'),
+ os.path.join('source-files', 'cn.wildfirechat.chat', 'settings.gradle'),
+ ]
+ self.assertEqual(sorted(paths), sorted(a))
+
+ def test_get_gradle_subdir(self):
+ subdirs = {
+ 'cn.wildfirechat.chat': 'chat',
+ 'com.anpmech.launcher': 'app',
+ 'org.tasks': 'app',
+ 'ut.ewh.audiometrytest': 'app',
+ }
+ for f in ('cn.wildfirechat.chat', 'com.anpmech.launcher', 'org.tasks', 'ut.ewh.audiometrytest'):
+ build_dir = os.path.join('source-files', f)
+ paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir)
+ logging.info(paths)
+ subdir = fdroidserver.common.get_gradle_subdir(build_dir, paths)
+ self.assertEqual(subdirs[f], subdir)
+
+ def test_bad_urls(self):
+ for url in ('asdf',
+ 'file://thing.git',
+ 'https:///github.com/my/project',
+ 'git:///so/many/slashes',
+ 'ssh:/notabug.org/missing/a/slash',
+ 'git:notabug.org/missing/some/slashes',
+ 'https//github.com/bar/baz'):
+ with self.assertRaises(ValueError):
+ fdroidserver.common.get_app_from_url(url)
+
def test_remove_signing_keys(self):
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
print(testdir)
diff --git a/tests/import.TestCase b/tests/import.TestCase
index 30b660aa..70fe83df 100755
--- a/tests/import.TestCase
+++ b/tests/import.TestCase
@@ -49,53 +49,11 @@ class ImportTest(unittest.TestCase):
print('Skipping ImportTest!')
return
- app = import_proxy.get_app_from_url(url)
+ app = fdroidserver.common.get_app_from_url(url)
import_proxy.clone_to_tmp_dir(app)
self.assertEqual(app.RepoType, 'git')
self.assertEqual(app.Repo, 'https://gitlab.com/fdroid/ci-test-app.git')
- def test_get_all_gradle_and_manifests(self):
- a = import_proxy.get_all_gradle_and_manifests(os.path.join('source-files', 'cn.wildfirechat.chat'))
- paths = [
- os.path.join('source-files', 'cn.wildfirechat.chat', 'avenginekit', 'build.gradle'),
- os.path.join('source-files', 'cn.wildfirechat.chat', 'build.gradle'),
- os.path.join('source-files', 'cn.wildfirechat.chat', 'chat', 'build.gradle'),
- os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'build.gradle'),
- os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'src', 'main', 'AndroidManifest.xml'),
- os.path.join('source-files', 'cn.wildfirechat.chat', 'emojilibrary', 'build.gradle'),
- os.path.join('source-files', 'cn.wildfirechat.chat', 'gradle', 'build_libraries.gradle'),
- os.path.join('source-files', 'cn.wildfirechat.chat', 'imagepicker', 'build.gradle'),
- os.path.join('source-files', 'cn.wildfirechat.chat', 'mars-core-release', 'build.gradle'),
- os.path.join('source-files', 'cn.wildfirechat.chat', 'push', 'build.gradle'),
- os.path.join('source-files', 'cn.wildfirechat.chat', 'settings.gradle'),
- ]
- self.assertEqual(sorted(paths), sorted(a))
-
- def test_get_gradle_subdir(self):
- subdirs = {
- 'cn.wildfirechat.chat': 'chat',
- 'com.anpmech.launcher': 'app',
- 'org.tasks': 'app',
- 'ut.ewh.audiometrytest': 'app',
- }
- for f in ('cn.wildfirechat.chat', 'com.anpmech.launcher', 'org.tasks', 'ut.ewh.audiometrytest'):
- build_dir = os.path.join('source-files', f)
- paths = import_proxy.get_all_gradle_and_manifests(build_dir)
- logging.info(paths)
- subdir = import_proxy.get_gradle_subdir(build_dir, paths)
- self.assertEqual(subdirs[f], subdir)
-
- def test_bad_urls(self):
- for url in ('asdf',
- 'file://thing.git',
- 'https:///github.com/my/project',
- 'git:///so/many/slashes',
- 'ssh:/notabug.org/missing/a/slash',
- 'git:notabug.org/missing/some/slashes',
- 'https//github.com/bar/baz'):
- with self.assertRaises(ValueError):
- import_proxy.get_app_from_url(url)
-
def test_get_app_from_url(self):
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
os.chdir(testdir)
@@ -111,7 +69,7 @@ class ImportTest(unittest.TestCase):
shutil.copytree(os.path.join(self.basedir, 'source-files', appid),
tmp_importer)
- app = import_proxy.get_app_from_url(url)
+ app = fdroidserver.common.get_app_from_url(url)
with mock.patch('fdroidserver.common.getvcs',
lambda a, b, c: fdroidserver.common.vcs(url, testdir)):
with mock.patch('fdroidserver.common.vcs.gotorevision',
@@ -122,7 +80,7 @@ class ImportTest(unittest.TestCase):
self.assertEqual(url, app.Repo)
self.assertEqual(url, app.SourceCode)
logging.info(build_dir)
- paths = import_proxy.get_all_gradle_and_manifests(build_dir)
+ paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir)
self.assertNotEqual(paths, [])
versionName, versionCode, package = fdroidserver.common.parse_androidmanifests(paths, app)
self.assertEqual(vn, versionName)
diff --git a/tests/import_proxy.py b/tests/import_proxy.py
index afe9544e..f230fdb1 100644
--- a/tests/import_proxy.py
+++ b/tests/import_proxy.py
@@ -19,9 +19,6 @@ module = __import__('fdroidserver.import')
for name, obj in inspect.getmembers(module):
if name == 'import':
clone_to_tmp_dir = obj.clone_to_tmp_dir
- get_all_gradle_and_manifests = obj.get_all_gradle_and_manifests
- get_app_from_url = obj.get_app_from_url
- get_gradle_subdir = obj.get_gradle_subdir
obj.options = Options()
options = obj.options
break
diff --git a/tests/server.TestCase b/tests/server.TestCase
index 5d058153..8bd769ea 100755
--- a/tests/server.TestCase
+++ b/tests/server.TestCase
@@ -142,6 +142,12 @@ class ServerTest(unittest.TestCase):
repo_section)
self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
+ @unittest.skipIf(not os.getenv('VIRUSTOTAL_API_KEY'), 'VIRUSTOTAL_API_KEY is not set')
+ def test_upload_to_virustotal(self):
+ fdroidserver.server.options.verbose = True
+ virustotal_apikey = os.getenv('VIRUSTOTAL_API_KEY')
+ fdroidserver.server.upload_to_virustotal('repo', virustotal_apikey)
+
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))