From ab2291475bee32c22a00630bcd62cbef97513566 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 26 Feb 2020 17:07:12 +0100 Subject: [PATCH 1/6] import: mv reusable functions to common.py to avoid import_proxy.py import is a strict keyword in Python, so it is not possible to import a module called 'import', even with things like: * import fdroidserver.import * from fdroidserver import import --- fdroidserver/common.py | 151 +++++++++++++++++++++++++++++++++++++++ fdroidserver/import.py | 155 ++--------------------------------------- tests/common.TestCase | 42 +++++++++++ tests/import.TestCase | 48 +------------ tests/import_proxy.py | 3 - 5 files changed, 200 insertions(+), 199 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 7b256476..b1b4f1c1 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -37,6 +37,8 @@ import logging import hashlib import socket import base64 +import urllib.parse +import urllib.request import zipfile import tempfile import json @@ -84,6 +86,9 @@ VALID_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-z_]+(?:\d*[a-zA-Z_]*)*)(?:\.[ re.IGNORECASE) ANDROID_PLUGIN_REGEX = re.compile(r'''\s*(:?apply plugin:|id)\(?\s*['"](android|com\.android\.application)['"]\s*\)?''') +SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?') +GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:([^'"]+)['"]''') + MAX_VERSION_CODE = 0x7fffffff # Java's Integer.MAX_VALUE (2147483647) XMLNS_ANDROID = '{http://schemas.android.com/apk/res/android}' @@ -1653,6 +1658,152 @@ def is_strict_application_id(name): and '.' in name +def get_all_gradle_and_manifests(build_dir): + paths = [] + for root, dirs, files in os.walk(build_dir): + for f in sorted(files): + if f == 'AndroidManifest.xml' \ + or f.endswith('.gradle') or f.endswith('.gradle.kts'): + full = os.path.join(root, f) + paths.append(full) + return paths + + +def get_gradle_subdir(build_dir, paths): + """get the subdir where the gradle build is based""" + first_gradle_dir = None + for path in paths: + if not first_gradle_dir: + first_gradle_dir = os.path.relpath(os.path.dirname(path), build_dir) + if os.path.exists(path) and SETTINGS_GRADLE_REGEX.match(os.path.basename(path)): + with open(path) as fp: + for m in GRADLE_SUBPROJECT_REGEX.finditer(fp.read()): + for f in glob.glob(os.path.join(os.path.dirname(path), m.group(1), 'build.gradle*')): + with open(f) as fp: + while True: + line = fp.readline() + if not line: + break + if ANDROID_PLUGIN_REGEX.match(line): + return os.path.relpath(os.path.dirname(f), build_dir) + if first_gradle_dir and first_gradle_dir != '.': + return first_gradle_dir + + return '' + + +def getrepofrompage(url): + """Get the repo type and address from the given web page. + + The page is scanned in a rather naive manner for 'git clone xxxx', + 'hg clone xxxx', etc, and when one of these is found it's assumed + that's the information we want. Returns repotype, address, or + None, reason + + """ + if not url.startswith('http'): + return (None, _('{url} does not start with "http"!'.format(url=url))) + req = urllib.request.urlopen(url) # nosec B310 non-http URLs are filtered out + if req.getcode() != 200: + return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode())) + page = req.read().decode(req.headers.get_content_charset()) + + # Works for BitBucket + m = re.search('data-fetch-url="(.*)"', page) + if m is not None: + repo = m.group(1) + + if repo.endswith('.git'): + return ('git', repo) + + return ('hg', repo) + + # Works for BitBucket (obsolete) + index = page.find('hg clone') + if index != -1: + repotype = 'hg' + repo = page[index + 9:] + index = repo.find('<') + if index == -1: + return (None, _("Error while getting repo address")) + repo = repo[:index] + repo = repo.split('"')[0] + return (repotype, repo) + + # Works for BitBucket (obsolete) + index = page.find('git clone') + if index != -1: + repotype = 'git' + repo = page[index + 10:] + index = repo.find('<') + if index == -1: + return (None, _("Error while getting repo address")) + repo = repo[:index] + repo = repo.split('"')[0] + return (repotype, repo) + + return (None, _("No information found.") + page) + + +def get_app_from_url(url): + """Guess basic app metadata from the URL. + + The URL must include a network hostname, unless it is an lp:, + file:, or git/ssh URL. This throws ValueError on bad URLs to + match urlparse(). + + """ + + parsed = urllib.parse.urlparse(url) + invalid_url = False + if not parsed.scheme or not parsed.path: + invalid_url = True + + app = fdroidserver.metadata.App() + app.Repo = url + if url.startswith('git://') or url.startswith('git@'): + app.RepoType = 'git' + elif parsed.netloc == 'github.com': + app.RepoType = 'git' + app.SourceCode = url + app.IssueTracker = url + '/issues' + elif parsed.netloc == 'gitlab.com': + # git can be fussy with gitlab URLs unless they end in .git + if url.endswith('.git'): + url = url[:-4] + app.Repo = url + '.git' + app.RepoType = 'git' + app.SourceCode = url + app.IssueTracker = url + '/issues' + elif parsed.netloc == 'notabug.org': + if url.endswith('.git'): + url = url[:-4] + app.Repo = url + '.git' + app.RepoType = 'git' + app.SourceCode = url + app.IssueTracker = url + '/issues' + elif parsed.netloc == 'bitbucket.org': + if url.endswith('/'): + url = url[:-1] + app.SourceCode = url + '/src' + app.IssueTracker = url + '/issues' + # Figure out the repo type and adddress... + app.RepoType, app.Repo = getrepofrompage(url) + elif url.startswith('https://') and url.endswith('.git'): + app.RepoType = 'git' + + if not parsed.netloc and parsed.scheme in ('git', 'http', 'https', 'ssh'): + invalid_url = True + + if invalid_url: + raise ValueError(_('"{url}" is not a valid URL!'.format(url=url))) + + if not app.RepoType: + raise FDroidException("Unable to determine vcs type. " + app.Repo) + + return app + + def getsrclib(spec, srclib_dir, subdir=None, basepath=False, raw=False, prepare=True, preponly=False, refresh=True, build=None): diff --git a/fdroidserver/import.py b/fdroidserver/import.py index 42ba55e3..cca3063c 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -18,14 +18,10 @@ # along with this program. If not, see . import git -import glob import json import os -import re import shutil import sys -import urllib.parse -import urllib.request import yaml from argparse import ArgumentParser import logging @@ -40,121 +36,12 @@ from . import common from . import metadata from .exception import FDroidException -SETTINGS_GRADLE = re.compile(r'settings\.gradle(?:\.kts)?') -GRADLE_SUBPROJECT = re.compile(r'''['"]:([^'"]+)['"]''') - - -# Get the repo type and address from the given web page. The page is scanned -# in a rather naive manner for 'git clone xxxx', 'hg clone xxxx', etc, and -# when one of these is found it's assumed that's the information we want. -# Returns repotype, address, or None, reason -def getrepofrompage(url): - if not url.startswith('http'): - return (None, _('{url} does not start with "http"!'.format(url=url))) - req = urllib.request.urlopen(url) # nosec B310 non-http URLs are filtered out - if req.getcode() != 200: - return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode())) - page = req.read().decode(req.headers.get_content_charset()) - - # Works for BitBucket - m = re.search('data-fetch-url="(.*)"', page) - if m is not None: - repo = m.group(1) - - if repo.endswith('.git'): - return ('git', repo) - - return ('hg', repo) - - # Works for BitBucket (obsolete) - index = page.find('hg clone') - if index != -1: - repotype = 'hg' - repo = page[index + 9:] - index = repo.find('<') - if index == -1: - return (None, _("Error while getting repo address")) - repo = repo[:index] - repo = repo.split('"')[0] - return (repotype, repo) - - # Works for BitBucket (obsolete) - index = page.find('git clone') - if index != -1: - repotype = 'git' - repo = page[index + 10:] - index = repo.find('<') - if index == -1: - return (None, _("Error while getting repo address")) - repo = repo[:index] - repo = repo.split('"')[0] - return (repotype, repo) - - return (None, _("No information found.") + page) - config = None options = None -def get_app_from_url(url): - """Guess basic app metadata from the URL. - - The URL must include a network hostname, unless it is an lp:, - file:, or git/ssh URL. This throws ValueError on bad URLs to - match urlparse(). - - """ - - parsed = urllib.parse.urlparse(url) - invalid_url = False - if not parsed.scheme or not parsed.path: - invalid_url = True - - app = metadata.App() - app.Repo = url - if url.startswith('git://') or url.startswith('git@'): - app.RepoType = 'git' - elif parsed.netloc == 'github.com': - app.RepoType = 'git' - app.SourceCode = url - app.IssueTracker = url + '/issues' - elif parsed.netloc == 'gitlab.com': - # git can be fussy with gitlab URLs unless they end in .git - if url.endswith('.git'): - url = url[:-4] - app.Repo = url + '.git' - app.RepoType = 'git' - app.SourceCode = url - app.IssueTracker = url + '/issues' - elif parsed.netloc == 'notabug.org': - if url.endswith('.git'): - url = url[:-4] - app.Repo = url + '.git' - app.RepoType = 'git' - app.SourceCode = url - app.IssueTracker = url + '/issues' - elif parsed.netloc == 'bitbucket.org': - if url.endswith('/'): - url = url[:-1] - app.SourceCode = url + '/src' - app.IssueTracker = url + '/issues' - # Figure out the repo type and adddress... - app.RepoType, app.Repo = getrepofrompage(url) - elif url.startswith('https://') and url.endswith('.git'): - app.RepoType = 'git' - - if not parsed.netloc and parsed.scheme in ('git', 'http', 'https', 'ssh'): - invalid_url = True - - if invalid_url: - raise ValueError(_('"{url}" is not a valid URL!'.format(url=url))) - - if not app.RepoType: - raise FDroidException("Unable to determine vcs type. " + app.Repo) - - return app - +# WARNING! This cannot be imported as a Python module, so reuseable functions need to go into common.py! def clone_to_tmp_dir(app): tmp_dir = 'tmp' @@ -171,40 +58,6 @@ def clone_to_tmp_dir(app): return tmp_dir -def get_all_gradle_and_manifests(build_dir): - paths = [] - for root, dirs, files in os.walk(build_dir): - for f in sorted(files): - if f == 'AndroidManifest.xml' \ - or f.endswith('.gradle') or f.endswith('.gradle.kts'): - full = os.path.join(root, f) - paths.append(full) - return paths - - -def get_gradle_subdir(build_dir, paths): - """get the subdir where the gradle build is based""" - first_gradle_dir = None - for path in paths: - if not first_gradle_dir: - first_gradle_dir = os.path.relpath(os.path.dirname(path), build_dir) - if os.path.exists(path) and SETTINGS_GRADLE.match(os.path.basename(path)): - with open(path) as fp: - for m in GRADLE_SUBPROJECT.finditer(fp.read()): - for f in glob.glob(os.path.join(os.path.dirname(path), m.group(1), 'build.gradle*')): - with open(f) as fp: - while True: - line = fp.readline() - if not line: - break - if common.ANDROID_PLUGIN_REGEX.match(line): - return os.path.relpath(os.path.dirname(f), build_dir) - if first_gradle_dir and first_gradle_dir != '.': - return first_gradle_dir - - return '' - - def main(): global config, options @@ -256,7 +109,7 @@ def main(): break write_local_file = True elif options.url: - app = get_app_from_url(options.url) + app = common.get_app_from_url(options.url) tmp_importer_dir = clone_to_tmp_dir(app) git_repo = git.repo.Repo(tmp_importer_dir) build.disable = 'Generated by import.py - check/set version fields and commit id' @@ -268,8 +121,8 @@ def main(): build.commit = common.get_head_commit_id(git_repo) # Extract some information... - paths = get_all_gradle_and_manifests(tmp_importer_dir) - subdir = get_gradle_subdir(tmp_importer_dir, paths) + paths = common.get_all_gradle_and_manifests(tmp_importer_dir) + subdir = common.get_gradle_subdir(tmp_importer_dir, paths) if paths: versionName, versionCode, package = common.parse_androidmanifests(paths, app) if not package: diff --git a/tests/common.TestCase b/tests/common.TestCase index 856b71e7..268d5dd4 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -983,6 +983,48 @@ class CommonTest(unittest.TestCase): self.assertEqual(('1.0-free', '1', 'com.kunzisoft.fdroidtest.applicationidsuffix'), fdroidserver.common.parse_androidmanifests(paths, app)) + def test_get_all_gradle_and_manifests(self): + a = fdroidserver.common.get_all_gradle_and_manifests(os.path.join('source-files', 'cn.wildfirechat.chat')) + paths = [ + os.path.join('source-files', 'cn.wildfirechat.chat', 'avenginekit', 'build.gradle'), + os.path.join('source-files', 'cn.wildfirechat.chat', 'build.gradle'), + os.path.join('source-files', 'cn.wildfirechat.chat', 'chat', 'build.gradle'), + os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'build.gradle'), + os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'src', 'main', 'AndroidManifest.xml'), + os.path.join('source-files', 'cn.wildfirechat.chat', 'emojilibrary', 'build.gradle'), + os.path.join('source-files', 'cn.wildfirechat.chat', 'gradle', 'build_libraries.gradle'), + os.path.join('source-files', 'cn.wildfirechat.chat', 'imagepicker', 'build.gradle'), + os.path.join('source-files', 'cn.wildfirechat.chat', 'mars-core-release', 'build.gradle'), + os.path.join('source-files', 'cn.wildfirechat.chat', 'push', 'build.gradle'), + os.path.join('source-files', 'cn.wildfirechat.chat', 'settings.gradle'), + ] + self.assertEqual(sorted(paths), sorted(a)) + + def test_get_gradle_subdir(self): + subdirs = { + 'cn.wildfirechat.chat': 'chat', + 'com.anpmech.launcher': 'app', + 'org.tasks': 'app', + 'ut.ewh.audiometrytest': 'app', + } + for f in ('cn.wildfirechat.chat', 'com.anpmech.launcher', 'org.tasks', 'ut.ewh.audiometrytest'): + build_dir = os.path.join('source-files', f) + paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir) + logging.info(paths) + subdir = fdroidserver.common.get_gradle_subdir(build_dir, paths) + self.assertEqual(subdirs[f], subdir) + + def test_bad_urls(self): + for url in ('asdf', + 'file://thing.git', + 'https:///github.com/my/project', + 'git:///so/many/slashes', + 'ssh:/notabug.org/missing/a/slash', + 'git:notabug.org/missing/some/slashes', + 'https//github.com/bar/baz'): + with self.assertRaises(ValueError): + fdroidserver.common.get_app_from_url(url) + def test_remove_signing_keys(self): testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) print(testdir) diff --git a/tests/import.TestCase b/tests/import.TestCase index 30b660aa..70fe83df 100755 --- a/tests/import.TestCase +++ b/tests/import.TestCase @@ -49,53 +49,11 @@ class ImportTest(unittest.TestCase): print('Skipping ImportTest!') return - app = import_proxy.get_app_from_url(url) + app = fdroidserver.common.get_app_from_url(url) import_proxy.clone_to_tmp_dir(app) self.assertEqual(app.RepoType, 'git') self.assertEqual(app.Repo, 'https://gitlab.com/fdroid/ci-test-app.git') - def test_get_all_gradle_and_manifests(self): - a = import_proxy.get_all_gradle_and_manifests(os.path.join('source-files', 'cn.wildfirechat.chat')) - paths = [ - os.path.join('source-files', 'cn.wildfirechat.chat', 'avenginekit', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'chat', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'src', 'main', 'AndroidManifest.xml'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'emojilibrary', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'gradle', 'build_libraries.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'imagepicker', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'mars-core-release', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'push', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'settings.gradle'), - ] - self.assertEqual(sorted(paths), sorted(a)) - - def test_get_gradle_subdir(self): - subdirs = { - 'cn.wildfirechat.chat': 'chat', - 'com.anpmech.launcher': 'app', - 'org.tasks': 'app', - 'ut.ewh.audiometrytest': 'app', - } - for f in ('cn.wildfirechat.chat', 'com.anpmech.launcher', 'org.tasks', 'ut.ewh.audiometrytest'): - build_dir = os.path.join('source-files', f) - paths = import_proxy.get_all_gradle_and_manifests(build_dir) - logging.info(paths) - subdir = import_proxy.get_gradle_subdir(build_dir, paths) - self.assertEqual(subdirs[f], subdir) - - def test_bad_urls(self): - for url in ('asdf', - 'file://thing.git', - 'https:///github.com/my/project', - 'git:///so/many/slashes', - 'ssh:/notabug.org/missing/a/slash', - 'git:notabug.org/missing/some/slashes', - 'https//github.com/bar/baz'): - with self.assertRaises(ValueError): - import_proxy.get_app_from_url(url) - def test_get_app_from_url(self): testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) os.chdir(testdir) @@ -111,7 +69,7 @@ class ImportTest(unittest.TestCase): shutil.copytree(os.path.join(self.basedir, 'source-files', appid), tmp_importer) - app = import_proxy.get_app_from_url(url) + app = fdroidserver.common.get_app_from_url(url) with mock.patch('fdroidserver.common.getvcs', lambda a, b, c: fdroidserver.common.vcs(url, testdir)): with mock.patch('fdroidserver.common.vcs.gotorevision', @@ -122,7 +80,7 @@ class ImportTest(unittest.TestCase): self.assertEqual(url, app.Repo) self.assertEqual(url, app.SourceCode) logging.info(build_dir) - paths = import_proxy.get_all_gradle_and_manifests(build_dir) + paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir) self.assertNotEqual(paths, []) versionName, versionCode, package = fdroidserver.common.parse_androidmanifests(paths, app) self.assertEqual(vn, versionName) diff --git a/tests/import_proxy.py b/tests/import_proxy.py index afe9544e..f230fdb1 100644 --- a/tests/import_proxy.py +++ b/tests/import_proxy.py @@ -19,9 +19,6 @@ module = __import__('fdroidserver.import') for name, obj in inspect.getmembers(module): if name == 'import': clone_to_tmp_dir = obj.clone_to_tmp_dir - get_all_gradle_and_manifests = obj.get_all_gradle_and_manifests - get_app_from_url = obj.get_app_from_url - get_gradle_subdir = obj.get_gradle_subdir obj.options = Options() options = obj.options break From 733e7be1b3cb774c1d4589f9085593d6b7f0d5da Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 3 Mar 2020 14:37:17 +0100 Subject: [PATCH 2/6] set F-Droid HTTP Headers globally --- fdroidserver/net.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fdroidserver/net.py b/fdroidserver/net.py index b9ddf72b..3f497999 100644 --- a/fdroidserver/net.py +++ b/fdroidserver/net.py @@ -17,16 +17,17 @@ # along with this program. If not, see . import os - import requests +HEADERS = {'User-Agent': 'F-Droid'} + def download_file(url, local_filename=None, dldir='tmp'): filename = url.split('/')[-1] if local_filename is None: local_filename = os.path.join(dldir, filename) # the stream=True parameter keeps memory usage low - r = requests.get(url, stream=True, allow_redirects=True) + r = requests.get(url, stream=True, allow_redirects=True, headers=HEADERS) r.raise_for_status() with open(local_filename, 'wb') as f: for chunk in r.iter_content(chunk_size=1024): @@ -48,16 +49,15 @@ def http_get(url, etag=None, timeout=600): - The raw content that was downloaded or None if it did not change - The new eTag as returned by the HTTP request """ - headers = {'User-Agent': 'F-Droid'} # TODO disable TLS Session IDs and TLS Session Tickets # (plain text cookie visible to anyone who can see the network traffic) if etag: - r = requests.head(url, headers=headers, timeout=timeout) + r = requests.head(url, headers=HEADERS, timeout=timeout) r.raise_for_status() if 'ETag' in r.headers and etag == r.headers['ETag']: return None, etag - r = requests.get(url, headers=headers, timeout=timeout) + r = requests.get(url, headers=HEADERS, timeout=timeout) r.raise_for_status() new_etag = None From b7901952a18dcc1b387e6102e5ed6ded5760090c Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 3 Mar 2020 14:38:14 +0100 Subject: [PATCH 3/6] deploy: make androidobservatory and virustotal functions reusable This should not change the logic at all, just make the loop runs into standalone functions. --- .gitignore | 1 + fdroidserver/server.py | 232 ++++++++++++++++++++++------------------- tests/server.TestCase | 6 ++ 3 files changed, 134 insertions(+), 105 deletions(-) diff --git a/.gitignore b/.gitignore index 25aab3b3..57da9fb8 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ makebuildserver.config.py /tests/repo/obb.mainpatch.current/en-US/icon_WI0pkO3LsklrsTAnRr-OQSxkkoMY41lYe2-fAvXLiLg=.png /tests/repo/org.videolan.vlc/en-US/icon_yAfSvPRJukZzMMfUzvbYqwaD1XmHXNtiPBtuPVHW-6s=.png /tests/urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk +/tests/virustotal/ /unsigned/ # generated by gettext diff --git a/fdroidserver/server.py b/fdroidserver/server.py index c01c1f26..57acd9ca 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -19,6 +19,7 @@ import sys import glob import hashlib +import json import os import paramiko import pwd @@ -447,9 +448,8 @@ def update_servergitmirrors(servergitmirrors, repo_section): def upload_to_android_observatory(repo_section): - # depend on requests and lxml only if users enable AO import requests - from lxml.html import fromstring + requests # stop unused import warning if options.verbose: logging.getLogger("requests").setLevel(logging.INFO) @@ -460,44 +460,53 @@ def upload_to_android_observatory(repo_section): if repo_section == 'repo': for f in sorted(glob.glob(os.path.join(repo_section, '*.apk'))): - fpath = f - fname = os.path.basename(f) - r = requests.post('https://androidobservatory.org/', - data={'q': update.sha256sum(f), 'searchby': 'hash'}) - if r.status_code == 200: - # from now on XPath will be used to retrieve the message in the HTML - # androidobservatory doesn't have a nice API to talk with - # so we must scrape the page content - tree = fromstring(r.text) + upload_apk_to_android_observatory(f) - href = None - for element in tree.xpath("//html/body/div/div/table/tbody/tr/td/a"): - a = element.attrib.get('href') - if a: - m = re.match(r'^/app/[0-9A-F]{40}$', a) - if m: - href = m.group() - page = 'https://androidobservatory.org' - message = '' - if href: - message = (_('Found {apkfilename} at {url}') - .format(apkfilename=fname, url=(page + href))) - if message: - logging.debug(message) - continue +def upload_apk_to_android_observatory(path): + # depend on requests and lxml only if users enable AO + import requests + from . import net + from lxml.html import fromstring - # upload the file with a post request - logging.info(_('Uploading {apkfilename} to androidobservatory.org') - .format(apkfilename=fname)) - r = requests.post('https://androidobservatory.org/upload', - files={'apk': (fname, open(fpath, 'rb'))}, - allow_redirects=False) + apkfilename = os.path.basename(path) + r = requests.post('https://androidobservatory.org/', + data={'q': update.sha256sum(path), 'searchby': 'hash'}, + headers=net.HEADERS) + if r.status_code == 200: + # from now on XPath will be used to retrieve the message in the HTML + # androidobservatory doesn't have a nice API to talk with + # so we must scrape the page content + tree = fromstring(r.text) + + href = None + for element in tree.xpath("//html/body/div/div/table/tbody/tr/td/a"): + a = element.attrib.get('href') + if a: + m = re.match(r'^/app/[0-9A-F]{40}$', a) + if m: + href = m.group() + + page = 'https://androidobservatory.org' + message = '' + if href: + message = (_('Found {apkfilename} at {url}') + .format(apkfilename=apkfilename, url=(page + href))) + if message: + logging.debug(message) + + # upload the file with a post request + logging.info(_('Uploading {apkfilename} to androidobservatory.org') + .format(apkfilename=apkfilename)) + r = requests.post('https://androidobservatory.org/upload', + files={'apk': (apkfilename, open(path, 'rb'))}, + headers=net.HEADERS, + allow_redirects=False) def upload_to_virustotal(repo_section, virustotal_apikey): - import json import requests + requests # stop unused import warning logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("requests").setLevel(logging.WARNING) @@ -514,82 +523,95 @@ def upload_to_virustotal(repo_section, virustotal_apikey): for packageName, packages in data['packages'].items(): for package in packages: - outputfilename = os.path.join('virustotal', - packageName + '_' + str(package.get('versionCode')) - + '_' + package['hash'] + '.json') - if os.path.exists(outputfilename): - logging.debug(package['apkName'] + ' results are in ' + outputfilename) - continue - filename = package['apkName'] - repofilename = os.path.join(repo_section, filename) - logging.info('Checking if ' + repofilename + ' is on virustotal') + upload_apk_to_virustotal(virustotal_apikey, **package) - headers = { - "User-Agent": "F-Droid" - } - data = { - 'apikey': virustotal_apikey, - 'resource': package['hash'], - } - needs_file_upload = False - while True: - r = requests.get('https://www.virustotal.com/vtapi/v2/file/report?' - + urllib.parse.urlencode(data), headers=headers) - if r.status_code == 200: - response = r.json() - if response['response_code'] == 0: - needs_file_upload = True - else: - response['filename'] = filename - response['packageName'] = packageName - response['versionCode'] = package.get('versionCode') - response['versionName'] = package.get('versionName') - with open(outputfilename, 'w') as fp: - json.dump(response, fp, indent=2, sort_keys=True) - if response.get('positives', 0) > 0: - logging.warning(repofilename + ' has been flagged by virustotal ' - + str(response['positives']) + ' times:' - + '\n\t' + response['permalink']) - break - elif r.status_code == 204: - time.sleep(10) # wait for public API rate limiting +def upload_apk_to_virustotal(virustotal_apikey, packageName, apkName, hash, + versionCode, **kwargs): + import requests - upload_url = None - if needs_file_upload: - manual_url = 'https://www.virustotal.com/' - size = os.path.getsize(repofilename) - if size > 200000000: - # VirusTotal API 200MB hard limit - logging.error(_('{path} more than 200MB, manually upload: {url}') - .format(path=repofilename, url=manual_url)) - elif size > 32000000: - # VirusTotal API requires fetching a URL to upload bigger files - r = requests.get('https://www.virustotal.com/vtapi/v2/file/scan/upload_url?' - + urllib.parse.urlencode(data), headers=headers) - if r.status_code == 200: - upload_url = r.json().get('upload_url') - elif r.status_code == 403: - logging.error(_('VirusTotal API key cannot upload files larger than 32MB, ' - + 'use {url} to upload {path}.') - .format(path=repofilename, url=manual_url)) - else: - r.raise_for_status() - else: - upload_url = 'https://www.virustotal.com/vtapi/v2/file/scan' + outputfilename = os.path.join('virustotal', + packageName + '_' + str(versionCode) + + '_' + hash + '.json') + if os.path.exists(outputfilename): + logging.debug(apkName + ' results are in ' + outputfilename) + return outputfilename + repofilename = os.path.join('repo', apkName) + logging.info('Checking if ' + repofilename + ' is on virustotal') - if upload_url: - logging.info(_('Uploading {apkfilename} to virustotal') - .format(apkfilename=repofilename)) - files = { - 'file': (filename, open(repofilename, 'rb')) - } - r = requests.post(upload_url, data=data, headers=headers, files=files) - logging.debug(_('If this upload fails, try manually uploading to {url}') - .format(url=manual_url)) - r.raise_for_status() - response = r.json() - logging.info(response['verbose_msg'] + " " + response['permalink']) + headers = { + "User-Agent": "F-Droid" + } + if 'headers' in kwargs: + for k, v in kwargs['headers'].items(): + headers[k] = v + + data = { + 'apikey': virustotal_apikey, + 'resource': hash, + } + needs_file_upload = False + while True: + r = requests.get('https://www.virustotal.com/vtapi/v2/file/report?' + + urllib.parse.urlencode(data), headers=headers) + if r.status_code == 200: + response = r.json() + if response['response_code'] == 0: + needs_file_upload = True + else: + response['filename'] = apkName + response['packageName'] = packageName + response['versionCode'] = versionCode + if kwargs.get('versionName'): + response['versionName'] = kwargs.get('versionName') + with open(outputfilename, 'w') as fp: + json.dump(response, fp, indent=2, sort_keys=True) + + if response.get('positives', 0) > 0: + logging.warning(repofilename + ' has been flagged by virustotal ' + + str(response['positives']) + ' times:' + + '\n\t' + response['permalink']) + break + elif r.status_code == 204: + time.sleep(10) # wait for public API rate limiting + + upload_url = None + if needs_file_upload: + manual_url = 'https://www.virustotal.com/' + size = os.path.getsize(repofilename) + if size > 200000000: + # VirusTotal API 200MB hard limit + logging.error(_('{path} more than 200MB, manually upload: {url}') + .format(path=repofilename, url=manual_url)) + elif size > 32000000: + # VirusTotal API requires fetching a URL to upload bigger files + r = requests.get('https://www.virustotal.com/vtapi/v2/file/scan/upload_url?' + + urllib.parse.urlencode(data), headers=headers) + if r.status_code == 200: + upload_url = r.json().get('upload_url') + elif r.status_code == 403: + logging.error(_('VirusTotal API key cannot upload files larger than 32MB, ' + + 'use {url} to upload {path}.') + .format(path=repofilename, url=manual_url)) + else: + r.raise_for_status() + else: + upload_url = 'https://www.virustotal.com/vtapi/v2/file/scan' + + if upload_url: + logging.info(_('Uploading {apkfilename} to virustotal') + .format(apkfilename=repofilename)) + files = { + 'file': (apkName, open(repofilename, 'rb')) + } + r = requests.post(upload_url, data=data, headers=headers, files=files) + logging.debug(_('If this upload fails, try manually uploading to {url}') + .format(url=manual_url)) + r.raise_for_status() + response = r.json() + logging.info(response['verbose_msg'] + " " + response['permalink']) + + return outputfilename def push_binary_transparency(git_repo_path, git_remote): diff --git a/tests/server.TestCase b/tests/server.TestCase index 5d058153..8bd769ea 100755 --- a/tests/server.TestCase +++ b/tests/server.TestCase @@ -142,6 +142,12 @@ class ServerTest(unittest.TestCase): repo_section) self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call') + @unittest.skipIf(not os.getenv('VIRUSTOTAL_API_KEY'), 'VIRUSTOTAL_API_KEY is not set') + def test_upload_to_virustotal(self): + fdroidserver.server.options.verbose = True + virustotal_apikey = os.getenv('VIRUSTOTAL_API_KEY') + fdroidserver.server.upload_to_virustotal('repo', virustotal_apikey) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From a1e17d2b2c048cb16130ee6414f2fde2ba60458b Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 3 Mar 2020 14:41:38 +0100 Subject: [PATCH 4/6] deploy: retry virustotal 30 seconds after rate limiting, and log --- fdroidserver/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 57acd9ca..eb1d8236 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -573,7 +573,8 @@ def upload_apk_to_virustotal(virustotal_apikey, packageName, apkName, hash, + '\n\t' + response['permalink']) break elif r.status_code == 204: - time.sleep(10) # wait for public API rate limiting + logging.warning(_('virustotal.com is rate limiting, waiting to retry...')) + time.sleep(30) # wait for public API rate limiting upload_url = None if needs_file_upload: From 1bb9cf43e1bff9b9ce9e5c7a52c0da072fefa174 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 3 Mar 2020 21:32:04 +0100 Subject: [PATCH 5/6] import: ensure gradle: yes is added for detected Gradle builds --- fdroidserver/import.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fdroidserver/import.py b/fdroidserver/import.py index cca3063c..772220ec 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -143,8 +143,10 @@ def main(): build.versionCode = versionCode or '0' # TODO heinous but this is still a str if options.subdir: build.subdir = options.subdir + build.gradle = ['yes'] elif subdir: build.subdir = subdir + build.gradle = ['yes'] if options.license: app.License = options.license From 138bc7366821d4159529b8e01d3787fe10fa10b4 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 3 Mar 2020 21:33:23 +0100 Subject: [PATCH 6/6] import: --omit-disable option to easily work with generated files --- fdroidserver/import.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fdroidserver/import.py b/fdroidserver/import.py index 772220ec..23d06d57 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -73,6 +73,8 @@ def main(): help=_("Comma separated list of categories.")) parser.add_argument("-l", "--license", default=None, help=_("Overall license of the project.")) + parser.add_argument("--omit-disable", default=False, + help=_("Do not add 'disable:' to the generated build entries")) parser.add_argument("--rev", default=None, help=_("Allows a different revision (or git branch) to be specified for the initial import")) metadata.add_metadata_arguments(parser) @@ -112,7 +114,8 @@ def main(): app = common.get_app_from_url(options.url) tmp_importer_dir = clone_to_tmp_dir(app) git_repo = git.repo.Repo(tmp_importer_dir) - build.disable = 'Generated by import.py - check/set version fields and commit id' + if not options.omit_disable: + build.disable = 'Generated by import.py - check/set version fields and commit id' write_local_file = False else: raise FDroidException("Specify project url.")