From f2b48575e6a949e35e856534c6ff4c6a86b8f010 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 6 Nov 2019 09:02:53 +0100 Subject: [PATCH 1/5] deploy: github/gitlab use bare git repos, only size the .git/ dir needs test: generate giant APKs by including AndroidManifest.xml and and large file from /dev/urandom, then sign it. Then add those to the git repo. --- fdroidserver/server.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 964edc3b..229aa13f 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -352,10 +352,13 @@ def update_servergitmirrors(servergitmirrors, repo_section): git_repodir = os.path.join(git_mirror_path, 'fdroid', repo_section) if not os.path.isdir(git_repodir): os.makedirs(git_repodir) - if os.path.isdir(dotgit) and _get_size(git_mirror_path) > 1000000000: + # github/gitlab use bare git repos, so only count the .git folder + # test: generate giant APKs by including AndroidManifest.xml and and large + # file from /dev/urandom, then sign it. Then add those to the git repo. + if os.path.isdir(dotgit) and _get_size(dotgit) > 1000000000: logging.warning('Deleting git-mirror history, repo is too big (1 gig max)') shutil.rmtree(dotgit) - if options.no_keep_git_mirror_archive and _get_size(git_mirror_path) > 1000000000: + if options.no_keep_git_mirror_archive and _get_size(dotgit) > 1000000000: logging.warning('Deleting archive, repo is too big (1 gig max)') archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive') shutil.rmtree(archive_path, ignore_errors=True) From 058b47e4843f74f7b8ffe97d8adea5fdb1288179 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 19 Dec 2019 10:34:48 +0100 Subject: [PATCH 2/5] deploy: fix virustotal report fetching, use GET and query string --- fdroidserver/server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 229aa13f..d4a1b784 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -476,7 +476,7 @@ def upload_to_android_observatory(repo_section): logging.info(message) -def upload_to_virustotal(repo_section, vt_apikey): +def upload_to_virustotal(repo_section, virustotal_apikey): import json import requests @@ -509,13 +509,13 @@ def upload_to_virustotal(repo_section, vt_apikey): "User-Agent": "F-Droid" } data = { - 'apikey': vt_apikey, + 'apikey': virustotal_apikey, 'resource': package['hash'], } needs_file_upload = False while True: - r = requests.post('https://www.virustotal.com/vtapi/v2/file/report', - data=data, headers=headers) + r = requests.get('https://www.virustotal.com/vtapi/v2/file/report?' + + urllib.parse.urlencode(data), headers=headers) if r.status_code == 200: response = r.json() if response['response_code'] == 0: From 4fa11ef4fc2b72d746e0be3f0937768b5d61f96c Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 19 Dec 2019 10:36:15 +0100 Subject: [PATCH 3/5] deploy: detect virustotal size limits while uploading This prevents the size limits from blocking the whole deploy. --- fdroidserver/server.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/fdroidserver/server.py b/fdroidserver/server.py index d4a1b784..a69ed030 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -25,6 +25,7 @@ import pwd import re import subprocess import time +import urllib from argparse import ArgumentParser import logging import shutil @@ -536,18 +537,40 @@ def upload_to_virustotal(repo_section, virustotal_apikey): elif r.status_code == 204: time.sleep(10) # wait for public API rate limiting + upload_url = None if needs_file_upload: - logging.info('Uploading ' + repofilename + ' to virustotal') + manual_url = 'https://www.virustotal.com/' + size = os.path.getsize(repofilename) + if size > 200000000: + # VirusTotal API 200MB hard limit + logging.error(_('{path} more than 200MB, manually upload: {url}') + .format(path=repofilename, url=manual_url)) + elif size > 32000000: + # VirusTotal API requires fetching a URL to upload bigger files + r = requests.get('https://www.virustotal.com/vtapi/v2/file/scan/upload_url?' + + urllib.parse.urlencode(data), headers=headers) + if r.status_code == 200: + upload_url = r.json().get('upload_url') + elif r.status_code == 403: + logging.error(_('VirusTotal API key cannot upload files larger than 32MB, ' + + 'use {url} to upload {path}.') + .format(path=repofilename, url=manual_url)) + else: + r.raise_for_status() + else: + upload_url = 'https://www.virustotal.com/vtapi/v2/file/scan' + + if upload_url: + logging.info(_('Uploading {apkfilename} to virustotal') + .format(apkfilename=repofilename)) files = { 'file': (filename, open(repofilename, 'rb')) } - r = requests.post('https://www.virustotal.com/vtapi/v2/file/scan', - data=data, headers=headers, files=files) - logging.debug('If this upload fails, try manually uploading here:\n' - + 'https://www.virustotal.com/') + r = requests.post(upload_url, data=data, headers=headers, files=files) + logging.debug(_('If this upload fails, try manually uploading to {url}') + .format(url=manual_url)) r.raise_for_status() response = r.json() - logging.info(response['verbose_msg'] + " " + response['permalink']) From e76a0c9d6afc35474b1b7773978e7b46278210f2 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 13 Jan 2020 11:48:23 +0100 Subject: [PATCH 4/5] git_mirror_size_limit config option to set max git mirror size GitHub and GitLab have some kinds of limits on how big a git repo can be, this makes that option configurable. This also is very useful for tests. --- examples/config.py | 7 ++++++ fdroidserver/common.py | 23 ++++++++++++++++++ fdroidserver/server.py | 12 ++++++---- tests/common.TestCase | 9 +++++++ tests/run-tests | 53 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 100 insertions(+), 4 deletions(-) diff --git a/examples/config.py b/examples/config.py index b400af6c..9035baa5 100644 --- a/examples/config.py +++ b/examples/config.py @@ -190,6 +190,13 @@ The repository of older versions of applications from the main demo repository. # 'https://gitlab.com/user/repo', # } +# Most git hosting services have hard size limits for each git repo. +# `fdroid deploy` will delete the git history when the git mirror repo +# approaches this limit to ensure that the repo will still fit when +# pushed. GitHub recommends 1GB, gitlab.com recommends 10GB. +# +# git_mirror_size_limit = '10GB' + # Any mirrors of this repo, for example all of the servers declared in # serverwebroot and all the servers declared in servergitmirrors, # will automatically be used by the client. If one diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 05060658..a18d49f1 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -148,6 +148,7 @@ default_config = { 'archive_description': _('These are the apps that have been archived from the main repo.'), 'archive_older': 0, 'lint_licenses': fdroidserver.lint.APPROVED_LICENSES, + 'git_mirror_size_limit': 10000000000, } @@ -354,9 +355,31 @@ def read_config(opts, config_file='config.py'): raise TypeError(_('only accepts strings, lists, and tuples')) config['servergitmirrors'] = roots + limit = config['git_mirror_size_limit'] + config['git_mirror_size_limit'] = parse_human_readable_size(limit) + return config +def parse_human_readable_size(size): + units = { + 'b': 1, + 'kb': 1000, 'mb': 1000**2, 'gb': 1000**3, 'tb': 1000**4, + 'kib': 1024, 'mib': 1024**2, 'gib': 1024**3, 'tib': 1024**4, + } + try: + return int(float(size)) + except (ValueError, TypeError): + if type(size) != str: + raise ValueError(_('Could not parse size "{size}", wrong type "{type}"') + .format(size=size, type=type(size))) + s = size.lower().replace(' ', '') + m = re.match(r'^(?P[0-9][0-9.]+) *(?P' + r'|'.join(units.keys()) + r')$', s) + if not m: + raise ValueError(_('Not a valid size definition: "{}"').format(size)) + return int(float(m.group("value")) * units[m.group("unit")]) + + def assert_config_keystore(config): """Check weather keystore is configured correctly and raise exception if not.""" diff --git a/fdroidserver/server.py b/fdroidserver/server.py index a69ed030..18b30f7c 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -356,11 +356,15 @@ def update_servergitmirrors(servergitmirrors, repo_section): # github/gitlab use bare git repos, so only count the .git folder # test: generate giant APKs by including AndroidManifest.xml and and large # file from /dev/urandom, then sign it. Then add those to the git repo. - if os.path.isdir(dotgit) and _get_size(dotgit) > 1000000000: - logging.warning('Deleting git-mirror history, repo is too big (1 gig max)') + dotgit_size = _get_size(dotgit) + dotgit_over_limit = dotgit_size > config['git_mirror_size_limit'] + if os.path.isdir(dotgit) and dotgit_over_limit: + logging.warning(_('Deleting git-mirror history, repo is too big ({size} max {limit})') + .format(size=dotgit_size, limit=config['git_mirror_size_limit'])) shutil.rmtree(dotgit) - if options.no_keep_git_mirror_archive and _get_size(dotgit) > 1000000000: - logging.warning('Deleting archive, repo is too big (1 gig max)') + if options.no_keep_git_mirror_archive and dotgit_over_limit: + logging.warning(_('Deleting archive, repo is too big ({size} max {limit})') + .format(size=dotgit_size, limit=config['git_mirror_size_limit'])) archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive') shutil.rmtree(archive_path, ignore_errors=True) diff --git a/tests/common.TestCase b/tests/common.TestCase index 33db7283..3d90707a 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -45,6 +45,15 @@ class CommonTest(unittest.TestCase): os.makedirs(self.tmpdir) os.chdir(self.basedir) + def test_parse_human_readable_size(self): + for k, v in ((9827, 9827), (123.456, 123), ('123b', 123), ('1.2', 1), + ('10.43 KiB', 10680), ('11GB', 11000000000), ('59kb', 59000), + ('343.1 mb', 343100000), ('99.9GiB', 107266808217)): + self.assertEqual(fdroidserver.common.parse_human_readable_size(k), v) + for v in ((12, 123), '0xfff', [], None, '12,123', '123GG', '982374bb', self): + with self.assertRaises(ValueError): + fdroidserver.common.parse_human_readable_size(v) + def test_assert_config_keystore(self): with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): with self.assertRaises(FDroidException): diff --git a/tests/run-tests b/tests/run-tests index 7ae22fe9..d769686e 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -1096,6 +1096,59 @@ $fdroid update --create-key test -e $KEYSTORE +#------------------------------------------------------------------------------# +echo_header "setup a new repo from scratch using ANDROID_HOME with git mirror" + +# fake git remote server for repo mirror +SERVER_GIT_MIRROR=`create_test_dir` +cd $SERVER_GIT_MIRROR +git init +git config receive.denyCurrentBranch updateInstead + +REPOROOT=`create_test_dir` +GIT_MIRROR=$REPOROOT/git-mirror +cd $REPOROOT +fdroid_init_with_prebuilt_keystore +echo "servergitmirrors = '$SERVER_GIT_MIRROR'" >> config.py + +cp $WORKSPACE/tests/repo/com.politedroid_[345].apk repo/ +$fdroid update --create-metadata +$fdroid deploy +test -e $GIT_MIRROR/fdroid/repo/com.politedroid_3.apk +test -e $GIT_MIRROR/fdroid/repo/com.politedroid_4.apk +test -e $GIT_MIRROR/fdroid/repo/com.politedroid_5.apk +test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_3.apk +test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_4.apk +test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_5.apk +date > $GIT_MIRROR/.git/test-stamp + +# add one more APK to trigger archiving +cp $WORKSPACE/tests/repo/com.politedroid_6.apk repo/ +$fdroid update +$fdroid deploy +test -e $REPOROOT/archive/com.politedroid_3.apk +! test -e $GIT_MIRROR/fdroid/archive/com.politedroid_3.apk +! test -e $SERVER_GIT_MIRROR/fdroid/archive/com.politedroid_3.apk +test -e $GIT_MIRROR/fdroid/repo/com.politedroid_4.apk +test -e $GIT_MIRROR/fdroid/repo/com.politedroid_5.apk +test -e $GIT_MIRROR/fdroid/repo/com.politedroid_6.apk +test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_4.apk +test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_5.apk +test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_6.apk +before=`du -s --bytes $GIT_MIRROR/.git/ | awk '{print $1}'` + +echo "git_mirror_size_limit = '60kb'" >> config.py +$fdroid update +$fdroid deploy +test -e $REPOROOT/archive/com.politedroid_3.apk +! test -e $SERVER_GIT_MIRROR/fdroid/archive/com.politedroid_3.apk +after=`du -s --bytes $GIT_MIRROR/.git/ | awk '{print $1}'` +! test -e $GIT_MIRROR/.git/test-stamp +git -C $GIT_MIRROR gc +git -C $SERVER_GIT_MIRROR gc +test $before -gt $after + + #------------------------------------------------------------------------------# echo_header "sign binary repo in offline box, then publishing from online box" From df7d7adf78ceb5c29e5e6dee5d0b1c0b37790a96 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 13 Jan 2020 15:06:02 +0100 Subject: [PATCH 5/5] mirror: optionally fetch build logs and src tarballs --- fdroidserver/mirror.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/fdroidserver/mirror.py b/fdroidserver/mirror.py index 0aa43722..920c9acf 100644 --- a/fdroidserver/mirror.py +++ b/fdroidserver/mirror.py @@ -48,6 +48,10 @@ def main(): + 'using the query string: ?fingerprint=')) parser.add_argument("--archive", action='store_true', default=False, help=_("Also mirror the full archive section")) + parser.add_argument("--build-logs", action='store_true', default=False, + help=_("Include the build logs in the mirror")) + parser.add_argument("--src-tarballs", action='store_true', default=False, + help=_("Include the source tarballs in the mirror")) parser.add_argument("--output-dir", default=None, help=_("The directory to write the mirror to")) options = parser.parse_args() @@ -135,7 +139,10 @@ def main(): for packageName, packageList in data['packages'].items(): for package in packageList: to_fetch = [] - for k in ('apkName', 'srcname'): + keys = ['apkName', ] + if options.src_tarballs: + keys.append('srcname') + for k in keys: if k in package: to_fetch.append(package[k]) elif k == 'apkName': @@ -146,6 +153,9 @@ def main(): or (f.endswith('.apk') and os.path.getsize(f) != package['size']): urls.append(_append_to_url_path(section, f)) urls.append(_append_to_url_path(section, f + '.asc')) + if options.build_logs and f.endswith('.apk'): + urls.append(_append_to_url_path(section, f[:-4] + '.log.gz')) + _run_wget(sectiondir, urls) for app in data['apps']: