diff --git a/examples/config.py b/examples/config.py index b400af6c..9035baa5 100644 --- a/examples/config.py +++ b/examples/config.py @@ -190,6 +190,13 @@ The repository of older versions of applications from the main demo repository. # 'https://gitlab.com/user/repo', # } +# Most git hosting services have hard size limits for each git repo. +# `fdroid deploy` will delete the git history when the git mirror repo +# approaches this limit to ensure that the repo will still fit when +# pushed. GitHub recommends 1GB, gitlab.com recommends 10GB. +# +# git_mirror_size_limit = '10GB' + # Any mirrors of this repo, for example all of the servers declared in # serverwebroot and all the servers declared in servergitmirrors, # will automatically be used by the client. If one diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 05060658..a18d49f1 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -148,6 +148,7 @@ default_config = { 'archive_description': _('These are the apps that have been archived from the main repo.'), 'archive_older': 0, 'lint_licenses': fdroidserver.lint.APPROVED_LICENSES, + 'git_mirror_size_limit': 10000000000, } @@ -354,9 +355,31 @@ def read_config(opts, config_file='config.py'): raise TypeError(_('only accepts strings, lists, and tuples')) config['servergitmirrors'] = roots + limit = config['git_mirror_size_limit'] + config['git_mirror_size_limit'] = parse_human_readable_size(limit) + return config +def parse_human_readable_size(size): + units = { + 'b': 1, + 'kb': 1000, 'mb': 1000**2, 'gb': 1000**3, 'tb': 1000**4, + 'kib': 1024, 'mib': 1024**2, 'gib': 1024**3, 'tib': 1024**4, + } + try: + return int(float(size)) + except (ValueError, TypeError): + if type(size) != str: + raise ValueError(_('Could not parse size "{size}", wrong type "{type}"') + .format(size=size, type=type(size))) + s = size.lower().replace(' ', '') + m = re.match(r'^(?P[0-9][0-9.]+) *(?P' + r'|'.join(units.keys()) + r')$', s) + if not m: + raise ValueError(_('Not a valid size definition: "{}"').format(size)) + return int(float(m.group("value")) * units[m.group("unit")]) + + def assert_config_keystore(config): """Check weather keystore is configured correctly and raise exception if not.""" diff --git a/fdroidserver/mirror.py b/fdroidserver/mirror.py index 0aa43722..920c9acf 100644 --- a/fdroidserver/mirror.py +++ b/fdroidserver/mirror.py @@ -48,6 +48,10 @@ def main(): + 'using the query string: ?fingerprint=')) parser.add_argument("--archive", action='store_true', default=False, help=_("Also mirror the full archive section")) + parser.add_argument("--build-logs", action='store_true', default=False, + help=_("Include the build logs in the mirror")) + parser.add_argument("--src-tarballs", action='store_true', default=False, + help=_("Include the source tarballs in the mirror")) parser.add_argument("--output-dir", default=None, help=_("The directory to write the mirror to")) options = parser.parse_args() @@ -135,7 +139,10 @@ def main(): for packageName, packageList in data['packages'].items(): for package in packageList: to_fetch = [] - for k in ('apkName', 'srcname'): + keys = ['apkName', ] + if options.src_tarballs: + keys.append('srcname') + for k in keys: if k in package: to_fetch.append(package[k]) elif k == 'apkName': @@ -146,6 +153,9 @@ def main(): or (f.endswith('.apk') and os.path.getsize(f) != package['size']): urls.append(_append_to_url_path(section, f)) urls.append(_append_to_url_path(section, f + '.asc')) + if options.build_logs and f.endswith('.apk'): + urls.append(_append_to_url_path(section, f[:-4] + '.log.gz')) + _run_wget(sectiondir, urls) for app in data['apps']: diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 964edc3b..18b30f7c 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -25,6 +25,7 @@ import pwd import re import subprocess import time +import urllib from argparse import ArgumentParser import logging import shutil @@ -352,11 +353,18 @@ def update_servergitmirrors(servergitmirrors, repo_section): git_repodir = os.path.join(git_mirror_path, 'fdroid', repo_section) if not os.path.isdir(git_repodir): os.makedirs(git_repodir) - if os.path.isdir(dotgit) and _get_size(git_mirror_path) > 1000000000: - logging.warning('Deleting git-mirror history, repo is too big (1 gig max)') + # github/gitlab use bare git repos, so only count the .git folder + # test: generate giant APKs by including AndroidManifest.xml and and large + # file from /dev/urandom, then sign it. Then add those to the git repo. + dotgit_size = _get_size(dotgit) + dotgit_over_limit = dotgit_size > config['git_mirror_size_limit'] + if os.path.isdir(dotgit) and dotgit_over_limit: + logging.warning(_('Deleting git-mirror history, repo is too big ({size} max {limit})') + .format(size=dotgit_size, limit=config['git_mirror_size_limit'])) shutil.rmtree(dotgit) - if options.no_keep_git_mirror_archive and _get_size(git_mirror_path) > 1000000000: - logging.warning('Deleting archive, repo is too big (1 gig max)') + if options.no_keep_git_mirror_archive and dotgit_over_limit: + logging.warning(_('Deleting archive, repo is too big ({size} max {limit})') + .format(size=dotgit_size, limit=config['git_mirror_size_limit'])) archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive') shutil.rmtree(archive_path, ignore_errors=True) @@ -473,7 +481,7 @@ def upload_to_android_observatory(repo_section): logging.info(message) -def upload_to_virustotal(repo_section, vt_apikey): +def upload_to_virustotal(repo_section, virustotal_apikey): import json import requests @@ -506,13 +514,13 @@ def upload_to_virustotal(repo_section, vt_apikey): "User-Agent": "F-Droid" } data = { - 'apikey': vt_apikey, + 'apikey': virustotal_apikey, 'resource': package['hash'], } needs_file_upload = False while True: - r = requests.post('https://www.virustotal.com/vtapi/v2/file/report', - data=data, headers=headers) + r = requests.get('https://www.virustotal.com/vtapi/v2/file/report?' + + urllib.parse.urlencode(data), headers=headers) if r.status_code == 200: response = r.json() if response['response_code'] == 0: @@ -533,18 +541,40 @@ def upload_to_virustotal(repo_section, vt_apikey): elif r.status_code == 204: time.sleep(10) # wait for public API rate limiting + upload_url = None if needs_file_upload: - logging.info('Uploading ' + repofilename + ' to virustotal') + manual_url = 'https://www.virustotal.com/' + size = os.path.getsize(repofilename) + if size > 200000000: + # VirusTotal API 200MB hard limit + logging.error(_('{path} more than 200MB, manually upload: {url}') + .format(path=repofilename, url=manual_url)) + elif size > 32000000: + # VirusTotal API requires fetching a URL to upload bigger files + r = requests.get('https://www.virustotal.com/vtapi/v2/file/scan/upload_url?' + + urllib.parse.urlencode(data), headers=headers) + if r.status_code == 200: + upload_url = r.json().get('upload_url') + elif r.status_code == 403: + logging.error(_('VirusTotal API key cannot upload files larger than 32MB, ' + + 'use {url} to upload {path}.') + .format(path=repofilename, url=manual_url)) + else: + r.raise_for_status() + else: + upload_url = 'https://www.virustotal.com/vtapi/v2/file/scan' + + if upload_url: + logging.info(_('Uploading {apkfilename} to virustotal') + .format(apkfilename=repofilename)) files = { 'file': (filename, open(repofilename, 'rb')) } - r = requests.post('https://www.virustotal.com/vtapi/v2/file/scan', - data=data, headers=headers, files=files) - logging.debug('If this upload fails, try manually uploading here:\n' - + 'https://www.virustotal.com/') + r = requests.post(upload_url, data=data, headers=headers, files=files) + logging.debug(_('If this upload fails, try manually uploading to {url}') + .format(url=manual_url)) r.raise_for_status() response = r.json() - logging.info(response['verbose_msg'] + " " + response['permalink']) diff --git a/tests/common.TestCase b/tests/common.TestCase index 33db7283..3d90707a 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -45,6 +45,15 @@ class CommonTest(unittest.TestCase): os.makedirs(self.tmpdir) os.chdir(self.basedir) + def test_parse_human_readable_size(self): + for k, v in ((9827, 9827), (123.456, 123), ('123b', 123), ('1.2', 1), + ('10.43 KiB', 10680), ('11GB', 11000000000), ('59kb', 59000), + ('343.1 mb', 343100000), ('99.9GiB', 107266808217)): + self.assertEqual(fdroidserver.common.parse_human_readable_size(k), v) + for v in ((12, 123), '0xfff', [], None, '12,123', '123GG', '982374bb', self): + with self.assertRaises(ValueError): + fdroidserver.common.parse_human_readable_size(v) + def test_assert_config_keystore(self): with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): with self.assertRaises(FDroidException): diff --git a/tests/run-tests b/tests/run-tests index 7ae22fe9..d769686e 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -1096,6 +1096,59 @@ $fdroid update --create-key test -e $KEYSTORE +#------------------------------------------------------------------------------# +echo_header "setup a new repo from scratch using ANDROID_HOME with git mirror" + +# fake git remote server for repo mirror +SERVER_GIT_MIRROR=`create_test_dir` +cd $SERVER_GIT_MIRROR +git init +git config receive.denyCurrentBranch updateInstead + +REPOROOT=`create_test_dir` +GIT_MIRROR=$REPOROOT/git-mirror +cd $REPOROOT +fdroid_init_with_prebuilt_keystore +echo "servergitmirrors = '$SERVER_GIT_MIRROR'" >> config.py + +cp $WORKSPACE/tests/repo/com.politedroid_[345].apk repo/ +$fdroid update --create-metadata +$fdroid deploy +test -e $GIT_MIRROR/fdroid/repo/com.politedroid_3.apk +test -e $GIT_MIRROR/fdroid/repo/com.politedroid_4.apk +test -e $GIT_MIRROR/fdroid/repo/com.politedroid_5.apk +test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_3.apk +test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_4.apk +test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_5.apk +date > $GIT_MIRROR/.git/test-stamp + +# add one more APK to trigger archiving +cp $WORKSPACE/tests/repo/com.politedroid_6.apk repo/ +$fdroid update +$fdroid deploy +test -e $REPOROOT/archive/com.politedroid_3.apk +! test -e $GIT_MIRROR/fdroid/archive/com.politedroid_3.apk +! test -e $SERVER_GIT_MIRROR/fdroid/archive/com.politedroid_3.apk +test -e $GIT_MIRROR/fdroid/repo/com.politedroid_4.apk +test -e $GIT_MIRROR/fdroid/repo/com.politedroid_5.apk +test -e $GIT_MIRROR/fdroid/repo/com.politedroid_6.apk +test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_4.apk +test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_5.apk +test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_6.apk +before=`du -s --bytes $GIT_MIRROR/.git/ | awk '{print $1}'` + +echo "git_mirror_size_limit = '60kb'" >> config.py +$fdroid update +$fdroid deploy +test -e $REPOROOT/archive/com.politedroid_3.apk +! test -e $SERVER_GIT_MIRROR/fdroid/archive/com.politedroid_3.apk +after=`du -s --bytes $GIT_MIRROR/.git/ | awk '{print $1}'` +! test -e $GIT_MIRROR/.git/test-stamp +git -C $GIT_MIRROR gc +git -C $SERVER_GIT_MIRROR gc +test $before -gt $after + + #------------------------------------------------------------------------------# echo_header "sign binary repo in offline box, then publishing from online box"