diff --git a/completion/bash-completion b/completion/bash-completion index 8bdd333f..d18edd35 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -264,6 +264,12 @@ __complete_btlog() { __complete_options } +__complete_mirror() { + opts="-v" + lopts="--archive --output-dir" + __complete_options +} + __complete_nightly() { opts="-v -q" lopts="--show-secret-var" @@ -316,6 +322,7 @@ import \ init \ install \ lint \ +mirror \ nightly \ publish \ readmeta \ diff --git a/fdroid b/fdroid index a07a4ecf..f5e6c92b 100755 --- a/fdroid +++ b/fdroid @@ -48,6 +48,7 @@ commands = OrderedDict([ ("btlog", _("Update the binary transparency log for a URL")), ("signatures", _("Extract signatures from APKs")), ("nightly", _("Set up an app build for a nightly build repo")), + ("mirror", _("Download complete mirrors of small repos")), ]) diff --git a/fdroidserver/mirror.py b/fdroidserver/mirror.py new file mode 100644 index 00000000..06595a44 --- /dev/null +++ b/fdroidserver/mirror.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 + +import ipaddress +import logging +import os +import posixpath +import socket +import subprocess +import sys +from argparse import ArgumentParser +import urllib.parse + +from . import _ +from . import common +from . import index +from . import update + +options = None + + +def _run_wget(path, urls): + if options.verbose: + verbose = '--verbose' + else: + verbose = '--no-verbose' + + if not urls: + return + logging.debug(_('Running wget in {path}').format(path=path)) + os.makedirs(path, exist_ok=True) + os.chdir(path) + urls_file = '.fdroid-mirror-wget-input-file' + with open(urls_file, 'w') as fp: + for url in urls: + fp.write(url.split('?')[0] + '\n') # wget puts query string in the filename + subprocess.call(['wget', verbose, '--continue', '--user-agent="fdroid mirror"', + '--input-file=' + urls_file]) + os.remove(urls_file) + + +def main(): + global options + + parser = ArgumentParser(usage=_("%(prog)s [options] url")) + common.setup_global_opts(parser) + parser.add_argument("url", nargs='?', + help=_('Base URL to mirror, can include the index signing key ' + + 'using the query string: ?fingerprint=')) + parser.add_argument("--archive", action='store_true', default=False, + help=_("Also mirror the full archive section")) + parser.add_argument("--output-dir", default=None, + help=_("The directory to write the mirror to")) + options = parser.parse_args() + + if options.url is None: + logging.error(_('A URL is required as an argument!') + '\n') + parser.print_help() + sys.exit(1) + + scheme, hostname, path, params, query, fragment = urllib.parse.urlparse(options.url) + fingerprint = urllib.parse.parse_qs(query).get('fingerprint') + + def _append_to_url_path(*args): + '''Append the list of path components to URL, keeping the rest the same''' + newpath = posixpath.join(path, *args) + return urllib.parse.urlunparse((scheme, hostname, newpath, params, query, fragment)) + + if fingerprint: + config = common.read_config(options) + if not ('jarsigner' in config or 'apksigner' in config): + logging.error(_('Java JDK not found! Install in standard location or set java_paths!')) + sys.exit(1) + + def _get_index(section, etag=None): + url = _append_to_url_path(section) + return index.download_repo_index(url, etag=etag) + else: + def _get_index(section, etag=None): + import io + import json + import zipfile + from . import net + url = _append_to_url_path(section, 'index-v1.jar') + content, etag = net.http_get(url) + with zipfile.ZipFile(io.BytesIO(content)) as zip: + jsoncontents = zip.open('index-v1.json').read() + data = json.loads(jsoncontents.decode('utf-8')) + return data, etag + + ip = None + try: + ip = ipaddress.ip_address(hostname) + except ValueError: + pass + if hostname == 'f-droid.org' \ + or (ip is not None and hostname in socket.gethostbyname_ex('f-droid.org')[2]): + print(_('ERROR: this command should never be used to mirror f-droid.org!\n' + 'A full mirror of f-droid.org requires more than 200GB.')) + sys.exit(1) + + path = path.rstrip('/') + if path.endswith('repo') or path.endswith('archive'): + logging.warning(_('Do not include "{path}" in URL!') + .format(path=path.split('/')[-1])) + elif not path.endswith('fdroid'): + logging.warning(_('{url} does not end with "fdroid", check the URL path!') + .format(url=options.url)) + + icondirs = ['icons', ] + for density in update.screen_densities: + icondirs.append('icons-' + density) + + if options.output_dir: + basedir = options.output_dir + else: + basedir = os.path.join(os.getcwd(), hostname, path.strip('/')) + os.makedirs(basedir, exist_ok=True) + + if options.archive: + sections = ('repo', 'archive') + else: + sections = ('repo', ) + + for section in sections: + sectiondir = os.path.join(basedir, section) + + data, etag = _get_index(section) + + os.makedirs(sectiondir, exist_ok=True) + os.chdir(sectiondir) + for icondir in icondirs: + os.makedirs(os.path.join(sectiondir, icondir), exist_ok=True) + + urls = [] + for packageName, packageList in data['packages'].items(): + for package in packageList: + to_fetch = [] + for k in ('apkName', 'srcname'): + if k in package: + to_fetch.append(package[k]) + elif k == 'apkName': + logging.error(_('{appid} is missing {name}') + .format(appid=package['packageName'], name=k)) + for f in to_fetch: + if not os.path.exists(f) \ + or (f.endswith('.apk') and os.path.getsize(f) != package['size']): + urls.append(_append_to_url_path(section, f)) + urls.append(_append_to_url_path(section, f + '.asc')) + _run_wget(sectiondir, urls) + + for app in data['apps']: + localized = app.get('localized') + if localized: + for locale, d in localized.items(): + urls = [] + components = (section, app['packageName'], locale) + for k in update.GRAPHIC_NAMES: + f = d.get(k) + if f: + urls.append(_append_to_url_path(*components, f)) + _run_wget(os.path.join(basedir, *components), urls) + for k in update.SCREENSHOT_DIRS: + urls = [] + filelist = d.get(k) + if filelist: + components = (section, app['packageName'], locale, k) + for f in filelist: + urls.append(_append_to_url_path(*components, f)) + _run_wget(os.path.join(basedir, *components), urls) + + urls = dict() + for app in data['apps']: + if 'icon' not in app: + logging.error(_('no "icon" in {appid}').format(appid=app['packageName'])) + continue + icon = app['icon'] + for icondir in icondirs: + url = _append_to_url_path(section, icondir, icon) + if icondir not in urls: + urls[icondir] = [] + urls[icondir].append(url) + + for icondir in icondirs: + _run_wget(os.path.join(basedir, section, icondir), urls[icondir]) + + +if __name__ == "__main__": + main()