From eadcd1372344c3b2e746bd5a239ce57fde66c430 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 27 Mar 2017 09:58:09 +0200 Subject: [PATCH 1/7] move make_binary_transparency_log to common for easy reuse --- fdroidserver/common.py | 77 +++++++++++++++++++++++++++++++++++++++++ fdroidserver/update.py | 78 +----------------------------------------- 2 files changed, 78 insertions(+), 77 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 0c12c3f9..00f22c7a 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -20,12 +20,15 @@ # common.py is imported by all modules, so do not import third-party # libraries here as they will become a requirement for all commands. +import collections import io import os import sys import re import shutil import glob +import json +import platform import stat import subprocess import time @@ -2348,3 +2351,77 @@ def is_repo_file(filename): b'index-v1.json', b'categories.txt', ] + + +def make_binary_transparency_log(repodirs): + '''Log the indexes in a standalone git repo to serve as a "binary + transparency" log. + + see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies + + ''' + + import git + btrepo = 'binary_transparency' + if os.path.exists(os.path.join(btrepo, '.git')): + gitrepo = git.Repo(btrepo) + else: + if not os.path.exists(btrepo): + os.mkdir(btrepo) + gitrepo = git.Repo.init(btrepo) + + gitconfig = gitrepo.config_writer() + gitconfig.set_value('user', 'name', 'fdroid update') + gitconfig.set_value('user', 'email', 'fdroid@' + platform.node()) + + url = config['repo_url'].rstrip('/') + with open(os.path.join(btrepo, 'README.md'), 'w') as fp: + fp.write(""" +# Binary Transparency Log for %s + +""" % url[:url.rindex('/')]) # strip '/repo' + gitrepo.index.add(['README.md', ]) + gitrepo.index.commit('add README') + + for repodir in repodirs: + cpdir = os.path.join(btrepo, repodir) + if not os.path.exists(cpdir): + os.mkdir(cpdir) + for f in ('index.xml', 'index-v1.json'): + dest = os.path.join(cpdir, f) + shutil.copyfile(os.path.join(repodir, f), dest) + gitrepo.index.add([os.path.join(repodir, f), ]) + for f in ('index.jar', 'index-v1.jar'): + repof = os.path.join(repodir, f) + dest = os.path.join(cpdir, f) + jarin = ZipFile(repof, 'r') + jarout = ZipFile(dest, 'w') + for info in jarin.infolist(): + if info.filename.startswith('META-INF/'): + jarout.writestr(info, jarin.read(info.filename)) + jarout.close() + jarin.close() + gitrepo.index.add([repof, ]) + + files = [] + for root, dirs, filenames in os.walk(repodir): + for f in filenames: + files.append(os.path.relpath(os.path.join(root, f), repodir)) + output = collections.OrderedDict() + for f in sorted(files): + repofile = os.path.join(repodir, f) + stat = os.stat(repofile) + output[f] = ( + stat.st_size, + stat.st_ctime_ns, + stat.st_mtime_ns, + stat.st_mode, + stat.st_uid, + stat.st_gid, + ) + fslogfile = os.path.join(cpdir, 'filesystemlog.json') + with open(fslogfile, 'w') as fp: + json.dump(output, fp, indent=2) + gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ]) + + gitrepo.index.commit('fdroid update') diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 0e42d684..943d3b8e 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -23,13 +23,11 @@ import sys import os import shutil import glob -import json import re import socket import zipfile import hashlib import pickle -import platform from datetime import datetime, timedelta from argparse import ArgumentParser @@ -1212,80 +1210,6 @@ def add_apks_to_per_app_repos(repodir, apks): shutil.copy(apkascpath, apk['per_app_repo']) -def make_binary_transparency_log(repodirs): - '''Log the indexes in a standalone git repo to serve as a "binary - transparency" log. - - see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies - - ''' - - import git - btrepo = 'binary_transparency' - if os.path.exists(os.path.join(btrepo, '.git')): - gitrepo = git.Repo(btrepo) - else: - if not os.path.exists(btrepo): - os.mkdir(btrepo) - gitrepo = git.Repo.init(btrepo) - - gitconfig = gitrepo.config_writer() - gitconfig.set_value('user', 'name', 'fdroid update') - gitconfig.set_value('user', 'email', 'fdroid@' + platform.node()) - - url = config['repo_url'].rstrip('/') - with open(os.path.join(btrepo, 'README.md'), 'w') as fp: - fp.write(""" -# Binary Transparency Log for %s - -""" % url[:url.rindex('/')]) # strip '/repo' - gitrepo.index.add(['README.md', ]) - gitrepo.index.commit('add README') - - for repodir in repodirs: - cpdir = os.path.join(btrepo, repodir) - if not os.path.exists(cpdir): - os.mkdir(cpdir) - for f in ('index.xml', 'index-v1.json'): - dest = os.path.join(cpdir, f) - shutil.copyfile(os.path.join(repodir, f), dest) - gitrepo.index.add([os.path.join(repodir, f), ]) - for f in ('index.jar', 'index-v1.jar'): - repof = os.path.join(repodir, f) - dest = os.path.join(cpdir, f) - jarin = zipfile.ZipFile(repof, 'r') - jarout = zipfile.ZipFile(dest, 'w') - for info in jarin.infolist(): - if info.filename.startswith('META-INF/'): - jarout.writestr(info, jarin.read(info.filename)) - jarout.close() - jarin.close() - gitrepo.index.add([repof, ]) - - files = [] - for root, dirs, filenames in os.walk(repodir): - for f in filenames: - files.append(os.path.relpath(os.path.join(root, f), repodir)) - output = collections.OrderedDict() - for f in sorted(files): - repofile = os.path.join(repodir, f) - stat = os.stat(repofile) - output[f] = ( - stat.st_size, - stat.st_ctime_ns, - stat.st_mtime_ns, - stat.st_mode, - stat.st_uid, - stat.st_gid, - ) - fslogfile = os.path.join(cpdir, 'filesystemlog.json') - with open(fslogfile, 'w') as fp: - json.dump(output, fp, indent=2) - gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ]) - - gitrepo.index.commit('fdroid update') - - config = None options = None @@ -1484,7 +1408,7 @@ def main(): index.make(apps, sortedids, archapks, repodirs[1], True) if config.get('binary_transparency_remote'): - make_binary_transparency_log(repodirs) + common.make_binary_transparency_log(repodirs) if config['update_stats']: # Update known apks info... From c591a4cd89d8d345fd6d163f496676f59db149b4 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 27 Mar 2017 21:52:15 +0200 Subject: [PATCH 2/7] `fdroid btlog` external binary transparency logger This complements the binary transparency logging that happens on the server side !226. Anyone can set up an efficient tracker of any F-Droid repo which stores all index files that it sees. It uses HEAD requests and ETag checking to be as efficient as possible, so that this can be automatically run at a frequent pace. --- completion/bash-completion | 6 ++ fdroid | 1 + fdroidserver/btlog.py | 120 +++++++++++++++++++++++++++++++++++++ fdroidserver/common.py | 24 +++++--- 4 files changed, 144 insertions(+), 7 deletions(-) create mode 100755 fdroidserver/btlog.py diff --git a/completion/bash-completion b/completion/bash-completion index bdaa2cbb..48352447 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -236,6 +236,12 @@ __complete_verify() { esac } +__complete_btlog() { + opts="-u" + lopts="--git-remote --git-repo --url" + __complete_options +} + __complete_stats() { opts="-v -q -d" lopts="--verbose --quiet --download" diff --git a/fdroid b/fdroid index feea104a..bc1655b9 100755 --- a/fdroid +++ b/fdroid @@ -42,6 +42,7 @@ commands = { "stats": "Update the stats of the repo", "server": "Interact with the repo HTTP server", "signindex": "Sign indexes created using update --nosign", + "btlog": "Update the binary transparency log for a URL", } diff --git a/fdroidserver/btlog.py b/fdroidserver/btlog.py new file mode 100755 index 00000000..1911a093 --- /dev/null +++ b/fdroidserver/btlog.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# +# btlog.py - part of the FDroid server tools +# Copyright (C) 2017, Hans-Christoph Steiner +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# This is for creating a binary transparency log in a git repo for any +# F-Droid repo accessible via HTTP. It is meant to run very often, +# even once a minute in a cronjob, so it uses HEAD requests and the +# HTTP ETag to check if the file has changed. HEAD requests should +# not count against the download counts. This pattern of a HEAD then +# a GET is what fdroidclient uses to avoid ETags being abused as +# cookies. This also uses the same HTTP User Agent as the F-Droid +# client app so its not easy for the server to distinguish this from +# the F-Droid client. + +import os +import json +import logging +import requests +import shutil +import sys +import tempfile +from argparse import ArgumentParser + +from . import common + + +options = None + + +def main(): + global options + + parser = ArgumentParser(usage="%(prog)s [options]") + common.setup_global_opts(parser) + parser.add_argument("--git-repo", + default=os.path.join(os.getcwd(), 'binary_transparency'), + help="Path to the git repo to use as the log") + parser.add_argument("-u", "--url", default='https://f-droid.org', + help="The base URL for the repo to log (default: https://f-droid.org)") + parser.add_argument("--git-remote", default=None, + help="Create a repo signing key in a keystore") + options = parser.parse_args() + + if options.verbose: + logging.getLogger("requests").setLevel(logging.INFO) + logging.getLogger("urllib3").setLevel(logging.INFO) + else: + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + + if not os.path.exists(options.git_repo): + logging.error('"' + options.git_repo + '/" does not exist! Create it, or use --git-repo') + sys.exit(1) + + session = requests.Session() + + new_files = False + repodirs = ('repo', 'archive') + tempdirbase = tempfile.mkdtemp(prefix='.fdroid-btlog-') + for repodir in repodirs: + # TODO read HTTP headers for etag from git repo + tempdir = os.path.join(tempdirbase, repodir) + os.makedirs(tempdir, exist_ok=True) + gitrepodir = os.path.join(options.git_repo, repodir) + os.makedirs(gitrepodir, exist_ok=True) + for f in ('index.jar', 'index.xml', 'index-v1.jar', 'index-v1.json'): + dlfile = os.path.join(tempdir, f) + dlurl = options.url + '/' + repodir + '/' + f + http_headers_file = os.path.join(gitrepodir, f + '.HTTP-headers.json') + + headers = { + 'User-Agent': 'F-Droid 0.102.3' + } + if os.path.exists(http_headers_file): + with open(http_headers_file) as fp: + etag = json.load(fp)['ETag'] + + r = session.head(dlurl, headers=headers, allow_redirects=False) + if r.status_code != 200: + logging.debug('HTTP Response (' + str(r.status_code) + '), did not download ' + dlurl) + continue + if etag and etag == r.headers.get('ETag'): + logging.debug('ETag matches, did not download ' + dlurl) + continue + + r = session.get(dlurl, headers=headers, allow_redirects=False) + if r.status_code == 200: + with open(dlfile, 'wb') as f: + for chunk in r: + f.write(chunk) + + dump = dict() + for k, v in r.headers.items(): + dump[k] = v + with open(http_headers_file, 'w') as fp: + json.dump(dump, fp, indent=2, sort_keys=True) + new_files = True + + if new_files: + os.chdir(tempdirbase) + common.make_binary_transparency_log(repodirs, options.git_repo, options.url, + 'fdroid btlog') + shutil.rmtree(tempdirbase, ignore_errors=True) + +if __name__ == "__main__": + main() diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 00f22c7a..be902c47 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -2353,7 +2353,9 @@ def is_repo_file(filename): ] -def make_binary_transparency_log(repodirs): +def make_binary_transparency_log(repodirs, btrepo='binary_transparency', + url=None, + commit_title='fdroid update'): '''Log the indexes in a standalone git repo to serve as a "binary transparency" log. @@ -2362,7 +2364,6 @@ def make_binary_transparency_log(repodirs): ''' import git - btrepo = 'binary_transparency' if os.path.exists(os.path.join(btrepo, '.git')): gitrepo = git.Repo(btrepo) else: @@ -2371,10 +2372,11 @@ def make_binary_transparency_log(repodirs): gitrepo = git.Repo.init(btrepo) gitconfig = gitrepo.config_writer() - gitconfig.set_value('user', 'name', 'fdroid update') + gitconfig.set_value('user', 'name', commit_title) gitconfig.set_value('user', 'email', 'fdroid@' + platform.node()) - url = config['repo_url'].rstrip('/') + if not url: + url = config['repo_url'].rstrip('/') with open(os.path.join(btrepo, 'README.md'), 'w') as fp: fp.write(""" # Binary Transparency Log for %s @@ -2388,11 +2390,16 @@ def make_binary_transparency_log(repodirs): if not os.path.exists(cpdir): os.mkdir(cpdir) for f in ('index.xml', 'index-v1.json'): + repof = os.path.join(repodir, f) + if not os.path.exists(repof): + continue dest = os.path.join(cpdir, f) - shutil.copyfile(os.path.join(repodir, f), dest) - gitrepo.index.add([os.path.join(repodir, f), ]) + shutil.copyfile(repof, dest) + gitrepo.index.add([repof, ]) for f in ('index.jar', 'index-v1.jar'): repof = os.path.join(repodir, f) + if not os.path.exists(repof): + continue dest = os.path.join(cpdir, f) jarin = ZipFile(repof, 'r') jarout = ZipFile(dest, 'w') @@ -2424,4 +2431,7 @@ def make_binary_transparency_log(repodirs): json.dump(output, fp, indent=2) gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ]) - gitrepo.index.commit('fdroid update') + for f in glob.glob(os.path.join(cpdir, '*.HTTP-headers.json')): + gitrepo.index.add([os.path.join(repodir, os.path.basename(f)), ]) + + gitrepo.index.commit(commit_title) From 6f71465ec1008e55b287fa9f9a9ad56215a6795e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Sun, 2 Apr 2017 21:58:34 +0200 Subject: [PATCH 3/7] support pretty output in JSON and for binary transparency logs This makes make_index_v1() support `fdroid update --pretty`, then also uses pretty output for the binary transparency logs, so that the git history has nice, readable diffs between commits. --- fdroidserver/common.py | 13 ++++++++++++- fdroidserver/index.py | 5 ++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index be902c47..cc4567dc 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -2364,6 +2364,8 @@ def make_binary_transparency_log(repodirs, btrepo='binary_transparency', ''' import git + import xml.dom.minidom + if os.path.exists(os.path.join(btrepo, '.git')): gitrepo = git.Repo(btrepo) else: @@ -2394,7 +2396,16 @@ def make_binary_transparency_log(repodirs, btrepo='binary_transparency', if not os.path.exists(repof): continue dest = os.path.join(cpdir, f) - shutil.copyfile(repof, dest) + if f.endswith('.xml'): + doc = xml.dom.minidom.parse(repof) + output = doc.toprettyxml(encoding='utf-8') + with open(dest, 'wb') as f: + f.write(output) + elif f.endswith('.json'): + with open(repof) as fp: + output = json.load(fp, object_pairs_hook=collections.OrderedDict) + with open(dest, 'w') as fp: + json.dump(output, fp, indent=2) gitrepo.index.add([repof, ]) for f in ('index.jar', 'index-v1.jar'): repof = os.path.join(repodir, f) diff --git a/fdroidserver/index.py b/fdroidserver/index.py index b42c63d1..e029fc50 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -220,7 +220,10 @@ def make_v1(apps, packages, repodir, repodict, requestsdict): json_name = 'index-v1.json' index_file = os.path.join(repodir, json_name) with open(index_file, 'w') as fp: - json.dump(output, fp, default=_index_encoder_default) + if common.options.pretty: + json.dump(output, fp, default=_index_encoder_default, indent=2) + else: + json.dump(output, fp, default=_index_encoder_default) if common.options.nosign: logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!') From 896c6496b4591ead0223c5c913cec23b061bb6f8 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Sun, 2 Apr 2017 23:07:05 +0200 Subject: [PATCH 4/7] remove setting git name/email for binary transparency logs The machine running the steps should include the git config, so that it is specific to that machines. --- fdroidserver/common.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index cc4567dc..e07c46c3 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -28,7 +28,6 @@ import re import shutil import glob import json -import platform import stat import subprocess import time @@ -2373,16 +2372,19 @@ def make_binary_transparency_log(repodirs, btrepo='binary_transparency', os.mkdir(btrepo) gitrepo = git.Repo.init(btrepo) - gitconfig = gitrepo.config_writer() - gitconfig.set_value('user', 'name', commit_title) - gitconfig.set_value('user', 'email', 'fdroid@' + platform.node()) - if not url: url = config['repo_url'].rstrip('/') with open(os.path.join(btrepo, 'README.md'), 'w') as fp: fp.write(""" # Binary Transparency Log for %s +This is a log of the signed app index metadata. This is stored in a +git repo, which serves as an imperfect append-only storage mechanism. +People can then check that any file that they received from that +F-Droid repository was a publicly released file. + +For more info on this idea: +* https://wiki.mozilla.org/Security/Binary_Transparency """ % url[:url.rindex('/')]) # strip '/repo' gitrepo.index.add(['README.md', ]) gitrepo.index.commit('add README') From 40290fc5e05718afbb7a63be87282930fbccc0a2 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 3 Apr 2017 16:02:07 +0200 Subject: [PATCH 5/7] move make_binary_transparency_log to btlog This keeps the code more organized, and reduces the number of things that are loaded for every command via common. --- fdroidserver/btlog.py | 104 ++++++++++++++++++++++++++++++++++++++++- fdroidserver/common.py | 100 --------------------------------------- fdroidserver/update.py | 3 +- 3 files changed, 104 insertions(+), 103 deletions(-) diff --git a/fdroidserver/btlog.py b/fdroidserver/btlog.py index 1911a093..64f4f262 100755 --- a/fdroidserver/btlog.py +++ b/fdroidserver/btlog.py @@ -26,6 +26,10 @@ # client app so its not easy for the server to distinguish this from # the F-Droid client. + +import collections +import git +import glob import os import json import logging @@ -33,6 +37,8 @@ import requests import shutil import sys import tempfile +import xml.dom.minidom +import zipfile from argparse import ArgumentParser from . import common @@ -41,6 +47,101 @@ from . import common options = None +def make_binary_transparency_log(repodirs, btrepo='binary_transparency', + url=None, + commit_title='fdroid update'): + '''Log the indexes in a standalone git repo to serve as a "binary + transparency" log. + + see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies + + ''' + + if os.path.exists(os.path.join(btrepo, '.git')): + gitrepo = git.Repo(btrepo) + else: + if not os.path.exists(btrepo): + os.mkdir(btrepo) + gitrepo = git.Repo.init(btrepo) + + if not url: + url = common.config['repo_url'].rstrip('/') + with open(os.path.join(btrepo, 'README.md'), 'w') as fp: + fp.write(""" +# Binary Transparency Log for %s + +This is a log of the signed app index metadata. This is stored in a +git repo, which serves as an imperfect append-only storage mechanism. +People can then check that any file that they received from that +F-Droid repository was a publicly released file. + +For more info on this idea: +* https://wiki.mozilla.org/Security/Binary_Transparency +""" % url[:url.rindex('/')]) # strip '/repo' + gitrepo.index.add(['README.md', ]) + gitrepo.index.commit('add README') + + for repodir in repodirs: + cpdir = os.path.join(btrepo, repodir) + if not os.path.exists(cpdir): + os.mkdir(cpdir) + for f in ('index.xml', 'index-v1.json'): + repof = os.path.join(repodir, f) + if not os.path.exists(repof): + continue + dest = os.path.join(cpdir, f) + if f.endswith('.xml'): + doc = xml.dom.minidom.parse(repof) + output = doc.toprettyxml(encoding='utf-8') + with open(dest, 'wb') as f: + f.write(output) + elif f.endswith('.json'): + with open(repof) as fp: + output = json.load(fp, object_pairs_hook=collections.OrderedDict) + with open(dest, 'w') as fp: + json.dump(output, fp, indent=2) + gitrepo.index.add([repof, ]) + for f in ('index.jar', 'index-v1.jar'): + repof = os.path.join(repodir, f) + if not os.path.exists(repof): + continue + dest = os.path.join(cpdir, f) + jarin = zipfile.ZipFile(repof, 'r') + jarout = zipfile.ZipFile(dest, 'w') + for info in jarin.infolist(): + if info.filename.startswith('META-INF/'): + jarout.writestr(info, jarin.read(info.filename)) + jarout.close() + jarin.close() + gitrepo.index.add([repof, ]) + + files = [] + for root, dirs, filenames in os.walk(repodir): + for f in filenames: + files.append(os.path.relpath(os.path.join(root, f), repodir)) + output = collections.OrderedDict() + for f in sorted(files): + repofile = os.path.join(repodir, f) + stat = os.stat(repofile) + output[f] = ( + stat.st_size, + stat.st_ctime_ns, + stat.st_mtime_ns, + stat.st_mode, + stat.st_uid, + stat.st_gid, + ) + fslogfile = os.path.join(cpdir, 'filesystemlog.json') + with open(fslogfile, 'w') as fp: + json.dump(output, fp, indent=2) + gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ]) + + for f in glob.glob(os.path.join(cpdir, '*.HTTP-headers.json')): + gitrepo.index.add([os.path.join(repodir, os.path.basename(f)), ]) + + gitrepo.index.commit(commit_title) + + def main(): global options @@ -112,8 +213,7 @@ def main(): if new_files: os.chdir(tempdirbase) - common.make_binary_transparency_log(repodirs, options.git_repo, options.url, - 'fdroid btlog') + make_binary_transparency_log(repodirs, options.git_repo, options.url, 'fdroid btlog') shutil.rmtree(tempdirbase, ignore_errors=True) if __name__ == "__main__": diff --git a/fdroidserver/common.py b/fdroidserver/common.py index e07c46c3..0c12c3f9 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -20,14 +20,12 @@ # common.py is imported by all modules, so do not import third-party # libraries here as they will become a requirement for all commands. -import collections import io import os import sys import re import shutil import glob -import json import stat import subprocess import time @@ -2350,101 +2348,3 @@ def is_repo_file(filename): b'index-v1.json', b'categories.txt', ] - - -def make_binary_transparency_log(repodirs, btrepo='binary_transparency', - url=None, - commit_title='fdroid update'): - '''Log the indexes in a standalone git repo to serve as a "binary - transparency" log. - - see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies - - ''' - - import git - import xml.dom.minidom - - if os.path.exists(os.path.join(btrepo, '.git')): - gitrepo = git.Repo(btrepo) - else: - if not os.path.exists(btrepo): - os.mkdir(btrepo) - gitrepo = git.Repo.init(btrepo) - - if not url: - url = config['repo_url'].rstrip('/') - with open(os.path.join(btrepo, 'README.md'), 'w') as fp: - fp.write(""" -# Binary Transparency Log for %s - -This is a log of the signed app index metadata. This is stored in a -git repo, which serves as an imperfect append-only storage mechanism. -People can then check that any file that they received from that -F-Droid repository was a publicly released file. - -For more info on this idea: -* https://wiki.mozilla.org/Security/Binary_Transparency -""" % url[:url.rindex('/')]) # strip '/repo' - gitrepo.index.add(['README.md', ]) - gitrepo.index.commit('add README') - - for repodir in repodirs: - cpdir = os.path.join(btrepo, repodir) - if not os.path.exists(cpdir): - os.mkdir(cpdir) - for f in ('index.xml', 'index-v1.json'): - repof = os.path.join(repodir, f) - if not os.path.exists(repof): - continue - dest = os.path.join(cpdir, f) - if f.endswith('.xml'): - doc = xml.dom.minidom.parse(repof) - output = doc.toprettyxml(encoding='utf-8') - with open(dest, 'wb') as f: - f.write(output) - elif f.endswith('.json'): - with open(repof) as fp: - output = json.load(fp, object_pairs_hook=collections.OrderedDict) - with open(dest, 'w') as fp: - json.dump(output, fp, indent=2) - gitrepo.index.add([repof, ]) - for f in ('index.jar', 'index-v1.jar'): - repof = os.path.join(repodir, f) - if not os.path.exists(repof): - continue - dest = os.path.join(cpdir, f) - jarin = ZipFile(repof, 'r') - jarout = ZipFile(dest, 'w') - for info in jarin.infolist(): - if info.filename.startswith('META-INF/'): - jarout.writestr(info, jarin.read(info.filename)) - jarout.close() - jarin.close() - gitrepo.index.add([repof, ]) - - files = [] - for root, dirs, filenames in os.walk(repodir): - for f in filenames: - files.append(os.path.relpath(os.path.join(root, f), repodir)) - output = collections.OrderedDict() - for f in sorted(files): - repofile = os.path.join(repodir, f) - stat = os.stat(repofile) - output[f] = ( - stat.st_size, - stat.st_ctime_ns, - stat.st_mtime_ns, - stat.st_mode, - stat.st_uid, - stat.st_gid, - ) - fslogfile = os.path.join(cpdir, 'filesystemlog.json') - with open(fslogfile, 'w') as fp: - json.dump(output, fp, indent=2) - gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ]) - - for f in glob.glob(os.path.join(cpdir, '*.HTTP-headers.json')): - gitrepo.index.add([os.path.join(repodir, os.path.basename(f)), ]) - - gitrepo.index.commit(commit_title) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 943d3b8e..db107cae 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -37,6 +37,7 @@ from binascii import hexlify from PIL import Image import logging +from . import btlog from . import common from . import index from . import metadata @@ -1408,7 +1409,7 @@ def main(): index.make(apps, sortedids, archapks, repodirs[1], True) if config.get('binary_transparency_remote'): - common.make_binary_transparency_log(repodirs) + btlog.make_binary_transparency_log(repodirs) if config['update_stats']: # Update known apks info... From 4b7084f779f9df02734ac08d5144fb30cc576e8c Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 3 Apr 2017 20:31:25 +0200 Subject: [PATCH 6/7] btlog: if git remote is specified, push commits to that remote This makes fdroid automatically push the new binary transparency commits if there is a git_remote specified in either config.py or from a CLI arg. --- fdroidserver/btlog.py | 20 +++++++++++++++++--- fdroidserver/update.py | 2 +- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/fdroidserver/btlog.py b/fdroidserver/btlog.py index 64f4f262..94d4a2cb 100755 --- a/fdroidserver/btlog.py +++ b/fdroidserver/btlog.py @@ -49,7 +49,8 @@ options = None def make_binary_transparency_log(repodirs, btrepo='binary_transparency', url=None, - commit_title='fdroid update'): + commit_title='fdroid update', + git_remote=None): '''Log the indexes in a standalone git repo to serve as a "binary transparency" log. @@ -57,6 +58,7 @@ def make_binary_transparency_log(repodirs, btrepo='binary_transparency', ''' + logging.info('Committing indexes to ' + btrepo) if os.path.exists(os.path.join(btrepo, '.git')): gitrepo = git.Repo(btrepo) else: @@ -140,6 +142,17 @@ For more info on this idea: gitrepo.index.add([os.path.join(repodir, os.path.basename(f)), ]) gitrepo.index.commit(commit_title) + if git_remote: + logging.info('Pushing binary transparency log to ' + git_remote) + origin = git.remote.Remote(gitrepo, 'origin') + if origin in gitrepo.remotes: + origin = gitrepo.remote('origin') + if 'set_url' in dir(origin): # added in GitPython 2.x + origin.set_url(git_remote) + else: + origin = gitrepo.create_remote('origin', git_remote) + origin.fetch() + origin.push('master') def main(): @@ -153,7 +166,7 @@ def main(): parser.add_argument("-u", "--url", default='https://f-droid.org', help="The base URL for the repo to log (default: https://f-droid.org)") parser.add_argument("--git-remote", default=None, - help="Create a repo signing key in a keystore") + help="Push the log to this git remote repository") options = parser.parse_args() if options.verbose: @@ -213,7 +226,8 @@ def main(): if new_files: os.chdir(tempdirbase) - make_binary_transparency_log(repodirs, options.git_repo, options.url, 'fdroid btlog') + make_binary_transparency_log(repodirs, options.git_repo, options.url, 'fdroid btlog', + git_remote=options.git_remote) shutil.rmtree(tempdirbase, ignore_errors=True) if __name__ == "__main__": diff --git a/fdroidserver/update.py b/fdroidserver/update.py index db107cae..652dd1fb 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1409,7 +1409,7 @@ def main(): index.make(apps, sortedids, archapks, repodirs[1], True) if config.get('binary_transparency_remote'): - btlog.make_binary_transparency_log(repodirs) + btlog.make_binary_transparency_log(repodirs, git_remote=config['binary_transparency_remote']) if config['update_stats']: # Update known apks info... From 136d58dbae5dd32e87752a2bd8a881294146293d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 3 Apr 2017 23:16:39 +0200 Subject: [PATCH 7/7] btlog: test with `fdroid update` without requiring ssh access Yay git! Just use a filesystem git remote instead of ssh. --- fdroidserver/update.py | 5 +++-- tests/run-tests | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 652dd1fb..e26388b6 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1408,8 +1408,9 @@ def main(): if len(repodirs) > 1: index.make(apps, sortedids, archapks, repodirs[1], True) - if config.get('binary_transparency_remote'): - btlog.make_binary_transparency_log(repodirs, git_remote=config['binary_transparency_remote']) + git_remote = config.get('binary_transparency_remote') + if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')): + btlog.make_binary_transparency_log(repodirs, git_remote=git_remote) if config['update_stats']: # Update known apks info... diff --git a/tests/run-tests b/tests/run-tests index 743cc766..d0ccf083 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -592,12 +592,15 @@ echo_header "copy tests/repo, update with binary transparency log" REPOROOT=`create_test_dir` GNUPGHOME=$REPOROOT/gnupghome KEYSTORE=$WORKSPACE/tests/keystore.jks +mkdir $REPOROOT/git_remote +cd $REPOROOT/git_remote +git init --bare cd $REPOROOT $fdroid init --keystore $KEYSTORE --repo-keyalias=sova cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $REPOROOT/ echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py -echo 'binary_transparency_remote = "git@gitlab.com:fdroid-continuous-integration/binary-transparency.git"' >> config.py +echo "binary_transparency_remote = '$REPOROOT/git_remote'" >> config.py echo "accepted_formats = ['json', 'txt', 'yml']" >> config.py $fdroid update --verbose --pretty test -e repo/index.xml @@ -606,6 +609,8 @@ test -e repo/index-v1.jar grep -F '