new repo-wide config option for a Binary Transparency Log

A Binary Transparency Log is a append only log of all binaries published by
a repo.  This is useful for people to find whether the binary they have
matches what F-Droid has published, and also makes it more difficult for
the published history to be changed without notice, or for a server to give
specific users custom malware binaries.

https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies

examples/config.py

@@ -238,6 +238,13 @@ The repository of older versions of applications from the main demo repository.
 # wiki_user = "login"
 # wiki_password = "1234"
 
+# Keep a log of all generated index files in a git repo to provide a
+# "binary transparency" log for anyone to check the history of the
+# binaries that are published.  This is in the form of a "git remote",
+# which this machine where `fdroid update` is run has already been
+# configured to allow push access (e.g. ssh key, username/password, etc)
+# binary_transparency_remote = "git@gitlab.com:fdroid/binary-transparency-log.git"
+
 # Only set this to true when running a repository where you want to generate
 # stats, and only then on the master build servers, not a development
 # machine.

fdroidserver/server.py

@@ -279,6 +279,22 @@ def upload_to_virustotal(repo_section, vt_apikey):
             logging.info(response['verbose_msg'] + " " + response['permalink'])
 
 
+def push_binary_transparency(binary_transparency_remote):
+    '''push the binary transparency git repo to the specifed remote'''
+    import git
+
+    repo = git.Repo('binary_transparency_log')
+    pushremote = None
+    for remote in repo.remotes:
+        if remote.url == binary_transparency_remote:
+            pushremote = remote
+            break
+
+    if not pushremote:
+        pushremote = repo.create_remote('fdroid_server_update', binary_transparency_remote)
+    pushremote.push('master')
+
+
 def main():
     global config, options
 
@@ -414,6 +430,11 @@ def main():
                 upload_to_android_observatory(repo_section)
             if config.get('virustotal_apikey'):
                 upload_to_virustotal(repo_section, config.get('virustotal_apikey'))
+
+            binary_transparency_remote = config.get('binary_transparency_remote')
+            if binary_transparency_remote:
+                push_binary_transparency(binary_transparency_remote)
+
     sys.exit(0)
 
 

fdroidserver/update.py

@@ -23,6 +23,7 @@ import sys
 import os
 import shutil
 import glob
+import json
 import re
 import socket
 import zipfile
@@ -1439,6 +1440,75 @@ def add_apks_to_per_app_repos(repodir, apks):
             shutil.copy(apkascpath, apk['per_app_repo'])
 
 
+def make_binary_transparency_log(repodirs):
+    '''Log the indexes in a standalone git repo to serve as a "binary
+    transparency" log.
+
+    see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies
+
+    '''
+
+    import git
+    btrepo = 'binary_transparency'
+    if os.path.exists(os.path.join(btrepo, '.git')):
+        gitrepo = git.Repo(btrepo)
+    else:
+        if not os.path.exists(btrepo):
+            os.mkdir(btrepo)
+        gitrepo = git.Repo.init(btrepo)
+        url = config['repo_url'].rstrip('/')
+        with open(os.path.join(btrepo, 'README.md'), 'w') as fp:
+            fp.write("""
+# Binary Transparency Log for %s
+
+""" % url[:url.rindex('/')])  # strip '/repo'
+        gitrepo.index.add(['README.md', ])
+        gitrepo.index.commit('add README')
+
+    for repodir in repodirs:
+        cpdir = os.path.join(btrepo, repodir)
+        if not os.path.exists(cpdir):
+            os.mkdir(cpdir)
+        for f in ('index.xml', ):
+            dest = os.path.join(cpdir, f)
+            shutil.copyfile(os.path.join(repodir, f), dest)
+            gitrepo.index.add([os.path.join(repodir, f), ])
+        for f in ('index.jar', ):
+            repof = os.path.join(repodir, f)
+            dest = os.path.join(cpdir, f)
+            jarin = zipfile.ZipFile(repof, 'r')
+            jarout = zipfile.ZipFile(dest, 'w')
+            for info in jarin.infolist():
+                if info.filename.startswith('META-INF/'):
+                    jarout.writestr(info, jarin.read(info.filename))
+            jarout.close()
+            jarin.close()
+            gitrepo.index.add([repof, ])
+
+        files = []
+        for root, dirs, filenames in os.walk(repodir):
+            for f in filenames:
+                files.append(os.path.relpath(os.path.join(root, f), repodir))
+        output = collections.OrderedDict()
+        for f in sorted(files):
+            repofile = os.path.join(repodir, f)
+            stat = os.stat(repofile)
+            output[f] = (
+                stat.st_size,
+                stat.st_ctime_ns,
+                stat.st_mtime_ns,
+                stat.st_mode,
+                stat.st_uid,
+                stat.st_gid,
+            )
+        fslogfile = os.path.join(cpdir, 'filesystemlog.json')
+        with open(fslogfile, 'w') as fp:
+            json.dump(output, fp, indent=2)
+        gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ])
+
+    gitrepo.index.commit('fdroid update')
+
+
 config = None
 options = None
 
@@ -1678,6 +1748,9 @@ def main():
     if len(repodirs) > 1:
         make_index(apps, sortedids, archapks, repodirs[1], True)
 
+    if config.get('binary_transparency_remote'):
+        make_binary_transparency_log(repodirs)
+
     if config['update_stats']:
 
         # Update known apks info...