mirror of
https://github.com/f-droid/fdroidserver.git
synced 2025-10-09 02:41:06 +03:00
move make_binary_transparency_log to btlog
This keeps the code more organized, and reduces the number of things that are loaded for every command via common.
This commit is contained in:
Hans-Christoph Steiner
parent
896c6496b4
commit
40290fc5e0
3 changed files with 104 additions and 103 deletions
|
|
@ -26,6 +26,10 @@
|
||||||
# client app so its not easy for the server to distinguish this from
|
# client app so its not easy for the server to distinguish this from
|
||||||
# the F-Droid client.
|
# the F-Droid client.
|
||||||
|
|
||||||
|
|
||||||
|
import collections
|
||||||
|
import git
|
||||||
|
import glob
|
||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
|
@ -33,6 +37,8 @@ import requests
|
||||||
import shutil
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import xml.dom.minidom
|
||||||
|
import zipfile
|
||||||
from argparse import ArgumentParser
|
from argparse import ArgumentParser
|
||||||
|
|
||||||
from . import common
|
from . import common
|
||||||
|
|
@ -41,6 +47,101 @@ from . import common
|
||||||
options = None
|
options = None
|
||||||
|
|
||||||
|
|
||||||
|
def make_binary_transparency_log(repodirs, btrepo='binary_transparency',
|
||||||
|
url=None,
|
||||||
|
commit_title='fdroid update'):
|
||||||
|
'''Log the indexes in a standalone git repo to serve as a "binary
|
||||||
|
transparency" log.
|
||||||
|
|
||||||
|
see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies
|
||||||
|
|
||||||
|
'''
|
||||||
|
|
||||||
|
if os.path.exists(os.path.join(btrepo, '.git')):
|
||||||
|
gitrepo = git.Repo(btrepo)
|
||||||
|
else:
|
||||||
|
if not os.path.exists(btrepo):
|
||||||
|
os.mkdir(btrepo)
|
||||||
|
gitrepo = git.Repo.init(btrepo)
|
||||||
|
|
||||||
|
if not url:
|
||||||
|
url = common.config['repo_url'].rstrip('/')
|
||||||
|
with open(os.path.join(btrepo, 'README.md'), 'w') as fp:
|
||||||
|
fp.write("""
|
||||||
|
# Binary Transparency Log for %s
|
||||||
|
|
||||||
|
This is a log of the signed app index metadata. This is stored in a
|
||||||
|
git repo, which serves as an imperfect append-only storage mechanism.
|
||||||
|
People can then check that any file that they received from that
|
||||||
|
F-Droid repository was a publicly released file.
|
||||||
|
|
||||||
|
For more info on this idea:
|
||||||
|
* https://wiki.mozilla.org/Security/Binary_Transparency
|
||||||
|
""" % url[:url.rindex('/')]) # strip '/repo'
|
||||||
|
gitrepo.index.add(['README.md', ])
|
||||||
|
gitrepo.index.commit('add README')
|
||||||
|
|
||||||
|
for repodir in repodirs:
|
||||||
|
cpdir = os.path.join(btrepo, repodir)
|
||||||
|
if not os.path.exists(cpdir):
|
||||||
|
os.mkdir(cpdir)
|
||||||
|
for f in ('index.xml', 'index-v1.json'):
|
||||||
|
repof = os.path.join(repodir, f)
|
||||||
|
if not os.path.exists(repof):
|
||||||
|
continue
|
||||||
|
dest = os.path.join(cpdir, f)
|
||||||
|
if f.endswith('.xml'):
|
||||||
|
doc = xml.dom.minidom.parse(repof)
|
||||||
|
output = doc.toprettyxml(encoding='utf-8')
|
||||||
|
with open(dest, 'wb') as f:
|
||||||
|
f.write(output)
|
||||||
|
elif f.endswith('.json'):
|
||||||
|
with open(repof) as fp:
|
||||||
|
output = json.load(fp, object_pairs_hook=collections.OrderedDict)
|
||||||
|
with open(dest, 'w') as fp:
|
||||||
|
json.dump(output, fp, indent=2)
|
||||||
|
gitrepo.index.add([repof, ])
|
||||||
|
for f in ('index.jar', 'index-v1.jar'):
|
||||||
|
repof = os.path.join(repodir, f)
|
||||||
|
if not os.path.exists(repof):
|
||||||
|
continue
|
||||||
|
dest = os.path.join(cpdir, f)
|
||||||
|
jarin = zipfile.ZipFile(repof, 'r')
|
||||||
|
jarout = zipfile.ZipFile(dest, 'w')
|
||||||
|
for info in jarin.infolist():
|
||||||
|
if info.filename.startswith('META-INF/'):
|
||||||
|
jarout.writestr(info, jarin.read(info.filename))
|
||||||
|
jarout.close()
|
||||||
|
jarin.close()
|
||||||
|
gitrepo.index.add([repof, ])
|
||||||
|
|
||||||
|
files = []
|
||||||
|
for root, dirs, filenames in os.walk(repodir):
|
||||||
|
for f in filenames:
|
||||||
|
files.append(os.path.relpath(os.path.join(root, f), repodir))
|
||||||
|
output = collections.OrderedDict()
|
||||||
|
for f in sorted(files):
|
||||||
|
repofile = os.path.join(repodir, f)
|
||||||
|
stat = os.stat(repofile)
|
||||||
|
output[f] = (
|
||||||
|
stat.st_size,
|
||||||
|
stat.st_ctime_ns,
|
||||||
|
stat.st_mtime_ns,
|
||||||
|
stat.st_mode,
|
||||||
|
stat.st_uid,
|
||||||
|
stat.st_gid,
|
||||||
|
)
|
||||||
|
fslogfile = os.path.join(cpdir, 'filesystemlog.json')
|
||||||
|
with open(fslogfile, 'w') as fp:
|
||||||
|
json.dump(output, fp, indent=2)
|
||||||
|
gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ])
|
||||||
|
|
||||||
|
for f in glob.glob(os.path.join(cpdir, '*.HTTP-headers.json')):
|
||||||
|
gitrepo.index.add([os.path.join(repodir, os.path.basename(f)), ])
|
||||||
|
|
||||||
|
gitrepo.index.commit(commit_title)
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
global options
|
global options
|
||||||
|
|
||||||
|
|
@ -112,8 +213,7 @@ def main():
|
||||||
|
|
||||||
if new_files:
|
if new_files:
|
||||||
os.chdir(tempdirbase)
|
os.chdir(tempdirbase)
|
||||||
common.make_binary_transparency_log(repodirs, options.git_repo, options.url,
|
make_binary_transparency_log(repodirs, options.git_repo, options.url, 'fdroid btlog')
|
||||||
'fdroid btlog')
|
|
||||||
shutil.rmtree(tempdirbase, ignore_errors=True)
|
shutil.rmtree(tempdirbase, ignore_errors=True)
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
|
||||||
|
|
@ -20,14 +20,12 @@
|
||||||
# common.py is imported by all modules, so do not import third-party
|
# common.py is imported by all modules, so do not import third-party
|
||||||
# libraries here as they will become a requirement for all commands.
|
# libraries here as they will become a requirement for all commands.
|
||||||
|
|
||||||
import collections
|
|
||||||
import io
|
import io
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import glob
|
import glob
|
||||||
import json
|
|
||||||
import stat
|
import stat
|
||||||
import subprocess
|
import subprocess
|
||||||
import time
|
import time
|
||||||
|
|
@ -2350,101 +2348,3 @@ def is_repo_file(filename):
|
||||||
b'index-v1.json',
|
b'index-v1.json',
|
||||||
b'categories.txt',
|
b'categories.txt',
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def make_binary_transparency_log(repodirs, btrepo='binary_transparency',
|
|
||||||
url=None,
|
|
||||||
commit_title='fdroid update'):
|
|
||||||
'''Log the indexes in a standalone git repo to serve as a "binary
|
|
||||||
transparency" log.
|
|
||||||
|
|
||||||
see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies
|
|
||||||
|
|
||||||
'''
|
|
||||||
|
|
||||||
import git
|
|
||||||
import xml.dom.minidom
|
|
||||||
|
|
||||||
if os.path.exists(os.path.join(btrepo, '.git')):
|
|
||||||
gitrepo = git.Repo(btrepo)
|
|
||||||
else:
|
|
||||||
if not os.path.exists(btrepo):
|
|
||||||
os.mkdir(btrepo)
|
|
||||||
gitrepo = git.Repo.init(btrepo)
|
|
||||||
|
|
||||||
if not url:
|
|
||||||
url = config['repo_url'].rstrip('/')
|
|
||||||
with open(os.path.join(btrepo, 'README.md'), 'w') as fp:
|
|
||||||
fp.write("""
|
|
||||||
# Binary Transparency Log for %s
|
|
||||||
|
|
||||||
This is a log of the signed app index metadata. This is stored in a
|
|
||||||
git repo, which serves as an imperfect append-only storage mechanism.
|
|
||||||
People can then check that any file that they received from that
|
|
||||||
F-Droid repository was a publicly released file.
|
|
||||||
|
|
||||||
For more info on this idea:
|
|
||||||
* https://wiki.mozilla.org/Security/Binary_Transparency
|
|
||||||
""" % url[:url.rindex('/')]) # strip '/repo'
|
|
||||||
gitrepo.index.add(['README.md', ])
|
|
||||||
gitrepo.index.commit('add README')
|
|
||||||
|
|
||||||
for repodir in repodirs:
|
|
||||||
cpdir = os.path.join(btrepo, repodir)
|
|
||||||
if not os.path.exists(cpdir):
|
|
||||||
os.mkdir(cpdir)
|
|
||||||
for f in ('index.xml', 'index-v1.json'):
|
|
||||||
repof = os.path.join(repodir, f)
|
|
||||||
if not os.path.exists(repof):
|
|
||||||
continue
|
|
||||||
dest = os.path.join(cpdir, f)
|
|
||||||
if f.endswith('.xml'):
|
|
||||||
doc = xml.dom.minidom.parse(repof)
|
|
||||||
output = doc.toprettyxml(encoding='utf-8')
|
|
||||||
with open(dest, 'wb') as f:
|
|
||||||
f.write(output)
|
|
||||||
elif f.endswith('.json'):
|
|
||||||
with open(repof) as fp:
|
|
||||||
output = json.load(fp, object_pairs_hook=collections.OrderedDict)
|
|
||||||
with open(dest, 'w') as fp:
|
|
||||||
json.dump(output, fp, indent=2)
|
|
||||||
gitrepo.index.add([repof, ])
|
|
||||||
for f in ('index.jar', 'index-v1.jar'):
|
|
||||||
repof = os.path.join(repodir, f)
|
|
||||||
if not os.path.exists(repof):
|
|
||||||
continue
|
|
||||||
dest = os.path.join(cpdir, f)
|
|
||||||
jarin = ZipFile(repof, 'r')
|
|
||||||
jarout = ZipFile(dest, 'w')
|
|
||||||
for info in jarin.infolist():
|
|
||||||
if info.filename.startswith('META-INF/'):
|
|
||||||
jarout.writestr(info, jarin.read(info.filename))
|
|
||||||
jarout.close()
|
|
||||||
jarin.close()
|
|
||||||
gitrepo.index.add([repof, ])
|
|
||||||
|
|
||||||
files = []
|
|
||||||
for root, dirs, filenames in os.walk(repodir):
|
|
||||||
for f in filenames:
|
|
||||||
files.append(os.path.relpath(os.path.join(root, f), repodir))
|
|
||||||
output = collections.OrderedDict()
|
|
||||||
for f in sorted(files):
|
|
||||||
repofile = os.path.join(repodir, f)
|
|
||||||
stat = os.stat(repofile)
|
|
||||||
output[f] = (
|
|
||||||
stat.st_size,
|
|
||||||
stat.st_ctime_ns,
|
|
||||||
stat.st_mtime_ns,
|
|
||||||
stat.st_mode,
|
|
||||||
stat.st_uid,
|
|
||||||
stat.st_gid,
|
|
||||||
)
|
|
||||||
fslogfile = os.path.join(cpdir, 'filesystemlog.json')
|
|
||||||
with open(fslogfile, 'w') as fp:
|
|
||||||
json.dump(output, fp, indent=2)
|
|
||||||
gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ])
|
|
||||||
|
|
||||||
for f in glob.glob(os.path.join(cpdir, '*.HTTP-headers.json')):
|
|
||||||
gitrepo.index.add([os.path.join(repodir, os.path.basename(f)), ])
|
|
||||||
|
|
||||||
gitrepo.index.commit(commit_title)
|
|
||||||
|
|
|
||||||
|
|
@ -37,6 +37,7 @@ from binascii import hexlify
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
|
from . import btlog
|
||||||
from . import common
|
from . import common
|
||||||
from . import index
|
from . import index
|
||||||
from . import metadata
|
from . import metadata
|
||||||
|
|
@ -1408,7 +1409,7 @@ def main():
|
||||||
index.make(apps, sortedids, archapks, repodirs[1], True)
|
index.make(apps, sortedids, archapks, repodirs[1], True)
|
||||||
|
|
||||||
if config.get('binary_transparency_remote'):
|
if config.get('binary_transparency_remote'):
|
||||||
common.make_binary_transparency_log(repodirs)
|
btlog.make_binary_transparency_log(repodirs)
|
||||||
|
|
||||||
if config['update_stats']:
|
if config['update_stats']:
|
||||||
# Update known apks info...
|
# Update known apks info...
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue