From 67a0f3ae5b3702c460421faebd7f5f53e03bc18f Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Mon, 12 Apr 2021 19:30:45 +0200 Subject: [PATCH 01/10] use subclass hack for better ZIP cloning See https://bugs.python.org/issue43547 for more info on the details. thanks to @obfusk for the technique --- fdroidserver/common.py | 26 ++++++++++++++++++++++++-- tests/common.TestCase | 7 +++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 45109513..926d407c 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -3,6 +3,7 @@ # common.py - part of the FDroid server tools # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com # Copyright (C) 2013-2014 Daniel Martí +# Copyright (C) 2021 Felix C. Stegerman # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -3007,6 +3008,27 @@ def metadata_find_developer_signing_files(appid, vercode): return None +class ClonedZipInfo(zipfile.ZipInfo): + """Hack to allow fully cloning ZipInfo instances + + The zipfile library has some bugs that prevent it from fully + cloning ZipInfo entries. https://bugs.python.org/issue43547 + + """ + def __init__(self, zinfo): + self.original = zinfo + for k in self.__slots__: + try: + setattr(self, k, getattr(zinfo, k)) + except AttributeError: + pass + + def __getattribute__(self, name): + if name in ("date_time", "external_attr", "flag_bits"): + return getattr(self.original, name) + return object.__getattribute__(self, name) + + def apk_strip_v1_signatures(signed_apk, strip_manifest=False): """Removes signatures from APK. @@ -3024,10 +3046,10 @@ def apk_strip_v1_signatures(signed_apk, strip_manifest=False): if strip_manifest: if info.filename != 'META-INF/MANIFEST.MF': buf = in_apk.read(info.filename) - out_apk.writestr(info, buf) + out_apk.writestr(ClonedZipInfo(info), buf) else: buf = in_apk.read(info.filename) - out_apk.writestr(info, buf) + out_apk.writestr(ClonedZipInfo(info), buf) def _zipalign(unsigned_apk, aligned_apk): diff --git a/tests/common.TestCase b/tests/common.TestCase index e6316b7e..07e3d95e 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -1754,6 +1754,13 @@ class CommonTest(unittest.TestCase): fdroidserver.common.read_pkg_args(appid_versionCode_pairs, allow_vercodes) ) + def test_apk_strip_v1_signatures(self): + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + before = os.path.join(self.basedir, 'no_targetsdk_minsdk1_unsigned.apk') + after = os.path.join(testdir, 'after.apk') + shutil.copy(before, after) + fdroidserver.common.apk_strip_v1_signatures(after, strip_manifest=False) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From 202fd8b25afc9dfeffd2c144c0acb52d1eae0408 Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Wed, 14 Apr 2021 21:06:20 +0200 Subject: [PATCH 02/10] vendor & use apksigcopier v0.4.0-12-g93d8e14 --- fdroidserver/apksigcopier.py | 597 +++++++++++++++++++++++++++++++++++ fdroidserver/common.py | 125 ++++---- fdroidserver/publish.py | 8 +- tests/common.TestCase | 2 +- 4 files changed, 655 insertions(+), 77 deletions(-) create mode 100644 fdroidserver/apksigcopier.py diff --git a/fdroidserver/apksigcopier.py b/fdroidserver/apksigcopier.py new file mode 100644 index 00000000..74e7e7df --- /dev/null +++ b/fdroidserver/apksigcopier.py @@ -0,0 +1,597 @@ +#!/usr/bin/python3 +# encoding: utf-8 + +# -- ; {{{1 +# +# File : apksigcopier +# Maintainer : Felix C. Stegerman +# Date : 2021-04-14 +# +# Copyright : Copyright (C) 2021 Felix C. Stegerman +# Version : v0.4.0 +# License : GPLv3+ +# +# -- ; }}}1 + +""" +copy/extract/patch apk signatures + +apksigcopier is a tool for copying APK signatures from a signed APK to an +unsigned one (in order to verify reproducible builds). + + +CLI +=== + +$ apksigcopier extract [OPTIONS] SIGNED_APK OUTPUT_DIR +$ apksigcopier patch [OPTIONS] METADATA_DIR UNSIGNED_APK OUTPUT_APK +$ apksigcopier copy [OPTIONS] SIGNED_APK UNSIGNED_APK OUTPUT_APK + +The following environment variables can be set to 1, yes, or true to +overide the default behaviour: + +* set APKSIGCOPIER_EXCLUDE_ALL_META=1 to exclude all metadata files +* set APKSIGCOPIER_COPY_EXTRA_BYTES=1 to copy extra bytes after data (e.g. a v2 sig) + + +API +=== + +>> from apksigcopier import do_extract, do_patch, do_copy +>> do_extract(signed_apk, output_dir, v1_only=NO) +>> do_patch(metadata_dir, unsigned_apk, output_apk, v1_only=NO) +>> do_copy(signed_apk, unsigned_apk, output_apk, v1_only=NO) + +You can use False, None, and True instead of NO, AUTO, and YES respectively. + +The following global variables (which default to False), can be set to +override the default behaviour: + +* set exclude_all_meta=True to exclude all metadata files +* set copy_extra_bytes=True to copy extra bytes after data (e.g. a v2 sig) +""" + +import glob +import os +import re +import struct +import sys +import zipfile +import zlib + +from collections import namedtuple +from typing import Any, Dict + +__version__ = "0.4.0" +NAME = "apksigcopier" + +SIGBLOCK, SIGOFFSET = "APKSigningBlock", "APKSigningBlockOffset" +NOAUTOYES = NO, AUTO, YES = ("no", "auto", "yes") +APK_META = re.compile(r"^META-INF/([0-9A-Za-z_-]+\.(SF|RSA|DSA|EC)|MANIFEST\.MF)$") +META_EXT = ("SF", "RSA|DSA|EC", "MF") +COPY_EXCLUDE = ("META-INF/MANIFEST.MF",) +DATETIMEZERO = (1980, 0, 0, 0, 0, 0) + +ZipData = namedtuple("ZipData", ("cd_offset", "eocd_offset", "cd_and_eocd")) + +exclude_all_meta = False # exclude all metadata files in copy_apk() +copy_extra_bytes = False # copy extra bytes after data in copy_apk() + + +class APKSigCopierError(Exception): + """Base class for errors.""" + + +class APKSigningBlockError(APKSigCopierError): + """Something wrong with the APK Signing Block.""" + + +class NoAPKSigningBlock(APKSigningBlockError): + """APK Signing Block Missing.""" + + +class ZipError(APKSigCopierError): + """Something wrong with ZIP file.""" + + +# FIXME: is there a better alternative? +class ReproducibleZipInfo(zipfile.ZipInfo): + """Reproducible ZipInfo hack.""" + + _override = {} # type: Dict[str, Any] + + def __init__(self, zinfo, **override): + if override: + self._override = {**self._override, **override} + for k in self.__slots__: + if hasattr(zinfo, k): + setattr(self, k, getattr(zinfo, k)) + + def __getattribute__(self, name): + if name != "_override": + try: + return self._override[name] + except KeyError: + pass + return object.__getattribute__(self, name) + + +class APKZipInfo(ReproducibleZipInfo): + """Reproducible ZipInfo for APK files.""" + + _override = dict( + compress_type=8, + create_system=0, + create_version=20, + date_time=DATETIMEZERO, + external_attr=0, + extract_version=20, + flag_bits=0x800, + ) + + +def noautoyes(value): + """ + Turns False into NO, None into AUTO, and True into YES. + + >>> from apksigcopier import noautoyes, NO, AUTO, YES + >>> noautoyes(False) == NO == noautoyes(NO) + True + >>> noautoyes(None) == AUTO == noautoyes(AUTO) + True + >>> noautoyes(True) == YES == noautoyes(YES) + True + + """ + if isinstance(value, str): + if value not in NOAUTOYES: + raise ValueError("expected NO, AUTO, or YES") + return value + try: + return {False: NO, None: AUTO, True: YES}[value] + except KeyError: + raise ValueError("expected False, None, or True") + + +def is_meta(filename): + """ + Returns whether filename is a v1 (JAR) signature file (.SF), signature block + file (.RSA, .DSA, or .EC), or manifest (MANIFEST.MF). + + See https://docs.oracle.com/javase/tutorial/deployment/jar/intro.html + """ + return APK_META.fullmatch(filename) is not None + + +def exclude_from_copying(filename): + """ + Returns whether to exclude a file during copy_apk(). + + Excludes filenames in COPY_EXCLUDE (i.e. MANIFEST.MF) by default; when + exclude_all_meta is set to True instead, excludes all metadata files as + matched by is_meta(). + """ + return is_meta(filename) if exclude_all_meta else filename in COPY_EXCLUDE + + +################################################################################ +# +# https://en.wikipedia.org/wiki/ZIP_(file_format) +# https://source.android.com/security/apksigning/v2#apk-signing-block-format +# +# ================================= +# | Contents of ZIP entries | +# ================================= +# | APK Signing Block | +# | ----------------------------- | +# | | size (w/o this) uint64 LE | | +# | | ... | | +# | | size (again) uint64 LE | | +# | | "APK Sig Block 42" (16B) | | +# | ----------------------------- | +# ================================= +# | ZIP Central Directory | +# ================================= +# | ZIP End of Central Directory | +# | ----------------------------- | +# | | 0x06054b50 ( 4B) | | +# | | ... (12B) | | +# | | CD Offset ( 4B) | | +# | | ... | | +# | ----------------------------- | +# ================================= +# +################################################################################ + + +# FIXME: makes certain assumptions and doesn't handle all valid ZIP files! +def copy_apk(unsigned_apk, output_apk): + """ + Copy APK like apksigner would, excluding files matched by + exclude_from_copying(). + + Returns max date_time. + + The following global variables (which default to False), can be set to + override the default behaviour: + + * set exclude_all_meta=True to exclude all metadata files + * set copy_extra_bytes=True to copy extra bytes after data (e.g. a v2 sig) + """ + with zipfile.ZipFile(unsigned_apk, "r") as zf: + infos = zf.infolist() + zdata = zip_data(unsigned_apk) + offsets = {} + with open(unsigned_apk, "rb") as fhi, open(output_apk, "w+b") as fho: + for info in sorted(infos, key=lambda info: info.header_offset): + off_i = fhi.tell() + if info.header_offset > off_i: + # copy extra bytes + fho.write(fhi.read(info.header_offset - off_i)) + hdr = fhi.read(30) + if hdr[:4] != b"\x50\x4b\x03\x04": + raise ZipError("Expected local file header signature") + n, m = struct.unpack("= 4: + hdr_id, size = struct.unpack(" len(old_xtr) - 4: + break + if not (hdr_id == 0 and size == 0): + if hdr_id == 0xd935: + if size >= 2: + align = int.from_bytes(old_xtr[4:6], "little") + else: + new_xtr += old_xtr[:size + 4] + old_xtr = old_xtr[size + 4:] + if old_off % align == 0 and new_off % align != 0: + pad = (align - (new_off - m + len(new_xtr) + 6) % align) % align + xtr = new_xtr + struct.pack(" 0: + data = fhi.read(min(size, blocksize)) + if not data: + break + size -= len(data) + fho.write(data) + if size != 0: + raise ZipError("Unexpected EOF") + + +def extract_meta(signed_apk): + """ + Extract v1 signature metadata files from signed APK. + + Yields (ZipInfo, data) pairs. + """ + with zipfile.ZipFile(signed_apk, "r") as zf_sig: + for info in zf_sig.infolist(): + if is_meta(info.filename): + yield info, zf_sig.read(info.filename) + + +def patch_meta(extracted_meta, output_apk, date_time=DATETIMEZERO): + """Add v1 signature metadata to APK (removes v2 sig block, if any).""" + with zipfile.ZipFile(output_apk, "r") as zf_out: + for info in zf_out.infolist(): + if is_meta(info.filename): + raise ZipError("Unexpected metadata") + with zipfile.ZipFile(output_apk, "a") as zf_out: + info_data = [(APKZipInfo(info, date_time=date_time), data) + for info, data in extracted_meta] + _write_to_zip(info_data, zf_out) + + +if sys.version_info >= (3, 7): + def _write_to_zip(info_data, zf_out): + for info, data in info_data: + zf_out.writestr(info, data, compresslevel=9) +else: + def _write_to_zip(info_data, zf_out): + old = zipfile._get_compressor + zipfile._get_compressor = lambda _: zlib.compressobj(9, 8, -15) + try: + for info, data in info_data: + zf_out.writestr(info, data) + finally: + zipfile._get_compressor = old + + +def extract_v2_sig(apkfile, expected=True): + """ + Extract APK Signing Block and offset from APK. + + When successful, returns (sb_offset, sig_block); otherwise raises + NoAPKSigningBlock when expected is True, else returns None. + """ + cd_offset = zip_data(apkfile).cd_offset + with open(apkfile, "rb") as fh: + fh.seek(cd_offset - 16) + if fh.read(16) != b"APK Sig Block 42": + if expected: + raise NoAPKSigningBlock("No APK Signing Block") + return None + fh.seek(-24, os.SEEK_CUR) + sb_size2 = int.from_bytes(fh.read(8), "little") + fh.seek(-sb_size2 + 8, os.SEEK_CUR) + sb_size1 = int.from_bytes(fh.read(8), "little") + if sb_size1 != sb_size2: + raise APKSigningBlockError("APK Signing Block sizes not equal") + fh.seek(-8, os.SEEK_CUR) + sb_offset = fh.tell() + sig_block = fh.read(sb_size2 + 8) + return sb_offset, sig_block + + +def zip_data(apkfile, count=1024): + """ + Extract central directory, EOCD, and offsets from ZIP. + + Returns ZipData. + """ + with open(apkfile, "rb") as fh: + fh.seek(-count, os.SEEK_END) + data = fh.read() + pos = data.rfind(b"\x50\x4b\x05\x06") + if pos == -1: + raise ZipError("Expected end of central directory record (EOCD)") + fh.seek(pos - len(data), os.SEEK_CUR) + eocd_offset = fh.tell() + fh.seek(16, os.SEEK_CUR) + cd_offset = int.from_bytes(fh.read(4), "little") + fh.seek(cd_offset) + cd_and_eocd = fh.read() + return ZipData(cd_offset, eocd_offset, cd_and_eocd) + + +# FIXME: can we determine signed_sb_offset? +def patch_v2_sig(extracted_v2_sig, output_apk): + """Implant extracted v2/v3 signature into APK.""" + signed_sb_offset, signed_sb = extracted_v2_sig + data_out = zip_data(output_apk) + if signed_sb_offset < data_out.cd_offset: + raise APKSigningBlockError("APK Signing Block offset < central directory offset") + padding = b"\x00" * (signed_sb_offset - data_out.cd_offset) + offset = len(signed_sb) + len(padding) + with open(output_apk, "r+b") as fh: + fh.seek(data_out.cd_offset) + fh.write(padding) + fh.write(signed_sb) + fh.write(data_out.cd_and_eocd) + fh.seek(data_out.eocd_offset + offset + 16) + fh.write(int.to_bytes(data_out.cd_offset + offset, 4, "little")) + + +def patch_apk(extracted_meta, extracted_v2_sig, unsigned_apk, output_apk): + """ + Patch extracted_meta + extracted_v2_sig (if not None) onto unsigned_apk and + save as output_apk. + """ + date_time = copy_apk(unsigned_apk, output_apk) + patch_meta(extracted_meta, output_apk, date_time=date_time) + if extracted_v2_sig is not None: + patch_v2_sig(extracted_v2_sig, output_apk) + + +def do_extract(signed_apk, output_dir, v1_only=NO): + """ + Extract signatures from signed_apk and save in output_dir. + + The v1_only parameter controls whether the absence of a v1 signature is + considered an error or not: + * use v1_only=NO (or v1_only=False) to only accept (v1+)v2/v3 signatures; + * use v1_only=AUTO (or v1_only=None) to automatically detect v2/v3 signatures; + * use v1_only=YES (or v1_only=True) to ignore any v2/v3 signatures. + """ + v1_only = noautoyes(v1_only) + extracted_meta = tuple(extract_meta(signed_apk)) + if len(extracted_meta) not in (len(META_EXT), 0): + raise APKSigCopierError("Unexpected or missing metadata files in signed_apk") + for info, data in extracted_meta: + name = os.path.basename(info.filename) + with open(os.path.join(output_dir, name), "wb") as fh: + fh.write(data) + if v1_only == YES: + if not extracted_meta: + raise APKSigCopierError("Expected v1 signature") + return + expected = v1_only == NO + extracted_v2_sig = extract_v2_sig(signed_apk, expected=expected) + if extracted_v2_sig is None: + if not extracted_meta: + raise APKSigCopierError("Expected v1 and/or v2/v3 signature, found neither") + return + signed_sb_offset, signed_sb = extracted_v2_sig + with open(os.path.join(output_dir, SIGOFFSET), "w") as fh: + fh.write(str(signed_sb_offset) + "\n") + with open(os.path.join(output_dir, SIGBLOCK), "wb") as fh: + fh.write(signed_sb) + + +def do_patch(metadata_dir, unsigned_apk, output_apk, v1_only=NO): + """ + Patch signatures from metadata_dir onto unsigned_apk and save as output_apk. + + The v1_only parameter controls whether the absence of a v1 signature is + considered an error or not: + * use v1_only=NO (or v1_only=False) to only accept (v1+)v2/v3 signatures; + * use v1_only=AUTO (or v1_only=None) to automatically detect v2/v3 signatures; + * use v1_only=YES (or v1_only=True) to ignore any v2/v3 signatures. + """ + v1_only = noautoyes(v1_only) + extracted_meta = [] + for pat in META_EXT: + files = [fn for ext in pat.split("|") for fn in + glob.glob(os.path.join(metadata_dir, "*." + ext))] + if len(files) != 1: + continue + info = zipfile.ZipInfo("META-INF/" + os.path.basename(files[0])) + with open(files[0], "rb") as fh: + extracted_meta.append((info, fh.read())) + if len(extracted_meta) not in (len(META_EXT), 0): + raise APKSigCopierError("Unexpected or missing files in metadata_dir") + if v1_only == YES: + extracted_v2_sig = None + else: + sigoffset_file = os.path.join(metadata_dir, SIGOFFSET) + sigblock_file = os.path.join(metadata_dir, SIGBLOCK) + if v1_only == AUTO and not os.path.exists(sigblock_file): + extracted_v2_sig = None + else: + with open(sigoffset_file, "r") as fh: + signed_sb_offset = int(fh.read()) + with open(sigblock_file, "rb") as fh: + signed_sb = fh.read() + extracted_v2_sig = signed_sb_offset, signed_sb + if not extracted_meta and extracted_v2_sig is None: + raise APKSigCopierError("Expected v1 and/or v2/v3 signature, found neither") + patch_apk(extracted_meta, extracted_v2_sig, unsigned_apk, output_apk) + + +def do_copy(signed_apk, unsigned_apk, output_apk, v1_only=NO): + """ + Copy signatures from signed_apk onto unsigned_apk and save as output_apk. + + The v1_only parameter controls whether the absence of a v1 signature is + considered an error or not: + * use v1_only=NO (or v1_only=False) to only accept (v1+)v2/v3 signatures; + * use v1_only=AUTO (or v1_only=None) to automatically detect v2/v3 signatures; + * use v1_only=YES (or v1_only=True) to ignore any v2/v3 signatures. + """ + v1_only = noautoyes(v1_only) + extracted_meta = extract_meta(signed_apk) + if v1_only == YES: + extracted_v2_sig = None + else: + extracted_v2_sig = extract_v2_sig(signed_apk, expected=v1_only == NO) + patch_apk(extracted_meta, extracted_v2_sig, unsigned_apk, output_apk) + + +def main(): + """CLI; requires click.""" + + global exclude_all_meta, copy_extra_bytes + exclude_all_meta = os.environ.get("APKSIGCOPIER_EXCLUDE_ALL_META") in ("1", "yes", "true") + copy_extra_bytes = os.environ.get("APKSIGCOPIER_COPY_EXTRA_BYTES") in ("1", "yes", "true") + + import click + + NAY = click.Choice(NOAUTOYES) + + @click.group(help=""" + apksigcopier - copy/extract/patch apk signatures + """) + @click.version_option(__version__) + def cli(): + pass + + @cli.command(help=""" + Extract APK signatures from signed APK. + """) + @click.option("--v1-only", type=NAY, default=NO, show_default=True, + envvar="APKSIGCOPIER_V1_ONLY") + @click.argument("signed_apk", type=click.Path(exists=True, dir_okay=False)) + @click.argument("output_dir", type=click.Path(exists=True, file_okay=False)) + def extract(*args, **kwargs): + do_extract(*args, **kwargs) + + @cli.command(help=""" + Patch extracted APK signatures onto unsigned APK. + """) + @click.option("--v1-only", type=NAY, default=NO, show_default=True, + envvar="APKSIGCOPIER_V1_ONLY") + @click.argument("metadata_dir", type=click.Path(exists=True, file_okay=False)) + @click.argument("unsigned_apk", type=click.Path(exists=True, dir_okay=False)) + @click.argument("output_apk", type=click.Path(dir_okay=False)) + def patch(*args, **kwargs): + do_patch(*args, **kwargs) + + @cli.command(help=""" + Copy (extract & patch) signatures from signed to unsigned APK. + """) + @click.option("--v1-only", type=NAY, default=NO, show_default=True, + envvar="APKSIGCOPIER_V1_ONLY") + @click.argument("signed_apk", type=click.Path(exists=True, dir_okay=False)) + @click.argument("unsigned_apk", type=click.Path(exists=True, dir_okay=False)) + @click.argument("output_apk", type=click.Path(dir_okay=False)) + def copy(*args, **kwargs): + do_copy(*args, **kwargs) + + # FIXME + if click.__version__.startswith("7."): + def autocomplete_path(ctx=None, args=(), incomplete=""): # pylint: disable=W0613 + head, tail = os.path.split(incomplete) + return sorted( + (e.path if head else e.path[2:]) + ("/" if e.is_dir() else "") + for e in os.scandir(head or ".") if e.name.startswith(tail) + ) + + for command in cli.commands.values(): + for param in command.params: + if isinstance(param.type, click.Path): + param.autocompletion = autocomplete_path + + try: + cli(prog_name=NAME) + except APKSigCopierError as e: + click.echo("Error: {}.".format(e), err=True) + sys.exit(1) + + +if __name__ == "__main__": + main() + +# vim: set tw=80 sw=4 sts=4 et fdm=marker : diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 926d407c..7bf4c158 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -68,6 +68,9 @@ from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesEx BuildException, VerificationException, MetaDataException from .asynchronousfilereader import AsynchronousFileReader +from . import apksigcopier +apksigcopier.exclude_all_meta = True # remove v1 signatures too + # The path to this fdroidserver distribution FDROID_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..')) @@ -2977,8 +2980,9 @@ def metadata_find_signing_files(appid, vercode): :param appid: app id string :param vercode: app version code - :returns: a list of triplets for each signing key with following paths: - (signature_file, singed_file, manifest_file) + :returns: a list of 4-tuples for each signing key with following paths: + (signature_file, singed_file, manifest_file, v2_files), where v2_files + is either a (sb_offset_file, sb_file) pair or None """ ret = [] sigdir = metadata_get_sigdir(appid, vercode) @@ -2986,12 +2990,18 @@ def metadata_find_signing_files(appid, vercode): glob.glob(os.path.join(sigdir, '*.EC')) + \ glob.glob(os.path.join(sigdir, '*.RSA')) extre = re.compile(r'(\.DSA|\.EC|\.RSA)$') + apk_sb = os.path.isfile(os.path.join(sigdir, "APKSigningBlock")) + apk_sbo = os.path.isfile(os.path.join(sigdir, "APKSigningBlockOffset")) + if os.path.isfile(apk_sb) and os.path.isfile(apk_sbo): + v2_files = apk_sb, apk_sbo + else: + v2_files = None for sig in sigs: sf = extre.sub('.SF', sig) if os.path.isfile(sf): mf = os.path.join(sigdir, 'MANIFEST.MF') if os.path.isfile(mf): - ret.append((sig, sf, mf)) + ret.append((sig, sf, mf, v2_files)) return ret @@ -3029,6 +3039,15 @@ class ClonedZipInfo(zipfile.ZipInfo): return object.__getattribute__(self, name) +def apk_has_v1_signatures(apkfile): + """Test whether an APK has v1 signature files.""" + with ZipFile(apkfile, 'r') as apk: + for info in apk.infolist(): + if APK_SIGNATURE_FILES.match(info.filename): + return True + return False + + def apk_strip_v1_signatures(signed_apk, strip_manifest=False): """Removes signatures from APK. @@ -3064,49 +3083,26 @@ def _zipalign(unsigned_apk, aligned_apk): raise BuildException("Failed to align application") -def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest): +def apk_implant_signatures(apkpath, outpath, manifest, v2_files): """Implants a signature from metadata into an APK. - Note: this changes there supplied APK in place. So copy it if you - need the original to be preserved. - - :param apkpath: location of the apk + :param apkpath: location of the unsigned apk + :param outpath: location of the output apk """ - # get list of available signature files in metadata - with tempfile.TemporaryDirectory() as tmpdir: - apkwithnewsig = os.path.join(tmpdir, 'newsig.apk') - with ZipFile(apkpath, 'r') as in_apk: - with ZipFile(apkwithnewsig, 'w') as out_apk: - for sig_file in [signaturefile, signedfile, manifest]: - with open(sig_file, 'rb') as fp: - buf = fp.read() - info = zipfile.ZipInfo('META-INF/' + os.path.basename(sig_file)) - info.compress_type = zipfile.ZIP_DEFLATED - info.create_system = 0 # "Windows" aka "FAT", what Android SDK uses - out_apk.writestr(info, buf) - for info in in_apk.infolist(): - if not APK_SIGNATURE_FILES.match(info.filename): - if info.filename != 'META-INF/MANIFEST.MF': - buf = in_apk.read(info.filename) - out_apk.writestr(info, buf) - os.remove(apkpath) - _zipalign(apkwithnewsig, apkpath) + + sigdir = os.path.dirname(manifest) # FIXME + apksigcopier.do_patch(sigdir, apkpath, outpath, v1_only=v2_files is None) -def apk_extract_signatures(apkpath, outdir, manifest=True): +def apk_extract_signatures(apkpath, outdir, v1_only=None): """Extracts a signature files from APK and puts them into target directory. :param apkpath: location of the apk :param outdir: folder where the extracted signature files will be stored - :param manifest: (optionally) disable extracting manifest file + :param v1_only: True for v1-only signatures, False for v1 and v2 signatures, + or None for autodetection """ - with ZipFile(apkpath, 'r') as in_apk: - for f in in_apk.infolist(): - if APK_SIGNATURE_FILES.match(f.filename) or \ - (manifest and f.filename == 'META-INF/MANIFEST.MF'): - newpath = os.path.join(outdir, os.path.basename(f.filename)) - with open(newpath, 'wb') as out_file: - out_file.write(in_apk.read(f.filename)) + apksigcopier.do_extract(apkpath, outdir, v1_only=v1_only) def get_min_sdk_version(apk): @@ -3169,11 +3165,11 @@ def sign_apk(unsigned_path, signed_path, keyalias): os.remove(unsigned_path) -def verify_apks(signed_apk, unsigned_apk, tmp_dir): +def verify_apks(signed_apk, unsigned_apk, tmp_dir, v1_only=None): """Verify that two apks are the same One of the inputs is signed, the other is unsigned. The signature metadata - is transferred from the signed to the unsigned apk, and then jarsigner is + is transferred from the signed to the unsigned apk, and then apksigner is used to verify that the signature from the signed APK is also valid for the unsigned one. If the APK given as unsigned actually does have a signature, it will be stripped out and ignored. @@ -3181,53 +3177,38 @@ def verify_apks(signed_apk, unsigned_apk, tmp_dir): :param signed_apk: Path to a signed APK file :param unsigned_apk: Path to an unsigned APK file expected to match it :param tmp_dir: Path to directory for temporary files + :param v1_only: True for v1-only signatures, False for v1 and v2 signatures, + or None for autodetection :returns: None if the verification is successful, otherwise a string describing what went wrong. """ + if not verify_apk_signature(signed_apk): + logging.info('...NOT verified - {0}'.format(signed_apk)) + return 'verification of signed APK failed' + if not os.path.isfile(signed_apk): return 'can not verify: file does not exists: {}'.format(signed_apk) - if not os.path.isfile(unsigned_apk): return 'can not verify: file does not exists: {}'.format(unsigned_apk) - with ZipFile(signed_apk, 'r') as signed: - meta_inf_files = ['META-INF/MANIFEST.MF'] - for f in signed.namelist(): - if APK_SIGNATURE_FILES.match(f): - meta_inf_files.append(f) - if len(meta_inf_files) < 3: - return "Signature files missing from {0}".format(signed_apk) + tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk)) - tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk)) - with ZipFile(unsigned_apk, 'r') as unsigned: - # only read the signature from the signed APK, everything else from unsigned - with ZipFile(tmp_apk, 'w') as tmp: - for filename in meta_inf_files: - tmp.writestr(signed.getinfo(filename), signed.read(filename)) - for info in unsigned.infolist(): - if info.filename in meta_inf_files: - logging.warning('Ignoring %s from %s', info.filename, unsigned_apk) - continue - if info.filename in tmp.namelist(): - return "duplicate filename found: " + info.filename - tmp.writestr(info, unsigned.read(info.filename)) - - # Use jarsigner to verify the v1 signature on the reproduced APK, as - # apksigner will reject the reproduced APK if the original also had a v2 - # signature try: - verify_jar_signature(tmp_apk) - verified = True - except Exception: - verified = False + apksigcopier.do_copy(signed_apk, unsigned_apk, tmp_apk, v1_only=v1_only) + except apksigcopier.APKSigCopierError as e: + logging.info('...NOT verified - {0}'.format(tmp_apk)) + return 'signature copying failed: {}'.format(str(e)) - if not verified: - logging.info("...NOT verified - {0}".format(tmp_apk)) - return compare_apks(signed_apk, tmp_apk, tmp_dir, - os.path.dirname(unsigned_apk)) + if not verify_apk_signature(tmp_apk): + logging.info('...NOT verified - {0}'.format(tmp_apk)) + result = compare_apks(signed_apk, tmp_apk, tmp_dir, + os.path.dirname(unsigned_apk)) + if result is not None: + return result + return 'verification of APK with copied signature failed' - logging.info("...successfully verified") + logging.info('...successfully verified') return None diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py index 71b5729d..445c15c9 100644 --- a/fdroidserver/publish.py +++ b/fdroidserver/publish.py @@ -3,6 +3,7 @@ # publish.py - part of the FDroid server tools # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com # Copyright (C) 2013-2014 Daniel Martí +# Copyright (C) 2021 Felix C. Stegerman # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -345,16 +346,15 @@ def main(): # metadata. This means we're going to prepare both a locally # signed APK and a version signed with the developers key. - signaturefile, signedfile, manifest = signingfiles + signaturefile, signedfile, manifest, v2_files = signingfiles with open(signaturefile, 'rb') as f: devfp = common.signer_fingerprint_short(common.get_certificate(f.read())) devsigned = '{}_{}_{}.apk'.format(appid, vercode, devfp) devsignedtmp = os.path.join(tmp_dir, devsigned) - shutil.copy(apkfile, devsignedtmp) - common.apk_implant_signatures(devsignedtmp, signaturefile, - signedfile, manifest) + common.apk_implant_signatures(apkfile, devsignedtmp, manifest=manifest, + v2_files=v2_files) if common.verify_apk_signature(devsignedtmp): shutil.move(devsignedtmp, os.path.join(output_dir, devsigned)) else: diff --git a/tests/common.TestCase b/tests/common.TestCase index 07e3d95e..996f3b8f 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -473,7 +473,7 @@ class CommonTest(unittest.TestCase): for info in apk.infolist(): testapk.writestr(info, apk.read(info.filename)) if info.filename.startswith('META-INF/'): - testapk.writestr(info, otherapk.read(info.filename)) + testapk.writestr(info.filename, otherapk.read(info.filename)) otherapk.close() self.assertFalse(fdroidserver.common.verify_apk_signature(twosigapk)) self.assertIsNone(fdroidserver.common.verify_apks(sourceapk, twosigapk, self.tmpdir)) From 44d481768f1b7da2ea94b8216c384d40cb52e9f6 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 14 Apr 2021 16:07:50 +0200 Subject: [PATCH 03/10] publish: add test for reproduble builds with signatures --- tests/publish.TestCase | 67 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/tests/publish.TestCase b/tests/publish.TestCase index f57dc955..eb1948be 100755 --- a/tests/publish.TestCase +++ b/tests/publish.TestCase @@ -30,6 +30,8 @@ if localmodule not in sys.path: from fdroidserver import publish from fdroidserver import common +from fdroidserver import metadata +from fdroidserver import signatures from fdroidserver.exception import FDroidException @@ -250,6 +252,71 @@ class PublishTest(unittest.TestCase): self.assertEqual(publish.config['jarsigner'], data['jarsigner']) self.assertEqual(publish.config['keytool'], data['keytool']) + def test_sign_then_implant_signature(self): + + class Options: + verbose = False + + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + + config = common.read_config(Options) + if 'apksigner' not in config: + self.skipTest('SKIPPING test_sign_then_implant_signature, apksigner not installed!') + config['repo_keyalias'] = 'sova' + config['keystorepass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=' + config['keypass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=' + shutil.copy(os.path.join(self.basedir, 'keystore.jks'), testdir) + config['keystore'] = 'keystore.jks' + config['keydname'] = 'CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US' + publish.config = config + common.config = config + + app = metadata.App() + app.id = 'org.fdroid.ci' + versionCode = 1 + build = metadata.Build( + { + 'versionCode': versionCode, + 'versionName': '1.0', + } + ) + app.Builds = [build] + os.mkdir('metadata') + metadata.write_metadata(os.path.join('metadata', '%s.yml' % app.id), app) + + os.mkdir('unsigned') + testapk = os.path.join(self.basedir, 'no_targetsdk_minsdk1_unsigned.apk') + unsigned = os.path.join('unsigned', common.get_release_filename(app, build)) + signed = os.path.join('repo', common.get_release_filename(app, build)) + shutil.copy(testapk, unsigned) + + # sign the unsigned APK + self.assertTrue(os.path.exists(unsigned)) + self.assertFalse(os.path.exists(signed)) + with mock.patch('sys.argv', ['fdroid publish', '%s:%d' % (app.id, versionCode)]): + publish.main() + self.assertFalse(os.path.exists(unsigned)) + self.assertTrue(os.path.exists(signed)) + + with mock.patch('sys.argv', ['fdroid signatures', signed]): + signatures.main() + self.assertTrue( + os.path.exists( + os.path.join('metadata', 'org.fdroid.ci', 'signatures', '1', 'MANIFEST.MF') + ) + ) + os.remove(signed) + + # implant the signature into the unsigned APK + shutil.copy(testapk, unsigned) + self.assertTrue(os.path.exists(unsigned)) + self.assertFalse(os.path.exists(signed)) + with mock.patch('sys.argv', ['fdroid publish', '%s:%d' % (app.id, versionCode)]): + publish.main() + self.assertFalse(os.path.exists(unsigned)) + self.assertTrue(os.path.exists(signed)) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From d28c1c04418ede624029cf1eae3585f565d9c163 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 14 Apr 2021 21:02:59 +0200 Subject: [PATCH 04/10] force apksigcopier "AUTO" to select sigs by available files --- fdroidserver/common.py | 11 +++++------ fdroidserver/publish.py | 3 +-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 7bf4c158..4c000078 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -3083,7 +3083,7 @@ def _zipalign(unsigned_apk, aligned_apk): raise BuildException("Failed to align application") -def apk_implant_signatures(apkpath, outpath, manifest, v2_files): +def apk_implant_signatures(apkpath, outpath, manifest): """Implants a signature from metadata into an APK. :param apkpath: location of the unsigned apk @@ -3091,18 +3091,17 @@ def apk_implant_signatures(apkpath, outpath, manifest, v2_files): """ sigdir = os.path.dirname(manifest) # FIXME - apksigcopier.do_patch(sigdir, apkpath, outpath, v1_only=v2_files is None) + apksigcopier.do_patch(sigdir, apkpath, outpath, v1_only=None) -def apk_extract_signatures(apkpath, outdir, v1_only=None): +def apk_extract_signatures(apkpath, outdir): """Extracts a signature files from APK and puts them into target directory. :param apkpath: location of the apk :param outdir: folder where the extracted signature files will be stored - :param v1_only: True for v1-only signatures, False for v1 and v2 signatures, - or None for autodetection + """ - apksigcopier.do_extract(apkpath, outdir, v1_only=v1_only) + apksigcopier.do_extract(apkpath, outdir, v1_only=None) def get_min_sdk_version(apk): diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py index 445c15c9..29e90439 100644 --- a/fdroidserver/publish.py +++ b/fdroidserver/publish.py @@ -353,8 +353,7 @@ def main(): devsigned = '{}_{}_{}.apk'.format(appid, vercode, devfp) devsignedtmp = os.path.join(tmp_dir, devsigned) - common.apk_implant_signatures(apkfile, devsignedtmp, manifest=manifest, - v2_files=v2_files) + common.apk_implant_signatures(apkfile, devsignedtmp, manifest=manifest) if common.verify_apk_signature(devsignedtmp): shutil.move(devsignedtmp, os.path.join(output_dir, devsigned)) else: From f5792d4f4937451175ab470667fac137ef01852a Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 14 Apr 2021 14:23:54 +0200 Subject: [PATCH 05/10] apksigcopier: purge main() to avoid confusion --- fdroidserver/apksigcopier.py | 76 ------------------------------------ 1 file changed, 76 deletions(-) diff --git a/fdroidserver/apksigcopier.py b/fdroidserver/apksigcopier.py index 74e7e7df..215dc5d0 100644 --- a/fdroidserver/apksigcopier.py +++ b/fdroidserver/apksigcopier.py @@ -60,7 +60,6 @@ import zipfile import zlib from collections import namedtuple -from typing import Any, Dict __version__ = "0.4.0" NAME = "apksigcopier" @@ -519,79 +518,4 @@ def do_copy(signed_apk, unsigned_apk, output_apk, v1_only=NO): extracted_v2_sig = extract_v2_sig(signed_apk, expected=v1_only == NO) patch_apk(extracted_meta, extracted_v2_sig, unsigned_apk, output_apk) - -def main(): - """CLI; requires click.""" - - global exclude_all_meta, copy_extra_bytes - exclude_all_meta = os.environ.get("APKSIGCOPIER_EXCLUDE_ALL_META") in ("1", "yes", "true") - copy_extra_bytes = os.environ.get("APKSIGCOPIER_COPY_EXTRA_BYTES") in ("1", "yes", "true") - - import click - - NAY = click.Choice(NOAUTOYES) - - @click.group(help=""" - apksigcopier - copy/extract/patch apk signatures - """) - @click.version_option(__version__) - def cli(): - pass - - @cli.command(help=""" - Extract APK signatures from signed APK. - """) - @click.option("--v1-only", type=NAY, default=NO, show_default=True, - envvar="APKSIGCOPIER_V1_ONLY") - @click.argument("signed_apk", type=click.Path(exists=True, dir_okay=False)) - @click.argument("output_dir", type=click.Path(exists=True, file_okay=False)) - def extract(*args, **kwargs): - do_extract(*args, **kwargs) - - @cli.command(help=""" - Patch extracted APK signatures onto unsigned APK. - """) - @click.option("--v1-only", type=NAY, default=NO, show_default=True, - envvar="APKSIGCOPIER_V1_ONLY") - @click.argument("metadata_dir", type=click.Path(exists=True, file_okay=False)) - @click.argument("unsigned_apk", type=click.Path(exists=True, dir_okay=False)) - @click.argument("output_apk", type=click.Path(dir_okay=False)) - def patch(*args, **kwargs): - do_patch(*args, **kwargs) - - @cli.command(help=""" - Copy (extract & patch) signatures from signed to unsigned APK. - """) - @click.option("--v1-only", type=NAY, default=NO, show_default=True, - envvar="APKSIGCOPIER_V1_ONLY") - @click.argument("signed_apk", type=click.Path(exists=True, dir_okay=False)) - @click.argument("unsigned_apk", type=click.Path(exists=True, dir_okay=False)) - @click.argument("output_apk", type=click.Path(dir_okay=False)) - def copy(*args, **kwargs): - do_copy(*args, **kwargs) - - # FIXME - if click.__version__.startswith("7."): - def autocomplete_path(ctx=None, args=(), incomplete=""): # pylint: disable=W0613 - head, tail = os.path.split(incomplete) - return sorted( - (e.path if head else e.path[2:]) + ("/" if e.is_dir() else "") - for e in os.scandir(head or ".") if e.name.startswith(tail) - ) - - for command in cli.commands.values(): - for param in command.params: - if isinstance(param.type, click.Path): - param.autocompletion = autocomplete_path - - try: - cli(prog_name=NAME) - except APKSigCopierError as e: - click.echo("Error: {}.".format(e), err=True) - sys.exit(1) - - -if __name__ == "__main__": - main() - # vim: set tw=80 sw=4 sts=4 et fdm=marker : From 4308a48717bd1b48e0e7863efae4db103363b57c Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 14 Apr 2021 22:16:48 +0200 Subject: [PATCH 06/10] apksigcopier: remove exclude_all_meta logic This option is unneeded and unused in fdroidserver, and confusing. https://github.com/obfusk/apksigcopier/pull/36 # Conflicts: # fdroidserver/apksigcopier.py # fdroidserver/common.py --- fdroidserver/apksigcopier.py | 11 ++--------- fdroidserver/common.py | 2 +- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/fdroidserver/apksigcopier.py b/fdroidserver/apksigcopier.py index 215dc5d0..48b72ce1 100644 --- a/fdroidserver/apksigcopier.py +++ b/fdroidserver/apksigcopier.py @@ -73,7 +73,6 @@ DATETIMEZERO = (1980, 0, 0, 0, 0, 0) ZipData = namedtuple("ZipData", ("cd_offset", "eocd_offset", "cd_and_eocd")) -exclude_all_meta = False # exclude all metadata files in copy_apk() copy_extra_bytes = False # copy extra bytes after data in copy_apk() @@ -163,14 +162,8 @@ def is_meta(filename): def exclude_from_copying(filename): - """ - Returns whether to exclude a file during copy_apk(). - - Excludes filenames in COPY_EXCLUDE (i.e. MANIFEST.MF) by default; when - exclude_all_meta is set to True instead, excludes all metadata files as - matched by is_meta(). - """ - return is_meta(filename) if exclude_all_meta else filename in COPY_EXCLUDE + """fdroidserver always wants JAR Signature files to be excluded""" + return is_meta(filename) ################################################################################ diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 4c000078..3a57a26e 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -69,7 +69,7 @@ from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesEx from .asynchronousfilereader import AsynchronousFileReader from . import apksigcopier -apksigcopier.exclude_all_meta = True # remove v1 signatures too + # The path to this fdroidserver distribution FDROID_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..')) From 416790306f8eb880dc825d6994630bfe50c67184 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 26 Mar 2021 13:24:26 +0100 Subject: [PATCH 07/10] common.py: update copyrights --- fdroidserver/common.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 3a57a26e..851e7738 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1,9 +1,16 @@ #!/usr/bin/env python3 # # common.py - part of the FDroid server tools -# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com -# Copyright (C) 2013-2014 Daniel Martí -# Copyright (C) 2021 Felix C. Stegerman +# +# Copyright (C) 2010-2016, Ciaran Gultnieks, ciaran@ciarang.com +# Copyright (C) 2013-2017, Daniel Martí +# Copyright (C) 2013-2021, Hans-Christoph Steiner +# Copyright (C) 2017-2018, Torsten Grote +# Copyright (C) 2017, tobiasKaminsky +# Copyright (C) 2017-2021, Michael Pöhn +# Copyright (C) 2017,2021, mimi89999 +# Copyright (C) 2019-2021, Jochen Sprickerhof +# Copyright (C) 2021, Felix C. Stegerman # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by From 2946c90dd4fc98020248b55b65f6d64ecd9a85a9 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 26 Mar 2021 12:00:01 +0100 Subject: [PATCH 08/10] publish: rename vars to match naming in JAR Signature docs https://docs.oracle.com/javase/tutorial/deployment/jar/intro.html closes #892 --- fdroidserver/common.py | 76 ++++++++++++++++++++++++++--------------- fdroidserver/publish.py | 4 +-- tests/common.TestCase | 29 ++++++++++++++++ 3 files changed, 80 insertions(+), 29 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 851e7738..c388fa63 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -2943,13 +2943,13 @@ def apk_signer_fingerprint_short(apk_path): def metadata_get_sigdir(appid, vercode=None): """Get signature directory for app""" if vercode: - return os.path.join('metadata', appid, 'signatures', vercode) + return os.path.join('metadata', appid, 'signatures', str(vercode)) else: return os.path.join('metadata', appid, 'signatures') def metadata_find_developer_signature(appid, vercode=None): - """Tires to find the developer signature for given appid. + """Tries to find the developer signature for given appid. This picks the first signature file found in metadata an returns its signature. @@ -2971,52 +2971,63 @@ def metadata_find_developer_signature(appid, vercode=None): appversigdirs.append(appversigdir) for sigdir in appversigdirs: - sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \ - glob.glob(os.path.join(sigdir, '*.EC')) + \ - glob.glob(os.path.join(sigdir, '*.RSA')) - if len(sigs) > 1: + signature_block_files = ( + glob.glob(os.path.join(sigdir, '*.DSA')) + + glob.glob(os.path.join(sigdir, '*.EC')) + + glob.glob(os.path.join(sigdir, '*.RSA')) + ) + if len(signature_block_files) > 1: raise FDroidException('ambiguous signatures, please make sure there is only one signature in \'{}\'. (The signature has to be the App maintainers signature for version of the APK.)'.format(sigdir)) - for sig in sigs: - with open(sig, 'rb') as f: + for signature_block_file in signature_block_files: + with open(signature_block_file, 'rb') as f: return signer_fingerprint(get_certificate(f.read())) return None def metadata_find_signing_files(appid, vercode): - """Gets a list of singed manifests and signatures. + """Gets a list of signed manifests and signatures. + + https://docs.oracle.com/javase/tutorial/deployment/jar/intro.html + https://source.android.com/security/apksigning/v2 + https://source.android.com/security/apksigning/v3 :param appid: app id string :param vercode: app version code :returns: a list of 4-tuples for each signing key with following paths: - (signature_file, singed_file, manifest_file, v2_files), where v2_files - is either a (sb_offset_file, sb_file) pair or None + (signature_file, signature_block_file, manifest, v2_files), where v2_files + is either a (apk_signing_block_offset_file, apk_signing_block_file) pair or None + """ ret = [] sigdir = metadata_get_sigdir(appid, vercode) - sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \ - glob.glob(os.path.join(sigdir, '*.EC')) + \ - glob.glob(os.path.join(sigdir, '*.RSA')) - extre = re.compile(r'(\.DSA|\.EC|\.RSA)$') - apk_sb = os.path.isfile(os.path.join(sigdir, "APKSigningBlock")) - apk_sbo = os.path.isfile(os.path.join(sigdir, "APKSigningBlockOffset")) - if os.path.isfile(apk_sb) and os.path.isfile(apk_sbo): - v2_files = apk_sb, apk_sbo + signature_block_files = ( + glob.glob(os.path.join(sigdir, '*.DSA')) + + glob.glob(os.path.join(sigdir, '*.EC')) + + glob.glob(os.path.join(sigdir, '*.RSA')) + ) + signature_block_pat = re.compile(r'(\.DSA|\.EC|\.RSA)$') + apk_signing_block = os.path.isfile(os.path.join(sigdir, "APKSigningBlock")) + apk_signing_block_offset = os.path.isfile(os.path.join(sigdir, "APKSigningBlockOffset")) + if os.path.isfile(apk_signing_block) and os.path.isfile(apk_signing_block_offset): + v2_files = apk_signing_block, apk_signing_block_offset else: v2_files = None - for sig in sigs: - sf = extre.sub('.SF', sig) - if os.path.isfile(sf): - mf = os.path.join(sigdir, 'MANIFEST.MF') - if os.path.isfile(mf): - ret.append((sig, sf, mf, v2_files)) + for signature_block_file in signature_block_files: + signature_file = signature_block_pat.sub('.SF', signature_block_file) + if os.path.isfile(signature_file): + manifest = os.path.join(sigdir, 'MANIFEST.MF') + if os.path.isfile(manifest): + ret.append((signature_block_file, signature_file, manifest, v2_files)) return ret def metadata_find_developer_signing_files(appid, vercode): """Get developer signature files for specified app from metadata. - :returns: A triplet of paths for signing files from metadata: - (signature_file, singed_file, manifest_file) + :returns: a list of 4-tuples for each signing key with following paths: + (signature_file, signature_block_file, manifest, v2_files), where v2_files + is either a (apk_signing_block_offset_file, apk_signing_block_file) pair or None + """ allsigningfiles = metadata_find_signing_files(appid, vercode) if allsigningfiles and len(allsigningfiles) == 1: @@ -3093,6 +3104,13 @@ def _zipalign(unsigned_apk, aligned_apk): def apk_implant_signatures(apkpath, outpath, manifest): """Implants a signature from metadata into an APK. + Note: this changes there supplied APK in place. So copy it if you + need the original to be preserved. + + https://docs.oracle.com/javase/tutorial/deployment/jar/intro.html + https://source.android.com/security/apksigning/v2 + https://source.android.com/security/apksigning/v3 + :param apkpath: location of the unsigned apk :param outpath: location of the output apk """ @@ -3104,6 +3122,10 @@ def apk_implant_signatures(apkpath, outpath, manifest): def apk_extract_signatures(apkpath, outdir): """Extracts a signature files from APK and puts them into target directory. + https://docs.oracle.com/javase/tutorial/deployment/jar/intro.html + https://source.android.com/security/apksigning/v2 + https://source.android.com/security/apksigning/v3 + :param apkpath: location of the apk :param outdir: folder where the extracted signature files will be stored diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py index 29e90439..56b6016c 100644 --- a/fdroidserver/publish.py +++ b/fdroidserver/publish.py @@ -346,9 +346,9 @@ def main(): # metadata. This means we're going to prepare both a locally # signed APK and a version signed with the developers key. - signaturefile, signedfile, manifest, v2_files = signingfiles + signature_file, _ignored, manifest, v2_files = signingfiles - with open(signaturefile, 'rb') as f: + with open(signature_file, 'rb') as f: devfp = common.signer_fingerprint_short(common.get_certificate(f.read())) devsigned = '{}_{}_{}.apk'.format(appid, vercode, devfp) devsignedtmp = os.path.join(tmp_dir, devsigned) diff --git a/tests/common.TestCase b/tests/common.TestCase index 996f3b8f..ddb2f07d 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -1761,6 +1761,35 @@ class CommonTest(unittest.TestCase): shutil.copy(before, after) fdroidserver.common.apk_strip_v1_signatures(after, strip_manifest=False) + def test_metadata_find_developer_signing_files(self): + appid = 'org.smssecure.smssecure' + + self.assertIsNone( + fdroidserver.common.metadata_find_developer_signing_files(appid, 133) + ) + + vc = '135' + self.assertEqual( + ( + os.path.join('metadata', appid, 'signatures', vc, '28969C09.RSA'), + os.path.join('metadata', appid, 'signatures', vc, '28969C09.SF'), + os.path.join('metadata', appid, 'signatures', vc, 'MANIFEST.MF'), + None + ), + fdroidserver.common.metadata_find_developer_signing_files(appid, vc) + ) + + vc = '134' + self.assertEqual( + ( + os.path.join('metadata', appid, 'signatures', vc, '28969C09.RSA'), + os.path.join('metadata', appid, 'signatures', vc, '28969C09.SF'), + os.path.join('metadata', appid, 'signatures', vc, 'MANIFEST.MF'), + None + ), + fdroidserver.common.metadata_find_developer_signing_files(appid, vc) + ) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From 0c807275a27df7097b7105ee9d175b2eb1321fad Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 12 Apr 2021 11:52:21 +0200 Subject: [PATCH 09/10] jenkins-build-all: refocus on building reproducible apps first This also deletes the reproducible APKs each time to test that they are still being built reproducibly. !893 fdroidserver#891 --- jenkins-build-all | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/jenkins-build-all b/jenkins-build-all index 9e9c76e5..282a4ab6 100755 --- a/jenkins-build-all +++ b/jenkins-build-all @@ -95,10 +95,25 @@ else sed -i '/^wiki_/d' config.yml fi +printf '\n@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\nbuild all with reproducible signatures\n' +for f in metadata/*/signatures/*; do + appid=$(basename $(dirname $(dirname $f))) + versionCode=$(basename $f) + rm -f repo/${appid}_* archive/${appid}_* unsigned/${appid}_* + $WORKSPACE/fdroid build --verbose --latest --no-tarball ${appid}:$versionCode +done + +printf '\n@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\nbuild all with Binaries:\n' +for appid in `grep '^Binaries: ' metadata/*.yml --files-with-match | sed 's,^metadata/\(.*\)\.yml$,\1,'`; do + rm -f repo/${appid}_* archive/${appid}_* unsigned/${appid}_* + $WORKSPACE/fdroid build --verbose --latest --no-tarball ${appid} +done + # force global timeout to 6 hours sed -Ei 's,^(\s+endtime\s*=\s*time\.time\(\))\s*.*,\1 + 6 * 60 * 60 # 6 hours,' \ $WORKSPACE/fdroidserver/build.py +printf '\n@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\nbuild all\n' $WORKSPACE/fdroid build --verbose --latest --no-tarball --all $wikiflag vagrant global-status From 5346ea3c19279f667c1c933fcdf29aa0f665c8b1 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 14 Apr 2021 22:54:48 +0200 Subject: [PATCH 10/10] gitlab-ci: prevent dualing linters: pyflakes vs mypy * pyflakes says: "typing.Dict' imported but unused" * mypy says: "Name 'Dict' is not defined" --- .gitlab-ci.yml | 3 ++- fdroidserver/apksigcopier.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3310b410..1f02ba77 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -193,7 +193,8 @@ lint_mypy: script: - pip install mypy - pip install -e .[test] - - mypy + # exclude vendored file + - mypy --exclude fdroidserver/apksigcopier.py fedora_latest: image: fedora:latest diff --git a/fdroidserver/apksigcopier.py b/fdroidserver/apksigcopier.py index 48b72ce1..bb4e9b22 100644 --- a/fdroidserver/apksigcopier.py +++ b/fdroidserver/apksigcopier.py @@ -96,7 +96,7 @@ class ZipError(APKSigCopierError): class ReproducibleZipInfo(zipfile.ZipInfo): """Reproducible ZipInfo hack.""" - _override = {} # type: Dict[str, Any] + _override = {} def __init__(self, zinfo, **override): if override: