diff --git a/fdroidserver/apksigcopier.py b/fdroidserver/apksigcopier.py index 50a26945..321aec52 100644 --- a/fdroidserver/apksigcopier.py +++ b/fdroidserver/apksigcopier.py @@ -1,22 +1,25 @@ #!/usr/bin/python3 # encoding: utf-8 +# SPDX-FileCopyrightText: 2022 FC Stegerman +# SPDX-License-Identifier: GPL-3.0-or-later # -- ; {{{1 # # File : apksigcopier -# Maintainer : Felix C. Stegerman -# Date : 2021-04-14 +# Maintainer : FC Stegerman +# Date : 2022-11-01 # -# Copyright : Copyright (C) 2021 Felix C. Stegerman -# Version : v0.4.0 +# Copyright : Copyright (C) 2022 FC Stegerman +# Version : v1.1.0 # License : GPLv3+ # # -- ; }}}1 -"""Copy/extract/patch apk signatures. +""" +Copy/extract/patch android apk signatures. -apksigcopier is a tool for copying APK signatures from a signed APK to an -unsigned one (in order to verify reproducible builds). +apksigcopier is a tool for copying android APK signatures from a signed APK to +an unsigned one (in order to verify reproducible builds). CLI @@ -27,9 +30,8 @@ $ apksigcopier patch [OPTIONS] METADATA_DIR UNSIGNED_APK OUTPUT_APK $ apksigcopier copy [OPTIONS] SIGNED_APK UNSIGNED_APK OUTPUT_APK The following environment variables can be set to 1, yes, or true to -overide the default behaviour: +override the default behaviour: -* set APKSIGCOPIER_EXCLUDE_ALL_META=1 to exclude all metadata files * set APKSIGCOPIER_COPY_EXTRA_BYTES=1 to copy extra bytes after data (e.g. a v2 sig) @@ -46,30 +48,62 @@ You can use False, None, and True instead of NO, AUTO, and YES respectively. The following global variables (which default to False), can be set to override the default behaviour: -* set exclude_all_meta=True to exclude all metadata files * set copy_extra_bytes=True to copy extra bytes after data (e.g. a v2 sig) """ import glob +import json import os import re import struct -import sys import zipfile import zlib from collections import namedtuple -from typing import Dict, Tuple, Union +from typing import Any, BinaryIO, Dict, Iterable, Iterator, Literal, Optional, Tuple, Union -__version__ = "0.4.0" +__version__ = "1.1.0" NAME = "apksigcopier" +DateTime = Tuple[int, int, int, int, int, int] +NoAutoYes = Literal["no", "auto", "yes"] +NoAutoYesBoolNone = Union[NoAutoYes, bool, None] +ZipInfoDataPairs = Iterable[Tuple[zipfile.ZipInfo, bytes]] + SIGBLOCK, SIGOFFSET = "APKSigningBlock", "APKSigningBlockOffset" -NOAUTOYES = NO, AUTO, YES = ("no", "auto", "yes") +NOAUTOYES: Tuple[NoAutoYes, NoAutoYes, NoAutoYes] = ("no", "auto", "yes") +NO, AUTO, YES = NOAUTOYES APK_META = re.compile(r"^META-INF/([0-9A-Za-z_-]+\.(SF|RSA|DSA|EC)|MANIFEST\.MF)$") -META_EXT = ("SF", "RSA|DSA|EC", "MF") -COPY_EXCLUDE = ("META-INF/MANIFEST.MF",) -DATETIMEZERO = (1980, 0, 0, 0, 0, 0) +META_EXT: Tuple[str, ...] = ("SF", "RSA|DSA|EC", "MF") +DATETIMEZERO: DateTime = (1980, 0, 0, 0, 0, 0) + +################################################################################ +# +# NB: these values are all from apksigner (the first element of each tuple, same +# as APKZipInfo) or signflinger/zipflinger, except for external_attr w/ 0664 +# permissions and flag_bits 0x08, added for completeness. +# +# NB: zipflinger changed from 0666 to 0644 in commit 895ba5fba6ab84617dd67e38f456a8f96aa37ff0 +# +# https://android.googlesource.com/platform/tools/apksig +# src/main/java/com/android/apksig/internal/zip/{CentralDirectoryRecord,LocalFileRecord,ZipUtils}.java +# https://android.googlesource.com/platform/tools/base +# signflinger/src/com/android/signflinger/SignedApk.java +# zipflinger/src/com/android/zipflinger/{CentralDirectoryRecord,LocalFileHeader,Source}.java +# +################################################################################ + +VALID_ZIP_META = dict( + compresslevel=(9, 1), # best compression, best speed + create_system=(0, 3), # fat, unx + create_version=(20, 0), # 2.0, 0.0 + external_attr=(0, # N/A + 0o100644 << 16, # regular file rw-r--r-- + 0o100664 << 16, # regular file rw-rw-r-- + 0o100666 << 16), # regular file rw-rw-rw- + extract_version=(20, 0), # 2.0, 0.0 + flag_bits=(0x800, 0, 0x08, 0x808), # 0x800 = utf8, 0x08 = data_descriptor +) ZipData = namedtuple("ZipData", ("cd_offset", "eocd_offset", "cd_and_eocd")) @@ -96,10 +130,9 @@ class ZipError(APKSigCopierError): class ReproducibleZipInfo(zipfile.ZipInfo): """Reproducible ZipInfo hack.""" - _override = {} # type: Dict[str, Union[int, Tuple[int, ...]]] + _override: Dict[str, Any] = {} - def __init__(self, zinfo, **override): - super().__init__() + def __init__(self, zinfo, **override): # pylint: disable=W0231 if override: self._override = {**self._override, **override} for k in self.__slots__: @@ -115,9 +148,12 @@ class ReproducibleZipInfo(zipfile.ZipInfo): return object.__getattribute__(self, name) +# See VALID_ZIP_META class APKZipInfo(ReproducibleZipInfo): """Reproducible ZipInfo for APK files.""" + COMPRESSLEVEL = 9 + _override = dict( compress_type=8, create_system=0, @@ -129,8 +165,9 @@ class APKZipInfo(ReproducibleZipInfo): ) -def noautoyes(value): - """Turn False into NO, None into AUTO, and True into YES. +def noautoyes(value: NoAutoYesBoolNone) -> NoAutoYes: + """ + Turn False into NO, None into AUTO, and True into YES. >>> from apksigcopier import noautoyes, NO, AUTO, YES >>> noautoyes(False) == NO == noautoyes(NO) @@ -147,30 +184,73 @@ def noautoyes(value): return value try: return {False: NO, None: AUTO, True: YES}[value] - except KeyError as exc: - raise ValueError("expected False, None, or True") from exc + except KeyError: + raise ValueError("expected False, None, or True") # pylint: disable=W0707 -def is_meta(filename): - """No summary. +def is_meta(filename: str) -> bool: + """ + Check whether filename is a JAR metadata file. - Returns whether filename is a v1 (JAR) signature file (.SF), signature block - file (.RSA, .DSA, or .EC), or manifest (MANIFEST.MF). + This is true when filename is a v1 (JAR) signature file (.SF), signature + block file (.RSA, .DSA, or .EC), or manifest (MANIFEST.MF). See https://docs.oracle.com/javase/tutorial/deployment/jar/intro.html """ return APK_META.fullmatch(filename) is not None -def exclude_from_copying(filename): - """Fdroidserver always wants JAR Signature files to be excluded.""" +def exclude_from_copying(filename: str) -> bool: + """ + Check whether to exclude a file during copy_apk(). + + Fdroidserver always wants JAR Signature files to be excluded, so + it excludes all metadata files as matched by is_meta(). + """ return is_meta(filename) +################################################################################ +# +# There is usually a 132-byte virtual entry at the start of an APK signed with a +# v1 signature by signflinger/zipflinger; almost certainly this is a default +# manifest ZIP entry created at initialisation, deleted (from the CD but not +# from the file) during v1 signing, and eventually replaced by a virtual entry. +# +# >>> (30 + len("META-INF/MANIFEST.MF") + +# ... len("Manifest-Version: 1.0\r\n" +# ... "Created-By: Android Gradle 7.1.3\r\n" +# ... "Built-By: Signflinger\r\n\r\n")) +# 132 +# +# NB: they could be a different size, depending on Created-By and Built-By. +# +# FIXME: could virtual entries occur elsewhere as well? +# +# https://android.googlesource.com/platform/tools/base +# signflinger/src/com/android/signflinger/SignedApk.java +# zipflinger/src/com/android/zipflinger/{LocalFileHeader,ZipArchive}.java +# +################################################################################ + +def zipflinger_virtual_entry(size: int) -> bytes: + """Create zipflinger virtual entry.""" + if size < 30: + raise ValueError("Minimum size for virtual entries is 30 bytes") + return ( + # header extract_version flag_bits + b"\x50\x4b\x03\x04" b"\x00\x00" b"\x00\x00" + # compress_type (1981,1,1,1,1,2) crc32 + b"\x00\x00" b"\x21\x08\x21\x02" b"\x00\x00\x00\x00" + # compress_size file_size filename length + b"\x00\x00\x00\x00" b"\x00\x00\x00\x00" b"\x00\x00" + ) + int.to_bytes(size - 30, 2, "little") + b"\x00" * (size - 30) + + ################################################################################ # # https://en.wikipedia.org/wiki/ZIP_(file_format) -# https://source.android.com/security/apksigning/v2#apk-signing-block-format +# https://source.android.com/docs/security/features/apksigning/v2#apk-signing-block-format # # ================================= # | Contents of ZIP entries | @@ -198,24 +278,36 @@ def exclude_from_copying(filename): # FIXME: makes certain assumptions and doesn't handle all valid ZIP files! -def copy_apk(unsigned_apk, output_apk): - """Copy APK like apksigner would, excluding files matched by exclude_from_copying(). +# FIXME: support zip64? +# FIXME: handle utf8 filenames w/o utf8 flag (as produced by zipflinger)? +# https://android.googlesource.com/platform/tools/apksig +# src/main/java/com/android/apksig/ApkSigner.java +def copy_apk(unsigned_apk: str, output_apk: str, *, zfe_size: Optional[int] = None) -> DateTime: + """ + Copy APK like apksigner would, excluding files matched by exclude_from_copying(). + + Adds a zipflinger virtual entry of zfe_size bytes if one is not already + present and zfe_size is not None. The following global variables (which default to False), can be set to override the default behaviour: - * set exclude_all_meta=True to exclude all metadata files * set copy_extra_bytes=True to copy extra bytes after data (e.g. a v2 sig) Returns ------- - max date_time. + max date_time """ with zipfile.ZipFile(unsigned_apk, "r") as zf: infos = zf.infolist() zdata = zip_data(unsigned_apk) offsets = {} with open(unsigned_apk, "rb") as fhi, open(output_apk, "w+b") as fho: + if zfe_size: + zfe = zipflinger_virtual_entry(zfe_size) + if fhi.read(zfe_size) != zfe: + fho.write(zfe) + fhi.seek(0) for info in sorted(infos, key=lambda info: info.header_offset): off_i = fhi.tell() if info.header_offset > off_i: @@ -231,7 +323,7 @@ def copy_apk(unsigned_apk, output_apk): fhi.seek(info.compress_size, os.SEEK_CUR) else: if info.filename in offsets: - raise ZipError("Duplicate ZIP entry: " + info.filename) + raise ZipError(f"Duplicate ZIP entry: {info.filename!r}") offsets[info.filename] = off_o = fho.tell() if info.compress_type == 0 and off_o != info.header_offset: hdr = _realign_zip_entry(info, hdr, n, m, off_o) @@ -268,7 +360,7 @@ def copy_apk(unsigned_apk, output_apk): # NB: doesn't sync local & CD headers! -def _realign_zip_entry(info, hdr, n, m, off_o): +def _realign_zip_entry(info: zipfile.ZipInfo, hdr: bytes, n: int, m: int, off_o: int) -> bytes: align = 4096 if info.filename.endswith(".so") else 4 old_off = 30 + n + m + info.header_offset new_off = 30 + n + m + off_o @@ -293,7 +385,7 @@ def _realign_zip_entry(info, hdr, n, m, off_o): return hdr -def _copy_bytes(fhi, fho, size, blocksize=4096): +def _copy_bytes(fhi: BinaryIO, fho: BinaryIO, size: int, blocksize: int = 4096) -> None: while size > 0: data = fhi.read(min(size, blocksize)) if not data: @@ -304,7 +396,7 @@ def _copy_bytes(fhi, fho, size, blocksize=4096): raise ZipError("Unexpected EOF") -def extract_meta(signed_apk): +def extract_meta(signed_apk: str) -> Iterator[Tuple[zipfile.ZipInfo, bytes]]: """ Extract v1 signature metadata files from signed APK. @@ -316,34 +408,97 @@ def extract_meta(signed_apk): yield info, zf_sig.read(info.filename) -def patch_meta(extracted_meta, output_apk, date_time=DATETIMEZERO): +def extract_differences(signed_apk: str, extracted_meta: ZipInfoDataPairs) \ + -> Optional[Dict[str, Any]]: + """Extract ZIP metadata differences from signed APK.""" + differences: Dict[str, Any] = {} + files = {} + for info, data in extracted_meta: + diffs = {} + for k in VALID_ZIP_META.keys(): + if k != "compresslevel": + v = getattr(info, k) + if v != APKZipInfo._override[k]: + if v not in VALID_ZIP_META[k]: + raise ZipError(f"Unsupported {k}") + diffs[k] = v + level = _get_compresslevel(info, data) + if level != APKZipInfo.COMPRESSLEVEL: + diffs["compresslevel"] = level + if diffs: + files[info.filename] = diffs + if files: + differences["files"] = files + with open(signed_apk, "rb") as fh: + zfe_start = zipflinger_virtual_entry(30)[:28] # w/o len(extra) + if fh.read(28) == zfe_start: + zfe_size = 30 + int.from_bytes(fh.read(2), "little") + if not (30 <= zfe_size <= 4096): + raise ZipError("Unsupported virtual entry size") + if not fh.read(zfe_size - 30) == b"\x00" * (zfe_size - 30): + raise ZipError("Unsupported virtual entry data") + differences["zipflinger_virtual_entry"] = zfe_size + return differences or None + + +def validate_differences(differences: Dict[str, Any]) -> Optional[str]: + """ + Validate differences dict. + + Returns None if valid, error otherwise. + """ + if set(differences.keys()) - {"files", "zipflinger_virtual_entry"}: + return "contains unknown key(s)" + if "zipflinger_virtual_entry" in differences: + if type(differences["zipflinger_virtual_entry"]) is not int: + return ".zipflinger_virtual_entry is not an int" + if not (30 <= differences["zipflinger_virtual_entry"] <= 4096): + return ".zipflinger_virtual_entry is < 30 or > 4096" + if "files" in differences: + if not isinstance(differences["files"], dict): + return ".files is not a dict" + for name, info in differences["files"].items(): + if not isinstance(info, dict): + return f".files[{name!r}] is not a dict" + if set(info.keys()) - set(VALID_ZIP_META.keys()): + return f".files[{name!r}] contains unknown key(s)" + for k, v in info.items(): + if v not in VALID_ZIP_META[k]: + return f".files[{name!r}].{k} has an unexpected value" + return None + + +# FIXME: false positives on same compressed size? compare actual data? +def _get_compresslevel(info: zipfile.ZipInfo, data: bytes) -> int: + if info.compress_type != 8: + raise ZipError("Unsupported compress_type") + for level in VALID_ZIP_META["compresslevel"]: + comp = zlib.compressobj(level, 8, -15) + if len(comp.compress(data) + comp.flush()) == info.compress_size: + return level + raise ZipError("Unsupported compresslevel") + + +def patch_meta(extracted_meta: ZipInfoDataPairs, output_apk: str, + date_time: DateTime = DATETIMEZERO, *, + differences: Optional[Dict[str, Any]] = None) -> None: """Add v1 signature metadata to APK (removes v2 sig block, if any).""" with zipfile.ZipFile(output_apk, "r") as zf_out: for info in zf_out.infolist(): if is_meta(info.filename): raise ZipError("Unexpected metadata") with zipfile.ZipFile(output_apk, "a") as zf_out: - info_data = [(APKZipInfo(info, date_time=date_time), data) - for info, data in extracted_meta] - _write_to_zip(info_data, zf_out) + for info, data in extracted_meta: + if differences and "files" in differences: + more = differences["files"].get(info.filename, {}).copy() + else: + more = {} + level = more.pop("compresslevel", APKZipInfo.COMPRESSLEVEL) + zinfo = APKZipInfo(info, date_time=date_time, **more) + zf_out.writestr(zinfo, data, compresslevel=level) -if sys.version_info >= (3, 7): - def _write_to_zip(info_data, zf_out): - for info, data in info_data: - zf_out.writestr(info, data, compresslevel=9) -else: - def _write_to_zip(info_data, zf_out): - old = zipfile._get_compressor - zipfile._get_compressor = lambda _: zlib.compressobj(9, 8, -15) - try: - for info, data in info_data: - zf_out.writestr(info, data) - finally: - zipfile._get_compressor = old - - -def extract_v2_sig(apkfile, expected=True): +def extract_v2_sig(apkfile: str, expected: bool = True) -> Optional[Tuple[int, bytes]]: """ Extract APK Signing Block and offset from APK. @@ -369,7 +524,8 @@ def extract_v2_sig(apkfile, expected=True): return sb_offset, sig_block -def zip_data(apkfile, count=1024): +# FIXME: OSError for APKs < 1024 bytes [wontfix] +def zip_data(apkfile: str, count: int = 1024) -> ZipData: """ Extract central directory, EOCD, and offsets from ZIP. @@ -393,7 +549,7 @@ def zip_data(apkfile, count=1024): # FIXME: can we determine signed_sb_offset? -def patch_v2_sig(extracted_v2_sig, output_apk): +def patch_v2_sig(extracted_v2_sig: Tuple[int, bytes], output_apk: str) -> None: """Implant extracted v2/v3 signature into APK.""" signed_sb_offset, signed_sb = extracted_v2_sig data_out = zip_data(output_apk) @@ -410,24 +566,28 @@ def patch_v2_sig(extracted_v2_sig, output_apk): fh.write(int.to_bytes(data_out.cd_offset + offset, 4, "little")) -def patch_apk(extracted_meta, extracted_v2_sig, unsigned_apk, output_apk): - """Patch extracted_meta + extracted_v2_sig. - - Patches extracted_meta + extracted_v2_sig (if not None) - onto unsigned_apk and save as output_apk. - """ - date_time = copy_apk(unsigned_apk, output_apk) - patch_meta(extracted_meta, output_apk, date_time=date_time) +def patch_apk(extracted_meta: ZipInfoDataPairs, extracted_v2_sig: Optional[Tuple[int, bytes]], + unsigned_apk: str, output_apk: str, *, + differences: Optional[Dict[str, Any]] = None) -> None: + """Patch extracted_meta + extracted_v2_sig (if not None) onto unsigned_apk and save as output_apk.""" + if differences and "zipflinger_virtual_entry" in differences: + zfe_size = differences["zipflinger_virtual_entry"] + else: + zfe_size = None + date_time = copy_apk(unsigned_apk, output_apk, zfe_size=zfe_size) + patch_meta(extracted_meta, output_apk, date_time=date_time, differences=differences) if extracted_v2_sig is not None: patch_v2_sig(extracted_v2_sig, output_apk) -def do_extract(signed_apk, output_dir, v1_only=NO): - """Extract signatures from signed_apk and save in output_dir. +# FIXME: support multiple signers? +def do_extract(signed_apk: str, output_dir: str, v1_only: NoAutoYesBoolNone = NO, + *, ignore_differences: bool = False) -> None: + """ + Extract signatures from signed_apk and save in output_dir. The v1_only parameter controls whether the absence of a v1 signature is considered an error or not: - * use v1_only=NO (or v1_only=False) to only accept (v1+)v2/v3 signatures; * use v1_only=AUTO (or v1_only=None) to automatically detect v2/v3 signatures; * use v1_only=YES (or v1_only=True) to ignore any v2/v3 signatures. @@ -455,20 +615,29 @@ def do_extract(signed_apk, output_dir, v1_only=NO): fh.write(str(signed_sb_offset) + "\n") with open(os.path.join(output_dir, SIGBLOCK), "wb") as fh: fh.write(signed_sb) + if not ignore_differences: + differences = extract_differences(signed_apk, extracted_meta) + if differences: + with open(os.path.join(output_dir, "differences.json"), "w") as fh: + json.dump(differences, fh, sort_keys=True, indent=2) + fh.write("\n") -def do_patch(metadata_dir, unsigned_apk, output_apk, v1_only=NO): - """Patch signatures from metadata_dir onto unsigned_apk and save as output_apk. +# FIXME: support multiple signers? +def do_patch(metadata_dir: str, unsigned_apk: str, output_apk: str, + v1_only: NoAutoYesBoolNone = NO, *, ignore_differences: bool = False) -> None: + """ + Patch signatures from metadata_dir onto unsigned_apk and save as output_apk. The v1_only parameter controls whether the absence of a v1 signature is considered an error or not: - * use v1_only=NO (or v1_only=False) to only accept (v1+)v2/v3 signatures; * use v1_only=AUTO (or v1_only=None) to automatically detect v2/v3 signatures; * use v1_only=YES (or v1_only=True) to ignore any v2/v3 signatures. """ v1_only = noautoyes(v1_only) extracted_meta = [] + differences = None for pat in META_EXT: files = [fn for ext in pat.split("|") for fn in glob.glob(os.path.join(metadata_dir, "*." + ext))] @@ -492,27 +661,43 @@ def do_patch(metadata_dir, unsigned_apk, output_apk, v1_only=NO): with open(sigblock_file, "rb") as fh: signed_sb = fh.read() extracted_v2_sig = signed_sb_offset, signed_sb + differences_file = os.path.join(metadata_dir, "differences.json") + if not ignore_differences and os.path.exists(differences_file): + with open(differences_file, "r") as fh: + try: + differences = json.load(fh) + except json.JSONDecodeError as e: + raise APKSigCopierError(f"Invalid differences.json: {e}") # pylint: disable=W0707 + error = validate_differences(differences) + if error: + raise APKSigCopierError(f"Invalid differences.json: {error}") if not extracted_meta and extracted_v2_sig is None: raise APKSigCopierError("Expected v1 and/or v2/v3 signature, found neither") - patch_apk(extracted_meta, extracted_v2_sig, unsigned_apk, output_apk) + patch_apk(extracted_meta, extracted_v2_sig, unsigned_apk, output_apk, + differences=differences) -def do_copy(signed_apk, unsigned_apk, output_apk, v1_only=NO): - """Copy signatures from signed_apk onto unsigned_apk and save as output_apk. +def do_copy(signed_apk: str, unsigned_apk: str, output_apk: str, + v1_only: NoAutoYesBoolNone = NO, *, ignore_differences: bool = False) -> None: + """ + Copy signatures from signed_apk onto unsigned_apk and save as output_apk. The v1_only parameter controls whether the absence of a v1 signature is considered an error or not: - * use v1_only=NO (or v1_only=False) to only accept (v1+)v2/v3 signatures; * use v1_only=AUTO (or v1_only=None) to automatically detect v2/v3 signatures; * use v1_only=YES (or v1_only=True) to ignore any v2/v3 signatures. """ v1_only = noautoyes(v1_only) - extracted_meta = extract_meta(signed_apk) + extracted_meta = tuple(extract_meta(signed_apk)) + differences = None if v1_only == YES: extracted_v2_sig = None else: extracted_v2_sig = extract_v2_sig(signed_apk, expected=v1_only == NO) - patch_apk(extracted_meta, extracted_v2_sig, unsigned_apk, output_apk) + if extracted_v2_sig is not None and not ignore_differences: + differences = extract_differences(signed_apk, extracted_meta) + patch_apk(extracted_meta, extracted_v2_sig, unsigned_apk, output_apk, + differences=differences) # vim: set tw=80 sw=4 sts=4 et fdm=marker :