From 11d21d6b181d252a83635f7c2adfbadb9a9308a8 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 11 Jan 2024 11:32:39 +0100 Subject: [PATCH 01/14] gitlab-ci: bump base commit in metadata_v0 job to get rev-parse fix !1427 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 17425b8a..188a2b90 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,7 +42,7 @@ metadata_v0: image: registry.gitlab.com/fdroid/fdroidserver:buildserver variables: GIT_DEPTH: 1000 - RELEASE_COMMIT_ID: a1c4f803de8d4dc92ebd6b571a493183d14a00bf # after ArchivePolicy: 0 + RELEASE_COMMIT_ID: 50aa35772b058e76b950c01e16019c072c191b73 # after switching to `git rev-parse` script: - git fetch https://gitlab.com/fdroid/fdroidserver.git $RELEASE_COMMIT_ID - cd tests From 77daf6feb686a93acaca0e6216faaa7c4cde1787 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Mon, 5 Dec 2022 14:58:08 +0100 Subject: [PATCH 02/14] Add Apple ipa support (Closes: #1067) --- fdroidserver/common.py | 2 ++ fdroidserver/update.py | 46 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index e97cedf5..cfe60150 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -102,6 +102,7 @@ STRICT_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-zA-Z]+(?:\d*[a-zA-Z_]*)*)(?: VALID_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-z_]+(?:\d*[a-zA-Z_]*)*)(?:\.[a-z_]+(?:\d*[a-zA-Z_]*)*)*$''', re.IGNORECASE) ANDROID_PLUGIN_REGEX = re.compile(r'''\s*(:?apply plugin:|id)\(?\s*['"](android|com\.android\.application)['"]\s*\)?''') +APPLE_BUNDLEiDENTIFIER_REGEX = re.compile(r'''^[a-zA-Z-.]*''') SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?') GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:?([^'"]+)['"]''') @@ -2015,6 +2016,7 @@ def is_valid_package_name(name): """ return VALID_APPLICATION_ID_REGEX.match(name) is not None \ + or APPLE_BUNDLEiDENTIFIER_REGEX.match(name) is not None \ or FDROID_PACKAGE_NAME_REGEX.match(name) is not None diff --git a/fdroidserver/update.py b/fdroidserver/update.py index a1d4dbc4..8aad6af3 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -46,6 +46,7 @@ except ImportError: import collections from binascii import hexlify +from biplist import readPlist from . import _ from . import common @@ -524,6 +525,48 @@ def insert_obbs(repodir, apps, apks): break +def process_ipa(repodir, apks): + """Scan the .ipa files in a given repo directory + + Parameters + ---------- + repodir + repo directory to scan + apps + list of current, valid apps + apks + current information on all APKs + """ + def ipaWarnDelete(f, msg): + logging.warning(msg + ' ' + f) + if options.delete_unknown: + logging.error(_("Deleting unknown file: {path}").format(path=f)) + os.remove(f) + + for f in glob.glob(os.path.join(repodir, '*.ipa')): + ipa = {} + apks.append(ipa) + + ipa["apkName"] = os.path.basename(f) + ipa["hash"] = common.sha256sum(f) + ipa["hashType"] = "sha256" + ipa["size"] = os.path.getsize(f) + + with zipfile.ZipFile(f) as ipa_zip: + for info in ipa_zip.infolist(): + if re.match("Payload/[^/]*.app/Info.plist", info.filename): + with ipa_zip.open(info) as plist_file: + plist = readPlist(plist_file) + # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleversion + version = plist["CFBundleVersion"].split('.') + major = int(version.pop(0)) + minor = int(version.pop(0)) if version else 0 + patch = int(version.pop(0)) if version else 0 + ipa["packageName"] = plist["CFBundleIdentifier"] + ipa["versionCode"] = major * 10**12 + minor * 10**6 + patch + ipa["versionName"] = plist["CFBundleShortVersionString"] + + def translate_per_build_anti_features(apps, apks): """Grab the anti-features list from the build metadata. @@ -1139,7 +1182,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): repodir = repodir.encode() for name in os.listdir(repodir): file_extension = common.get_file_extension(name) - if file_extension in ('apk', 'obb'): + if file_extension in ('apk', 'obb', 'ipa'): continue filename = os.path.join(repodir, name) name_utf8 = name.decode() @@ -2128,6 +2171,7 @@ def prepare_apps(apps, apks, repodir): ------- the relevant subset of apps (as a deepcopy) """ + process_ipa(repodir, apks) apps_with_packages = get_apps_with_packages(apps, apks) apply_info_from_latest_apk(apps_with_packages, apks) insert_funding_yml_donation_links(apps) From a987341c377707a91e2703f1dc00399c56cd436b Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Fri, 9 Dec 2022 11:47:06 +0100 Subject: [PATCH 03/14] ipa: add Usage permissions --- fdroidserver/update.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 8aad6af3..d8c04619 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -565,6 +565,7 @@ def process_ipa(repodir, apks): ipa["packageName"] = plist["CFBundleIdentifier"] ipa["versionCode"] = major * 10**12 + minor * 10**6 + patch ipa["versionName"] = plist["CFBundleShortVersionString"] + ipa["usage"] = {k: v for k, v in plist.items() if 'Usage' in k} def translate_per_build_anti_features(apps, apks): From dfbb2df8397ec22f7cb5da18780a0be7a74dce72 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Fri, 9 Dec 2022 16:04:51 +0100 Subject: [PATCH 04/14] Use CFBundleShortVersionString for version code --- fdroidserver/update.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index d8c04619..38b11b2c 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -557,8 +557,8 @@ def process_ipa(repodir, apks): if re.match("Payload/[^/]*.app/Info.plist", info.filename): with ipa_zip.open(info) as plist_file: plist = readPlist(plist_file) - # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleversion - version = plist["CFBundleVersion"].split('.') + # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring + version = plist["CFBundleShortVersionString"].split('.') major = int(version.pop(0)) minor = int(version.pop(0)) if version else 0 patch = int(version.pop(0)) if version else 0 From 7d066085314bc5b37006cc44d0412bac52bd5a62 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Fri, 13 Jan 2023 16:00:19 +0100 Subject: [PATCH 05/14] Move version_string_to_int into separate function --- fdroidserver/update.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 38b11b2c..30f2f5ea 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -525,8 +525,20 @@ def insert_obbs(repodir, apps, apks): break +def version_string_to_int(version): + """Approximately convert a [Major].[Minor].[Patch] version string + consisting of numeric characters (0-9) and periods to a number. The + exponents are chosen such that it still fits in the 64bit index.yaml range. + """ + version = version.split('.') + major = int(version.pop(0)) if version else 0 + minor = int(version.pop(0)) if version else 0 + patch = int(version.pop(0)) if version else 0 + return major * 10**12 + minor * 10**6 + patch + + def process_ipa(repodir, apks): - """Scan the .ipa files in a given repo directory + """Scan the .ipa files in a given repo directory. Parameters ---------- @@ -557,13 +569,9 @@ def process_ipa(repodir, apks): if re.match("Payload/[^/]*.app/Info.plist", info.filename): with ipa_zip.open(info) as plist_file: plist = readPlist(plist_file) - # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring - version = plist["CFBundleShortVersionString"].split('.') - major = int(version.pop(0)) - minor = int(version.pop(0)) if version else 0 - patch = int(version.pop(0)) if version else 0 ipa["packageName"] = plist["CFBundleIdentifier"] - ipa["versionCode"] = major * 10**12 + minor * 10**6 + patch + # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring + ipa["versionCode"] = version_string_to_int(plist["CFBundleShortVersionString"]) ipa["versionName"] = plist["CFBundleShortVersionString"] ipa["usage"] = {k: v for k, v in plist.items() if 'Usage' in k} From e3d319f30b8f53549c1d1449572d7f44e8bb216b Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Fri, 13 Jan 2023 18:00:29 +0100 Subject: [PATCH 06/14] Update with suggestions --- fdroidserver/common.py | 4 ++-- fdroidserver/update.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index cfe60150..d52e61fd 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -102,7 +102,7 @@ STRICT_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-zA-Z]+(?:\d*[a-zA-Z_]*)*)(?: VALID_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-z_]+(?:\d*[a-zA-Z_]*)*)(?:\.[a-z_]+(?:\d*[a-zA-Z_]*)*)*$''', re.IGNORECASE) ANDROID_PLUGIN_REGEX = re.compile(r'''\s*(:?apply plugin:|id)\(?\s*['"](android|com\.android\.application)['"]\s*\)?''') -APPLE_BUNDLEiDENTIFIER_REGEX = re.compile(r'''^[a-zA-Z-.]*''') +APPLE_BUNDLEIDENTIFIER_REGEX = re.compile(r'''^[a-zA-Z-.]*''') SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?') GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:?([^'"]+)['"]''') @@ -2016,7 +2016,7 @@ def is_valid_package_name(name): """ return VALID_APPLICATION_ID_REGEX.match(name) is not None \ - or APPLE_BUNDLEiDENTIFIER_REGEX.match(name) is not None \ + or APPLE_BUNDLEIDENTIFIER_REGEX.match(name) is not None \ or FDROID_PACKAGE_NAME_REGEX.match(name) is not None diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 30f2f5ea..307fd5ec 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -528,7 +528,7 @@ def insert_obbs(repodir, apps, apks): def version_string_to_int(version): """Approximately convert a [Major].[Minor].[Patch] version string consisting of numeric characters (0-9) and periods to a number. The - exponents are chosen such that it still fits in the 64bit index.yaml range. + exponents are chosen such that it still fits in the 64bit JSON/Android range. """ version = version.split('.') major = int(version.pop(0)) if version else 0 From 60371093e243c0d6a6606e5a5d6168d95ee5b1d1 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Fri, 13 Jan 2023 18:11:34 +0100 Subject: [PATCH 07/14] Make python3-biplist optional --- fdroidserver/update.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 307fd5ec..9dadfd2a 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -46,7 +46,6 @@ except ImportError: import collections from binascii import hexlify -from biplist import readPlist from . import _ from . import common @@ -555,25 +554,28 @@ def process_ipa(repodir, apks): logging.error(_("Deleting unknown file: {path}").format(path=f)) os.remove(f) - for f in glob.glob(os.path.join(repodir, '*.ipa')): - ipa = {} - apks.append(ipa) + ipas = glob.glob(os.path.join(repodir, '*.ipa')) + if ipas: + from biplist import readPlist + for f in ipas: + ipa = {} + apks.append(ipa) - ipa["apkName"] = os.path.basename(f) - ipa["hash"] = common.sha256sum(f) - ipa["hashType"] = "sha256" - ipa["size"] = os.path.getsize(f) + ipa["apkName"] = os.path.basename(f) + ipa["hash"] = common.sha256sum(f) + ipa["hashType"] = "sha256" + ipa["size"] = os.path.getsize(f) - with zipfile.ZipFile(f) as ipa_zip: - for info in ipa_zip.infolist(): - if re.match("Payload/[^/]*.app/Info.plist", info.filename): - with ipa_zip.open(info) as plist_file: - plist = readPlist(plist_file) - ipa["packageName"] = plist["CFBundleIdentifier"] - # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring - ipa["versionCode"] = version_string_to_int(plist["CFBundleShortVersionString"]) - ipa["versionName"] = plist["CFBundleShortVersionString"] - ipa["usage"] = {k: v for k, v in plist.items() if 'Usage' in k} + with zipfile.ZipFile(f) as ipa_zip: + for info in ipa_zip.infolist(): + if re.match("Payload/[^/]*.app/Info.plist", info.filename): + with ipa_zip.open(info) as plist_file: + plist = readPlist(plist_file) + ipa["packageName"] = plist["CFBundleIdentifier"] + # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring + ipa["versionCode"] = version_string_to_int(plist["CFBundleShortVersionString"]) + ipa["versionName"] = plist["CFBundleShortVersionString"] + ipa["usage"] = {k: v for k, v in plist.items() if 'Usage' in k} def translate_per_build_anti_features(apps, apks): From ea9374ecf633cebbe7faf6694b9d33050111ce85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 30 Nov 2023 16:27:09 +0100 Subject: [PATCH 08/14] =?UTF-8?q?=F0=9F=9B=A0=EF=B8=8F=20=20update.py:=20f?= =?UTF-8?q?inish=20minimal=20IPA=20support?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This add a few missing pieces to get IPA support working. (added and lastUpdated dates + caching for ipa files) --- fdroidserver/update.py | 113 ++++++++++++++++++++++++++++------------- 1 file changed, 79 insertions(+), 34 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 9dadfd2a..0304507c 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -525,7 +525,10 @@ def insert_obbs(repodir, apps, apks): def version_string_to_int(version): - """Approximately convert a [Major].[Minor].[Patch] version string + """ + Convert sermver version designation to version code. + + Approximately convert a [Major].[Minor].[Patch] version string consisting of numeric characters (0-9) and periods to a number. The exponents are chosen such that it still fits in the 64bit JSON/Android range. """ @@ -536,46 +539,73 @@ def version_string_to_int(version): return major * 10**12 + minor * 10**6 + patch -def process_ipa(repodir, apks): - """Scan the .ipa files in a given repo directory. +def parse_ipa(ipa_path, file_size, sha256): + from biplist import readPlist + + ipa = { + "apkName": os.path.basename(ipa_path), + "hash": sha256, + "hashType": "sha256", + "size": file_size, + } + + with zipfile.ZipFile(ipa_path) as ipa_zip: + for info in ipa_zip.infolist(): + if re.match("Payload/[^/]*.app/Info.plist", info.filename): + with ipa_zip.open(info) as plist_file: + plist = readPlist(plist_file) + ipa["packageName"] = plist["CFBundleIdentifier"] + # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring + ipa["versionCode"] = version_string_to_int(plist["CFBundleShortVersionString"]) + ipa["versionName"] = plist["CFBundleShortVersionString"] + ipa["usage"] = {k: v for k, v in plist.items() if 'Usage' in k} + return ipa + + +def scan_repo_for_ipas(apkcache, repodir, knownapks): + """Scan for IPA files in a given repo directory. Parameters ---------- + apkcache + cache dictionary containting cached file infos from previous runs repodir repo directory to scan - apps - list of current, valid apps - apks - current information on all APKs + knownapks + list of all known files, as per metadata.read_metadata + + Returns + ------- + ipas + list of file infos for ipa files in ./repo folder + cachechanged + ture if new ipa files were found and added to `apkcache` """ - def ipaWarnDelete(f, msg): - logging.warning(msg + ' ' + f) - if options.delete_unknown: - logging.error(_("Deleting unknown file: {path}").format(path=f)) - os.remove(f) + cachechanged = False + ipas = [] + for ipa_path in glob.glob(os.path.join(repodir, '*.ipa')): + ipa_name = os.path.basename(ipa_path) - ipas = glob.glob(os.path.join(repodir, '*.ipa')) - if ipas: - from biplist import readPlist - for f in ipas: - ipa = {} - apks.append(ipa) + file_size = os.stat(ipa_path).st_size + if file_size == 0: + raise FDroidException(_('{path} is zero size!') + .format(path=ipa_path)) - ipa["apkName"] = os.path.basename(f) - ipa["hash"] = common.sha256sum(f) - ipa["hashType"] = "sha256" - ipa["size"] = os.path.getsize(f) + sha256 = common.sha256sum(ipa_path) + ipa = apkcache.get(ipa_name, {}) - with zipfile.ZipFile(f) as ipa_zip: - for info in ipa_zip.infolist(): - if re.match("Payload/[^/]*.app/Info.plist", info.filename): - with ipa_zip.open(info) as plist_file: - plist = readPlist(plist_file) - ipa["packageName"] = plist["CFBundleIdentifier"] - # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring - ipa["versionCode"] = version_string_to_int(plist["CFBundleShortVersionString"]) - ipa["versionName"] = plist["CFBundleShortVersionString"] - ipa["usage"] = {k: v for k, v in plist.items() if 'Usage' in k} + if ipa.get('hash') != sha256: + ipa = parse_ipa(ipa_path, file_size, sha256) + apkcache[ipa_name] = ipa + cachechanged = True + + added = knownapks.recordapk(ipa_name, ipa['packageName']) + if added: + ipa['added'] = added + + ipas.append(ipa) + + return ipas, cachechanged def translate_per_build_anti_features(apps, apks): @@ -1175,7 +1205,10 @@ def insert_localized_app_metadata(apps): def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): - """Scan a repo for all files with an extension except APK/OBB. + """Scan a repo for all files with an extension except APK/OBB/IPA. + + This allows putting all kinds of files into repostories. E.g. Media Files, + Zip archives, ... Parameters ---------- @@ -1192,22 +1225,29 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): repo_files = [] repodir = repodir.encode() for name in os.listdir(repodir): + # skip files based on file extensions, that are handled elsewhere file_extension = common.get_file_extension(name) if file_extension in ('apk', 'obb', 'ipa'): continue + + # skip source tarballs generated by fdroidserver filename = os.path.join(repodir, name) name_utf8 = name.decode() if filename.endswith(b'_src.tar.gz'): logging.debug(_('skipping source tarball: {path}') .format(path=filename.decode())) continue + + # skip all other files generated by fdroidserver if not common.is_repo_file(filename): continue + stat = os.stat(filename) if stat.st_size == 0: raise FDroidException(_('{path} is zero size!') .format(path=filename)) + # load file infos from cache if not stale shasum = common.sha256sum(filename) usecache = False if name_utf8 in apkcache: @@ -1220,6 +1260,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): logging.debug(_("Ignoring stale cache data for {apkfilename}") .format(apkfilename=name_utf8)) + # scan file if info wasn't in cache if not usecache: logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8)) repo_file = collections.OrderedDict() @@ -2182,7 +2223,6 @@ def prepare_apps(apps, apks, repodir): ------- the relevant subset of apps (as a deepcopy) """ - process_ipa(repodir, apks) apps_with_packages = get_apps_with_packages(apps, apks) apply_info_from_latest_apk(apps_with_packages, apks) insert_funding_yml_donation_links(apps) @@ -2309,6 +2349,11 @@ def main(): options.use_date_from_apk) cachechanged = cachechanged or fcachechanged apks += files + + ipas, icachechanged = scan_repo_for_ipas(apkcache, repodirs[0], knownapks) + cachechanged = cachechanged or icachechanged + apks += ipas + appid_has_apks = set() appid_has_repo_files = set() remove_apks = [] From 3ee91d17775a70eae68db0bfeff53fea03a7d28c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 30 Nov 2023 17:28:22 +0100 Subject: [PATCH 09/14] =?UTF-8?q?=F0=9F=A7=B4=20force=20android=20package?= =?UTF-8?q?=20names=20for=20IPAs=20for=20now?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fdroidserver/common.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index d52e61fd..e97cedf5 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -102,7 +102,6 @@ STRICT_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-zA-Z]+(?:\d*[a-zA-Z_]*)*)(?: VALID_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-z_]+(?:\d*[a-zA-Z_]*)*)(?:\.[a-z_]+(?:\d*[a-zA-Z_]*)*)*$''', re.IGNORECASE) ANDROID_PLUGIN_REGEX = re.compile(r'''\s*(:?apply plugin:|id)\(?\s*['"](android|com\.android\.application)['"]\s*\)?''') -APPLE_BUNDLEIDENTIFIER_REGEX = re.compile(r'''^[a-zA-Z-.]*''') SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?') GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:?([^'"]+)['"]''') @@ -2016,7 +2015,6 @@ def is_valid_package_name(name): """ return VALID_APPLICATION_ID_REGEX.match(name) is not None \ - or APPLE_BUNDLEIDENTIFIER_REGEX.match(name) is not None \ or FDROID_PACKAGE_NAME_REGEX.match(name) is not None From c288317530971f036c7170941f359040579441a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Mon, 18 Dec 2023 12:58:37 +0100 Subject: [PATCH 10/14] =?UTF-8?q?=F0=9F=AA=A8=20version=20string=20convers?= =?UTF-8?q?ion:=20error=20handling+tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fdroidserver/update.py | 13 +++++++++---- tests/update.TestCase | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 0304507c..7934ffa8 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -524,6 +524,9 @@ def insert_obbs(repodir, apps, apks): break +VERSION_STRING_RE = re.compile(r'^([0-9]+)\.([0-9]+)\.([0-9]+)$') + + def version_string_to_int(version): """ Convert sermver version designation to version code. @@ -532,10 +535,12 @@ def version_string_to_int(version): consisting of numeric characters (0-9) and periods to a number. The exponents are chosen such that it still fits in the 64bit JSON/Android range. """ - version = version.split('.') - major = int(version.pop(0)) if version else 0 - minor = int(version.pop(0)) if version else 0 - patch = int(version.pop(0)) if version else 0 + m = VERSION_STRING_RE.match(version) + if not m: + raise ValueError(f"invalid version string '{version}'") + major = int(m.group(1)) + minor = int(m.group(2)) + patch = int(m.group(3)) return major * 10**12 + minor * 10**6 + patch diff --git a/tests/update.TestCase b/tests/update.TestCase index abce3a30..0bc78462 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -1923,6 +1923,28 @@ class UpdateTest(unittest.TestCase): ) +class TestUpdateVersionStringToInt(unittest.TestCase): + + def test_version_string_to_int(self): + self.assertEqual(fdroidserver.update.version_string_to_int("1.2.3"), 1000002000003) + self.assertEqual(fdroidserver.update.version_string_to_int("0.0.0003"), 3) + self.assertEqual(fdroidserver.update.version_string_to_int("0.0.0"), 0) + self.assertEqual(fdroidserver.update.version_string_to_int("4321.321.21"), 4321000321000021) + self.assertEqual(fdroidserver.update.version_string_to_int("18446744.073709.551615"), 18446744073709551615) + + def test_version_string_to_int_value_errors(self): + with self.assertRaises(ValueError): + fdroidserver.update.version_string_to_int("1.2.3a") + with self.assertRaises(ValueError): + fdroidserver.update.version_string_to_int("asdfasdf") + with self.assertRaises(ValueError): + fdroidserver.update.version_string_to_int("1.2.-3") + with self.assertRaises(ValueError): + fdroidserver.update.version_string_to_int("-1.2.-3") + with self.assertRaises(ValueError): + fdroidserver.update.version_string_to_int("0.0.0x3") + + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) @@ -1938,4 +1960,5 @@ if __name__ == "__main__": newSuite = unittest.TestSuite() newSuite.addTest(unittest.makeSuite(UpdateTest)) + newSuite.addTest(unittest.makeSuite(TestUpdateVersionStringToInt)) unittest.main(failfast=False) From 995118bcd20a55fda6545fb087a181b48a3d98ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Mon, 18 Dec 2023 13:35:06 +0100 Subject: [PATCH 11/14] =?UTF-8?q?=F0=9F=A5=94=20add=20strapped=20IPA=20fil?= =?UTF-8?q?e=20and=20test=20for=20parse=5Fipa?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fdroidserver/update.py | 1 - tests/com.fake.IpaApp_1000000000001.ipa | Bin 0 -> 1722 bytes tests/update.TestCase | 13 +++++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 tests/com.fake.IpaApp_1000000000001.ipa diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 7934ffa8..b5b956a2 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -563,7 +563,6 @@ def parse_ipa(ipa_path, file_size, sha256): # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring ipa["versionCode"] = version_string_to_int(plist["CFBundleShortVersionString"]) ipa["versionName"] = plist["CFBundleShortVersionString"] - ipa["usage"] = {k: v for k, v in plist.items() if 'Usage' in k} return ipa diff --git a/tests/com.fake.IpaApp_1000000000001.ipa b/tests/com.fake.IpaApp_1000000000001.ipa new file mode 100644 index 0000000000000000000000000000000000000000..d392cb940a8585307deb50d6f791ebc18fbd7a37 GIT binary patch literal 1722 zcma)+c{J2(9L9fRY{wq5gvl}ySuVO)lCjJ%7ulN8SQ29i2{X2i%C#gSVT|QoGKiUs zFry@K#buCPOlWLlzho&YijJF3sy}qjec$uG@Av(l_j#W8uaBJ-n3E6K9lVL@XTKC* zH(o#lu){`N3c`A6IXDUcZVzByyXzfD9I*SW5(n?668qfTQ(54!CODiX7KhW?6YzeC zLFZBDmpB0c&IB@58B};Ov)7%$t%tY z*yzN7dp0iFzNeFUrtNp4{imKYa#iXV+h?a*e3`eDb7~kRl6f()H5Q^IC zVH!&xs7#c1t-QHsy6h_P-x1nQbjg(rP;?4QvU3Tb&8rJ5AD454hlCDQfk1&U5}0WHjF!=0b;9^^<2rg=&b|cbDNFAPguv> z{13u>;}iZ?#5>B3yjvG4ZiAC7RTAtqaYv*}+tY^C+{M%Q;F4ELz2zz$d_ngLDc#Ij zC+>pq@IsOUYG|$pHLr`3V&(T`NeT1Odq1wN0?qGmg0s);xi5C)!}%Mj5h67u(TnPn z%pF7~zGwl{?-Vvk3J@}255&EHl#4v|G)PUxN`eDo$g0qCge#>Neu`8fH%%3m@V=U7QrTUBbAo7|e-Nr4f~TRcRT&O`95<{Oob}%N z6TPjKdos&UT%?HSTc?1Ha0Kk#GB^>+`1o-&hE!AdyJHLnQ$Md}q`RL*p bpZ!C=)BXR&adGed3H*6VfB+zcYxnjahOgOe literal 0 HcmV?d00001 diff --git a/tests/update.TestCase b/tests/update.TestCase index 0bc78462..9d1dea23 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -1922,6 +1922,19 @@ class UpdateTest(unittest.TestCase): index['repo'][CATEGORIES_CONFIG_NAME], ) + def test_parse_ipa(self): + result = fdroidserver.update.parse_ipa('./com.fake.IpaApp_1000000000001.ipa', 'fake_size', 'fake_sha') + self.maxDiff = None + self.assertDictEqual(result, { + 'apkName': 'com.fake.IpaApp_1000000000001.ipa', + 'hash': 'fake_sha', + 'hashType': 'sha256', + 'packageName': 'org.onionshare.OnionShare', + 'size': 'fake_size', + 'versionCode': 1000000000001, + 'versionName': '1.0.1', + }) + class TestUpdateVersionStringToInt(unittest.TestCase): From 7211e9f9b43433154231ce25ef7f724ed9933fc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 20 Dec 2023 04:11:05 +0100 Subject: [PATCH 12/14] =?UTF-8?q?=F0=9F=8D=B2=20add=20unit=20test=20for=20?= =?UTF-8?q?update.scan=5Frepo=5Ffor=5Fipas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fdroidserver/update.py | 10 ++++----- tests/update.TestCase | 49 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index b5b956a2..26e248d5 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -49,10 +49,10 @@ from binascii import hexlify from . import _ from . import common -from . import index from . import metadata from .common import DEFAULT_LOCALE from .exception import BuildException, FDroidException, VerificationException +import fdroidserver.index from PIL import Image, PngImagePlugin @@ -599,7 +599,7 @@ def scan_repo_for_ipas(apkcache, repodir, knownapks): ipa = apkcache.get(ipa_name, {}) if ipa.get('hash') != sha256: - ipa = parse_ipa(ipa_path, file_size, sha256) + ipa = fdroidserver.update.parse_ipa(ipa_path, file_size, sha256) apkcache[ipa_name] = ipa cachechanged = True @@ -2433,7 +2433,7 @@ def main(): if len(repodirs) > 1: archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older']) archived_apps = prepare_apps(apps, archapks, repodirs[1]) - index.make(archived_apps, archapks, repodirs[1], True) + fdroidserver.index.make(archived_apps, archapks, repodirs[1], True) repoapps = prepare_apps(apps, apks, repodirs[0]) @@ -2446,13 +2446,13 @@ def main(): app_dict = dict() app_dict[appid] = app if os.path.isdir(repodir): - index.make(app_dict, apks, repodir, False) + fdroidserver.index.make(app_dict, apks, repodir, False) else: logging.info(_('Skipping index generation for {appid}').format(appid=appid)) return # Make the index for the main repo... - index.make(repoapps, apks, repodirs[0], False) + fdroidserver.index.make(repoapps, apks, repodirs[0], False) git_remote = config.get('binary_transparency_remote') if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')): diff --git a/tests/update.TestCase b/tests/update.TestCase index 9d1dea23..d8f278b6 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -1958,6 +1958,54 @@ class TestUpdateVersionStringToInt(unittest.TestCase): fdroidserver.update.version_string_to_int("0.0.0x3") +class TestScanRepoForIpas(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + + def test_scan_repo_for_ipas_no_cache(self): + self.maxDiff = None + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + os.mkdir("repo") + with open('repo/abc.Def_123.ipa', 'w') as f: + f.write('abc') + with open('repo/xyz.XXX_123.ipa', 'w') as f: + f.write('xyz') + + apkcache = mock.MagicMock() + # apkcache['a'] = 1 + repodir = "repo" + knownapks = mock.MagicMock() + + def mocked_parse(p, s, c): + return { + 'packageName': 'abc' if 'abc' in p else 'xyz' + } + + with mock.patch('fdroidserver.update.parse_ipa', mocked_parse): + ipas, checkchanged = fdroidserver.update.scan_repo_for_ipas(apkcache, repodir, knownapks) + + self.assertEqual(checkchanged, True) + self.assertEqual(len(ipas), 2) + self.assertEqual(ipas[0]['packageName'], 'xyz') + self.assertEqual(ipas[1]['packageName'], 'abc') + + self.assertEqual(apkcache.__setitem__.mock_calls[0].args[1]['packageName'], 'xyz') + self.assertEqual(apkcache.__setitem__.mock_calls[1].args[1]['packageName'], 'abc') + self.assertEqual(apkcache.__setitem__.call_count, 2) + + knownapks.recordapk.call_count = 2 + self.assertEqual( + knownapks.recordapk.mock_calls[0], + unittest.mock.call('xyz.XXX_123.ipa', 'xyz'), + ) + # skipping one call here, because accessing `if added:` shows up in mock_calls + self.assertEqual( + knownapks.recordapk.mock_calls[2], + unittest.mock.call('abc.Def_123.ipa', 'abc'), + ) + + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) @@ -1974,4 +2022,5 @@ if __name__ == "__main__": newSuite = unittest.TestSuite() newSuite.addTest(unittest.makeSuite(UpdateTest)) newSuite.addTest(unittest.makeSuite(TestUpdateVersionStringToInt)) + newSuite.addTest(unittest.makeSuite(TestScanRepoForIpas)) unittest.main(failfast=False) From 881943a0db430db07c4ad33f8bad73b00f5ab121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sat, 30 Dec 2023 00:06:16 +0100 Subject: [PATCH 13/14] =?UTF-8?q?=F0=9F=A5=94=20install=20biplist=20for=20?= =?UTF-8?q?ci=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit biplist is an optional python dependency required for processing iOS apps. (.ipa files) --- .gitlab-ci.yml | 6 +++--- setup.py | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 188a2b90..00fd097c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,7 +20,7 @@ buildserver run-tests: image: registry.gitlab.com/fdroid/fdroidserver:buildserver script: - apt-get update - - apt-get install gnupg-agent python3-babel python3-clint python3-pycountry + - apt-get install gnupg-agent python3-babel python3-biplist python3-clint python3-pycountry - ./tests/run-tests # make sure that translations do not cause stacktraces - cd $CI_PROJECT_DIR/locale @@ -154,8 +154,8 @@ ubuntu_jammy_pip: - $pip install sdkmanager - sdkmanager 'build-tools;33.0.0' - # pycountry is only for linting config/mirrors.yml, so its not in setup.py - - $pip install pycountry + # Install extras_require.optional from setup.py + - $pip install biplist pycountry - $pip install dist/fdroidserver-*.tar.gz - tar xzf dist/fdroidserver-*.tar.gz diff --git a/setup.py b/setup.py index 522c3377..afff96b4 100755 --- a/setup.py +++ b/setup.py @@ -108,7 +108,11 @@ setup( 'sdkmanager >= 0.6.4', 'yamllint', ], + # Some requires are only needed for very limited cases: + # * biplist is only used for parsing Apple .ipa files + # * pycountry is only for linting config/mirrors.yml extras_require={ + 'optional': ['biplist', 'pycountry'], 'test': ['pyjks', 'html5print'], 'docs': [ 'sphinx', From 8b5a61bb257fefeeadcc4d30141881294e656f6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sat, 30 Dec 2023 13:51:37 +0100 Subject: [PATCH 14/14] =?UTF-8?q?=E2=9B=B0=EF=B8=8F=20=20make=20ipa=20rela?= =?UTF-8?q?ted=20test=20cases=20more=20robust?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- MANIFEST.in | 1 + tests/update.TestCase | 25 +++++++++++-------------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 1aed9975..05a022b2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -543,6 +543,7 @@ include tests/build-tools/28.0.3/aapt-output-souch.smsbypass_9.txt include tests/build-tools/generate.sh include tests/check-fdroid-apk include tests/checkupdates.TestCase +include tests/com.fake.IpaApp_1000000000001.ipa include tests/common.TestCase include tests/config.py include tests/config/antiFeatures.yml diff --git a/tests/update.TestCase b/tests/update.TestCase index d8f278b6..f1c07fd9 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -1923,7 +1923,8 @@ class UpdateTest(unittest.TestCase): ) def test_parse_ipa(self): - result = fdroidserver.update.parse_ipa('./com.fake.IpaApp_1000000000001.ipa', 'fake_size', 'fake_sha') + ipa_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'com.fake.IpaApp_1000000000001.ipa') + result = fdroidserver.update.parse_ipa(ipa_path, 'fake_size', 'fake_sha') self.maxDiff = None self.assertDictEqual(result, { 'apkName': 'com.fake.IpaApp_1000000000001.ipa', @@ -1978,6 +1979,7 @@ class TestScanRepoForIpas(unittest.TestCase): knownapks = mock.MagicMock() def mocked_parse(p, s, c): + # pylint: disable=unused-argument return { 'packageName': 'abc' if 'abc' in p else 'xyz' } @@ -1987,23 +1989,18 @@ class TestScanRepoForIpas(unittest.TestCase): self.assertEqual(checkchanged, True) self.assertEqual(len(ipas), 2) - self.assertEqual(ipas[0]['packageName'], 'xyz') - self.assertEqual(ipas[1]['packageName'], 'abc') + package_names_in_ipas = [x['packageName'] for x in ipas] + self.assertTrue('abc' in package_names_in_ipas) + self.assertTrue('xyz' in package_names_in_ipas) - self.assertEqual(apkcache.__setitem__.mock_calls[0].args[1]['packageName'], 'xyz') - self.assertEqual(apkcache.__setitem__.mock_calls[1].args[1]['packageName'], 'abc') + apkcache_setter_package_name = [x.args[1]['packageName'] for x in apkcache.__setitem__.mock_calls] + self.assertTrue('abc' in apkcache_setter_package_name) + self.assertTrue('xyz' in apkcache_setter_package_name) self.assertEqual(apkcache.__setitem__.call_count, 2) knownapks.recordapk.call_count = 2 - self.assertEqual( - knownapks.recordapk.mock_calls[0], - unittest.mock.call('xyz.XXX_123.ipa', 'xyz'), - ) - # skipping one call here, because accessing `if added:` shows up in mock_calls - self.assertEqual( - knownapks.recordapk.mock_calls[2], - unittest.mock.call('abc.Def_123.ipa', 'abc'), - ) + self.assertTrue(unittest.mock.call('abc.Def_123.ipa', 'abc') in knownapks.recordapk.mock_calls) + self.assertTrue(unittest.mock.call('xyz.XXX_123.ipa', 'xyz') in knownapks.recordapk.mock_calls) if __name__ == "__main__":