Merge branch 'ipa-support' into 'master'

minimal IPA support

Closes #1067

See merge request fdroid/fdroidserver!1413

.gitlab-ci.yml

@@ -20,7 +20,7 @@ buildserver run-tests:
   image: registry.gitlab.com/fdroid/fdroidserver:buildserver
   script:
     - apt-get update
-    - apt-get install gnupg-agent python3-babel python3-clint python3-pycountry
+    - apt-get install gnupg-agent python3-babel python3-biplist python3-clint python3-pycountry
     - ./tests/run-tests
     # make sure that translations do not cause stacktraces
     - cd $CI_PROJECT_DIR/locale
@@ -42,7 +42,7 @@ metadata_v0:
   image: registry.gitlab.com/fdroid/fdroidserver:buildserver
   variables:
     GIT_DEPTH: 1000
-    RELEASE_COMMIT_ID: a1c4f803de8d4dc92ebd6b571a493183d14a00bf  # after ArchivePolicy: 0
+    RELEASE_COMMIT_ID: 50aa35772b058e76b950c01e16019c072c191b73  # after switching to `git rev-parse`
   script:
     - git fetch https://gitlab.com/fdroid/fdroidserver.git $RELEASE_COMMIT_ID
     - cd tests
@@ -154,8 +154,8 @@ ubuntu_jammy_pip:
     - $pip install sdkmanager
     - sdkmanager 'build-tools;33.0.0'
 
-    # pycountry is only for linting config/mirrors.yml, so its not in setup.py
+    # Install extras_require.optional from setup.py
-    - $pip install pycountry
+    - $pip install biplist pycountry
 
     - $pip install dist/fdroidserver-*.tar.gz
     - tar xzf dist/fdroidserver-*.tar.gz

MANIFEST.in

@@ -543,6 +543,7 @@ include tests/build-tools/28.0.3/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/generate.sh
 include tests/check-fdroid-apk
 include tests/checkupdates.TestCase
+include tests/com.fake.IpaApp_1000000000001.ipa
 include tests/common.TestCase
 include tests/config.py
 include tests/config/antiFeatures.yml

fdroidserver/update.py

@@ -49,10 +49,10 @@ from binascii import hexlify
 
 from . import _
 from . import common
-from . import index
 from . import metadata
 from .common import DEFAULT_LOCALE
 from .exception import BuildException, FDroidException, VerificationException
+import fdroidserver.index
 
 from PIL import Image, PngImagePlugin
 
@@ -524,6 +524,94 @@ def insert_obbs(repodir, apps, apks):
                 break
 
 
+VERSION_STRING_RE = re.compile(r'^([0-9]+)\.([0-9]+)\.([0-9]+)$')
+
+
+def version_string_to_int(version):
+    """
+    Convert sermver version designation to version code.
+
+    Approximately convert a [Major].[Minor].[Patch] version string
+    consisting of numeric characters (0-9) and periods to a number. The
+    exponents are chosen such that it still fits in the 64bit JSON/Android range.
+    """
+    m = VERSION_STRING_RE.match(version)
+    if not m:
+        raise ValueError(f"invalid version string '{version}'")
+    major = int(m.group(1))
+    minor = int(m.group(2))
+    patch = int(m.group(3))
+    return major * 10**12 + minor * 10**6 + patch
+
+
+def parse_ipa(ipa_path, file_size, sha256):
+    from biplist import readPlist
+
+    ipa = {
+        "apkName": os.path.basename(ipa_path),
+        "hash": sha256,
+        "hashType": "sha256",
+        "size": file_size,
+    }
+
+    with zipfile.ZipFile(ipa_path) as ipa_zip:
+        for info in ipa_zip.infolist():
+            if re.match("Payload/[^/]*.app/Info.plist", info.filename):
+                with ipa_zip.open(info) as plist_file:
+                    plist = readPlist(plist_file)
+                    ipa["packageName"] = plist["CFBundleIdentifier"]
+                    # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring
+                    ipa["versionCode"] = version_string_to_int(plist["CFBundleShortVersionString"])
+                    ipa["versionName"] = plist["CFBundleShortVersionString"]
+    return ipa
+
+
+def scan_repo_for_ipas(apkcache, repodir, knownapks):
+    """Scan for IPA files in a given repo directory.
+
+    Parameters
+    ----------
+    apkcache
+      cache dictionary containting cached file infos from previous runs
+    repodir
+      repo directory to scan
+    knownapks
+      list of all known files, as per metadata.read_metadata
+
+    Returns
+    -------
+    ipas
+      list of file infos for ipa files in ./repo folder
+    cachechanged
+      ture if new ipa files were found and added to `apkcache`
+    """
+    cachechanged = False
+    ipas = []
+    for ipa_path in glob.glob(os.path.join(repodir, '*.ipa')):
+        ipa_name = os.path.basename(ipa_path)
+
+        file_size = os.stat(ipa_path).st_size
+        if file_size == 0:
+            raise FDroidException(_('{path} is zero size!')
+                                  .format(path=ipa_path))
+
+        sha256 = common.sha256sum(ipa_path)
+        ipa = apkcache.get(ipa_name, {})
+
+        if ipa.get('hash') != sha256:
+            ipa = fdroidserver.update.parse_ipa(ipa_path, file_size, sha256)
+            apkcache[ipa_name] = ipa
+            cachechanged = True
+
+        added = knownapks.recordapk(ipa_name, ipa['packageName'])
+        if added:
+            ipa['added'] = added
+
+        ipas.append(ipa)
+
+    return ipas, cachechanged
+
+
 def translate_per_build_anti_features(apps, apks):
     """Grab the anti-features list from the build metadata.
 
@@ -1121,7 +1209,10 @@ def insert_localized_app_metadata(apps):
 
 
 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
-    """Scan a repo for all files with an extension except APK/OBB.
+    """Scan a repo for all files with an extension except APK/OBB/IPA.
+
+    This allows putting all kinds of files into repostories. E.g. Media Files,
+    Zip archives, ...
 
     Parameters
     ----------
@@ -1138,22 +1229,29 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
     repo_files = []
     repodir = repodir.encode()
     for name in os.listdir(repodir):
+        # skip files based on file extensions, that are handled elsewhere
         file_extension = common.get_file_extension(name)
-        if file_extension in ('apk', 'obb'):
+        if file_extension in ('apk', 'obb', 'ipa'):
             continue
+
+        # skip source tarballs generated by fdroidserver
         filename = os.path.join(repodir, name)
         name_utf8 = name.decode()
         if filename.endswith(b'_src.tar.gz'):
             logging.debug(_('skipping source tarball: {path}')
                           .format(path=filename.decode()))
             continue
+
+        # skip all other files generated by fdroidserver
         if not common.is_repo_file(filename):
             continue
+
         stat = os.stat(filename)
         if stat.st_size == 0:
             raise FDroidException(_('{path} is zero size!')
                                   .format(path=filename))
 
+        # load file infos from cache if not stale
         shasum = common.sha256sum(filename)
         usecache = False
         if name_utf8 in apkcache:
@@ -1166,6 +1264,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
                               .format(apkfilename=name_utf8))
 
+        # scan file if info wasn't in cache
         if not usecache:
             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
             repo_file = collections.OrderedDict()
@@ -2254,6 +2353,11 @@ def main():
                                            options.use_date_from_apk)
     cachechanged = cachechanged or fcachechanged
     apks += files
+
+    ipas, icachechanged = scan_repo_for_ipas(apkcache, repodirs[0], knownapks)
+    cachechanged = cachechanged or icachechanged
+    apks += ipas
+
     appid_has_apks = set()
     appid_has_repo_files = set()
     remove_apks = []
@@ -2329,7 +2433,7 @@ def main():
     if len(repodirs) > 1:
         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
         archived_apps = prepare_apps(apps, archapks, repodirs[1])
-        index.make(archived_apps, archapks, repodirs[1], True)
+        fdroidserver.index.make(archived_apps, archapks, repodirs[1], True)
 
     repoapps = prepare_apps(apps, apks, repodirs[0])
 
@@ -2342,13 +2446,13 @@ def main():
             app_dict = dict()
             app_dict[appid] = app
             if os.path.isdir(repodir):
-                index.make(app_dict, apks, repodir, False)
+                fdroidserver.index.make(app_dict, apks, repodir, False)
             else:
                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
         return
 
     # Make the index for the main repo...
-    index.make(repoapps, apks, repodirs[0], False)
+    fdroidserver.index.make(repoapps, apks, repodirs[0], False)
 
     git_remote = config.get('binary_transparency_remote')
     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):

setup.py

@@ -108,7 +108,11 @@ setup(
         'sdkmanager >= 0.6.4',
         'yamllint',
     ],
+    # Some requires are only needed for very limited cases:
+    # * biplist is only used for parsing Apple .ipa files
+    # * pycountry is only for linting config/mirrors.yml
     extras_require={
+        'optional': ['biplist', 'pycountry'],
         'test': ['pyjks', 'html5print'],
         'docs': [
             'sphinx',

tests/update.TestCase

@@ -1922,6 +1922,86 @@ class UpdateTest(unittest.TestCase):
             index['repo'][CATEGORIES_CONFIG_NAME],
         )
 
+    def test_parse_ipa(self):
+        ipa_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'com.fake.IpaApp_1000000000001.ipa')
+        result = fdroidserver.update.parse_ipa(ipa_path, 'fake_size', 'fake_sha')
+        self.maxDiff = None
+        self.assertDictEqual(result, {
+            'apkName': 'com.fake.IpaApp_1000000000001.ipa',
+            'hash': 'fake_sha',
+            'hashType': 'sha256',
+            'packageName': 'org.onionshare.OnionShare',
+            'size': 'fake_size',
+            'versionCode': 1000000000001,
+            'versionName': '1.0.1',
+        })
+
+
+class TestUpdateVersionStringToInt(unittest.TestCase):
+
+    def test_version_string_to_int(self):
+        self.assertEqual(fdroidserver.update.version_string_to_int("1.2.3"), 1000002000003)
+        self.assertEqual(fdroidserver.update.version_string_to_int("0.0.0003"), 3)
+        self.assertEqual(fdroidserver.update.version_string_to_int("0.0.0"), 0)
+        self.assertEqual(fdroidserver.update.version_string_to_int("4321.321.21"), 4321000321000021)
+        self.assertEqual(fdroidserver.update.version_string_to_int("18446744.073709.551615"), 18446744073709551615)
+
+    def test_version_string_to_int_value_errors(self):
+        with self.assertRaises(ValueError):
+            fdroidserver.update.version_string_to_int("1.2.3a")
+        with self.assertRaises(ValueError):
+            fdroidserver.update.version_string_to_int("asdfasdf")
+        with self.assertRaises(ValueError):
+            fdroidserver.update.version_string_to_int("1.2.-3")
+        with self.assertRaises(ValueError):
+            fdroidserver.update.version_string_to_int("-1.2.-3")
+        with self.assertRaises(ValueError):
+            fdroidserver.update.version_string_to_int("0.0.0x3")
+
+
+class TestScanRepoForIpas(unittest.TestCase):
+
+    def setUp(self):
+        self.maxDiff = None
+
+    def test_scan_repo_for_ipas_no_cache(self):
+        self.maxDiff = None
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            os.mkdir("repo")
+            with open('repo/abc.Def_123.ipa', 'w') as f:
+                f.write('abc')
+            with open('repo/xyz.XXX_123.ipa', 'w') as f:
+                f.write('xyz')
+
+            apkcache = mock.MagicMock()
+            # apkcache['a'] = 1
+            repodir = "repo"
+            knownapks = mock.MagicMock()
+
+            def mocked_parse(p, s, c):
+                # pylint: disable=unused-argument
+                return {
+                    'packageName': 'abc' if 'abc' in p else 'xyz'
+                }
+
+            with mock.patch('fdroidserver.update.parse_ipa', mocked_parse):
+                ipas, checkchanged = fdroidserver.update.scan_repo_for_ipas(apkcache, repodir, knownapks)
+
+            self.assertEqual(checkchanged, True)
+            self.assertEqual(len(ipas), 2)
+            package_names_in_ipas = [x['packageName'] for x in ipas]
+            self.assertTrue('abc' in package_names_in_ipas)
+            self.assertTrue('xyz' in package_names_in_ipas)
+
+            apkcache_setter_package_name = [x.args[1]['packageName'] for x in apkcache.__setitem__.mock_calls]
+            self.assertTrue('abc' in apkcache_setter_package_name)
+            self.assertTrue('xyz' in apkcache_setter_package_name)
+            self.assertEqual(apkcache.__setitem__.call_count, 2)
+
+            knownapks.recordapk.call_count = 2
+            self.assertTrue(unittest.mock.call('abc.Def_123.ipa', 'abc') in knownapks.recordapk.mock_calls)
+            self.assertTrue(unittest.mock.call('xyz.XXX_123.ipa', 'xyz') in knownapks.recordapk.mock_calls)
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))
@@ -1938,4 +2018,6 @@ if __name__ == "__main__":
 
     newSuite = unittest.TestSuite()
     newSuite.addTest(unittest.makeSuite(UpdateTest))
+    newSuite.addTest(unittest.makeSuite(TestUpdateVersionStringToInt))
+    newSuite.addTest(unittest.makeSuite(TestScanRepoForIpas))
     unittest.main(failfast=False)