Merge branch 'ios-metadata' into 'master'

support iOS fastlane metadata

See merge request fdroid/fdroidserver!1426

fdroidserver/update.py

@@ -1136,6 +1136,9 @@ def insert_localized_app_metadata(apps):
                         if base not in apps[packageName] or not isinstance(apps[packageName][base], collections.OrderedDict):
                             apps[packageName][base] = collections.OrderedDict()
                         apps[packageName][base][locale] = common.file_entry(dst)
+
+            # copy screenshots from local source code checkout into wellknown
+            # location in repo directory
             for d in dirs:
                 if d in SCREENSHOT_DIRS:
                     if locale == 'images':
@@ -1148,6 +1151,8 @@ def insert_localized_app_metadata(apps):
                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
                             _strip_and_copy_image(f, screenshotdestdir)
 
+
+def ingest_screenshots_from_repo_dir(apps):
     repodirs = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z]*')))
     for d in repodirs:
         if not os.path.isdir(d):
@@ -1208,6 +1213,127 @@ def insert_localized_app_metadata(apps):
                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 
 
+LANG_CODE = re.compile(r'^[a-z]{2}([-_][A-Z][a-zA-Z]{1,3})?$')
+
+
+FASTLANE_IOS_MAP = {
+    "name.txt": 'name',
+    "subtitle.txt": 'summary',
+    "description.txt": 'description',
+}
+
+
+def parse_ios_screenshot_name(path):
+    """
+    Infer type and categorization info from screenshot file name.
+
+    This is not really an exact algorithm, it's based on filenames observed in
+    the wild.
+    """
+    s = path.stem.split('@')
+    if len(s) >= 2:
+        if "iphone" in s[0].lower():
+            return ("phoneScreenshots", s[0].strip(), ('@'.join(s[1:])).split('-')[0].strip())
+        elif "ipad" in s[0].lower():
+            return ("tenInchScreenshots", s[0].strip(), ('@'.join(s[1:])).split('-')[0].strip())
+    else:
+        fragments = path.stem.lower().split("_")
+        device = "unknown"
+        os = "unknown"
+        screenshot_type = "phoneScreenshots"
+        for f in fragments:
+            if "iphone" in f:
+                device = f
+                continue
+            if "ipad" in f:
+                screenshot_type = "tenInchScreenshots"
+                device = f
+            if "ios" in f:
+                os = f
+        return (screenshot_type, device, os)
+
+    return ("phoneScreenshots", 'unknown', 'unknown')
+
+
+def discover_ios_screenshots(fastlane_dir):
+    """Traverse git checkouts in build dir, search for fastlane-screenshots and put findings into a dict."""
+    fastlane_screenshot_dir = fastlane_dir / 'screenshots'
+    screenshots = {}
+    if fastlane_screenshot_dir.is_dir():
+        for lang_sdir in fastlane_screenshot_dir.iterdir():
+            locale = lang_sdir.name
+            m = LANG_CODE.match(locale)
+            if m:
+                screenshots[locale] = {}
+                fifo_idevice = {}
+                fifo_ios = {}
+                for screenshot in lang_sdir.iterdir():
+                    if screenshot.suffix[1:] in ALLOWED_EXTENSIONS:
+                        screenshot_type, idevice_name, ios_name = parse_ios_screenshot_name(screenshot)
+
+                        # since there is no easy mapping here, we're just
+                        # resorting to fifo here, so ieg. if there's 2
+                        # screenshots categorized for more than one
+                        # iPhone/iOS combinations we just remember the
+                        # first combination, use them as screenshots in
+                        # F-Droid and ignore all other screenshots, for
+                        # this screenshot type
+                        if not fifo_idevice.get(screenshot_type):
+                            fifo_idevice[screenshot_type] = idevice_name
+                            fifo_ios[screenshot_type] = ios_name
+
+                        if fifo_idevice[screenshot_type] == idevice_name and fifo_ios[screenshot_type] == ios_name:
+                            if screenshot_type not in screenshots[locale]:
+                                screenshots[locale][screenshot_type] = []
+                            screenshots[locale][screenshot_type].append(screenshot)
+
+    # sort all found screenshots alphanumerically
+    for locale, translated_screenshots in screenshots.items():
+        for device in translated_screenshots.keys():
+            translated_screenshots[device].sort()
+
+    return screenshots
+
+
+def copy_ios_screenshots_to_repo(screenshots, package_name):
+    for locale, translated_screenshots in screenshots.items():
+        for device, translated_device_screenshots in translated_screenshots.items():
+            dest_dir = Path('repo') / package_name / locale / device
+            dest_dir.mkdir(mode=0o755, parents=True, exist_ok=True)
+            for path in translated_device_screenshots:
+                dest = dest_dir / (path.name.replace(" ", "_").replace("\t", "_"))
+                fdroidserver.update._strip_and_copy_image(str(path), str(dest))
+
+
+def insert_localized_ios_app_metadata(apps_with_packages):
+
+    if not any(Path('repo').glob('*.ipa')):
+        # no IPA files present in repo, nothing to do here, exiting early
+        return
+
+    for package_name, app in apps_with_packages.items():
+        if not any(Path('repo').glob(f'{package_name}*.ipa')):
+            # couldn't find any IPA files for this package_name
+            # so we don't have to look for fastlane data
+            continue
+
+        fastlane_dir = Path('build', package_name, 'fastlane')
+        fastlane_meta_dir = (fastlane_dir / "metadata")
+
+        if fastlane_meta_dir.is_dir():
+            for lang_dir in fastlane_meta_dir.iterdir():
+                locale = lang_dir.name
+                m = LANG_CODE.match(locale)
+                if m:
+                    for metadata_file in (lang_dir).iterdir():
+                        key = FASTLANE_IOS_MAP.get(metadata_file.name)
+                        if key:
+                            fdroidserver.update._set_localized_text_entry(app, locale, key, metadata_file)
+
+        screenshots = fdroidserver.update.discover_ios_screenshots(fastlane_dir)
+        fdroidserver.update.copy_ios_screenshots_to_repo(screenshots, package_name)
+
+
 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
     """Scan a repo for all files with an extension except APK/OBB/IPA.
 
@@ -2240,6 +2366,8 @@ def prepare_apps(apps, apks, repodir):
     translate_per_build_anti_features(apps_with_packages, apks)
     if repodir == 'repo':
         insert_localized_app_metadata(apps_with_packages)
+        insert_localized_ios_app_metadata(apps_with_packages)
+        ingest_screenshots_from_repo_dir(apps_with_packages)
     insert_missing_app_names_from_apks(apps_with_packages, apks)
     return apps_with_packages
 

tests/update.TestCase

@@ -165,6 +165,7 @@ class UpdateTest(unittest.TestCase):
         apps['eu.siacs.conversations']['Builds'] = [build_conversations]
 
         fdroidserver.update.insert_localized_app_metadata(apps)
+        fdroidserver.update.ingest_screenshots_from_repo_dir(apps)
 
         appdir = os.path.join('repo', 'info.guardianproject.urzip', 'en-US')
         self.assertTrue(
@@ -278,6 +279,7 @@ class UpdateTest(unittest.TestCase):
         knownapks = fdroidserver.common.KnownApks()
         apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False)
         fdroidserver.update.insert_localized_app_metadata(apps)
+        fdroidserver.update.ingest_screenshots_from_repo_dir(apps)
         fdroidserver.update.apply_info_from_latest_apk(apps, apks)
         app = apps['info.guardianproject.urzip']
         self.assertIsNone(app.Name)
@@ -2040,6 +2042,165 @@ class TestScanRepoForIpas(unittest.TestCase):
             )
 
 
+class TestParseIosScreenShotName(unittest.TestCase):
+    def setUp(self):
+        self.maxDiff = None
+
+    def test_parse_ios_screenshot_name_atforamt_iphone8(self):
+        self.assertEqual(
+            fdroidserver.update.parse_ios_screenshot_name(Path("iPhone 8+ @ iOS 16-1.png")),
+            ("phoneScreenshots", "iPhone 8+", "iOS 16",),
+        )
+
+    def test_parse_ios_screenshot_name_atforamt_ipad13(self):
+        self.assertEqual(
+            fdroidserver.update.parse_ios_screenshot_name(Path("iPad Pro 12.9\" 2gen @ iOS 16-1.png")),
+            ("tenInchScreenshots", "iPad Pro 12.9\" 2gen", "iOS 16",),
+        )
+
+    def test_parse_ios_screenshot_name_underscoreforamt_ipad(self):
+        self.assertEqual(
+            fdroidserver.update.parse_ios_screenshot_name(Path("1_ipadPro129_1.1.png")),
+            ("tenInchScreenshots", "ipadpro129", "unknown",),
+        )
+
+    def test_parse_ios_screenshot_name_underscoreforamt_iphone(self):
+        self.assertEqual(
+            fdroidserver.update.parse_ios_screenshot_name(Path("1_iphone6Plus_1.1.png")),
+            ("phoneScreenshots", "iphone6plus", "unknown",),
+        )
+
+
+class TestInsertLocalizedIosAppMetadata(unittest.TestCase):
+
+    def test_insert_localized_ios_app_metadata(self):
+        self.maxDiff = None
+
+        self.apps_with_packages = {
+            "org.fake": {}
+        }
+
+        def _mock_discover(fastlane_dir):
+            self.assertEqual(
+                fastlane_dir,
+                Path('build/org.fake/fastlane'),
+            )
+            return {"fake screenshots": "fake"}
+
+        def _mock_copy(screenshots, package_name):
+            self.assertEqual(screenshots, {"fake screenshots": "fake"})
+            self.assertEqual(package_name, "org.fake")
+
+        with mock.patch('fdroidserver.update.discover_ios_screenshots', _mock_discover):
+            self.set_localized_mock = mock.Mock()
+            with mock.patch('fdroidserver.update.copy_ios_screenshots_to_repo', _mock_copy):
+                with mock.patch("fdroidserver.update._set_localized_text_entry", self.set_localized_mock):
+                    return fdroidserver.update.insert_localized_ios_app_metadata(
+                        self.apps_with_packages
+                    )
+
+        self.assertListEqual(
+            self.set_localized_mock.call_args_list,
+            [
+                mock.call({}, 'en-US', 'name', Path('build/org.fake/fastlane/metadata/en-US/name.txt')),
+                mock.call({}, 'en-US', 'summary', Path('build/org.fake/fastlane/metadata/en-US/subtitle.txt')),
+                mock.call({}, 'en-US', 'description', Path('build/org.fake/fastlane/metadata/en-US/description.txt')),
+                mock.call({}, 'de-DE', 'name', Path('build/org.fake/fastlane/metadata/de-DE/name.txt')),
+                mock.call({}, 'de-DE', 'summary', Path('build/org.fake/fastlane/metadata/de-DE/subtitle.txt')),
+                mock.call({}, 'de-DE', 'description', Path('build/org.fake/fastlane/metadata/de-DE/description.txt')),
+            ],
+        )
+
+
+class TestDiscoverIosScreenshots(unittest.TestCase):
+    def test_discover_ios_screenshots(self):
+        self.maxDiff = None
+
+        with tempfile.TemporaryDirectory() as fastlane_dir:
+            fastlane_dir = Path(fastlane_dir)
+            (fastlane_dir / "screenshots/en-US").mkdir(parents=True)
+            with open(fastlane_dir / "screenshots/en-US/iPhone 8+ @ iOS 16-1.png", 'w') as f:
+                f.write("1")
+            with open(fastlane_dir / "screenshots/en-US/iPad Pro 12.9\" 2gen @ iOS 16-1.png", "w") as f:
+                f.write("2")
+            with open(fastlane_dir / "screenshots/en-US/iPad Pro 12.9\" 2gen @ iOS 16-2.png", "w") as f:
+                f.write("3")
+            (fastlane_dir / "screenshots/de-DE").mkdir(parents=True)
+            with open(fastlane_dir / "screenshots/de-DE/1_ipadPro129_1.1.png", "w") as f:
+                f.write("4")
+
+            screenshots = fdroidserver.update.discover_ios_screenshots(fastlane_dir)
+
+            self.assertDictEqual(
+                screenshots,
+                {
+                    "en-US": {
+                        "phoneScreenshots": [
+                            fastlane_dir / "screenshots/en-US/iPhone 8+ @ iOS 16-1.png",
+                        ],
+                        "tenInchScreenshots": [
+                            fastlane_dir / "screenshots/en-US/iPad Pro 12.9\" 2gen @ iOS 16-1.png",
+                            fastlane_dir / "screenshots/en-US/iPad Pro 12.9\" 2gen @ iOS 16-2.png",
+                        ],
+                    },
+                    "de-DE": {
+                        "tenInchScreenshots": [
+                            fastlane_dir / "screenshots/de-DE/1_ipadPro129_1.1.png",
+                        ],
+                    },
+                },
+            )
+
+
+class TestCopyIosScreenshotsToRepo(unittest.TestCase):
+    def test_copy_ios_screenshots_to_repo(self):
+        self.maxDiff = None
+
+        screenshot_dir_en = Path("build/org.fake/fastlane/screenshots/en-US")
+        s1 = screenshot_dir_en / "iPhone 8+ @ iOS 16-1.png"
+        s2 = screenshot_dir_en / "iPad Pro 12.9\" 2gen @ iOS 16-1.png"
+        s3 = screenshot_dir_en / "iPad Pro 12.9\" 2gen @ iOS 16-2.png"
+        screenshot_dir_de = Path("build/org.fake/fastlane/screenshots/de-DE")
+        s4 = screenshot_dir_de / "1_ipadPro129_1.1.png"
+
+        cmock = mock.Mock()
+        with mock.patch("fdroidserver.update._strip_and_copy_image", cmock):
+            fdroidserver.update.copy_ios_screenshots_to_repo(
+                {
+                    "en-US": {
+                        "phoneScreenshots": [s1],
+                        "tenInchScreenshots": [s2, s3],
+                    },
+                    "de-DE": {
+                        "tenInchScreenshots": [s4],
+                    },
+                },
+                "org.fake",
+            )
+
+        self.assertListEqual(
+            cmock.call_args_list,
+            [
+                mock.call(
+                    'build/org.fake/fastlane/screenshots/en-US/iPhone 8+ @ iOS 16-1.png',
+                    'repo/org.fake/en-US/phoneScreenshots/iPhone_8+_@_iOS_16-1.png',
+                ),
+                mock.call(
+                    'build/org.fake/fastlane/screenshots/en-US/iPad Pro 12.9" 2gen @ iOS 16-1.png',
+                    'repo/org.fake/en-US/tenInchScreenshots/iPad_Pro_12.9"_2gen_@_iOS_16-1.png',
+                ),
+                mock.call(
+                    'build/org.fake/fastlane/screenshots/en-US/iPad Pro 12.9" 2gen @ iOS 16-2.png',
+                    'repo/org.fake/en-US/tenInchScreenshots/iPad_Pro_12.9"_2gen_@_iOS_16-2.png',
+                ),
+                mock.call(
+                    'build/org.fake/fastlane/screenshots/de-DE/1_ipadPro129_1.1.png',
+                    'repo/org.fake/de-DE/tenInchScreenshots/1_ipadPro129_1.1.png',
+                ),
+            ],
+        )
+
+
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))
 
@@ -2057,4 +2218,7 @@ if __name__ == "__main__":
     newSuite.addTest(unittest.makeSuite(UpdateTest))
     newSuite.addTest(unittest.makeSuite(TestUpdateVersionStringToInt))
     newSuite.addTest(unittest.makeSuite(TestScanRepoForIpas))
+    newSuite.addTest(unittest.makeSuite(TestParseIosScreenShotName))
+    newSuite.addTest(unittest.makeSuite(TestInsertLocalizedIosAppMetadata))
+    newSuite.addTest(unittest.makeSuite(TestDiscoverIosScreenshots))
     unittest.main(failfast=False)