Merge branch 'scanner' into 'master'

scanner: fix catalog match

See merge request fdroid/fdroidserver!1692

MANIFEST.in

@@ -757,6 +757,9 @@ include tests/source-files/com.anpmech.launcher/settings.gradle
 include tests/source-files/com.github.jameshnsears.quoteunquote/build.gradle
 include tests/source-files/com.github.shadowsocks/core/build.gradle.kts
 include tests/source-files/com.github.shadowsocks/mobile/build.gradle.kts
+include tests/source-files/com.infomaniak.mail/Core/gradle/core.versions.toml
+include tests/source-files/com.infomaniak.mail/gradle/libs.versions.toml
+include tests/source-files/com.infomaniak.mail/settings.gradle
 include tests/source-files/com.integreight.onesheeld/build.gradle
 include tests/source-files/com.integreight.onesheeld/gradle/wrapper/gradle-wrapper.properties
 include tests/source-files/com.integreight.onesheeld/localeapi/build.gradle

fdroidserver/scanner.py

@@ -71,11 +71,8 @@ SCANNER_CACHE_VERSION = 1
 DEFAULT_CATALOG_PREFIX_REGEX = re.compile(
     r'''defaultLibrariesExtensionName\s*=\s*['"](\w+)['"]'''
 )
-GRADLE_KTS_CATALOG_FILE_REGEX = re.compile(
-    r'''create\("(\w+)"\)\s*\{[^}]*from\(files\(['"]([^"]+)['"]\)\)'''
-)
 GRADLE_CATALOG_FILE_REGEX = re.compile(
-    r'''(\w+)\s*\{[^}]*from\(files\(['"]([^"]+)['"]\)\)'''
+    r'''(?:create\()?['"]?(\w+)['"]?\)?\s*\{[^}]*from\(files\(['"]([^"]+)['"]\)\)'''
 )
 VERSION_CATALOG_REGEX = re.compile(r'versionCatalogs\s*\{')
 
@@ -192,7 +189,9 @@ class GradleVersionCatalog:
     def get_coordinate(self, accessor: str) -> list[str]:
         """Get the Gradle coordinate from the catalog with an accessor."""
         if accessor.startswith("plugins."):
-            return [self.plugins.get(accessor[8:], "")]
+            return [
+                self.plugins.get(accessor[8:].removesuffix(".asLibraryDependency"), "")
+            ]
         if accessor.startswith("bundles."):
             return self.bundles.get(accessor[8:], [])
         return [self.libraries.get(accessor, "")]
@@ -225,21 +224,18 @@ def get_catalogs(root: str) -> dict[str, GradleVersionCatalog]:
     groovy_file = root / "settings.gradle"
     kotlin_file = root / "settings.gradle.kts"
     if groovy_file.is_file():
-        s = groovy_file.read_text(encoding="utf-8")
+        gradle_file = groovy_file
+    elif kotlin_file.is_file():
+        gradle_file = kotlin_file
+    else:
+        return {}
+
+    s = gradle_file.read_text(encoding="utf-8")
     version_catalogs_m = VERSION_CATALOG_REGEX.search(s)
     if version_catalogs_m:
         start = version_catalogs_m.end()
         end = find_block_end(s, start)
         catalog_files_m = GRADLE_CATALOG_FILE_REGEX.finditer(s, start, end)
-    elif kotlin_file.is_file():
-        s = kotlin_file.read_text(encoding="utf-8")
-        version_catalogs_m = VERSION_CATALOG_REGEX.search(s)
-        if version_catalogs_m:
-            start = version_catalogs_m.end()
-            end = find_block_end(s, start)
-            catalog_files_m = GRADLE_KTS_CATALOG_FILE_REGEX.finditer(s, start, end)
-    else:
-        return {}
 
     m_default = DEFAULT_CATALOG_PREFIX_REGEX.search(s)
     if m_default:

tests/source-files/catalog.test/buildSrc/build.gradle.kts

@@ -3,3 +3,7 @@ plugins {
     alias(libs.plugins.firebase.crashlytics)
     alias(projectLibs.plugins.firebase.crashlytics)
 }
+
+dependencies {
+    implementation(libs.plugins.androidApplication.asLibraryDependency)
+}

tests/source-files/catalog.test/gradle/libs.versions.toml

@@ -1,16 +1,18 @@
 [versions]
 firebase = "1.1.1"
 gms = "1.2.1"
+androidGradlePlugin = "8.12.0"
 
 [libraries]
 firebase-crash = { module = "com.google.firebase:firebase-crash", version.ref = "firebase" }
 firebase_core = { module = "com.google.firebase:firebase-core", version = "2.2.2" }
-"play.service.ads" = { module = "com.google.android.gms:play-services-ads", version.ref = "gms"}
+"play.service.ads" = { module = "com.google.android.gms:play-services-ads", version.ref = "gms" }
 jacoco = "org.jacoco:org.jacoco.core:0.8.7"
 
 [plugins]
 google-services = { id = "com.google.gms.google-services", version.ref = "gms" }
 firebase-crashlytics = { id = "com.google.firebase.crashlytics", version.ref = "firebase" }
+androidApplication = { id = "com.android.application", version.ref = "androidGradlePlugin" }
 
 [bundles]
 firebase = ["firebase-crash", "firebase_core"]

tests/source-files/com.infomaniak.mail/settings.gradle

@@ -0,0 +1,44 @@
+pluginManagement {
+    repositories {
+        gradlePluginPortal()
+        google()
+        mavenCentral()
+    }
+}
+
+dependencyResolutionManagement {
+    repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS)
+    repositories {
+        google()
+        mavenCentral()
+        maven { url 'https://jitpack.io' }
+    }
+    versionCatalogs {
+        create("core") { from(files("Core/gradle/core.versions.toml")) }
+    }
+}
+
+rootProject.name = 'Infomaniak Mail'
+include ':app',
+        ':Core:AppIntegrity',
+        ':Core:Auth',
+        ':Core:Avatar',
+        ':Core:Coil',
+        ':Core:Compose:Basics',
+        ':Core:Compose:Margin',
+        ':Core:Compose:MaterialThemeFromXml',
+        ':Core:CrossAppLogin',
+        ':Core:CrossAppLoginUI',
+        ':Core:FragmentNavigation',
+        ':Core:Legacy',
+        ':Core:Legacy:AppLock',
+        ':Core:Legacy:BugTracker',
+        ':Core:Legacy:Confetti',
+        ':Core:Legacy:Stores',
+        ':Core:Matomo',
+        ':Core:MyKSuite',
+        ':Core:Network',
+        ':Core:Network:Models',
+        ':Core:Sentry',
+        ':EmojiComponents',
+        ':HtmlCleaner'

tests/test_scanner.py

@@ -165,6 +165,9 @@ class ScannerTest(unittest.TestCase):
                 'com.google.firebase:firebase-crash:1.1.1',
                 'com.google.firebase:firebase-core:2.2.2',
             ],
+            'plugins.androidApplication.asLibraryDependency': [
+                'com.android.application:8.12.0'
+            ],
         }
         with open('source-files/catalog.test/gradle/libs.versions.toml', 'rb') as f:
             catalog = fdroidserver.scanner.GradleVersionCatalog(tomllib.load(f))
@@ -176,6 +179,7 @@ class ScannerTest(unittest.TestCase):
             ('source-files/com.lolo.io.onelist/', 1),
             ('source-files/catalog.test/', 3),
             ('source-files/org.piepmeyer.gauguin/', 1),
+            ('source-files/com.infomaniak.mail/', 2),
         ]
 
         for root, count in test_files:
@@ -353,11 +357,15 @@ class ScannerTest(unittest.TestCase):
 
         with mock.patch('fdroidserver.common.replace_build_vars', wraps=make_fake_apk):
             with mock.patch('fdroidserver.common.get_native_code', return_value='x86'):
-                with mock.patch(
+                with (
+                    mock.patch(
                         'fdroidserver.common.get_apk_id',
                         return_value=(app.id, build.versionCode, build.versionName),
-                ), mock.patch(
+                    ),
-                    'fdroidserver.common.get_source_date_epoch', lambda f: '1234567890'
+                    mock.patch(
+                        'fdroidserver.common.get_source_date_epoch',
+                        lambda f: '1234567890',
+                    ),
                 ):
                     with mock.patch(
                         'fdroidserver.common.is_debuggable_or_testOnly',
@@ -671,15 +679,19 @@ class Test_SignatureDataController(unittest.TestCase):
         func_lfc = mock.Mock()
         func_vd = mock.Mock()
         func_clu = mock.Mock()
-        with mock.patch(
+        with (
+            mock.patch(
                 'fdroidserver.scanner.SignatureDataController.load_from_cache',
                 func_lfc,
-        ), mock.patch(
+            ),
+            mock.patch(
                 'fdroidserver.scanner.SignatureDataController.verify_data',
                 func_vd,
-        ), mock.patch(
+            ),
+            mock.patch(
                 'fdroidserver.scanner.SignatureDataController.check_last_updated',
                 func_clu,
+            ),
         ):
             sdc.load()
         func_lfc.assert_called_once_with()
@@ -694,12 +706,15 @@ class Test_SignatureDataController(unittest.TestCase):
             side_effect=fdroidserver.scanner.SignatureDataCacheMissException
         )
         func_lfd = mock.Mock()
-        with mock.patch(
+        with (
+            mock.patch(
                 'fdroidserver.scanner.SignatureDataController.load_from_cache',
                 func_lfc,
-        ), mock.patch(
+            ),
+            mock.patch(
                 'fdroidserver.scanner.SignatureDataController.load_from_defaults',
                 func_lfd,
+            ),
         ):
             sdc.load()
         func_lfc.assert_called_once_with()
@@ -716,21 +731,27 @@ class Test_SignatureDataController(unittest.TestCase):
         )
         func_fsfw = mock.Mock()
         func_wtc = mock.Mock()
-        with mock.patch(
+        with (
+            mock.patch(
                 'fdroidserver.scanner.SignatureDataController.load_from_cache',
                 func_lfc,
-        ), mock.patch(
+            ),
+            mock.patch(
                 'fdroidserver.scanner.SignatureDataController.verify_data',
                 func_vd,
-        ), mock.patch(
+            ),
+            mock.patch(
                 'fdroidserver.scanner.SignatureDataController.check_last_updated',
                 func_clu,
-        ), mock.patch(
+            ),
+            mock.patch(
                 'fdroidserver.scanner.SignatureDataController.fetch_signatures_from_web',
                 func_fsfw,
-        ), mock.patch(
+            ),
+            mock.patch(
                 'fdroidserver.scanner.SignatureDataController.write_to_cache',
                 func_wtc,
+            ),
         ):
             sdc.load()
         func_lfc.assert_called_once_with()
@@ -751,18 +772,23 @@ class Test_SignatureDataController(unittest.TestCase):
         )
         func_fsfw = mock.Mock()
         func_wtc = mock.Mock()
-        with mock.patch(
+        with (
+            mock.patch(
                 'fdroidserver.scanner.SignatureDataController.load_from_cache',
                 func_lfc,
-        ), mock.patch(
+            ),
+            mock.patch(
                 'fdroidserver.scanner.SignatureDataController.load_from_defaults',
                 func_lfd,
-        ), mock.patch(
+            ),
+            mock.patch(
                 'fdroidserver.scanner.SignatureDataController.fetch_signatures_from_web',
                 func_fsfw,
-        ), mock.patch(
+            ),
+            mock.patch(
                 'fdroidserver.scanner.SignatureDataController.write_to_cache',
                 func_wtc,
+            ),
         ):
             sdc.load()
         func_lfc.assert_called_once_with()
@@ -781,9 +807,12 @@ class Test_SignatureDataController(unittest.TestCase):
         )
         sdc.data = {"mocked": "data"}
 
-        with mock.patch("builtins.open", open_func), mock.patch(
+        with (
+            mock.patch("builtins.open", open_func),
+            mock.patch(
                 "fdroidserver.scanner._scanner_cachedir",
                 return_value=pathlib.Path('.'),
+            ),
         ):
             sdc.write_to_cache()