Merge branch 'import' into 'master'

import_subcommand.py: misc fixes and updates

See merge request fdroid/fdroidserver!1525

fdroidserver/common.py

@@ -101,9 +101,6 @@ VALID_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-z_]+(?:\d*[a-zA-Z_]*)*)(?:\.[
                                         re.IGNORECASE)
 ANDROID_PLUGIN_REGEX = re.compile(r'''\s*(:?apply plugin:|id)\(?\s*['"](android|com\.android\.application)['"]\s*\)?''')
 
-SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?')
-GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:?([^'"]+)['"]''')
-
 MAX_VERSION_CODE = 0x7fffffff  # Java's Integer.MAX_VALUE (2147483647)
 
 XMLNS_ANDROID = '{http://schemas.android.com/apk/res/android}'
@@ -2120,37 +2117,6 @@ def is_strict_application_id(name):
         and '.' in name
 
 
-def get_all_gradle_and_manifests(build_dir):
-    paths = []
-    # TODO: Python3.6: Accepts a path-like object.
-    for root, dirs, files in os.walk(str(build_dir)):
-        for f in sorted(files):
-            if f == 'AndroidManifest.xml' \
-               or f.endswith('.gradle') or f.endswith('.gradle.kts'):
-                full = Path(root) / f
-                paths.append(full)
-    return paths
-
-
-def get_gradle_subdir(build_dir, paths):
-    """Get the subdir where the gradle build is based."""
-    first_gradle_dir = None
-    for path in paths:
-        if not first_gradle_dir:
-            first_gradle_dir = path.parent.relative_to(build_dir)
-        if path.exists() and SETTINGS_GRADLE_REGEX.match(str(path.name)):
-            for m in GRADLE_SUBPROJECT_REGEX.finditer(path.read_text(encoding='utf-8')):
-                for f in (path.parent / m.group(1)).glob('build.gradle*'):
-                    with f.open(encoding='utf-8') as fp:
-                        for line in fp.readlines():
-                            if ANDROID_PLUGIN_REGEX.match(line):
-                                return f.parent.relative_to(build_dir)
-    if first_gradle_dir and first_gradle_dir != Path('.'):
-        return first_gradle_dir
-
-    return
-
-
 def parse_srclib_spec(spec):
 
     if type(spec) != str:

fdroidserver/import_subcommand.py

@@ -18,34 +18,64 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import json
+import logging
 import os
 import re
-import stat
-import urllib
-
-import git
-import json
 import shutil
+import stat
 import sys
-import yaml
+import urllib
 from argparse import ArgumentParser
-import logging
 from pathlib import Path
 from typing import Optional
 
+import git
+import yaml
+
 try:
     from yaml import CSafeLoader as SafeLoader
 except ImportError:
     from yaml import SafeLoader
 
-from . import _
+from . import _, common, metadata
-from . import common
-from . import metadata
 from .exception import FDroidException
 
-
 config = None
 
+SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?')
+GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:?([^'"]+)['"]''')
+APPLICATION_ID_REGEX = re.compile(r'''\s*applicationId\s=?\s?['"].*['"]''')
+
+
+def get_all_gradle_and_manifests(build_dir):
+    paths = []
+    for root, dirs, files in os.walk(build_dir):
+        for f in sorted(files):
+            if f == 'AndroidManifest.xml' or f.endswith(('.gradle', '.gradle.kts')):
+                full = Path(root) / f
+                paths.append(full)
+    return paths
+
+
+def get_gradle_subdir(build_dir, paths):
+    """Get the subdir where the gradle build is based."""
+    first_gradle_dir = None
+    for path in paths:
+        if not first_gradle_dir:
+            first_gradle_dir = path.parent.relative_to(build_dir)
+        if path.exists() and SETTINGS_GRADLE_REGEX.match(path.name):
+            for m in GRADLE_SUBPROJECT_REGEX.finditer(path.read_text(encoding='utf-8')):
+                for f in (path.parent / m.group(1)).glob('build.gradle*'):
+                    with f.open(encoding='utf-8') as fp:
+                        for line in fp:
+                            if common.ANDROID_PLUGIN_REGEX.match(
+                                line
+                            ) or APPLICATION_ID_REGEX.match(line):
+                                return f.parent.relative_to(build_dir)
+    if first_gradle_dir and first_gradle_dir != Path('.'):
+        return first_gradle_dir
+
 
 def handle_retree_error_on_windows(function, path, excinfo):
     """Python can't remove a readonly file on Windows so chmod first."""
@@ -100,6 +130,7 @@ def getrepofrompage(url: str) -> tuple[Optional[str], str]:
         The found repository type or None if an error occured.
     address_or_reason
         The address to the found repository or the reason if an error occured.
+
     """
     if not url.startswith('http'):
         return (None, _('{url} does not start with "http"!'.format(url=url)))
@@ -122,7 +153,7 @@ def getrepofrompage(url: str) -> tuple[Optional[str], str]:
     index = page.find('hg clone')
     if index != -1:
         repotype = 'hg'
-        repo = page[index + 9:]
+        repo = page[index + 9 :]
         index = repo.find('<')
         if index == -1:
             return (None, _("Error while getting repo address"))
@@ -134,7 +165,7 @@ def getrepofrompage(url: str) -> tuple[Optional[str], str]:
     index = page.find('git clone')
     if index != -1:
         repotype = 'git'
-        repo = page[index + 10:]
+        repo = page[index + 10 :]
         index = repo.find('<')
         if index == -1:
             return (None, _("Error while getting repo address"))
@@ -168,6 +199,7 @@ def get_app_from_url(url: str) -> metadata.App:
         If the VCS type could not be determined.
     :exc:`ValueError`
         If the URL is invalid.
+
     """
     parsed = urllib.parse.urlparse(url)
     invalid_url = False
@@ -243,18 +275,29 @@ def main():
     # Parse command line...
     parser = ArgumentParser()
     common.setup_global_opts(parser)
-    parser.add_argument("-u", "--url", default=None,
+    parser.add_argument("-u", "--url", help=_("Project URL to import from."))
-                        help=_("Project URL to import from."))
+    parser.add_argument(
-    parser.add_argument("-s", "--subdir", default=None,
+        "-s",
-                        help=_("Path to main Android project subdirectory, if not in root."))
+        "--subdir",
-    parser.add_argument("-c", "--categories", default=None,
+        help=_("Path to main Android project subdirectory, if not in root."),
-                        help=_("Comma separated list of categories."))
+    )
-    parser.add_argument("-l", "--license", default=None,
+    parser.add_argument(
-                        help=_("Overall license of the project."))
+        "-c",
-    parser.add_argument("--omit-disable", action="store_true", default=False,
+        "--categories",
-                        help=_("Do not add 'disable:' to the generated build entries"))
+        help=_("Comma separated list of categories."),
-    parser.add_argument("--rev", default=None,
+    )
-                        help=_("Allows a different revision (or git branch) to be specified for the initial import"))
+    parser.add_argument("-l", "--license", help=_("Overall license of the project."))
+    parser.add_argument(
+        "--omit-disable",
+        action="store_true",
+        help=_("Do not add 'disable:' to the generated build entries"),
+    )
+    parser.add_argument(
+        "--rev",
+        help=_(
+            "Allows a different revision (or git branch) to be specified for the initial import"
+        ),
+    )
     metadata.add_metadata_arguments(parser)
     options = common.parse_args(parser)
     metadata.warnings_action = options.W
@@ -268,24 +311,20 @@ def main():
 
     local_metadata_files = common.get_local_metadata_files()
     if local_metadata_files:
-        raise FDroidException(_("This repo already has local metadata: %s") % local_metadata_files[0])
+        raise FDroidException(
+            _("This repo already has local metadata: %s") % local_metadata_files[0]
+        )
 
     build = metadata.Build()
+    app = metadata.App()
     if options.url is None and Path('.git').is_dir():
-        app = metadata.App()
-        app.AutoName = Path.cwd().name
         app.RepoType = 'git'
-
+        tmp_importer_dir = Path.cwd()
-        if Path('build.gradle').exists() or Path('build.gradle.kts').exists():
+        git_repo = git.Repo(tmp_importer_dir)
-            build.gradle = ['yes']
-
-        git_repo = git.Repo(Path.cwd())
         for remote in git.Remote.iter_items(git_repo):
             if remote.name == 'origin':
                 url = git_repo.remotes.origin.url
-                if url.startswith('https://git'):  # github, gitlab
+                app = get_app_from_url(url)
-                    app.SourceCode = url.rstrip('.git')
-                app.Repo = url
                 break
         write_local_file = True
     elif options.url:
@@ -294,17 +333,20 @@ def main():
         git_repo = git.Repo(tmp_importer_dir)
 
         if not options.omit_disable:
-            build.disable = 'Generated by `fdroid import` - check version fields and commitid'
+            build.disable = (
+                'Generated by `fdroid import` - check version fields and commitid'
+            )
         write_local_file = False
     else:
         raise FDroidException("Specify project url.")
 
+    app.AutoUpdateMode = 'Version'
     app.UpdateCheckMode = 'Tags'
     build.commit = common.get_head_commit_id(git_repo)
 
     # Extract some information...
-    paths = common.get_all_gradle_and_manifests(tmp_importer_dir)
+    paths = get_all_gradle_and_manifests(tmp_importer_dir)
-    subdir = common.get_gradle_subdir(tmp_importer_dir, paths)
+    gradle_subdir = get_gradle_subdir(tmp_importer_dir, paths)
     if paths:
         versionName, versionCode, appid = common.parse_androidmanifests(paths, app)
         if not appid:
@@ -322,16 +364,15 @@ def main():
 
     # Create a build line...
     build.versionName = versionName or 'Unknown'
+    app.CurrentVersion = build.versionName
     build.versionCode = versionCode or 0
+    app.CurrentVersionCode = build.versionCode
     if options.subdir:
         build.subdir = options.subdir
-        build.gradle = ['yes']
+    elif gradle_subdir:
-    elif subdir:
+        build.subdir = gradle_subdir.as_posix()
-        build.subdir = subdir.as_posix()
+    # subdir might be None
-        build.gradle = ['yes']
+    subdir = Path(tmp_importer_dir / build.subdir) if build.subdir else tmp_importer_dir
-    else:
-        # subdir might be None
-        subdir = Path()
 
     if options.license:
         app.License = options.license
@@ -339,23 +380,23 @@ def main():
         app.Categories = options.categories.split(',')
     if (subdir / 'jni').exists():
         build.buildjni = ['yes']
-    if (subdir / 'build.gradle').exists() or (subdir / 'build.gradle').exists():
+    if (subdir / 'build.gradle').exists() or (subdir / 'build.gradle.kts').exists():
         build.gradle = ['yes']
 
+    app.AutoName = common.fetch_real_name(subdir, build.gradle)
+
     package_json = tmp_importer_dir / 'package.json'  # react-native
     pubspec_yaml = tmp_importer_dir / 'pubspec.yaml'  # flutter
     if package_json.exists():
         build.sudo = [
             'sysctl fs.inotify.max_user_watches=524288 || true',
-            'curl -Lo node.tar.gz https://nodejs.org/download/release/v19.3.0/node-v19.3.0-linux-x64.tar.gz',
+            'apt-get update',
-            'echo "b525028ae5bb71b5b32cb7fce903ccce261dbfef4c7dd0f3e0ffc27cd6fc0b3f node.tar.gz" | sha256sum -c -',
+            'apt-get install -y npm',
-            'tar xzf node.tar.gz --strip-components=1 -C /usr/local/',
-            'npm -g install yarn',
         ]
         build.init = ['npm install --build-from-source']
         with package_json.open() as fp:
             data = json.load(fp)
-        app.AutoName = data.get('name', app.AutoName)
+        app.AutoName = app.AutoName or data.get('name')
         app.License = data.get('license', app.License)
         app.Description = data.get('description', app.Description)
         app.WebSite = data.get('homepage', app.WebSite)
@@ -365,11 +406,11 @@ def main():
         if app_json.exists():
             with app_json.open() as fp:
                 data = json.load(fp)
-            app.AutoName = data.get('name', app.AutoName)
+            app.AutoName = app.AutoName or data.get('name')
     if pubspec_yaml.exists():
         with pubspec_yaml.open() as fp:
             data = yaml.load(fp, Loader=SafeLoader)
-        app.AutoName = data.get('name', app.AutoName)
+        app.AutoName = app.AutoName or data.get('name')
         app.License = data.get('license', app.License)
         app.Description = data.get('description', app.Description)
         app.UpdateCheckData = 'pubspec.yaml|version:\\s.+\\+(\\d+)|.|version:\\s(.+)\\+'
@@ -405,8 +446,11 @@ def main():
         Path('build').mkdir(exist_ok=True)
         build_dir = Path('build') / appid
         if build_dir.exists():
-            logging.warning(_('{path} already exists, ignoring import results!')
+            logging.warning(
-                            .format(path=build_dir))
+                _('{path} already exists, ignoring import results!').format(
+                    path=build_dir
+                )
+            )
             sys.exit(1)
         elif tmp_importer_dir:
             # For Windows: Close the repo or a git.exe instance holds handles to repo

pyproject.toml

@@ -1,4 +1,3 @@
-
 # We ignore the following PEP8 warnings
 # * E123: closing bracket does not match indentation of opening bracket's line
 #   - Broken if multiple indentation levels start on a single line
@@ -38,7 +37,6 @@ force-exclude = '''(
   | fdroidserver/build\.py
   | fdroidserver/checkupdates\.py
   | fdroidserver/common\.py
-  | fdroidserver/import_subcommand\.py
   | fdroidserver/index\.py
   | fdroidserver/metadata\.py
   | fdroidserver/nightly\.py

tests/common.TestCase

@@ -1522,45 +1522,6 @@ class CommonTest(unittest.TestCase):
         self.assertEqual(('2021-06-30', 34, 'de.varengold.activeTAN'),
                          fdroidserver.common.parse_androidmanifests(paths, app))
 
-    def test_get_all_gradle_and_manifests(self):
-        """Test whether the function works with relative and absolute paths"""
-        a = fdroidserver.common.get_all_gradle_and_manifests(Path('source-files/cn.wildfirechat.chat'))
-        paths = [
-            'avenginekit/build.gradle',
-            'build.gradle',
-            'chat/build.gradle',
-            'client/build.gradle',
-            'client/src/main/AndroidManifest.xml',
-            'emojilibrary/build.gradle',
-            'gradle/build_libraries.gradle',
-            'imagepicker/build.gradle',
-            'mars-core-release/build.gradle',
-            'push/build.gradle',
-            'settings.gradle',
-        ]
-        paths = [Path('source-files/cn.wildfirechat.chat') / path for path in paths]
-        self.assertEqual(sorted(paths), sorted(a))
-
-        abspath = Path(self.basedir) / 'source-files/realm'
-        p = fdroidserver.common.get_all_gradle_and_manifests(abspath)
-        self.assertEqual(1, len(p))
-        self.assertTrue(p[0].is_relative_to(abspath))
-
-    def test_get_gradle_subdir(self):
-        subdirs = {
-            'cn.wildfirechat.chat': 'chat',
-            'com.anpmech.launcher': 'app',
-            'org.tasks': 'app',
-            'ut.ewh.audiometrytest': 'app',
-            'org.noise_planet.noisecapture': 'app',
-        }
-        for k, v in subdirs.items():
-            build_dir = Path('source-files') / k
-            paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir)
-            logging.info(paths)
-            subdir = fdroidserver.common.get_gradle_subdir(build_dir, paths)
-            self.assertEqual(v, str(subdir))
-
     def test_parse_srclib_spec_good(self):
         self.assertEqual(fdroidserver.common.parse_srclib_spec('osmand-external-skia@android/oreo'),
                          ('osmand-external-skia', 'android/oreo', None, None))

tests/import_subcommand.TestCase

@@ -2,29 +2,30 @@
 
 # http://www.drdobbs.com/testing/unit-testing-with-python/240165163
 
-import git
 import logging
 import os
 import shutil
 import sys
 import tempfile
 import unittest
-import yaml
-from unittest import mock
 from pathlib import Path
+from unittest import mock
 
+import git
 import requests
+import yaml
 
 localmodule = Path(__file__).resolve().parent.parent
 print('localmodule: ' + str(localmodule))
 if localmodule not in sys.path:
     sys.path.insert(0, str(localmodule))
 
+from testcommon import TmpCwd, mkdtemp, parse_args_for_test
+
 import fdroidserver.common
 import fdroidserver.import_subcommand
 import fdroidserver.metadata
 from fdroidserver.exception import FDroidException
-from testcommon import TmpCwd, mkdtemp, parse_args_for_test
 
 
 class ImportTest(unittest.TestCase):
@@ -41,6 +42,49 @@ class ImportTest(unittest.TestCase):
         os.chdir(self.basedir)
         self._td.cleanup()
 
+    def test_get_all_gradle_and_manifests(self):
+        """Test whether the function works with relative and absolute paths"""
+        a = fdroidserver.import_subcommand.get_all_gradle_and_manifests(
+            Path('source-files/cn.wildfirechat.chat')
+        )
+        paths = [
+            'avenginekit/build.gradle',
+            'build.gradle',
+            'chat/build.gradle',
+            'client/build.gradle',
+            'client/src/main/AndroidManifest.xml',
+            'emojilibrary/build.gradle',
+            'gradle/build_libraries.gradle',
+            'imagepicker/build.gradle',
+            'mars-core-release/build.gradle',
+            'push/build.gradle',
+            'settings.gradle',
+        ]
+        paths = [Path('source-files/cn.wildfirechat.chat') / path for path in paths]
+        self.assertEqual(sorted(paths), sorted(a))
+
+        abspath = Path(self.basedir) / 'source-files/realm'
+        p = fdroidserver.import_subcommand.get_all_gradle_and_manifests(abspath)
+        self.assertEqual(1, len(p))
+        self.assertTrue(p[0].is_relative_to(abspath))
+
+    def test_get_gradle_subdir(self):
+        subdirs = {
+            'cn.wildfirechat.chat': 'chat',
+            'com.anpmech.launcher': 'app',
+            'org.tasks': 'app',
+            'ut.ewh.audiometrytest': 'app',
+            'org.noise_planet.noisecapture': 'app',
+        }
+        for k, v in subdirs.items():
+            build_dir = Path('source-files') / k
+            paths = fdroidserver.import_subcommand.get_all_gradle_and_manifests(
+                build_dir
+            )
+            logging.info(paths)
+            subdir = fdroidserver.import_subcommand.get_gradle_subdir(build_dir, paths)
+            self.assertEqual(v, str(subdir))
+
     def test_import_gitlab(self):
         with tempfile.TemporaryDirectory() as testdir, TmpCwd(testdir):
             # FDroidPopen needs some config to work
@@ -106,7 +150,9 @@ class ImportTest(unittest.TestCase):
                 self.assertEqual(url, app.Repo)
                 self.assertEqual(url, app.SourceCode)
                 logging.info(build_dir)
-                paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir)
+                paths = fdroidserver.import_subcommand.get_all_gradle_and_manifests(
+                    build_dir
+                )
                 self.assertNotEqual(paths, [])
                 (
                     versionName,