Merge branch 'modernize-ndk-handling' into 'master'

buildserver: standardize SDK install location /opt/android-sdk

Closes #902

See merge request fdroid/fdroidserver!927

buildserver/config.buildserver.yml

@@ -1,8 +1,4 @@
 sdk_path: /opt/android-sdk
-ndk_paths:
-  r10e: /opt/android-sdk/ndk/r10e
-  r21e: /opt/android-sdk/ndk/21.4.7075529
-  r22b: /opt/android-sdk/ndk/22.0.7026061
 
 java_paths:
   8: /usr/lib/jvm/java-8-openjdk-amd64

buildserver/provision-android-ndk

@@ -10,11 +10,6 @@ NDK_BASE=$1
 test -e $NDK_BASE || mkdir -p $NDK_BASE
 cd $NDK_BASE
 
-if [ ! -e $NDK_BASE/r10e ]; then
-    7zr x /vagrant/cache/android-ndk-r10e-linux-x86_64.bin > /dev/null
-    mv android-ndk-r10e r10e
-fi
-
 for version in r21e r22b; do
     if [ ! -e ${NDK_BASE}/${version} ]; then
         unzip /vagrant/cache/android-ndk-${version}-linux-x86_64.zip > /dev/null

buildserver/provision-apt-get-install

@@ -96,7 +96,6 @@ packages="
  openjdk-8-jre-headless
  openjdk-8-jdk-headless
  optipng
- p7zip
  pkg-config
  python-gnupg
  python-lxml

examples/config.yml

@@ -5,24 +5,20 @@
 # Custom path to the Android SDK, defaults to $ANDROID_HOME
 # sdk_path: $ANDROID_HOME
 
-# Paths to various installed versions of the Android NDK.  If a
+# Paths to installed versions of the Android NDK.  This will be
-# required version is missing in the buildserver VM, it will be
+# automatically filled out from well known sources like
-# automatically downloaded and installed into a temporary dir.
+# $ANDROID_HOME/ndk-bundle and $ANDROID_HOME/ndk/*.  If a required
+# version is missing in the buildserver VM, it will be automatically
+# downloaded and installed into the standard $ANDROID_HOME/ndk/
+# directory.  Manually setting it here will override the auto-detected
+# values.  The keys can either be the "release" (e.g. r21e) or the
+# "revision" (e.g. 21.4.7075529).
 #
 # ndk_paths:
-#   r10e: None
+#   r10e: $ANDROID_HOME/android-ndk-r10e
-#   r11c: None
+#   r17: ""
-#   r12b: None
+#   21.4.7075529: ~/Android/Ndk
-#   r13b: None
+#   r22b: null
-#   r14b: None
-#   r15c: None
-#   r16b: None
-#   r17c: None
-#   r18b: None
-#   r19c: None
-#   r20b: None
-#   r21e: None
-#   r22b: None
 
 # Directory to store downloaded tools in (i.e. gradle versions)
 # By default, these are stored in ~/.cache/fdroidserver

fdroidserver/build.py

@@ -237,7 +237,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
         try:
             cmd_stdout = chan.makefile('rb', 1024)
             output = bytes()
-            output += common.get_android_tools_version_log(build.ndk_path()).encode()
+            output += common.get_android_tools_version_log().encode()
             while not chan.exit_status_ready():
                 line = cmd_stdout.readline()
                 if line:
@@ -402,7 +402,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
         log_path = os.path.join(log_dir,
                                 common.get_toolsversion_logname(app, build))
         with open(log_path, 'w') as f:
-            f.write(common.get_android_tools_version_log(build.ndk_path()))
+            f.write(common.get_android_tools_version_log())
     else:
         if build.sudo:
             logging.warning('%s:%s runs this on the buildserver with sudo:\n\t%s\nThese commands were skipped because fdroid build is not running on a dedicated build server.'
@@ -1088,7 +1088,7 @@ def main():
             build_starttime = common.get_wiki_timestamp()
             tools_version_log = ''
             if not options.onserver:
-                tools_version_log = common.get_android_tools_version_log(build.ndk_path())
+                tools_version_log = common.get_android_tools_version_log()
                 common.write_running_status_json(status_output)
             try:
 

fdroidserver/common.py

@@ -214,6 +214,14 @@ def _add_java_paths_to_config(pathlist, thisconfig):
 
 
 def fill_config_defaults(thisconfig):
+    """Fill in the global config dict with relevant defaults
+
+    For config values that have a path that can be expanded, e.g. an
+    env var or a ~/, this will store the original value using "_orig"
+    appended to the key name so that if the config gets written out,
+    it will preserve the original, unexpanded string.
+
+    """
     for k, v in default_config.items():
         if k not in thisconfig:
             thisconfig[k] = v
@@ -281,6 +289,28 @@ def fill_config_defaults(thisconfig):
                 thisconfig[k][k2] = exp
                 thisconfig[k][k2 + '_orig'] = v
 
+    ndk_paths = thisconfig.get('ndk_paths', {})
+
+    ndk_bundle = os.path.join(thisconfig['sdk_path'], 'ndk-bundle')
+    if os.path.exists(ndk_bundle):
+        version = get_ndk_version(ndk_bundle)
+        if version not in ndk_paths:
+            ndk_paths[version] = ndk_bundle
+
+    ndk_dir = os.path.join(thisconfig['sdk_path'], 'ndk')
+    if os.path.exists(ndk_dir):
+        for ndk in glob.glob(os.path.join(ndk_dir, '*')):
+            version = get_ndk_version(ndk)
+            if version not in ndk_paths:
+                ndk_paths[version] = ndk
+
+    for k in list(ndk_paths.keys()):
+        if not re.match(r'r[1-9][0-9]*[a-z]?', k):
+            for ndkdict in NDKS:
+                if k == ndkdict.get('revision'):
+                    ndk_paths[ndkdict['release']] = ndk_paths.pop(k)
+                    break
+
 
 def regsub_file(pattern, repl, path):
     with open(path, 'rb') as f:
@@ -3842,7 +3872,7 @@ def get_wiki_timestamp(timestamp=None):
     return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp)
 
 
-def get_android_tools_versions(ndk_path=None):
+def get_android_tools_versions():
     '''get a list of the versions of all installed Android SDK/NDK components'''
 
     global config
@@ -3850,11 +3880,9 @@ def get_android_tools_versions(ndk_path=None):
     if sdk_path[-1] != '/':
         sdk_path += '/'
     components = []
-    if ndk_path:
+    for ndk_path in config.get('ndk_paths', []):
-        ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
+        version = get_ndk_version(ndk_path)
-        if os.path.isfile(ndk_release_txt):
+        components.append((os.path.basename(ndk_path), str(version)))
-            with open(ndk_release_txt, 'r') as fp:
-                components.append((os.path.basename(ndk_path), fp.read()[:-1]))
 
     pattern = re.compile(r'^Pkg.Revision *= *(.+)', re.MULTILINE)
     for root, dirs, files in os.walk(sdk_path):
@@ -3868,10 +3896,10 @@ def get_android_tools_versions(ndk_path=None):
     return components
 
 
-def get_android_tools_version_log(ndk_path=None):
+def get_android_tools_version_log():
     '''get a list of the versions of all installed Android SDK/NDK components'''
     log = '== Installed Android Tools ==\n\n'
-    components = get_android_tools_versions(ndk_path)
+    components = get_android_tools_versions()
     for name, version in sorted(components):
         log += '* ' + name + ' (' + version + ')\n'
 
@@ -3985,12 +4013,23 @@ def sha256base64(filename):
 
 
 def get_ndk_version(ndk_path):
+    """Get the version info from the metadata in the NDK package
+
+    Since r11, the info is nice and easy to find in
+    sources.properties.  Before, there was a kludgey format in
+    RELEASE.txt.  This is only needed for r10e.
+
+    """
     source_properties = os.path.join(ndk_path, 'source.properties')
+    release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
     if os.path.exists(source_properties):
         with open(source_properties) as fp:
             m = re.search(r'^Pkg.Revision *= *(.+)', fp.read(), flags=re.MULTILINE)
             if m:
                 return m.group(1)
+    elif os.path.exists(release_txt):
+        with open(release_txt) as fp:
+            return fp.read().split('-')[0]
 
 
 def auto_install_ndk(build):
@@ -4021,6 +4060,32 @@ def auto_install_ndk(build):
     ndk = build.get('ndk')
     if not ndk:
         return
+    if isinstance(ndk, str):
+        _install_ndk(ndk)
+    elif isinstance(ndk, list):
+        for n in ndk:
+            _install_ndk(n)
+    else:
+        BuildException(_('Invalid ndk: entry in build: "{ndk}"')
+                       .format(ndk=str(ndk)))
+
+
+def _install_ndk(ndk):
+    """Install specified NDK if it is not already installed
+
+    Parameters
+    ----------
+
+    ndk
+        The NDK version to install, either in "release" form (r21e) or
+        "revision" form (21.4.7075529).
+    """
+    if re.match(r'[1-9][0-9.]+[0-9]', ndk):
+        for ndkdict in NDKS:
+            if ndk == ndkdict['revision']:
+                ndk = ndkdict['release']
+                break
+
     ndk_path = config.get(ndk)
     if ndk_path and os.path.isdir(ndk_path):
         return
@@ -4062,6 +4127,11 @@ def auto_install_ndk(build):
 
 """Derived from https://gitlab.com/fdroid/android-sdk-transparency-log/-/blob/master/checksums.json"""
 NDKS = [
+    {
+        "release": "r10e",
+        "sha256": "ee5f405f3b57c4f5c3b3b8b5d495ae12b660e03d2112e4ed5c728d349f1e520c",
+        "url": "https://dl.google.com/android/repository/android-ndk-r10e-linux-x86_64.zip"
+    },
     {
         "release": "r11",
         "revision": "11.0.2655954",

fdroidserver/metadata.py

@@ -326,7 +326,11 @@ class Build(dict):
         return 'ant'
 
     def ndk_path(self):
-        return fdroidserver.common.config['ndk_paths'].get(self.ndk, '')
+        """Returns the path to the first configured NDK or an empty string"""
+        ndk = self.ndk
+        if isinstance(ndk, list):
+            ndk = self.ndk[0]
+        return fdroidserver.common.config['ndk_paths'].get(ndk, '')
 
 
 flagtypes = {

makebuildserver

@@ -291,8 +291,6 @@ CACHE_FILES = [
      'dccda8aa069563c8ba2f6cdfd0777df0e34a5b4d15138ca8b9757e94f4e8a8cb'),
     ('https://services.gradle.org/distributions/gradle-7.0.2-bin.zip',
      '0e46229820205440b48a5501122002842b82886e76af35f0f3a069243dca4b3c'),
-    ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
-     '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
     ('https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip',
      'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e'),
     ('https://dl.google.com/android/repository/android-ndk-r22b-linux-x86_64.zip',

tests/build.TestCase

@@ -362,7 +362,7 @@ class BuildTest(unittest.TestCase):
         with mock.patch(
             'fdroidserver.common.force_exit', lambda *args: None
         ) as a, mock.patch(
-            'fdroidserver.common.get_android_tools_version_log', lambda s: 'fake'
+            'fdroidserver.common.get_android_tools_version_log', lambda: 'fake'
         ) as b, mock.patch(
             'fdroidserver.common.FDroidPopen', FakeProcess
         ) as c, mock.patch(

tests/common.TestCase

@@ -1796,6 +1796,88 @@ class CommonTest(unittest.TestCase):
             fdroidserver.common.metadata_find_developer_signing_files(appid, vc)
         )
 
+    def test_auto_install_ndk(self):
+        """Test all possible field data types for build.ndk"""
+        build = fdroidserver.metadata.Build()
+
+        none_entry = mock.Mock()
+        with mock.patch('fdroidserver.common._install_ndk', none_entry):
+            fdroidserver.common.auto_install_ndk(build)
+            none_entry.assert_not_called()
+
+        empty_list = mock.Mock()
+        build.ndk = []
+        with mock.patch('fdroidserver.common._install_ndk', empty_list):
+            fdroidserver.common.auto_install_ndk(build)
+            empty_list.assert_not_called()
+
+        release_entry = mock.Mock()
+        build.ndk = 'r21e'
+        with mock.patch('fdroidserver.common._install_ndk', release_entry):
+            fdroidserver.common.auto_install_ndk(build)
+            release_entry.assert_called_once_with('r21e')
+
+        revision_entry = mock.Mock()
+        build.ndk = '21.4.7075529'
+        with mock.patch('fdroidserver.common._install_ndk', revision_entry):
+            fdroidserver.common.auto_install_ndk(build)
+            revision_entry.assert_called_once_with('21.4.7075529')
+
+        list_entry = mock.Mock()
+        calls = []
+        build.ndk = ['r10e', '11.0.2655954', 'r12b', 'r21e']
+        for n in build.ndk:
+            calls.append(mock.call(n))
+        with mock.patch('fdroidserver.common._install_ndk', list_entry):
+            fdroidserver.common.auto_install_ndk(build)
+            list_entry.assert_has_calls(calls)
+
+    @unittest.skip("This test downloads and unzips a 1GB file.")
+    def test_install_ndk(self):
+        """NDK r10e is a special case since its missing source.properties"""
+        sdk_path = tempfile.mkdtemp(
+            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
+        )
+        config = {'sdk_path': sdk_path}
+        fdroidserver.common.config = config
+        fdroidserver.common._install_ndk('r10e')
+        r10e = os.path.join(sdk_path, 'ndk', 'r10e')
+        self.assertEqual('r10e', fdroidserver.common.get_ndk_version(r10e))
+        fdroidserver.common.fill_config_defaults(config)
+        self.assertEqual({'r10e': r10e}, config['ndk_paths'])
+
+    def test_fill_config_defaults(self):
+        """Test the auto-detection of NDKs installed in standard paths"""
+        sdk_path = tempfile.mkdtemp(
+            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
+        )
+
+        ndk_bundle = os.path.join(sdk_path, 'ndk-bundle')
+        os.makedirs(ndk_bundle)
+        with open(os.path.join(ndk_bundle, 'source.properties'), 'w') as fp:
+            fp.write('Pkg.Desc = Android NDK\nPkg.Revision = 17.2.4988734\n')
+        config = {'sdk_path': sdk_path}
+        fdroidserver.common.fill_config_defaults(config)
+        self.assertEqual({'r17c': ndk_bundle}, config['ndk_paths'])
+
+        r21e = os.path.join(sdk_path, 'ndk', '21.4.7075529')
+        os.makedirs(r21e)
+        with open(os.path.join(r21e, 'source.properties'), 'w') as fp:
+            fp.write('Pkg.Desc = Android NDK\nPkg.Revision = 21.4.7075529\n')
+        config = {'sdk_path': sdk_path}
+        fdroidserver.common.fill_config_defaults(config)
+        self.assertEqual({'r17c': ndk_bundle, 'r21e': r21e}, config['ndk_paths'])
+
+        r10e = os.path.join(sdk_path, 'ndk', 'r10e')
+        os.makedirs(r10e)
+        with open(os.path.join(r10e, 'RELEASE.TXT'), 'w') as fp:
+            fp.write('r10e-rc4 (64-bit)\n')
+        config = {'sdk_path': sdk_path}
+        fdroidserver.common.fill_config_defaults(config)
+        self.assertEqual(
+            {'r10e': r10e, 'r17c': ndk_bundle, 'r21e': r21e}, config['ndk_paths']
+        )
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))