diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index de48b3c8..20a82096 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -153,6 +153,7 @@ ubuntu_jammy_pip: - $pip install dist/fdroidserver-*.tar.gz - tar xzf dist/fdroidserver-*.tar.gz - cd fdroidserver-* + - export PATH=$PATH:$ANDROID_HOME/build-tools/33.0.0 - fdroid=`which fdroid` ./tests/run-tests diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 2ce83557..d1f69e40 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -48,6 +48,11 @@ DISTINGUISHED_NAME = 'CN=Android Debug,O=Android,C=US' NIGHTLY = '-nightly' +def _get_keystore_secret_var(keystore): + with open(keystore, 'rb') as fp: + return base64.standard_b64encode(fp.read()).decode('ascii') + + def _ssh_key_from_debug_keystore(keystore=None): if keystore is None: # set this here so it can be overridden in the tests @@ -450,8 +455,7 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, shutil.rmtree(os.path.dirname(privkey)) if options.show_secret_var: - with open(options.keystore, 'rb') as fp: - debug_keystore = base64.standard_b64encode(fp.read()).decode('ascii') + debug_keystore = _get_keystore_secret_var(options.keystore) print( _('\n{path} encoded for the DEBUG_KEYSTORE secret variable:').format( path=options.keystore diff --git a/tests/nightly.TestCase b/tests/nightly.TestCase index c5fbf0cd..ab854faf 100755 --- a/tests/nightly.TestCase +++ b/tests/nightly.TestCase @@ -6,10 +6,12 @@ import optparse import os import requests import shutil +import subprocess import sys import tempfile import time import unittest +import yaml from pathlib import Path from unittest.mock import patch @@ -21,9 +23,15 @@ print('localmodule: ' + localmodule) if localmodule not in sys.path: sys.path.insert(0, localmodule) -from fdroidserver import common, nightly +from fdroidserver import common, index, nightly +DEBUG_KEYSTORE = '/u3+7QAAAAIAAAABAAAAAQAPYW5kcm9pZGRlYnVna2V5AAABNYhAuskAAAK8MIICuDAOBgorBgEEASoCEQEBBQAEggKkqRnFlhidQmVff83bsAeewXPIsF0jiymzJnvrnUAQtCK0MV9uZonu37Mrj/qKLn56mf6QcvEoKvpCstZxzftgYYpAHWMVLM+hy2Z707QZEHlY7Ukppt8DItj+dXkeqGt7f8KzOb2AQwDbt9lm1fJb+MefLowTaubtvrLMcKIne43CbCu2D8HyN7RPWpEkVetA2Qgr5W4sa3tIUT80afqo9jzwJjKCspuxY9A1M8EIM3/kvyLo2B9r0cuWwRjYZXJ6gmTYI2ARNz0KQnCZUok14NDg+mZTb1B7AzRfb0lfjbA6grbzuAL+WaEpO8/LgGfuOh7QBZBT498TElOaFfQ9toQWA79wAmrQCm4OoFukpPIy2m/l6VjJSmlK5Q+CMOl/Au7OG1sUUCTvPaIr0XKnsiwDJ7a71n9garnPWHkvuWapSRCzCNgaUoGQjB+fTMJFFrwT8P1aLfM6onc3KNrDStoQZuYe5ngCLlNS56bENkVGvJBfdkboxtHZjqDXXON9jWGSOI527J3o2D5sjSVyx3T9XPrsL4TA/nBtdU+c/+M6aoASZR2VymzAKdMrGfj9kE5GXp8vv2vkJj9+OJ4Jm5yeczocc/Idtojjb1yg+sq1yY8kAQxgezpY1rpgi2jF3tSN01c23DNvAaSJLJX2ZuH8sD40ACc80Y1Qp1nUTdpwBZUeaeNruBwx4PHU8GnC71FwtiUpwNs0OoSl0pgDUJ3ODC5bs8B5QmW1wu1eg7I4mMSmCsNGW6VN3sFcu+WEqnmTxPoZombdFZKxsr2oq359Nn4bJ6Uc9PBz/sXsns7Zx1vND/oK/Jv5Y269UVAMeKX/eGpfnxzagW3tqGbOu12C2p9Azo5VxiU2fG/tmk2PjaG5hV/ywReco7I6C1p8OWM2fwAAAAEABVguNTA5AAAB6TCCAeUwggFOoAMCAQICBE89gTUwDQYJKoZIhvcNAQEFBQAwNzELMAkGA1UEBhMCVVMxEDAOBgNVBAoTB0FuZHJvaWQxFjAUBgNVBAMTDUFuZHJvaWQgRGVidWcwHhcNMTIwMjE2MjIyMDM3WhcNNDIwMjA4MjIyMDM3WjA3MQswCQYDVQQGEwJVUzEQMA4GA1UEChMHQW5kcm9pZDEWMBQGA1UEAxMNQW5kcm9pZCBEZWJ1ZzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA3AKU7S7JXhUjEwxWP1/LPHXieh61SaA/+xbpqsPA+yjGz1sAcGAyuG6bjNAVm56pq7nkjJzicX7Wi83nUBo58DEC/quxOLdy0C4PEOSAeTnTT1RJIwMDvOgiL1GFCErvQ7gCH6zuAID/JRFbN6nIkhDjs2DYnSBl7aJJf8wCLc0CAwEAATANBgkqhkiG9w0BAQUFAAOBgQAoq/TJffA0l+ZGf89xndmHdxrO6qi+TzSlByvLZ4eFfCovTh1iO+Edrd5V1yXGLxyyvdsadMAFZT8SaxMrP5xxhJ0nra0APWYLpA96M//auMhQBWPgqPntwgvEZuEH7f0kdItjBJ39yijbG8xfgwid6XqNUo0TDDkp/wNWKpJ9tJe+2PrGw1NAvrgSydoH2j8DI1Eq' +DEBUG_KEYSTORE_KEY_FILE_NAME = ( + 'debug_keystore_QW+xRCJDGHXyyFtgCW8QRajj+6uYmsLwGWpCfYqYQ5M_id_rsa' +) + +AOSP_TESTKEY_DEBUG_KEYSTORE = '/u3+7QAAAAIAAAABAAAAAQAPYW5kcm9pZGRlYnVna2V5AAABejjuIU0AAAUBMIIE/TAOBgorBgEEASoCEQEBBQAEggTpvqhdBtq9D3jRUZGnhKLbFH1LMtCKqwGg25ETAEhvK1GVRNuWAHAUUedCnarjgeUy/zx9OsHuZq18KjUI115kWq/jxkf00fIg7wrOmXoyJf5Dbc7NGKjU64rRmppQEkJ417Lq4Uola9EBJ/WweEu6UTjTn5HcNl4mVloWKMBKNPkVfhZhAkXUyjiZ9rCVHMjLOVKG5vyTWZLwXpYR00Xz6VyzSunTyDza5oUOT/Fh7Gw74V7iNHANydkBHmH+UJ100p0vNPRFvt/3ABfMjkNbRXKNERnyN7NeBmCAOceuXjme/n0XLUidP9/NYk1yAmRJgUnauKD6UPSZYaUPuNSSdf4dD5fCQ7OVDq95e7vmqRDfrKUoWmtpndN7hbVl+OHVZXk2ngvXbvoS+F7ShsEfbq7+c37dnOcVrIlrY+wlOWX2jN42T+AkGt3AfA8zdIPdNgLGk64Op+aP4vGyLQqbuUEzOTNG9uExjGlamogPKFf93GAF83xv7AChYLR/9H+B1E955FL58bRuYOXVWJfLRsO/jyjXsilhBggo3VD1omRuOp98AkKP+P9JXCTswK7IZgvbMK3GB6QIzD20vlT0eK6JGLeWE7cXVn6oT26zvnqAjJ94PjS+YckMOExhqwCivPp1VaX6JzpQ1wr52OsGDUvconcjYrBEHBiY+UnMUk0Wj4mhZlJd1lpybZcWZ3vhTIlM0uMt4udl7t+zsgZ6BW97/pkGaa+QoxeTvgNlHGYyDYp8hveM3bCLXTHULw8mXUHxOJawq/J3E6vZ5/h2nzfmQmWtZtBOGWCkq+gKusTFUsHghjvHsPcQ2+EVfMcePBb/FKvtzSgH59C3iNOHE29l3ceSqccgxlxfStzbf+QkP7gxGVGZ8rLnCn3s8WzkGHZE4LtS0Zm3Y+hV5igrClk940YZP1hmilt2y7adPE4gCyQjb44JXgc3/NxlkZJcmeZTfAGxMXT8HG6Use/Kti114phsF7GDrqk1kPbB51Hr3xF1NAJUWP3csg3jgTS3E6jgD5XjPPG9BEDE2MwnBlUUMe3TC8TIWkK+AlwjlsDr5B9nqy2Fevv62+k5Adplw+fsQ8VzZREZF+MllWO3vtkD6srdx9h4vPD3dp5urFCFXNRaoD3SMDk27z3EVCQZ4bPL5PsVpB/ZBotLGkUZ0yi+5oC+u7ByP1ihMXMsRgvXbQpyOonEqDy84EZiIPWbyzGd0tEAXLz3mMh1x/IqZ1wxyDT/vkxhNCFqlBNlRW6GbMN2cng4A9Cigj9eNu9ptL1tdgFTxwndjoNRQMJ0NAc6WnsQ1UeIu8nMsa8/kLDtnVFLVmPQv2ZBUM4mxLrwC1mxOiQrWBW2XJ1OIheimSkLHfQOef1mIH3Z0cBuLBKGkRYGaXiZ6RX7po+ch0WFGjBef3e3uczl1mT5WGKdIG4x1+aRAtJHL+9K7Z6wzG0ygoamdiX2Fd0xBrWjTU72DzYbceqc+uHrbcLKDa5w0ENhyYK0+XEzG5fXHjFgmawY1D7xZQOJZO3jxStcv+xzoiTnNSrIxbxog/0Fez/WhMM9H6gV4eeDjMWEg79cJLugCBNwqmp3Yoe5EDU2TxQlLT53tye3Aji3FbocuDWjLI3Jc5VDxd7lrbzeIbFzSNpoFG8DSgjSiq41WJVeuzXxmdl7HM4zQpGRAAAAAQAFWC41MDkAAASsMIIEqDCCA5CgAwIBAgIJAJNurL4H8gHfMA0GCSqGSIb3DQEBBQUAMIGUMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEQMA4GA1UEChMHQW5kcm9pZDEQMA4GA1UECxMHQW5kcm9pZDEQMA4GA1UEAxMHQW5kcm9pZDEiMCAGCSqGSIb3DQEJARYTYW5kcm9pZEBhbmRyb2lkLmNvbTAeFw0wODAyMjkwMTMzNDZaFw0zNTA3MTcwMTMzNDZaMIGUMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEQMA4GA1UEChMHQW5kcm9pZDEQMA4GA1UECxMHQW5kcm9pZDEQMA4GA1UEAxMHQW5kcm9pZDEiMCAGCSqGSIb3DQEJARYTYW5kcm9pZEBhbmRyb2lkLmNvbTCCASAwDQYJKoZIhvcNAQEBBQADggENADCCAQgCggEBANaTGQTexgskse3HYuDZ2CU+Ps1s6x3i/waMqOi8qM1r03hupwqnbOYOuw+ZNVn/2T53qUPn6D1LZLjk/qLT5lbx4meoG7+yMLV4wgRDvkxyGLhG9SEVhvA4oU6Jwr44f46+z4/Kw9oe4zDJ6pPQp8PcSvNQIg1QCAcy4ICXF+5qBTNZ5qaU7Cyz8oSgpGbIepTYOzEJOmc3Li9kEsBubULxWBjf/gOBzAzURNps3cO4JFgZSAGzJWQTT7/emMkod0jb9WdqVA2BVMi7yge54kdVMxHEa5r3b97szI5p58ii0I54JiCUP5lyfTwE/nKZHZnfm644oLIXf6MdW2r+6R8CAQOjgfwwgfkwHQYDVR0OBBYEFEhZAFY9JyxGrhGGBaR0GawJyowRMIHJBgNVHSMEgcEwgb6AFEhZAFY9JyxGrhGGBaR0GawJyowRoYGapIGXMIGUMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEQMA4GA1UEChMHQW5kcm9pZDEQMA4GA1UECxMHQW5kcm9pZDEQMA4GA1UEAxMHQW5kcm9pZDEiMCAGCSqGSIb3DQEJARYTYW5kcm9pZEBhbmRyb2lkLmNvbYIJAJNurL4H8gHfMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAHqvlozrUMRBBVEY0NqrrwFbinZaJ6cVosK0TyIUFf/azgMJWr+kLfcHCHJsIGnlw27drgQAvilFLAhLwn62oX6snb4YLCBOsVMR9FXYJLZW2+TcIkCRLXWG/oiVHQGo/rWuWkJgU134NDEFJCJGjDbiLCpe+ZTWHdcwauTJ9pUbo8EvHRkU3cYfGmLaLfgn9gP+pWA7LFQNvXwBnDa6sppCccEX31I828XzgXpJ4O+mDL1/dBd+ek8ZPUP0IgdyZm5MTYPhvVqGCHzzTy3sIeJFymwrsBbmg2OAUNLEMO6nwmocSdN2ClirfxqCzJOLSDE4QyS9BAH6EhY6UFcOaE21IJawTAEXnf52TqT7diFUlWRSnQ==' AOSP_TESTKEY_DEBUG_KEYSTORE_KEY_FILE_NAME = ( 'debug_keystore_k47SVrA85+oMZAexHc62PkgvIgO8TJBYN00U82xSlxc_id_rsa' ) @@ -45,6 +53,8 @@ class NightlyTest(unittest.TestCase): path = os.environ['PATH'] def setUp(self): + common.config = None + nightly.config = None logging.basicConfig(level=logging.WARNING) self.basedir = Path(localmodule) / 'tests' self.testroot = Path(localmodule) / '.testfiles' @@ -69,6 +79,11 @@ class NightlyTest(unittest.TestCase): self.dot_android / 'debug.keystore', ) + def _copy_debug_apk(self): + outputdir = Path('app/build/output/apk/debug') + outputdir.mkdir(parents=True) + shutil.copy(self.basedir / 'urzip.apk', outputdir / 'urzip-debug.apk') + def test_get_repo_base_url(self): for clone_url, repo_git_base, result in [ ( @@ -88,6 +103,14 @@ class NightlyTest(unittest.TestCase): # gitlab.com often returns 403 Forbidden from their cloudflare restrictions self.assertTrue(r.status_code in (200, 403), 'should not be a redirect') + def test_get_keystore_secret_var(self): + self.assertEqual( + AOSP_TESTKEY_DEBUG_KEYSTORE, + nightly._get_keystore_secret_var( + self.basedir / 'aosp_testkey_debug.keystore' + ), + ) + @patch.dict(os.environ, clear=True) def test_ssh_key_from_debug_keystore(self): os.environ['HOME'] = str(self.home) @@ -145,6 +168,153 @@ class NightlyTest(unittest.TestCase): assert (dot_ssh / AOSP_TESTKEY_DEBUG_KEYSTORE_KEY_FILE_NAME).exists() assert (dot_ssh / (AOSP_TESTKEY_DEBUG_KEYSTORE_KEY_FILE_NAME + '.pub')).exists() + def _put_fdroid_in_args(self, args): + """Find fdroid command that belongs to this source code tree""" + fdroid = os.path.join(localmodule, 'fdroid') + if not os.path.exists(fdroid): + fdroid = os.getenv('fdroid') + return [fdroid] + args[1:] + + @patch('sys.argv', ['fdroid nightly', '--verbose']) + @patch('platform.node', lambda: 'example.com') + def test_github_actions(self): + """Careful! If the test env is bad, it'll mess up the local SSH setup + + https://docs.github.com/en/actions/learn-github-actions/environment-variables + + """ + + called = [] + orig_check_call = subprocess.check_call + os.chdir(self.testdir) + os.makedirs('fdroid/git-mirror/fdroid/repo') # fake this to avoid cloning + self._copy_test_debug_keystore() + self._copy_debug_apk() + + def _subprocess_check_call(args, cwd=None, env=None): + if os.path.basename(args[0]) in ('keytool', 'openssl'): + orig_check_call(args, cwd=cwd, env=env) + elif args[:2] == ['fdroid', 'update']: + orig_check_call(self._put_fdroid_in_args(args), cwd=cwd, env=env) + else: + called.append(args[:2]) + return + + with patch.dict( + os.environ, + { + 'CI': 'true', + 'DEBUG_KEYSTORE': DEBUG_KEYSTORE, + 'GITHUB_ACTIONS': 'true', + 'GITHUB_ACTOR': 'username', + 'GITHUB_REPOSITORY': 'f-droid/test', + 'GITHUB_SERVER_URL': 'https://github.com', + 'HOME': str(self.testdir), + 'PATH': os.getenv('PATH'), + 'fdroid': os.getenv('fdroid', ''), + }, + clear=True, + ): + self.assertTrue(self.testroot == Path.home().parent) + with patch('subprocess.check_call', _subprocess_check_call): + nightly.main() + self.assertEqual(called, [['ssh', '-Tvi'], ['fdroid', 'deploy']]) + self.assertFalse(os.path.exists('config.py')) + git_url = 'git@github.com:f-droid/test-nightly' + mirror_url = index.get_mirror_service_urls(git_url)[0] + expected = { + 'archive_description': 'Old nightly builds that have been archived.', + 'archive_name': 'f-droid/test-nightly archive', + 'archive_older': 20, + 'archive_url': mirror_url + '/archive', + 'keydname': 'CN=Android Debug,O=Android,C=US', + 'keypass': 'android', + 'keystore': nightly.KEYSTORE_FILE, + 'keystorepass': 'android', + 'make_current_version_link': False, + 'repo_description': 'Nightly builds from username@example.com', + 'repo_keyalias': 'androiddebugkey', + 'repo_name': 'f-droid/test-nightly', + 'repo_url': mirror_url + '/repo', + 'servergitmirrors': git_url, + 'update_stats': True, + } + with open('config.yml') as fp: + config = yaml.safe_load(fp) + # .ssh is random tmpdir set in nightly.py, so test basename only + self.assertEqual( + os.path.basename(config['identity_file']), + DEBUG_KEYSTORE_KEY_FILE_NAME, + ) + del config['identity_file'] + self.assertEqual(expected, config) + + @patch('sys.argv', ['fdroid nightly', '--verbose']) + def test_gitlab_ci(self): + """Careful! If the test env is bad, it can mess up the local SSH setup""" + called = [] + orig_check_call = subprocess.check_call + os.chdir(self.testdir) + os.makedirs('fdroid/git-mirror/fdroid/repo') # fake this to avoid cloning + self._copy_test_debug_keystore() + self._copy_debug_apk() + + def _subprocess_check_call(args, cwd=None, env=None): + if os.path.basename(args[0]) in ('keytool', 'openssl'): + orig_check_call(args, cwd=cwd, env=env) + elif args[:2] == ['fdroid', 'update']: + orig_check_call(self._put_fdroid_in_args(args), cwd=cwd, env=env) + else: + called.append(args[:2]) + return + + with patch.dict( + os.environ, + { + 'CI': 'true', + 'CI_PROJECT_PATH': 'fdroid/test', + 'CI_PROJECT_URL': 'https://gitlab.com/fdroid/test', + 'DEBUG_KEYSTORE': DEBUG_KEYSTORE, + 'GITLAB_USER_NAME': 'username', + 'GITLAB_USER_EMAIL': 'username@example.com', + 'HOME': str(self.testdir), + 'PATH': os.getenv('PATH'), + 'fdroid': os.getenv('fdroid', ''), + }, + clear=True, + ): + self.assertTrue(self.testroot == Path.home().parent) + with patch('subprocess.check_call', _subprocess_check_call): + nightly.main() + self.assertEqual(called, [['ssh', '-Tvi'], ['fdroid', 'deploy']]) + self.assertFalse(os.path.exists('config.py')) + expected = { + 'archive_description': 'Old nightly builds that have been archived.', + 'archive_name': 'fdroid/test-nightly archive', + 'archive_older': 20, + 'archive_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/archive', + 'keydname': 'CN=Android Debug,O=Android,C=US', + 'keypass': 'android', + 'keystore': nightly.KEYSTORE_FILE, + 'keystorepass': 'android', + 'make_current_version_link': False, + 'repo_description': 'Nightly builds from username@example.com', + 'repo_keyalias': 'androiddebugkey', + 'repo_name': 'fdroid/test-nightly', + 'repo_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/repo', + 'servergitmirrors': 'git@gitlab.com:fdroid/test-nightly', + 'update_stats': True, + } + with open('config.yml') as fp: + config = yaml.safe_load(fp) + # .ssh is random tmpdir set in nightly.py, so test basename only + self.assertEqual( + os.path.basename(config['identity_file']), + DEBUG_KEYSTORE_KEY_FILE_NAME, + ) + del config['identity_file'] + self.assertEqual(expected, config) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__))