mirror of
https://github.com/f-droid/fdroidserver.git
synced 2025-11-03 22:20:28 +03:00
Merge branch 'nightly-fixes' into 'master'
fixes to deploy/nightly made while working on !1666 and reviewing !1650 See merge request fdroid/fdroidserver!1668
This commit is contained in:
Michael Pöhn
commit
720cbed8d5
7 changed files with 222 additions and 190 deletions
|
|
@ -156,7 +156,7 @@ def main():
|
|||
).git.describe(always=True, tags=True)
|
||||
)
|
||||
sys.exit(0)
|
||||
except git.exc.InvalidGitRepositoryError:
|
||||
except git.InvalidGitRepositoryError:
|
||||
print(_('No version information could be found.'))
|
||||
sys.exit(1)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -40,7 +40,6 @@ import fdroidserver.github
|
|||
from . import _, common, index
|
||||
from .exception import FDroidException
|
||||
|
||||
config = None
|
||||
start_timestamp = time.gmtime()
|
||||
|
||||
GIT_BRANCH = 'master'
|
||||
|
|
@ -144,6 +143,7 @@ def update_remote_storage_with_rclone(
|
|||
"""
|
||||
logging.debug(_('Using rclone to sync to "{name}"').format(name=awsbucket))
|
||||
|
||||
config = common.get_config()
|
||||
rclone_config = config.get('rclone_config', [])
|
||||
if rclone_config and isinstance(rclone_config, str):
|
||||
rclone_config = [rclone_config]
|
||||
|
|
@ -271,6 +271,7 @@ def update_serverwebroot(serverwebroot, repo_section):
|
|||
has a low resolution timestamp
|
||||
|
||||
"""
|
||||
config = common.get_config()
|
||||
try:
|
||||
subprocess.run(['rsync', '--version'], capture_output=True, check=True)
|
||||
except Exception as e:
|
||||
|
|
@ -431,6 +432,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
|||
"""
|
||||
from clint.textui import progress
|
||||
|
||||
config = common.get_config()
|
||||
if config.get('local_copy_dir') and not config.get('sync_from_local_copy_dir'):
|
||||
logging.debug(
|
||||
_('Offline machine, skipping git mirror generation until `fdroid deploy`')
|
||||
|
|
@ -530,6 +532,20 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
|||
progressbar.done()
|
||||
|
||||
|
||||
def _get_commit_author(git_repo):
|
||||
"""If the author is set locally, use it, otherwise use static info."""
|
||||
ret = {'name': 'servergitmirrors', 'email': 'fdroid@deploy'}
|
||||
with git_repo.config_reader() as cr:
|
||||
for option in ('name', 'email'):
|
||||
try:
|
||||
value = cr.get_value('user', option)
|
||||
except (configparser.NoSectionError, configparser.NoOptionError):
|
||||
value = os.getenv(f'GITLAB_USER_{option.upper()}')
|
||||
if value:
|
||||
ret[option] = value
|
||||
return git.Actor(ret['name'], ret['email'])
|
||||
|
||||
|
||||
def upload_to_servergitmirror(
|
||||
mirror_config: Dict[str, str],
|
||||
local_repo: Repo,
|
||||
|
|
@ -557,18 +573,23 @@ def upload_to_servergitmirror(
|
|||
logging.info('Mirroring to: ' + remote_url)
|
||||
|
||||
if is_index_only:
|
||||
logging.debug(_('Committing index files to git mirror'))
|
||||
files_to_upload = _get_index_file_paths(
|
||||
os.path.join(local_repo.working_tree_dir, 'fdroid', repo_section)
|
||||
)
|
||||
files_to_upload = _remove_missing_files(files_to_upload)
|
||||
local_repo.index.add(files_to_upload)
|
||||
local_repo.index.commit(
|
||||
"servergitmirrors: index-only in git-mirror",
|
||||
author=_get_commit_author(local_repo),
|
||||
)
|
||||
else:
|
||||
# sadly index.add don't allow the --all parameter
|
||||
logging.debug('Adding all files to git mirror')
|
||||
logging.debug(_('Adding all files to git mirror'))
|
||||
local_repo.git.add(all=True)
|
||||
|
||||
logging.debug('Committing files into git mirror')
|
||||
local_repo.index.commit("fdroidserver git-mirror")
|
||||
local_repo.index.commit(
|
||||
"servergitmirrors: in git-mirror", author=_get_commit_author(local_repo)
|
||||
)
|
||||
|
||||
# only deploy to GitLab Artifacts if too big for GitLab Pages
|
||||
if (
|
||||
|
|
@ -1029,8 +1050,6 @@ def upload_to_github_releases_repo(repo_conf, release_infos, global_gh_token):
|
|||
|
||||
|
||||
def main():
|
||||
global config
|
||||
|
||||
parser = ArgumentParser()
|
||||
common.setup_global_opts(parser)
|
||||
parser.add_argument(
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
|
||||
import base64
|
||||
import datetime
|
||||
import glob
|
||||
import hashlib
|
||||
import inspect
|
||||
import logging
|
||||
|
|
@ -236,11 +237,21 @@ def clone_git_repo(clone_url, git_mirror_path):
|
|||
'GIT_TERMINAL_PROMPT': '0',
|
||||
},
|
||||
)
|
||||
except git.exc.GitCommandError as e:
|
||||
except git.GitCommandError as e:
|
||||
logging.warning(_('WARNING: only public git repos are supported!'))
|
||||
raise VCSException(f'git clone {clone_url} failed:', str(e)) from e
|
||||
|
||||
|
||||
def _getenv(name):
|
||||
"""Return the value of an environment variable, printing an error if None."""
|
||||
value = os.getenv(name)
|
||||
if not value:
|
||||
logging.error(
|
||||
_('Environment variable "{name}" has an empty value!').format(name=name)
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
def main():
|
||||
"""Deploy to F-Droid repository or generate SSH private key from keystore.
|
||||
|
||||
|
|
@ -303,7 +314,7 @@ def main():
|
|||
umask = os.umask(0o077)
|
||||
|
||||
if 'CI' in os.environ:
|
||||
v = os.getenv('DEBUG_KEYSTORE')
|
||||
v = _getenv('DEBUG_KEYSTORE')
|
||||
debug_keystore = None
|
||||
if v:
|
||||
debug_keystore = base64.b64decode(v)
|
||||
|
|
@ -324,20 +335,19 @@ def main():
|
|||
# the 'master' branch is hardcoded in fdroidserver/deploy.py
|
||||
if 'CI_PROJECT_PATH' in os.environ and 'CI_PROJECT_URL' in os.environ:
|
||||
# we are in GitLab CI
|
||||
repo_git_base = os.getenv('CI_PROJECT_PATH') + NIGHTLY
|
||||
clone_url = os.getenv('CI_PROJECT_URL') + NIGHTLY
|
||||
repo_git_base = f"{_getenv('CI_PROJECT_PATH')}{NIGHTLY}"
|
||||
base_url = f"{_getenv('CI_PROJECT_URL')}{NIGHTLY}"
|
||||
clone_url = f'{base_url}.git' # avoid redirects while cloning
|
||||
repo_base = get_repo_base_url(
|
||||
clone_url, repo_git_base, force_type='gitlab.com'
|
||||
base_url, repo_git_base, force_type='gitlab.com'
|
||||
)
|
||||
servergitmirror = 'git@' + urlparse(clone_url).netloc + ':' + repo_git_base
|
||||
deploy_key_url = (
|
||||
f'{clone_url}/-/settings/repository#js-deploy-keys-settings'
|
||||
)
|
||||
git_user_name = os.getenv('GITLAB_USER_NAME')
|
||||
git_user_email = os.getenv('GITLAB_USER_EMAIL')
|
||||
servergitmirror = f'git@{urlparse(base_url).netloc}:{repo_git_base}.git'
|
||||
deploy_key_url = f'{base_url}/-/settings/repository#js-deploy-keys-settings'
|
||||
git_user_name = _getenv('GITLAB_USER_NAME')
|
||||
git_user_email = _getenv('GITLAB_USER_EMAIL')
|
||||
elif 'TRAVIS_REPO_SLUG' in os.environ:
|
||||
# we are in Travis CI
|
||||
repo_git_base = os.getenv('TRAVIS_REPO_SLUG') + NIGHTLY
|
||||
repo_git_base = _getenv('TRAVIS_REPO_SLUG') + NIGHTLY
|
||||
clone_url = 'https://github.com/' + repo_git_base
|
||||
repo_base = get_repo_base_url(
|
||||
clone_url, repo_git_base, force_type='github.com'
|
||||
|
|
@ -348,7 +358,7 @@ def main():
|
|||
+ '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys'
|
||||
)
|
||||
git_user_name = repo_git_base
|
||||
git_user_email = os.getenv('USER') + '@' + platform.node()
|
||||
git_user_email = _getenv('USER') + '@' + platform.node()
|
||||
elif (
|
||||
'CIRCLE_REPOSITORY_URL' in os.environ
|
||||
and 'CIRCLE_PROJECT_USERNAME' in os.environ
|
||||
|
|
@ -356,12 +366,12 @@ def main():
|
|||
):
|
||||
# we are in Circle CI
|
||||
repo_git_base = (
|
||||
os.getenv('CIRCLE_PROJECT_USERNAME')
|
||||
_getenv('CIRCLE_PROJECT_USERNAME')
|
||||
+ '/'
|
||||
+ os.getenv('CIRCLE_PROJECT_REPONAME')
|
||||
+ _getenv('CIRCLE_PROJECT_REPONAME')
|
||||
+ NIGHTLY
|
||||
)
|
||||
clone_url = os.getenv('CIRCLE_REPOSITORY_URL') + NIGHTLY
|
||||
clone_url = _getenv('CIRCLE_REPOSITORY_URL') + NIGHTLY
|
||||
repo_base = get_repo_base_url(
|
||||
clone_url, repo_git_base, force_type='github.com'
|
||||
)
|
||||
|
|
@ -370,12 +380,12 @@ def main():
|
|||
f'https://github.com/{repo_git_base}/settings/keys'
|
||||
+ '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys'
|
||||
)
|
||||
git_user_name = os.getenv('CIRCLE_USERNAME')
|
||||
git_user_name = _getenv('CIRCLE_USERNAME')
|
||||
git_user_email = git_user_name + '@' + platform.node()
|
||||
elif 'GITHUB_ACTIONS' in os.environ:
|
||||
# we are in Github actions
|
||||
repo_git_base = os.getenv('GITHUB_REPOSITORY') + NIGHTLY
|
||||
clone_url = os.getenv('GITHUB_SERVER_URL') + '/' + repo_git_base
|
||||
repo_git_base = _getenv('GITHUB_REPOSITORY') + NIGHTLY
|
||||
clone_url = _getenv('GITHUB_SERVER_URL') + '/' + repo_git_base
|
||||
repo_base = get_repo_base_url(
|
||||
clone_url, repo_git_base, force_type='github.com'
|
||||
)
|
||||
|
|
@ -384,7 +394,7 @@ def main():
|
|||
f'https://github.com/{repo_git_base}/settings/keys'
|
||||
+ '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys'
|
||||
)
|
||||
git_user_name = os.getenv('GITHUB_ACTOR')
|
||||
git_user_name = _getenv('GITHUB_ACTOR')
|
||||
git_user_email = git_user_name + '@' + platform.node()
|
||||
else:
|
||||
print(_('ERROR: unsupported CI type, patches welcome!'))
|
||||
|
|
@ -397,6 +407,8 @@ def main():
|
|||
git_mirror_metadatadir = os.path.join(git_mirror_fdroiddir, 'metadata')
|
||||
if not os.path.isdir(git_mirror_repodir):
|
||||
clone_git_repo(clone_url, git_mirror_path)
|
||||
for f in glob.glob(f'{git_mirror_repodir}/*.apk'):
|
||||
shutil.copy2(f, repodir)
|
||||
if not os.path.isdir(git_mirror_repodir):
|
||||
os.makedirs(git_mirror_repodir, mode=0o755)
|
||||
if os.path.exists('LICENSE'):
|
||||
|
|
@ -442,7 +454,7 @@ Last updated: {date}'''.format(
|
|||
|
||||
ssh_private_key_file = _ssh_key_from_debug_keystore()
|
||||
# this is needed for GitPython to find the SSH key
|
||||
ssh_dir = os.path.join(os.getenv('HOME'), '.ssh')
|
||||
ssh_dir = os.path.join(_getenv('HOME'), '.ssh')
|
||||
os.makedirs(ssh_dir, exist_ok=True)
|
||||
ssh_config = os.path.join(ssh_dir, 'config')
|
||||
logging.debug(_('adding IdentityFile to {path}').format(path=ssh_config))
|
||||
|
|
@ -587,7 +599,7 @@ Last updated: {date}'''.format(
|
|||
+ '\n -dname "CN=Android Debug,O=Android,C=US"'
|
||||
)
|
||||
sys.exit(1)
|
||||
ssh_dir = os.path.join(os.getenv('HOME'), '.ssh')
|
||||
ssh_dir = os.path.join(_getenv('HOME'), '.ssh')
|
||||
privkey = _ssh_key_from_debug_keystore(options.keystore)
|
||||
if os.path.exists(ssh_dir):
|
||||
ssh_private_key_file = os.path.join(ssh_dir, os.path.basename(privkey))
|
||||
|
|
|
|||
|
|
@ -2466,16 +2466,18 @@ def create_metadata_from_template(apk):
|
|||
|
||||
|
||||
def read_added_date_from_all_apks(apps, apks):
|
||||
"""No summary.
|
||||
"""Read the "added" date from all packages.
|
||||
|
||||
Added dates come from the repo/index-v2.json file but are
|
||||
read when scanning apks and thus need to be applied form apk
|
||||
level to app level for _all_ apps and not only from non-archived
|
||||
ones
|
||||
Added dates come from index-v2.json file but are read when scanning
|
||||
APKs and thus need to be applied from package-level to app-level for
|
||||
_all_ apps and not only from non-archived ones.
|
||||
|
||||
TODO: read the added dates directly from index-v2.json instead of
|
||||
going through apks that way it also works for for repos that
|
||||
don't keep an archive of apks.
|
||||
don't keep an archive of APKs.
|
||||
|
||||
https://gitlab.com/fdroid/wiki/-/wikis/PackageDateHandling
|
||||
|
||||
"""
|
||||
for appid, app in apps.items():
|
||||
for apk in apks:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import configparser
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
|
|
@ -26,15 +27,21 @@ def _mock_rclone_config_file(cmd, text): # pylint: disable=unused-argument
|
|||
class DeployTest(unittest.TestCase):
|
||||
'''fdroidserver/deploy.py'''
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
# suppress "WARNING:root:unsafe permissions on 'config.yml' (should be 0600)!"
|
||||
os.chmod(os.path.join(basedir, fdroidserver.common.CONFIG_FILE), 0o600)
|
||||
|
||||
def setUp(self):
|
||||
os.chdir(basedir)
|
||||
self._td = mkdtemp()
|
||||
self.testdir = self._td.name
|
||||
|
||||
fdroidserver.common.options = mock.Mock()
|
||||
fdroidserver.deploy.config = {}
|
||||
fdroidserver.common.get_config()
|
||||
|
||||
def tearDown(self):
|
||||
fdroidserver.common.config = None
|
||||
self._td.cleanup()
|
||||
|
||||
def test_update_serverwebroots_bad_None(self):
|
||||
|
|
@ -64,7 +71,6 @@ class DeployTest(unittest.TestCase):
|
|||
|
||||
# setup parameters for this test run
|
||||
fdroidserver.common.options.identity_file = None
|
||||
fdroidserver.deploy.config['make_current_version_link'] = False
|
||||
|
||||
dest_apk0 = url0 / fake_apk
|
||||
dest_apk1 = url1 / fake_apk
|
||||
|
|
@ -81,17 +87,17 @@ class DeployTest(unittest.TestCase):
|
|||
self.assertTrue(dest_apk1.is_file())
|
||||
|
||||
def test_update_serverwebroots_url_does_not_end_with_fdroid(self):
|
||||
with self.assertRaises(SystemExit):
|
||||
with self.assertRaises(SystemExit), self.assertLogs(level=logging.ERROR):
|
||||
fdroidserver.deploy.update_serverwebroots([{'url': 'url'}], 'repo')
|
||||
|
||||
def test_update_serverwebroots_bad_ssh_url(self):
|
||||
with self.assertRaises(SystemExit):
|
||||
with self.assertRaises(SystemExit), self.assertLogs(level=logging.ERROR):
|
||||
fdroidserver.deploy.update_serverwebroots(
|
||||
[{'url': 'f@b.ar::/path/to/fdroid'}], 'repo'
|
||||
)
|
||||
|
||||
def test_update_serverwebroots_unsupported_ssh_url(self):
|
||||
with self.assertRaises(SystemExit):
|
||||
with self.assertRaises(SystemExit), self.assertLogs(level=logging.ERROR):
|
||||
fdroidserver.deploy.update_serverwebroots([{'url': 'ssh://nope'}], 'repo')
|
||||
|
||||
@unittest.skipUnless(shutil.which('rclone'), 'requires rclone')
|
||||
|
|
@ -120,9 +126,9 @@ class DeployTest(unittest.TestCase):
|
|||
|
||||
# setup parameters for this test run
|
||||
awsbucket = 'test_bucket_folder'
|
||||
fdroidserver.deploy.config['awsbucket'] = awsbucket
|
||||
fdroidserver.deploy.config['rclone_config'] = 'test-local-config'
|
||||
fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
|
||||
fdroidserver.common.config['awsbucket'] = awsbucket
|
||||
fdroidserver.common.config['rclone_config'] = 'test-local-config'
|
||||
fdroidserver.common.config['path_to_custom_rclone_config'] = str(rclone_file)
|
||||
fdroidserver.common.options = VerboseFalseOptions
|
||||
|
||||
# write out destination path
|
||||
|
|
@ -163,9 +169,9 @@ class DeployTest(unittest.TestCase):
|
|||
|
||||
# setup parameters for this test run
|
||||
awsbucket = 'test_bucket_folder'
|
||||
fdroidserver.deploy.config['awsbucket'] = awsbucket
|
||||
fdroidserver.deploy.config['rclone_config'] = 'test-local-config'
|
||||
fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
|
||||
fdroidserver.common.config['awsbucket'] = awsbucket
|
||||
fdroidserver.common.config['rclone_config'] = 'test-local-config'
|
||||
fdroidserver.common.config['path_to_custom_rclone_config'] = str(rclone_file)
|
||||
fdroidserver.common.options = VerboseFalseOptions
|
||||
|
||||
# write out destination path
|
||||
|
|
@ -224,7 +230,7 @@ class DeployTest(unittest.TestCase):
|
|||
return 0
|
||||
|
||||
mock_call.side_effect = _mock_subprocess_call
|
||||
fdroidserver.deploy.config = {'awsbucket': awsbucket}
|
||||
fdroidserver.common.config = {'awsbucket': awsbucket}
|
||||
fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket)
|
||||
mock_call.assert_called()
|
||||
|
||||
|
|
@ -243,7 +249,7 @@ class DeployTest(unittest.TestCase):
|
|||
|
||||
mock_call.side_effect = _mock_subprocess_call
|
||||
|
||||
fdroidserver.deploy.config = {'awsbucket': awsbucket}
|
||||
fdroidserver.common.config = {'awsbucket': awsbucket}
|
||||
fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket)
|
||||
self.maxDiff = None
|
||||
self.assertEqual(
|
||||
|
|
@ -262,7 +268,6 @@ class DeployTest(unittest.TestCase):
|
|||
@mock.patch('subprocess.check_output', _mock_rclone_config_file)
|
||||
@mock.patch('subprocess.call')
|
||||
def test_update_remote_storage_with_rclone_mock_rclone_config(self, mock_call):
|
||||
awsbucket = 'test_bucket_folder'
|
||||
self.last_cmd = None
|
||||
|
||||
def _mock_subprocess_call(cmd):
|
||||
|
|
@ -271,10 +276,9 @@ class DeployTest(unittest.TestCase):
|
|||
|
||||
mock_call.side_effect = _mock_subprocess_call
|
||||
|
||||
fdroidserver.deploy.config = {
|
||||
'awsbucket': awsbucket,
|
||||
'rclone_config': 'test_local_config',
|
||||
}
|
||||
awsbucket = 'test_bucket_folder'
|
||||
fdroidserver.common.config['awsbucket'] = awsbucket
|
||||
fdroidserver.common.config['rclone_config'] = 'test_local_config'
|
||||
fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket)
|
||||
self.maxDiff = None
|
||||
self.assertEqual(
|
||||
|
|
@ -304,8 +308,8 @@ class DeployTest(unittest.TestCase):
|
|||
Path('rclone.conf').write_text('placeholder, contents ignored')
|
||||
|
||||
awsbucket = 'test_bucket_folder'
|
||||
fdroidserver.deploy.config['awsbucket'] = awsbucket
|
||||
fdroidserver.deploy.config['rclone_config'] = config_name
|
||||
fdroidserver.common.config['awsbucket'] = awsbucket
|
||||
fdroidserver.common.config['rclone_config'] = config_name
|
||||
fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket)
|
||||
self.maxDiff = None
|
||||
self.assertEqual(
|
||||
|
|
@ -339,7 +343,6 @@ class DeployTest(unittest.TestCase):
|
|||
fdroidserver.common.options = mock.Mock()
|
||||
fdroidserver.common.options.identity_file = None
|
||||
fdroidserver.common.options.identity_file = None
|
||||
fdroidserver.deploy.config['make_current_version_link'] = False
|
||||
|
||||
dest_apk = Path(url) / fake_apk
|
||||
dest_index = Path(url) / fake_index
|
||||
|
|
@ -366,7 +369,6 @@ class DeployTest(unittest.TestCase):
|
|||
# setup parameters for this test run
|
||||
fdroidserver.common.options = mock.Mock()
|
||||
fdroidserver.common.options.identity_file = None
|
||||
fdroidserver.deploy.config['make_current_version_link'] = False
|
||||
|
||||
dest_apk = Path(url) / fake_apk
|
||||
dest_index = Path(url) / fake_index
|
||||
|
|
@ -396,7 +398,7 @@ class DeployTest(unittest.TestCase):
|
|||
fdroidserver.common.options.verbose = False
|
||||
fdroidserver.common.options.quiet = True
|
||||
fdroidserver.common.options.index_only = False
|
||||
fdroidserver.deploy.config = {'make_current_version_link': True}
|
||||
fdroidserver.common.config['make_current_version_link'] = True
|
||||
url = "example.com:/var/www/fdroid"
|
||||
repo_section = 'repo'
|
||||
|
||||
|
|
@ -504,7 +506,7 @@ class DeployTest(unittest.TestCase):
|
|||
fdroidserver.common.options.verbose = False
|
||||
fdroidserver.common.options.quiet = True
|
||||
fdroidserver.common.options.identity_file = None
|
||||
fdroidserver.deploy.config['make_current_version_link'] = True
|
||||
fdroidserver.common.config['make_current_version_link'] = True
|
||||
url = "example.com:/var/www/fdroid"
|
||||
repo_section = 'repo'
|
||||
|
||||
|
|
@ -603,7 +605,7 @@ class DeployTest(unittest.TestCase):
|
|||
fdroidserver.common.options.quiet = False
|
||||
fdroidserver.common.options.identity_file = None
|
||||
fdroidserver.common.options.index_only = False
|
||||
fdroidserver.deploy.config = {'identity_file': './id_rsa'}
|
||||
fdroidserver.common.config = {'identity_file': './id_rsa'}
|
||||
url = "example.com:/var/www/fdroid"
|
||||
repo_section = 'archive'
|
||||
|
||||
|
|
@ -623,7 +625,7 @@ class DeployTest(unittest.TestCase):
|
|||
'--verbose',
|
||||
'-e',
|
||||
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
|
||||
+ fdroidserver.deploy.config['identity_file'],
|
||||
+ fdroidserver.common.config['identity_file'],
|
||||
'--exclude',
|
||||
'archive/altstore-index.json',
|
||||
'--exclude',
|
||||
|
|
@ -669,7 +671,7 @@ class DeployTest(unittest.TestCase):
|
|||
'--verbose',
|
||||
'-e',
|
||||
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
|
||||
+ fdroidserver.deploy.config['identity_file'],
|
||||
+ fdroidserver.common.config['identity_file'],
|
||||
'archive',
|
||||
url,
|
||||
],
|
||||
|
|
@ -690,8 +692,7 @@ class DeployTest(unittest.TestCase):
|
|||
fdroidserver.common.options.verbose = True
|
||||
fdroidserver.common.options.quiet = False
|
||||
fdroidserver.common.options.identity_file = None
|
||||
fdroidserver.deploy.config['identity_file'] = './id_rsa'
|
||||
fdroidserver.deploy.config['make_current_version_link'] = False
|
||||
fdroidserver.common.config['identity_file'] = './id_rsa'
|
||||
url = "example.com:/var/www/fdroid"
|
||||
repo_section = 'archive'
|
||||
|
||||
|
|
@ -711,7 +712,7 @@ class DeployTest(unittest.TestCase):
|
|||
'--verbose',
|
||||
'-e',
|
||||
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
|
||||
+ fdroidserver.deploy.config['identity_file'],
|
||||
+ fdroidserver.common.config['identity_file'],
|
||||
'archive/altstore-index.json',
|
||||
'archive/altstore-index.json.asc',
|
||||
'archive/entry.jar',
|
||||
|
|
@ -741,7 +742,7 @@ class DeployTest(unittest.TestCase):
|
|||
'--verbose',
|
||||
'-e',
|
||||
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
|
||||
+ fdroidserver.deploy.config['identity_file'],
|
||||
+ fdroidserver.common.config['identity_file'],
|
||||
"example.com:/var/www/fdroid/archive/",
|
||||
],
|
||||
)
|
||||
|
|
@ -785,7 +786,58 @@ class DeployTest(unittest.TestCase):
|
|||
name, fdroidserver.deploy.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
|
||||
)
|
||||
|
||||
def test_update_servergitmirrors(self):
|
||||
@mock.patch.dict(os.environ, clear=True)
|
||||
def test_get_commit_author_no_config(self):
|
||||
os.environ['HOME'] = self.testdir
|
||||
git_repo = git.Repo.init(self.testdir)
|
||||
self.assertEqual(
|
||||
git.Actor('servergitmirrors', 'fdroid@deploy'),
|
||||
fdroidserver.deploy._get_commit_author(git_repo),
|
||||
)
|
||||
|
||||
@mock.patch.dict(os.environ, clear=True)
|
||||
def test_get_commit_author_repo_config(self):
|
||||
os.environ['HOME'] = self.testdir
|
||||
git_repo = git.Repo.init(self.testdir)
|
||||
user_name = 'Foo Bar'
|
||||
user_email = 'foo@bar.com'
|
||||
with git_repo.config_writer() as cw:
|
||||
cw.set_value('user', 'name', user_name)
|
||||
cw.set_value('user', 'email', user_email)
|
||||
self.assertEqual(
|
||||
git.Actor(user_name, user_email),
|
||||
fdroidserver.deploy._get_commit_author(git_repo),
|
||||
)
|
||||
|
||||
@mock.patch.dict(os.environ, clear=True)
|
||||
def test_get_commit_author_repo_config_name_only(self):
|
||||
os.environ['HOME'] = self.testdir
|
||||
git_repo = git.Repo.init(self.testdir)
|
||||
user_name = 'Foo Bar'
|
||||
with git_repo.config_writer() as cw:
|
||||
cw.set_value('user', 'name', user_name)
|
||||
self.assertEqual(
|
||||
git.Actor(user_name, 'fdroid@deploy'),
|
||||
fdroidserver.deploy._get_commit_author(git_repo),
|
||||
)
|
||||
|
||||
@mock.patch.dict(os.environ, clear=True)
|
||||
def test_get_commit_author_repo_config_email_only(self):
|
||||
os.environ['HOME'] = self.testdir
|
||||
git_repo = git.Repo.init(self.testdir)
|
||||
user_email = 'foo@bar.com'
|
||||
with git_repo.config_writer() as cw:
|
||||
cw.set_value('user', 'email', user_email)
|
||||
self.assertEqual(
|
||||
git.Actor('servergitmirrors', user_email),
|
||||
fdroidserver.deploy._get_commit_author(git_repo),
|
||||
)
|
||||
|
||||
|
||||
class TestServerGitMirrors(unittest.TestCase):
|
||||
def setUp(self):
|
||||
fdroidserver.deploy.USER_RCLONE_CONF = False
|
||||
|
||||
# setup parameters for this test run
|
||||
fdroidserver.common.options = mock.Mock()
|
||||
fdroidserver.common.options.identity_file = None
|
||||
|
|
@ -793,130 +845,93 @@ class DeployTest(unittest.TestCase):
|
|||
fdroidserver.common.options.verbose = False
|
||||
fdroidserver.common.options.quiet = True
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
fdroidserver.deploy.config = config
|
||||
|
||||
self._td = mkdtemp()
|
||||
self.testdir = self._td.name
|
||||
os.chdir(self.testdir)
|
||||
|
||||
repo_section = 'repo'
|
||||
initial_branch = fdroidserver.deploy.GIT_BRANCH
|
||||
|
||||
remote_repo = Path(self.testdir) / 'remote'
|
||||
remote_repo.mkdir(parents=True)
|
||||
remote_git_repo = git.Repo.init(
|
||||
remote_repo, initial_branch=initial_branch, bare=True
|
||||
self.remote_git_repo = git.Repo.init(
|
||||
remote_repo, initial_branch=fdroidserver.deploy.GIT_BRANCH, bare=True
|
||||
)
|
||||
fdroidserver.deploy.config["servergitmirrors"] = [{"url": str(remote_repo)}]
|
||||
|
||||
os.chdir(self.testdir)
|
||||
repo = Path('repo')
|
||||
repo.mkdir(parents=True)
|
||||
fake_apk = 'Sym.apk'
|
||||
fake_files = fdroidserver.common.INDEX_FILES + [fake_apk]
|
||||
for filename in fake_files:
|
||||
fdroidserver.common.get_config()
|
||||
fdroidserver.common.config["servergitmirrors"] = [{"url": str(remote_repo)}]
|
||||
|
||||
self.repo_section = 'repo'
|
||||
repo = Path(self.repo_section)
|
||||
repo.mkdir()
|
||||
self.fake_apk = 'Sym.apk'
|
||||
self.fake_files = fdroidserver.common.INDEX_FILES + [self.fake_apk]
|
||||
for filename in self.fake_files:
|
||||
fake_file = repo / filename
|
||||
with fake_file.open('w') as fp:
|
||||
fp.write('not a real one, but has the right filename')
|
||||
|
||||
def tearDown(self):
|
||||
fdroidserver.common.config = None
|
||||
fdroidserver.common.options = None
|
||||
self._td.cleanup()
|
||||
|
||||
def test_update_servergitmirrors(self):
|
||||
fdroidserver.deploy.update_servergitmirrors(
|
||||
fdroidserver.deploy.config["servergitmirrors"], repo_section
|
||||
fdroidserver.common.config["servergitmirrors"], self.repo_section
|
||||
)
|
||||
|
||||
verify_repo = remote_git_repo.clone(
|
||||
Path(self.testdir) / 'verify',
|
||||
verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify')
|
||||
self.assertIsNotNone(verify_repo.working_tree_dir)
|
||||
for filename in self.fake_files:
|
||||
remote_file = f"fdroid/{self.repo_section}/{filename}"
|
||||
self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists())
|
||||
|
||||
def test_update_servergitmirrors_with_existing_git_repo(self):
|
||||
"""Confirm it works with clones done manually or with nightly."""
|
||||
fdroidserver.deploy.update_servergitmirrors(
|
||||
fdroidserver.common.config["servergitmirrors"], self.repo_section
|
||||
)
|
||||
|
||||
for filename in fake_files:
|
||||
remote_file = f"fdroid/{repo_section}/{filename}"
|
||||
# now delete the local setup, clone the remote, and add a new APK
|
||||
git_mirror = os.path.join(self.testdir, 'git-mirror')
|
||||
shutil.rmtree(git_mirror)
|
||||
self.remote_git_repo.clone(git_mirror)
|
||||
new_fake_apk = 'Sym2.apk'
|
||||
self.fake_files.append(new_fake_apk)
|
||||
(Path(self.repo_section) / new_fake_apk).write_text('a new fake APK')
|
||||
|
||||
self.assertIsNotNone(verify_repo.working_tree_dir)
|
||||
if verify_repo.working_tree_dir is not None:
|
||||
self.assertTrue(
|
||||
(Path(verify_repo.working_tree_dir) / remote_file).exists()
|
||||
)
|
||||
fdroidserver.deploy.update_servergitmirrors(
|
||||
fdroidserver.common.config["servergitmirrors"], self.repo_section
|
||||
)
|
||||
|
||||
verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify')
|
||||
self.assertIsNotNone(verify_repo.working_tree_dir)
|
||||
for filename in self.fake_files:
|
||||
remote_file = f"fdroid/{self.repo_section}/{filename}"
|
||||
self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists())
|
||||
|
||||
def test_update_servergitmirrors_in_index_only_mode(self):
|
||||
# setup parameters for this test run
|
||||
fdroidserver.common.options = mock.Mock()
|
||||
fdroidserver.common.options.identity_file = None
|
||||
fdroidserver.common.options.no_keep_git_mirror_archive = False
|
||||
fdroidserver.common.options.verbose = False
|
||||
fdroidserver.common.options.quiet = True
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
fdroidserver.deploy.config = config
|
||||
|
||||
os.chdir(self.testdir)
|
||||
|
||||
repo_section = 'repo'
|
||||
initial_branch = fdroidserver.deploy.GIT_BRANCH
|
||||
|
||||
remote_repo = Path(self.testdir) / 'remote'
|
||||
remote_repo.mkdir(parents=True)
|
||||
remote_git_repo = git.Repo.init(
|
||||
remote_repo, initial_branch=initial_branch, bare=True
|
||||
)
|
||||
fdroidserver.deploy.config["servergitmirrors"] = [
|
||||
{"url": str(remote_repo), "index_only": True}
|
||||
]
|
||||
|
||||
os.chdir(self.testdir)
|
||||
repo = Path('repo')
|
||||
repo.mkdir(parents=True)
|
||||
fake_apk = 'Sym.apk'
|
||||
fake_files = fdroidserver.common.INDEX_FILES + [fake_apk]
|
||||
for filename in fake_files:
|
||||
fake_file = repo / filename
|
||||
with fake_file.open('w') as fp:
|
||||
fp.write('not a real one, but has the right filename')
|
||||
fdroidserver.common.config["servergitmirrors"][0]["index_only"] = True
|
||||
|
||||
fdroidserver.deploy.update_servergitmirrors(
|
||||
fdroidserver.deploy.config["servergitmirrors"], repo_section
|
||||
)
|
||||
|
||||
verify_repo = remote_git_repo.clone(
|
||||
Path(self.testdir) / 'verify',
|
||||
fdroidserver.common.config["servergitmirrors"], self.repo_section
|
||||
)
|
||||
|
||||
verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify')
|
||||
self.assertIsNotNone(verify_repo.working_tree_dir)
|
||||
for filename in fdroidserver.common.INDEX_FILES:
|
||||
remote_file = f"fdroid/{repo_section}/{filename}"
|
||||
|
||||
self.assertIsNotNone(verify_repo.working_tree_dir)
|
||||
if verify_repo.working_tree_dir is not None:
|
||||
self.assertTrue(
|
||||
(Path(verify_repo.working_tree_dir) / remote_file).exists()
|
||||
)
|
||||
remote_file = f"fdroid/{self.repo_section}/{filename}"
|
||||
self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists())
|
||||
|
||||
# Should not have the APK file
|
||||
remote_file = f"fdroid/{repo_section}/{fake_apk}"
|
||||
if verify_repo.working_tree_dir is not None:
|
||||
self.assertFalse(
|
||||
(Path(verify_repo.working_tree_dir) / remote_file).exists()
|
||||
)
|
||||
remote_file = f"fdroid/{self.repo_section}/{self.fake_apk}"
|
||||
self.assertFalse((Path(verify_repo.working_tree_dir) / remote_file).exists())
|
||||
|
||||
def test_upload_to_servergitmirror_in_index_only_mode(self):
|
||||
# setup parameters for this test run
|
||||
fdroidserver.common.options = mock.Mock()
|
||||
fdroidserver.common.options.identity_file = None
|
||||
fdroidserver.common.options.no_keep_git_mirror_archive = False
|
||||
fdroidserver.common.options.verbose = False
|
||||
fdroidserver.common.options.quiet = True
|
||||
fdroidserver.common.options.identity_file = None
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
fdroidserver.deploy.config = config
|
||||
shutil.rmtree('repo') # the class-wide test files are not used here
|
||||
|
||||
repo_section = 'repo'
|
||||
initial_branch = fdroidserver.deploy.GIT_BRANCH
|
||||
|
||||
os.chdir(self.testdir)
|
||||
|
||||
local_git_repo_path = Path(self.testdir) / 'local'
|
||||
local_git_repo = git.Repo.init(
|
||||
local_git_repo_path, initial_branch=initial_branch
|
||||
local_git_repo_path, initial_branch=fdroidserver.deploy.GIT_BRANCH
|
||||
)
|
||||
|
||||
fdroid_dir = local_git_repo_path / 'fdroid'
|
||||
|
|
@ -929,13 +944,7 @@ class DeployTest(unittest.TestCase):
|
|||
with fake_file.open('w') as fp:
|
||||
fp.write('not a real one, but has the right filename')
|
||||
|
||||
# The remote repo must be a bare repo to allow being pushed to
|
||||
remote_git_repo_dir = Path(self.testdir) / 'remote'
|
||||
remote_git_repo = git.Repo.init(
|
||||
remote_git_repo_dir, initial_branch=initial_branch, bare=True
|
||||
)
|
||||
|
||||
mirror_config = {"url": str(remote_git_repo_dir), "index_only": True}
|
||||
mirror_config = {"url": str(self.remote_git_repo.git_dir), "index_only": True}
|
||||
enabled_remotes = []
|
||||
ssh_cmd = 'ssh -oBatchMode=yes'
|
||||
fdroidserver.deploy.upload_to_servergitmirror(
|
||||
|
|
@ -950,25 +959,15 @@ class DeployTest(unittest.TestCase):
|
|||
progress=git.RemoteProgress(),
|
||||
)
|
||||
|
||||
verify_repo = remote_git_repo.clone(
|
||||
Path(self.testdir) / 'verify',
|
||||
)
|
||||
|
||||
verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify')
|
||||
self.assertIsNotNone(verify_repo.working_tree_dir)
|
||||
for filename in fdroidserver.common.INDEX_FILES:
|
||||
remote_file = f"fdroid/{repo_section}/{filename}"
|
||||
|
||||
self.assertIsNotNone(verify_repo.working_tree_dir)
|
||||
if verify_repo.working_tree_dir is not None:
|
||||
self.assertTrue(
|
||||
(Path(verify_repo.working_tree_dir) / remote_file).exists()
|
||||
)
|
||||
self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists())
|
||||
|
||||
# Should not have the APK file
|
||||
remote_file = f"fdroid/{repo_section}/{fake_apk}"
|
||||
if verify_repo.working_tree_dir is not None:
|
||||
self.assertFalse(
|
||||
(Path(verify_repo.working_tree_dir) / remote_file).exists()
|
||||
)
|
||||
self.assertFalse((Path(verify_repo.working_tree_dir) / remote_file).exists())
|
||||
|
||||
|
||||
class GitHubReleasesTest(unittest.TestCase):
|
||||
|
|
|
|||
|
|
@ -75,7 +75,6 @@ class IntegrationTest(unittest.TestCase):
|
|||
self.testdir = mkdir_testfiles(WORKSPACE, self)
|
||||
self.tmp_repo_root = self.testdir / "fdroid"
|
||||
self.tmp_repo_root.mkdir(parents=True)
|
||||
deploy.config = {}
|
||||
os.chdir(self.tmp_repo_root)
|
||||
|
||||
def tearDown(self):
|
||||
|
|
@ -1611,10 +1610,11 @@ class IntegrationTest(unittest.TestCase):
|
|||
rclone_config.write(configfile)
|
||||
|
||||
# set up config for run
|
||||
common.get_config()
|
||||
awsbucket = "test-bucket"
|
||||
deploy.config['awsbucket'] = awsbucket
|
||||
deploy.config['rclone_config'] = "test-minio-config"
|
||||
deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
|
||||
common.config['awsbucket'] = awsbucket
|
||||
common.config['rclone_config'] = "test-minio-config"
|
||||
common.config['path_to_custom_rclone_config'] = str(rclone_file)
|
||||
common.options = VerboseFalseOptions
|
||||
|
||||
# call function
|
||||
|
|
@ -1667,9 +1667,9 @@ class IntegrationTest(unittest.TestCase):
|
|||
|
||||
# set up config for run
|
||||
awsbucket = "test-bucket"
|
||||
deploy.config['awsbucket'] = awsbucket
|
||||
deploy.config['rclone_config'] = "test-minio-config"
|
||||
deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
|
||||
common.config['awsbucket'] = awsbucket
|
||||
common.config['rclone_config'] = "test-minio-config"
|
||||
common.config['path_to_custom_rclone_config'] = str(rclone_file)
|
||||
common.options = VerboseFalseOptions
|
||||
|
||||
# call function
|
||||
|
|
|
|||
|
|
@ -359,7 +359,7 @@ class NightlyTest(unittest.TestCase):
|
|||
'repo_keyalias': 'androiddebugkey',
|
||||
'repo_name': 'fdroid/test-nightly',
|
||||
'repo_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/repo',
|
||||
'servergitmirrors': [{"url": 'git@gitlab.com:fdroid/test-nightly'}],
|
||||
'servergitmirrors': [{"url": 'git@gitlab.com:fdroid/test-nightly.git'}],
|
||||
}
|
||||
with open(common.CONFIG_FILE) as fp:
|
||||
config = yaml.safe_load(fp)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue