mirrors/fdroidserver
1
0
Fork 0
mirror of https://github.com/f-droid/fdroidserver.git synced 2025-11-04 14:30:30 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'nightly-fixes' into 'master'

Browse source 
fixes to deploy/nightly made while working on !1666 and reviewing !1650

See merge request fdroid/fdroidserver!1668
This commit is contained in:
Michael Pöhn 2025-09-29 12:21:54 +00:00
commit 720cbed8d5
7 changed files with 222 additions and 190 deletions
Download patch file Download diff file Expand all files Collapse all files

2
fdroidserver/__main__.py
View file

@ -156,7 +156,7 @@ def main():
).git.describe(always=True, tags=True) ).git.describe(always=True, tags=True)
) )
sys.exit(0) sys.exit(0)
except git.exc.InvalidGitRepositoryError: except git.InvalidGitRepositoryError:
print(_('No version information could be found.')) print(_('No version information could be found.'))
sys.exit(1) sys.exit(1)
else: else:

33
fdroidserver/deploy.py
View file

@ -40,7 +40,6 @@ import fdroidserver.github
from . import _, common, index from . import _, common, index
from .exception import FDroidException from .exception import FDroidException
config = None
start_timestamp = time.gmtime() start_timestamp = time.gmtime()
GIT_BRANCH = 'master' GIT_BRANCH = 'master'
@ -144,6 +143,7 @@ def update_remote_storage_with_rclone(
""" """
logging.debug(_('Using rclone to sync to "{name}"').format(name=awsbucket)) logging.debug(_('Using rclone to sync to "{name}"').format(name=awsbucket))
config = common.get_config()
rclone_config = config.get('rclone_config', []) rclone_config = config.get('rclone_config', [])
if rclone_config and isinstance(rclone_config, str): if rclone_config and isinstance(rclone_config, str):
rclone_config = [rclone_config] rclone_config = [rclone_config]
@ -271,6 +271,7 @@ def update_serverwebroot(serverwebroot, repo_section):
has a low resolution timestamp has a low resolution timestamp
""" """
config = common.get_config()
try: try:
subprocess.run(['rsync', '--version'], capture_output=True, check=True) subprocess.run(['rsync', '--version'], capture_output=True, check=True)
except Exception as e: except Exception as e:
@ -431,6 +432,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
""" """
from clint.textui import progress from clint.textui import progress
config = common.get_config()
if config.get('local_copy_dir') and not config.get('sync_from_local_copy_dir'): if config.get('local_copy_dir') and not config.get('sync_from_local_copy_dir'):
logging.debug( logging.debug(
_('Offline machine, skipping git mirror generation until `fdroid deploy`') _('Offline machine, skipping git mirror generation until `fdroid deploy`')
@ -530,6 +532,20 @@ def update_servergitmirrors(servergitmirrors, repo_section):
progressbar.done() progressbar.done()
def _get_commit_author(git_repo):
"""If the author is set locally, use it, otherwise use static info."""
ret = {'name': 'servergitmirrors', 'email': 'fdroid@deploy'}
with git_repo.config_reader() as cr:
for option in ('name', 'email'):
try:
value = cr.get_value('user', option)
except (configparser.NoSectionError, configparser.NoOptionError):
value = os.getenv(f'GITLAB_USER_{option.upper()}')
if value:
ret[option] = value
return git.Actor(ret['name'], ret['email'])
def upload_to_servergitmirror( def upload_to_servergitmirror(
mirror_config: Dict[str, str], mirror_config: Dict[str, str],
local_repo: Repo, local_repo: Repo,
@ -557,18 +573,23 @@ def upload_to_servergitmirror(
logging.info('Mirroring to: ' + remote_url) logging.info('Mirroring to: ' + remote_url)
if is_index_only: if is_index_only:
logging.debug(_('Committing index files to git mirror'))
files_to_upload = _get_index_file_paths( files_to_upload = _get_index_file_paths(
os.path.join(local_repo.working_tree_dir, 'fdroid', repo_section) os.path.join(local_repo.working_tree_dir, 'fdroid', repo_section)
) )
files_to_upload = _remove_missing_files(files_to_upload) files_to_upload = _remove_missing_files(files_to_upload)
local_repo.index.add(files_to_upload) local_repo.index.add(files_to_upload)
local_repo.index.commit(
"servergitmirrors: index-only in git-mirror",
author=_get_commit_author(local_repo),
)
else: else:
# sadly index.add don't allow the --all parameter # sadly index.add don't allow the --all parameter
logging.debug('Adding all files to git mirror') logging.debug(_('Adding all files to git mirror'))
local_repo.git.add(all=True) local_repo.git.add(all=True)
local_repo.index.commit(
logging.debug('Committing files into git mirror') "servergitmirrors: in git-mirror", author=_get_commit_author(local_repo)
local_repo.index.commit("fdroidserver git-mirror") )
# only deploy to GitLab Artifacts if too big for GitLab Pages # only deploy to GitLab Artifacts if too big for GitLab Pages
if ( if (
@ -1029,8 +1050,6 @@ def upload_to_github_releases_repo(repo_conf, release_infos, global_gh_token):
def main(): def main():
global config
parser = ArgumentParser() parser = ArgumentParser()
common.setup_global_opts(parser) common.setup_global_opts(parser)
parser.add_argument( parser.add_argument(

56
fdroidserver/nightly.py
View file

@ -19,6 +19,7 @@
import base64 import base64
import datetime import datetime
import glob
import hashlib import hashlib
import inspect import inspect
import logging import logging
@ -236,11 +237,21 @@ def clone_git_repo(clone_url, git_mirror_path):
'GIT_TERMINAL_PROMPT': '0', 'GIT_TERMINAL_PROMPT': '0',
}, },
) )
except git.exc.GitCommandError as e: except git.GitCommandError as e:
logging.warning(_('WARNING: only public git repos are supported!')) logging.warning(_('WARNING: only public git repos are supported!'))
raise VCSException(f'git clone {clone_url} failed:', str(e)) from e raise VCSException(f'git clone {clone_url} failed:', str(e)) from e
def _getenv(name):
"""Return the value of an environment variable, printing an error if None."""
value = os.getenv(name)
if not value:
logging.error(
_('Environment variable "{name}" has an empty value!').format(name=name)
)
return value
def main(): def main():
"""Deploy to F-Droid repository or generate SSH private key from keystore. """Deploy to F-Droid repository or generate SSH private key from keystore.
@ -303,7 +314,7 @@ def main():
umask = os.umask(0o077) umask = os.umask(0o077)
if 'CI' in os.environ: if 'CI' in os.environ:
v = os.getenv('DEBUG_KEYSTORE') v = _getenv('DEBUG_KEYSTORE')
debug_keystore = None debug_keystore = None
if v: if v:
debug_keystore = base64.b64decode(v) debug_keystore = base64.b64decode(v)
@ -324,20 +335,19 @@ def main():
# the 'master' branch is hardcoded in fdroidserver/deploy.py # the 'master' branch is hardcoded in fdroidserver/deploy.py
if 'CI_PROJECT_PATH' in os.environ and 'CI_PROJECT_URL' in os.environ: if 'CI_PROJECT_PATH' in os.environ and 'CI_PROJECT_URL' in os.environ:
# we are in GitLab CI # we are in GitLab CI
repo_git_base = os.getenv('CI_PROJECT_PATH') + NIGHTLY repo_git_base = f"{_getenv('CI_PROJECT_PATH')}{NIGHTLY}"
clone_url = os.getenv('CI_PROJECT_URL') + NIGHTLY base_url = f"{_getenv('CI_PROJECT_URL')}{NIGHTLY}"
clone_url = f'{base_url}.git' # avoid redirects while cloning
repo_base = get_repo_base_url( repo_base = get_repo_base_url(
clone_url, repo_git_base, force_type='gitlab.com' base_url, repo_git_base, force_type='gitlab.com'
) )
servergitmirror = 'git@' + urlparse(clone_url).netloc + ':' + repo_git_base servergitmirror = f'git@{urlparse(base_url).netloc}:{repo_git_base}.git'
deploy_key_url = ( deploy_key_url = f'{base_url}/-/settings/repository#js-deploy-keys-settings'
f'{clone_url}/-/settings/repository#js-deploy-keys-settings' git_user_name = _getenv('GITLAB_USER_NAME')
) git_user_email = _getenv('GITLAB_USER_EMAIL')
git_user_name = os.getenv('GITLAB_USER_NAME')
git_user_email = os.getenv('GITLAB_USER_EMAIL')
elif 'TRAVIS_REPO_SLUG' in os.environ: elif 'TRAVIS_REPO_SLUG' in os.environ:
# we are in Travis CI # we are in Travis CI
repo_git_base = os.getenv('TRAVIS_REPO_SLUG') + NIGHTLY repo_git_base = _getenv('TRAVIS_REPO_SLUG') + NIGHTLY
clone_url = 'https://github.com/' + repo_git_base clone_url = 'https://github.com/' + repo_git_base
repo_base = get_repo_base_url( repo_base = get_repo_base_url(
clone_url, repo_git_base, force_type='github.com' clone_url, repo_git_base, force_type='github.com'
@ -348,7 +358,7 @@ def main():
+ '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys' + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys'
) )
git_user_name = repo_git_base git_user_name = repo_git_base
git_user_email = os.getenv('USER') + '@' + platform.node() git_user_email = _getenv('USER') + '@' + platform.node()
elif ( elif (
'CIRCLE_REPOSITORY_URL' in os.environ 'CIRCLE_REPOSITORY_URL' in os.environ
and 'CIRCLE_PROJECT_USERNAME' in os.environ and 'CIRCLE_PROJECT_USERNAME' in os.environ
@ -356,12 +366,12 @@ def main():
): ):
# we are in Circle CI # we are in Circle CI
repo_git_base = ( repo_git_base = (
os.getenv('CIRCLE_PROJECT_USERNAME') _getenv('CIRCLE_PROJECT_USERNAME')
+ '/' + '/'
+ os.getenv('CIRCLE_PROJECT_REPONAME') + _getenv('CIRCLE_PROJECT_REPONAME')
+ NIGHTLY + NIGHTLY
) )
clone_url = os.getenv('CIRCLE_REPOSITORY_URL') + NIGHTLY clone_url = _getenv('CIRCLE_REPOSITORY_URL') + NIGHTLY
repo_base = get_repo_base_url( repo_base = get_repo_base_url(
clone_url, repo_git_base, force_type='github.com' clone_url, repo_git_base, force_type='github.com'
) )
@ -370,12 +380,12 @@ def main():
f'https://github.com/{repo_git_base}/settings/keys' f'https://github.com/{repo_git_base}/settings/keys'
+ '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys' + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys'
) )
git_user_name = os.getenv('CIRCLE_USERNAME') git_user_name = _getenv('CIRCLE_USERNAME')
git_user_email = git_user_name + '@' + platform.node() git_user_email = git_user_name + '@' + platform.node()
elif 'GITHUB_ACTIONS' in os.environ: elif 'GITHUB_ACTIONS' in os.environ:
# we are in Github actions # we are in Github actions
repo_git_base = os.getenv('GITHUB_REPOSITORY') + NIGHTLY repo_git_base = _getenv('GITHUB_REPOSITORY') + NIGHTLY
clone_url = os.getenv('GITHUB_SERVER_URL') + '/' + repo_git_base clone_url = _getenv('GITHUB_SERVER_URL') + '/' + repo_git_base
repo_base = get_repo_base_url( repo_base = get_repo_base_url(
clone_url, repo_git_base, force_type='github.com' clone_url, repo_git_base, force_type='github.com'
) )
@ -384,7 +394,7 @@ def main():
f'https://github.com/{repo_git_base}/settings/keys' f'https://github.com/{repo_git_base}/settings/keys'
+ '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys' + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys'
) )
git_user_name = os.getenv('GITHUB_ACTOR') git_user_name = _getenv('GITHUB_ACTOR')
git_user_email = git_user_name + '@' + platform.node() git_user_email = git_user_name + '@' + platform.node()
else: else:
print(_('ERROR: unsupported CI type, patches welcome!')) print(_('ERROR: unsupported CI type, patches welcome!'))
@ -397,6 +407,8 @@ def main():
git_mirror_metadatadir = os.path.join(git_mirror_fdroiddir, 'metadata') git_mirror_metadatadir = os.path.join(git_mirror_fdroiddir, 'metadata')
if not os.path.isdir(git_mirror_repodir): if not os.path.isdir(git_mirror_repodir):
clone_git_repo(clone_url, git_mirror_path) clone_git_repo(clone_url, git_mirror_path)
for f in glob.glob(f'{git_mirror_repodir}/*.apk'):
shutil.copy2(f, repodir)
if not os.path.isdir(git_mirror_repodir): if not os.path.isdir(git_mirror_repodir):
os.makedirs(git_mirror_repodir, mode=0o755) os.makedirs(git_mirror_repodir, mode=0o755)
if os.path.exists('LICENSE'): if os.path.exists('LICENSE'):
@ -442,7 +454,7 @@ Last updated: {date}'''.format(
ssh_private_key_file = _ssh_key_from_debug_keystore() ssh_private_key_file = _ssh_key_from_debug_keystore()
# this is needed for GitPython to find the SSH key # this is needed for GitPython to find the SSH key
ssh_dir = os.path.join(os.getenv('HOME'), '.ssh') ssh_dir = os.path.join(_getenv('HOME'), '.ssh')
os.makedirs(ssh_dir, exist_ok=True) os.makedirs(ssh_dir, exist_ok=True)
ssh_config = os.path.join(ssh_dir, 'config') ssh_config = os.path.join(ssh_dir, 'config')
logging.debug(_('adding IdentityFile to {path}').format(path=ssh_config)) logging.debug(_('adding IdentityFile to {path}').format(path=ssh_config))
@ -587,7 +599,7 @@ Last updated: {date}'''.format(
+ '\n -dname "CN=Android Debug,O=Android,C=US"' + '\n -dname "CN=Android Debug,O=Android,C=US"'
) )
sys.exit(1) sys.exit(1)
ssh_dir = os.path.join(os.getenv('HOME'), '.ssh') ssh_dir = os.path.join(_getenv('HOME'), '.ssh')
privkey = _ssh_key_from_debug_keystore(options.keystore) privkey = _ssh_key_from_debug_keystore(options.keystore)
if os.path.exists(ssh_dir): if os.path.exists(ssh_dir):
ssh_private_key_file = os.path.join(ssh_dir, os.path.basename(privkey)) ssh_private_key_file = os.path.join(ssh_dir, os.path.basename(privkey))

14
fdroidserver/update.py
View file

@ -2466,16 +2466,18 @@ def create_metadata_from_template(apk):
def read_added_date_from_all_apks(apps, apks): def read_added_date_from_all_apks(apps, apks):
"""No summary. """Read the "added" date from all packages.
Added dates come from the repo/index-v2.json file but are Added dates come from index-v2.json file but are read when scanning
read when scanning apks and thus need to be applied form apk APKs and thus need to be applied from package-level to app-level for
level to app level for _all_ apps and not only from non-archived _all_ apps and not only from non-archived ones.
ones
TODO: read the added dates directly from index-v2.json instead of TODO: read the added dates directly from index-v2.json instead of
going through apks that way it also works for for repos that going through apks that way it also works for for repos that
don't keep an archive of apks. don't keep an archive of APKs.
https://gitlab.com/fdroid/wiki/-/wikis/PackageDateHandling
""" """
for appid, app in apps.items(): for appid, app in apps.items():
for apk in apks: for apk in apks:

291
tests/test_deploy.py
View file

@ -1,6 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import configparser import configparser
import logging
import os import os
import shutil import shutil
import tempfile import tempfile
@ -26,15 +27,21 @@ def _mock_rclone_config_file(cmd, text): # pylint: disable=unused-argument
class DeployTest(unittest.TestCase): class DeployTest(unittest.TestCase):
'''fdroidserver/deploy.py''' '''fdroidserver/deploy.py'''
@classmethod
def setUpClass(cls):
# suppress "WARNING:root:unsafe permissions on 'config.yml' (should be 0600)!"
os.chmod(os.path.join(basedir, fdroidserver.common.CONFIG_FILE), 0o600)
def setUp(self): def setUp(self):
os.chdir(basedir) os.chdir(basedir)
self._td = mkdtemp() self._td = mkdtemp()
self.testdir = self._td.name self.testdir = self._td.name
fdroidserver.common.options = mock.Mock() fdroidserver.common.options = mock.Mock()
fdroidserver.deploy.config = {} fdroidserver.common.get_config()
def tearDown(self): def tearDown(self):
fdroidserver.common.config = None
self._td.cleanup() self._td.cleanup()
def test_update_serverwebroots_bad_None(self): def test_update_serverwebroots_bad_None(self):
@ -64,7 +71,6 @@ class DeployTest(unittest.TestCase):
# setup parameters for this test run # setup parameters for this test run
fdroidserver.common.options.identity_file = None fdroidserver.common.options.identity_file = None
fdroidserver.deploy.config['make_current_version_link'] = False
dest_apk0 = url0 / fake_apk dest_apk0 = url0 / fake_apk
dest_apk1 = url1 / fake_apk dest_apk1 = url1 / fake_apk
@ -81,17 +87,17 @@ class DeployTest(unittest.TestCase):
self.assertTrue(dest_apk1.is_file()) self.assertTrue(dest_apk1.is_file())
def test_update_serverwebroots_url_does_not_end_with_fdroid(self): def test_update_serverwebroots_url_does_not_end_with_fdroid(self):
with self.assertRaises(SystemExit): with self.assertRaises(SystemExit), self.assertLogs(level=logging.ERROR):
fdroidserver.deploy.update_serverwebroots([{'url': 'url'}], 'repo') fdroidserver.deploy.update_serverwebroots([{'url': 'url'}], 'repo')
def test_update_serverwebroots_bad_ssh_url(self): def test_update_serverwebroots_bad_ssh_url(self):
with self.assertRaises(SystemExit): with self.assertRaises(SystemExit), self.assertLogs(level=logging.ERROR):
fdroidserver.deploy.update_serverwebroots( fdroidserver.deploy.update_serverwebroots(
[{'url': 'f@b.ar::/path/to/fdroid'}], 'repo' [{'url': 'f@b.ar::/path/to/fdroid'}], 'repo'
) )
def test_update_serverwebroots_unsupported_ssh_url(self): def test_update_serverwebroots_unsupported_ssh_url(self):
with self.assertRaises(SystemExit): with self.assertRaises(SystemExit), self.assertLogs(level=logging.ERROR):
fdroidserver.deploy.update_serverwebroots([{'url': 'ssh://nope'}], 'repo') fdroidserver.deploy.update_serverwebroots([{'url': 'ssh://nope'}], 'repo')
@unittest.skipUnless(shutil.which('rclone'), 'requires rclone') @unittest.skipUnless(shutil.which('rclone'), 'requires rclone')
@ -120,9 +126,9 @@ class DeployTest(unittest.TestCase):
# setup parameters for this test run # setup parameters for this test run
awsbucket = 'test_bucket_folder' awsbucket = 'test_bucket_folder'
fdroidserver.deploy.config['awsbucket'] = awsbucket fdroidserver.common.config['awsbucket'] = awsbucket
fdroidserver.deploy.config['rclone_config'] = 'test-local-config' fdroidserver.common.config['rclone_config'] = 'test-local-config'
fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file) fdroidserver.common.config['path_to_custom_rclone_config'] = str(rclone_file)
fdroidserver.common.options = VerboseFalseOptions fdroidserver.common.options = VerboseFalseOptions
# write out destination path # write out destination path
@ -163,9 +169,9 @@ class DeployTest(unittest.TestCase):
# setup parameters for this test run # setup parameters for this test run
awsbucket = 'test_bucket_folder' awsbucket = 'test_bucket_folder'
fdroidserver.deploy.config['awsbucket'] = awsbucket fdroidserver.common.config['awsbucket'] = awsbucket
fdroidserver.deploy.config['rclone_config'] = 'test-local-config' fdroidserver.common.config['rclone_config'] = 'test-local-config'
fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file) fdroidserver.common.config['path_to_custom_rclone_config'] = str(rclone_file)
fdroidserver.common.options = VerboseFalseOptions fdroidserver.common.options = VerboseFalseOptions
# write out destination path # write out destination path
@ -224,7 +230,7 @@ class DeployTest(unittest.TestCase):
return 0 return 0
mock_call.side_effect = _mock_subprocess_call mock_call.side_effect = _mock_subprocess_call
fdroidserver.deploy.config = {'awsbucket': awsbucket} fdroidserver.common.config = {'awsbucket': awsbucket}
fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket) fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket)
mock_call.assert_called() mock_call.assert_called()
@ -243,7 +249,7 @@ class DeployTest(unittest.TestCase):
mock_call.side_effect = _mock_subprocess_call mock_call.side_effect = _mock_subprocess_call
fdroidserver.deploy.config = {'awsbucket': awsbucket} fdroidserver.common.config = {'awsbucket': awsbucket}
fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket) fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket)
self.maxDiff = None self.maxDiff = None
self.assertEqual( self.assertEqual(
@ -262,7 +268,6 @@ class DeployTest(unittest.TestCase):
@mock.patch('subprocess.check_output', _mock_rclone_config_file) @mock.patch('subprocess.check_output', _mock_rclone_config_file)
@mock.patch('subprocess.call') @mock.patch('subprocess.call')
def test_update_remote_storage_with_rclone_mock_rclone_config(self, mock_call): def test_update_remote_storage_with_rclone_mock_rclone_config(self, mock_call):
awsbucket = 'test_bucket_folder'
self.last_cmd = None self.last_cmd = None
def _mock_subprocess_call(cmd): def _mock_subprocess_call(cmd):
@ -271,10 +276,9 @@ class DeployTest(unittest.TestCase):
mock_call.side_effect = _mock_subprocess_call mock_call.side_effect = _mock_subprocess_call
fdroidserver.deploy.config = { awsbucket = 'test_bucket_folder'
'awsbucket': awsbucket, fdroidserver.common.config['awsbucket'] = awsbucket
'rclone_config': 'test_local_config', fdroidserver.common.config['rclone_config'] = 'test_local_config'
}
fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket) fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket)
self.maxDiff = None self.maxDiff = None
self.assertEqual( self.assertEqual(
@ -304,8 +308,8 @@ class DeployTest(unittest.TestCase):
Path('rclone.conf').write_text('placeholder, contents ignored') Path('rclone.conf').write_text('placeholder, contents ignored')
awsbucket = 'test_bucket_folder' awsbucket = 'test_bucket_folder'
fdroidserver.deploy.config['awsbucket'] = awsbucket fdroidserver.common.config['awsbucket'] = awsbucket
fdroidserver.deploy.config['rclone_config'] = config_name fdroidserver.common.config['rclone_config'] = config_name
fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket) fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket)
self.maxDiff = None self.maxDiff = None
self.assertEqual( self.assertEqual(
@ -339,7 +343,6 @@ class DeployTest(unittest.TestCase):
fdroidserver.common.options = mock.Mock() fdroidserver.common.options = mock.Mock()
fdroidserver.common.options.identity_file = None fdroidserver.common.options.identity_file = None
fdroidserver.common.options.identity_file = None fdroidserver.common.options.identity_file = None
fdroidserver.deploy.config['make_current_version_link'] = False
dest_apk = Path(url) / fake_apk dest_apk = Path(url) / fake_apk
dest_index = Path(url) / fake_index dest_index = Path(url) / fake_index
@ -366,7 +369,6 @@ class DeployTest(unittest.TestCase):
# setup parameters for this test run # setup parameters for this test run
fdroidserver.common.options = mock.Mock() fdroidserver.common.options = mock.Mock()
fdroidserver.common.options.identity_file = None fdroidserver.common.options.identity_file = None
fdroidserver.deploy.config['make_current_version_link'] = False
dest_apk = Path(url) / fake_apk dest_apk = Path(url) / fake_apk
dest_index = Path(url) / fake_index dest_index = Path(url) / fake_index
@ -396,7 +398,7 @@ class DeployTest(unittest.TestCase):
fdroidserver.common.options.verbose = False fdroidserver.common.options.verbose = False
fdroidserver.common.options.quiet = True fdroidserver.common.options.quiet = True
fdroidserver.common.options.index_only = False fdroidserver.common.options.index_only = False
fdroidserver.deploy.config = {'make_current_version_link': True} fdroidserver.common.config['make_current_version_link'] = True
url = "example.com:/var/www/fdroid" url = "example.com:/var/www/fdroid"
repo_section = 'repo' repo_section = 'repo'
@ -504,7 +506,7 @@ class DeployTest(unittest.TestCase):
fdroidserver.common.options.verbose = False fdroidserver.common.options.verbose = False
fdroidserver.common.options.quiet = True fdroidserver.common.options.quiet = True
fdroidserver.common.options.identity_file = None fdroidserver.common.options.identity_file = None
fdroidserver.deploy.config['make_current_version_link'] = True fdroidserver.common.config['make_current_version_link'] = True
url = "example.com:/var/www/fdroid" url = "example.com:/var/www/fdroid"
repo_section = 'repo' repo_section = 'repo'
@ -603,7 +605,7 @@ class DeployTest(unittest.TestCase):
fdroidserver.common.options.quiet = False fdroidserver.common.options.quiet = False
fdroidserver.common.options.identity_file = None fdroidserver.common.options.identity_file = None
fdroidserver.common.options.index_only = False fdroidserver.common.options.index_only = False
fdroidserver.deploy.config = {'identity_file': './id_rsa'} fdroidserver.common.config = {'identity_file': './id_rsa'}
url = "example.com:/var/www/fdroid" url = "example.com:/var/www/fdroid"
repo_section = 'archive' repo_section = 'archive'
@ -623,7 +625,7 @@ class DeployTest(unittest.TestCase):
'--verbose', '--verbose',
'-e', '-e',
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
+ fdroidserver.deploy.config['identity_file'], + fdroidserver.common.config['identity_file'],
'--exclude', '--exclude',
'archive/altstore-index.json', 'archive/altstore-index.json',
'--exclude', '--exclude',
@ -669,7 +671,7 @@ class DeployTest(unittest.TestCase):
'--verbose', '--verbose',
'-e', '-e',
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
+ fdroidserver.deploy.config['identity_file'], + fdroidserver.common.config['identity_file'],
'archive', 'archive',
url, url,
], ],
@ -690,8 +692,7 @@ class DeployTest(unittest.TestCase):
fdroidserver.common.options.verbose = True fdroidserver.common.options.verbose = True
fdroidserver.common.options.quiet = False fdroidserver.common.options.quiet = False
fdroidserver.common.options.identity_file = None fdroidserver.common.options.identity_file = None
fdroidserver.deploy.config['identity_file'] = './id_rsa' fdroidserver.common.config['identity_file'] = './id_rsa'
fdroidserver.deploy.config['make_current_version_link'] = False
url = "example.com:/var/www/fdroid" url = "example.com:/var/www/fdroid"
repo_section = 'archive' repo_section = 'archive'
@ -711,7 +712,7 @@ class DeployTest(unittest.TestCase):
'--verbose', '--verbose',
'-e', '-e',
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
+ fdroidserver.deploy.config['identity_file'], + fdroidserver.common.config['identity_file'],
'archive/altstore-index.json', 'archive/altstore-index.json',
'archive/altstore-index.json.asc', 'archive/altstore-index.json.asc',
'archive/entry.jar', 'archive/entry.jar',
@ -741,7 +742,7 @@ class DeployTest(unittest.TestCase):
'--verbose', '--verbose',
'-e', '-e',
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
+ fdroidserver.deploy.config['identity_file'], + fdroidserver.common.config['identity_file'],
"example.com:/var/www/fdroid/archive/", "example.com:/var/www/fdroid/archive/",
], ],
) )
@ -785,7 +786,58 @@ class DeployTest(unittest.TestCase):
name, fdroidserver.deploy.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url) name, fdroidserver.deploy.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
) )
def test_update_servergitmirrors(self): @mock.patch.dict(os.environ, clear=True)
def test_get_commit_author_no_config(self):
os.environ['HOME'] = self.testdir
git_repo = git.Repo.init(self.testdir)
self.assertEqual(
git.Actor('servergitmirrors', 'fdroid@deploy'),
fdroidserver.deploy._get_commit_author(git_repo),
)
@mock.patch.dict(os.environ, clear=True)
def test_get_commit_author_repo_config(self):
os.environ['HOME'] = self.testdir
git_repo = git.Repo.init(self.testdir)
user_name = 'Foo Bar'
user_email = 'foo@bar.com'
with git_repo.config_writer() as cw:
cw.set_value('user', 'name', user_name)
cw.set_value('user', 'email', user_email)
self.assertEqual(
git.Actor(user_name, user_email),
fdroidserver.deploy._get_commit_author(git_repo),
)
@mock.patch.dict(os.environ, clear=True)
def test_get_commit_author_repo_config_name_only(self):
os.environ['HOME'] = self.testdir
git_repo = git.Repo.init(self.testdir)
user_name = 'Foo Bar'
with git_repo.config_writer() as cw:
cw.set_value('user', 'name', user_name)
self.assertEqual(
git.Actor(user_name, 'fdroid@deploy'),
fdroidserver.deploy._get_commit_author(git_repo),
)
@mock.patch.dict(os.environ, clear=True)
def test_get_commit_author_repo_config_email_only(self):
os.environ['HOME'] = self.testdir
git_repo = git.Repo.init(self.testdir)
user_email = 'foo@bar.com'
with git_repo.config_writer() as cw:
cw.set_value('user', 'email', user_email)
self.assertEqual(
git.Actor('servergitmirrors', user_email),
fdroidserver.deploy._get_commit_author(git_repo),
)
class TestServerGitMirrors(unittest.TestCase):
def setUp(self):
fdroidserver.deploy.USER_RCLONE_CONF = False
# setup parameters for this test run # setup parameters for this test run
fdroidserver.common.options = mock.Mock() fdroidserver.common.options = mock.Mock()
fdroidserver.common.options.identity_file = None fdroidserver.common.options.identity_file = None
@ -793,130 +845,93 @@ class DeployTest(unittest.TestCase):
fdroidserver.common.options.verbose = False fdroidserver.common.options.verbose = False
fdroidserver.common.options.quiet = True fdroidserver.common.options.quiet = True
config = {} self._td = mkdtemp()
fdroidserver.common.fill_config_defaults(config) self.testdir = self._td.name
fdroidserver.deploy.config = config
os.chdir(self.testdir) os.chdir(self.testdir)
repo_section = 'repo'
initial_branch = fdroidserver.deploy.GIT_BRANCH
remote_repo = Path(self.testdir) / 'remote' remote_repo = Path(self.testdir) / 'remote'
remote_repo.mkdir(parents=True) remote_repo.mkdir(parents=True)
remote_git_repo = git.Repo.init( self.remote_git_repo = git.Repo.init(
remote_repo, initial_branch=initial_branch, bare=True remote_repo, initial_branch=fdroidserver.deploy.GIT_BRANCH, bare=True
) )
fdroidserver.deploy.config["servergitmirrors"] = [{"url": str(remote_repo)}]
os.chdir(self.testdir) fdroidserver.common.get_config()
repo = Path('repo') fdroidserver.common.config["servergitmirrors"] = [{"url": str(remote_repo)}]
repo.mkdir(parents=True)
fake_apk = 'Sym.apk' self.repo_section = 'repo'
fake_files = fdroidserver.common.INDEX_FILES + [fake_apk] repo = Path(self.repo_section)
for filename in fake_files: repo.mkdir()
self.fake_apk = 'Sym.apk'
self.fake_files = fdroidserver.common.INDEX_FILES + [self.fake_apk]
for filename in self.fake_files:
fake_file = repo / filename fake_file = repo / filename
with fake_file.open('w') as fp: with fake_file.open('w') as fp:
fp.write('not a real one, but has the right filename') fp.write('not a real one, but has the right filename')
def tearDown(self):
fdroidserver.common.config = None
fdroidserver.common.options = None
self._td.cleanup()
def test_update_servergitmirrors(self):
fdroidserver.deploy.update_servergitmirrors( fdroidserver.deploy.update_servergitmirrors(
fdroidserver.deploy.config["servergitmirrors"], repo_section fdroidserver.common.config["servergitmirrors"], self.repo_section
) )
verify_repo = remote_git_repo.clone( verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify')
Path(self.testdir) / 'verify', self.assertIsNotNone(verify_repo.working_tree_dir)
for filename in self.fake_files:
remote_file = f"fdroid/{self.repo_section}/{filename}"
self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists())
def test_update_servergitmirrors_with_existing_git_repo(self):
"""Confirm it works with clones done manually or with nightly."""
fdroidserver.deploy.update_servergitmirrors(
fdroidserver.common.config["servergitmirrors"], self.repo_section
) )
for filename in fake_files: # now delete the local setup, clone the remote, and add a new APK
remote_file = f"fdroid/{repo_section}/{filename}" git_mirror = os.path.join(self.testdir, 'git-mirror')
shutil.rmtree(git_mirror)
self.remote_git_repo.clone(git_mirror)
new_fake_apk = 'Sym2.apk'
self.fake_files.append(new_fake_apk)
(Path(self.repo_section) / new_fake_apk).write_text('a new fake APK')
self.assertIsNotNone(verify_repo.working_tree_dir) fdroidserver.deploy.update_servergitmirrors(
if verify_repo.working_tree_dir is not None: fdroidserver.common.config["servergitmirrors"], self.repo_section
self.assertTrue( )
(Path(verify_repo.working_tree_dir) / remote_file).exists()
) verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify')
self.assertIsNotNone(verify_repo.working_tree_dir)
for filename in self.fake_files:
remote_file = f"fdroid/{self.repo_section}/{filename}"
self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists())
def test_update_servergitmirrors_in_index_only_mode(self): def test_update_servergitmirrors_in_index_only_mode(self):
# setup parameters for this test run fdroidserver.common.config["servergitmirrors"][0]["index_only"] = True
fdroidserver.common.options = mock.Mock()
fdroidserver.common.options.identity_file = None
fdroidserver.common.options.no_keep_git_mirror_archive = False
fdroidserver.common.options.verbose = False
fdroidserver.common.options.quiet = True
config = {}
fdroidserver.common.fill_config_defaults(config)
fdroidserver.deploy.config = config
os.chdir(self.testdir)
repo_section = 'repo'
initial_branch = fdroidserver.deploy.GIT_BRANCH
remote_repo = Path(self.testdir) / 'remote'
remote_repo.mkdir(parents=True)
remote_git_repo = git.Repo.init(
remote_repo, initial_branch=initial_branch, bare=True
)
fdroidserver.deploy.config["servergitmirrors"] = [
{"url": str(remote_repo), "index_only": True}
]
os.chdir(self.testdir)
repo = Path('repo')
repo.mkdir(parents=True)
fake_apk = 'Sym.apk'
fake_files = fdroidserver.common.INDEX_FILES + [fake_apk]
for filename in fake_files:
fake_file = repo / filename
with fake_file.open('w') as fp:
fp.write('not a real one, but has the right filename')
fdroidserver.deploy.update_servergitmirrors( fdroidserver.deploy.update_servergitmirrors(
fdroidserver.deploy.config["servergitmirrors"], repo_section fdroidserver.common.config["servergitmirrors"], self.repo_section
)
verify_repo = remote_git_repo.clone(
Path(self.testdir) / 'verify',
) )
verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify')
self.assertIsNotNone(verify_repo.working_tree_dir)
for filename in fdroidserver.common.INDEX_FILES: for filename in fdroidserver.common.INDEX_FILES:
remote_file = f"fdroid/{repo_section}/{filename}" remote_file = f"fdroid/{self.repo_section}/{filename}"
self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists())
self.assertIsNotNone(verify_repo.working_tree_dir)
if verify_repo.working_tree_dir is not None:
self.assertTrue(
(Path(verify_repo.working_tree_dir) / remote_file).exists()
)
# Should not have the APK file # Should not have the APK file
remote_file = f"fdroid/{repo_section}/{fake_apk}" remote_file = f"fdroid/{self.repo_section}/{self.fake_apk}"
if verify_repo.working_tree_dir is not None: self.assertFalse((Path(verify_repo.working_tree_dir) / remote_file).exists())
self.assertFalse(
(Path(verify_repo.working_tree_dir) / remote_file).exists()
)
def test_upload_to_servergitmirror_in_index_only_mode(self): def test_upload_to_servergitmirror_in_index_only_mode(self):
# setup parameters for this test run shutil.rmtree('repo') # the class-wide test files are not used here
fdroidserver.common.options = mock.Mock()
fdroidserver.common.options.identity_file = None
fdroidserver.common.options.no_keep_git_mirror_archive = False
fdroidserver.common.options.verbose = False
fdroidserver.common.options.quiet = True
fdroidserver.common.options.identity_file = None
config = {}
fdroidserver.common.fill_config_defaults(config)
fdroidserver.deploy.config = config
repo_section = 'repo' repo_section = 'repo'
initial_branch = fdroidserver.deploy.GIT_BRANCH
os.chdir(self.testdir)
local_git_repo_path = Path(self.testdir) / 'local' local_git_repo_path = Path(self.testdir) / 'local'
local_git_repo = git.Repo.init( local_git_repo = git.Repo.init(
local_git_repo_path, initial_branch=initial_branch local_git_repo_path, initial_branch=fdroidserver.deploy.GIT_BRANCH
) )
fdroid_dir = local_git_repo_path / 'fdroid' fdroid_dir = local_git_repo_path / 'fdroid'
@ -929,13 +944,7 @@ class DeployTest(unittest.TestCase):
with fake_file.open('w') as fp: with fake_file.open('w') as fp:
fp.write('not a real one, but has the right filename') fp.write('not a real one, but has the right filename')
# The remote repo must be a bare repo to allow being pushed to mirror_config = {"url": str(self.remote_git_repo.git_dir), "index_only": True}
remote_git_repo_dir = Path(self.testdir) / 'remote'
remote_git_repo = git.Repo.init(
remote_git_repo_dir, initial_branch=initial_branch, bare=True
)
mirror_config = {"url": str(remote_git_repo_dir), "index_only": True}
enabled_remotes = [] enabled_remotes = []
ssh_cmd = 'ssh -oBatchMode=yes' ssh_cmd = 'ssh -oBatchMode=yes'
fdroidserver.deploy.upload_to_servergitmirror( fdroidserver.deploy.upload_to_servergitmirror(
@ -950,25 +959,15 @@ class DeployTest(unittest.TestCase):
progress=git.RemoteProgress(), progress=git.RemoteProgress(),
) )
verify_repo = remote_git_repo.clone( verify_repo = self.remote_git_repo.clone(Path(self.testdir) / 'verify')
Path(self.testdir) / 'verify', self.assertIsNotNone(verify_repo.working_tree_dir)
)
for filename in fdroidserver.common.INDEX_FILES: for filename in fdroidserver.common.INDEX_FILES:
remote_file = f"fdroid/{repo_section}/{filename}" remote_file = f"fdroid/{repo_section}/{filename}"
self.assertTrue((Path(verify_repo.working_tree_dir) / remote_file).exists())
self.assertIsNotNone(verify_repo.working_tree_dir)
if verify_repo.working_tree_dir is not None:
self.assertTrue(
(Path(verify_repo.working_tree_dir) / remote_file).exists()
)
# Should not have the APK file # Should not have the APK file
remote_file = f"fdroid/{repo_section}/{fake_apk}" remote_file = f"fdroid/{repo_section}/{fake_apk}"
if verify_repo.working_tree_dir is not None: self.assertFalse((Path(verify_repo.working_tree_dir) / remote_file).exists())
self.assertFalse(
(Path(verify_repo.working_tree_dir) / remote_file).exists()
)
class GitHubReleasesTest(unittest.TestCase): class GitHubReleasesTest(unittest.TestCase):

14
tests/test_integration.py
View file

@ -75,7 +75,6 @@ class IntegrationTest(unittest.TestCase):
self.testdir = mkdir_testfiles(WORKSPACE, self) self.testdir = mkdir_testfiles(WORKSPACE, self)
self.tmp_repo_root = self.testdir / "fdroid" self.tmp_repo_root = self.testdir / "fdroid"
self.tmp_repo_root.mkdir(parents=True) self.tmp_repo_root.mkdir(parents=True)
deploy.config = {}
os.chdir(self.tmp_repo_root) os.chdir(self.tmp_repo_root)
def tearDown(self): def tearDown(self):
@ -1611,10 +1610,11 @@ class IntegrationTest(unittest.TestCase):
rclone_config.write(configfile) rclone_config.write(configfile)
# set up config for run # set up config for run
common.get_config()
awsbucket = "test-bucket" awsbucket = "test-bucket"
deploy.config['awsbucket'] = awsbucket common.config['awsbucket'] = awsbucket
deploy.config['rclone_config'] = "test-minio-config" common.config['rclone_config'] = "test-minio-config"
deploy.config['path_to_custom_rclone_config'] = str(rclone_file) common.config['path_to_custom_rclone_config'] = str(rclone_file)
common.options = VerboseFalseOptions common.options = VerboseFalseOptions
# call function # call function
@ -1667,9 +1667,9 @@ class IntegrationTest(unittest.TestCase):
# set up config for run # set up config for run
awsbucket = "test-bucket" awsbucket = "test-bucket"
deploy.config['awsbucket'] = awsbucket common.config['awsbucket'] = awsbucket
deploy.config['rclone_config'] = "test-minio-config" common.config['rclone_config'] = "test-minio-config"
deploy.config['path_to_custom_rclone_config'] = str(rclone_file) common.config['path_to_custom_rclone_config'] = str(rclone_file)
common.options = VerboseFalseOptions common.options = VerboseFalseOptions
# call function # call function

2
tests/test_nightly.py
View file

@ -359,7 +359,7 @@ class NightlyTest(unittest.TestCase):
'repo_keyalias': 'androiddebugkey', 'repo_keyalias': 'androiddebugkey',
'repo_name': 'fdroid/test-nightly', 'repo_name': 'fdroid/test-nightly',
'repo_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/repo', 'repo_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/repo',
'servergitmirrors': [{"url": 'git@gitlab.com:fdroid/test-nightly'}], 'servergitmirrors': [{"url": 'git@gitlab.com:fdroid/test-nightly.git'}],
} }
with open(common.CONFIG_FILE) as fp: with open(common.CONFIG_FILE) as fp:
config = yaml.safe_load(fp) config = yaml.safe_load(fp)