mirror of
https://github.com/f-droid/fdroidserver.git
synced 2025-11-04 14:30:30 +03:00
Remove libcloud and s3cmd from fdroidserver
This commit is contained in:
paul mayero
Hans-Christoph Steiner
parent
a9856cfb92
commit
dbd769db9f
9 changed files with 417 additions and 819 deletions
|
|
@ -15,6 +15,12 @@ import fdroidserver
|
|||
from .shared_test_code import TmpCwd, VerboseFalseOptions, mkdtemp
|
||||
|
||||
basedir = Path(__file__).parent
|
||||
FILES = basedir
|
||||
|
||||
|
||||
def _mock_rclone_config_file(cmd, text): # pylint: disable=unused-argument
|
||||
"""Mock output from rclone 1.60.1 but with nonexistent conf file."""
|
||||
return "Configuration file doesn't exist, but rclone will use this path:\n/nonexistent/rclone.conf\n"
|
||||
|
||||
|
||||
class DeployTest(unittest.TestCase):
|
||||
|
|
@ -27,7 +33,6 @@ class DeployTest(unittest.TestCase):
|
|||
|
||||
fdroidserver.common.options = mock.Mock()
|
||||
fdroidserver.deploy.config = {}
|
||||
fdroidserver.deploy.USER_RCLONE_CONF = False
|
||||
|
||||
def tearDown(self):
|
||||
self._td.cleanup()
|
||||
|
|
@ -89,7 +94,7 @@ class DeployTest(unittest.TestCase):
|
|||
with self.assertRaises(SystemExit):
|
||||
fdroidserver.deploy.update_serverwebroots([{'url': 'ssh://nope'}], 'repo')
|
||||
|
||||
@unittest.skipUnless(shutil.which('rclone'), '/usr/bin/rclone')
|
||||
@unittest.skipUnless(shutil.which('rclone'), 'requires rclone')
|
||||
def test_update_remote_storage_with_rclone(self):
|
||||
os.chdir(self.testdir)
|
||||
repo = Path('repo')
|
||||
|
|
@ -114,26 +119,25 @@ class DeployTest(unittest.TestCase):
|
|||
rclone_config.write(configfile)
|
||||
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.config['awsbucket'] = 'test_bucket_folder'
|
||||
fdroidserver.deploy.config['rclone'] = True
|
||||
awsbucket = 'test_bucket_folder'
|
||||
fdroidserver.deploy.config['awsbucket'] = awsbucket
|
||||
fdroidserver.deploy.config['rclone_config'] = 'test-local-config'
|
||||
fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
|
||||
fdroidserver.common.options = VerboseFalseOptions
|
||||
|
||||
# write out destination path
|
||||
destination = Path('test_bucket_folder/fdroid')
|
||||
destination = Path(f'{awsbucket}/fdroid')
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
dest_apk = Path(destination) / fake_apk
|
||||
dest_index = Path(destination) / fake_index
|
||||
self.assertFalse(dest_apk.is_file())
|
||||
self.assertFalse(dest_index.is_file())
|
||||
repo_section = str(repo)
|
||||
# fdroidserver.deploy.USER_RCLONE_CONF = str(rclone_file)
|
||||
fdroidserver.deploy.update_remote_storage_with_rclone(repo_section)
|
||||
fdroidserver.deploy.update_remote_storage_with_rclone(repo_section, awsbucket)
|
||||
self.assertTrue(dest_apk.is_file())
|
||||
self.assertTrue(dest_index.is_file())
|
||||
|
||||
@unittest.skipUnless(shutil.which('rclone'), '/usr/bin/rclone')
|
||||
@unittest.skipUnless(shutil.which('rclone'), 'requires rclone')
|
||||
def test_update_remote_storage_with_rclone_in_index_only_mode(self):
|
||||
os.chdir(self.testdir)
|
||||
repo = Path('repo')
|
||||
|
|
@ -158,51 +162,131 @@ class DeployTest(unittest.TestCase):
|
|||
rclone_config.write(configfile)
|
||||
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.config['awsbucket'] = 'test_bucket_folder'
|
||||
fdroidserver.deploy.config['rclone'] = True
|
||||
awsbucket = 'test_bucket_folder'
|
||||
fdroidserver.deploy.config['awsbucket'] = awsbucket
|
||||
fdroidserver.deploy.config['rclone_config'] = 'test-local-config'
|
||||
fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
|
||||
fdroidserver.common.options = VerboseFalseOptions
|
||||
|
||||
# write out destination path
|
||||
destination = Path('test_bucket_folder/fdroid')
|
||||
destination = Path(f'{awsbucket}/fdroid')
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
dest_apk = Path(destination) / fake_apk
|
||||
dest_index = Path(destination) / fake_index
|
||||
self.assertFalse(dest_apk.is_file())
|
||||
self.assertFalse(dest_index.is_file())
|
||||
repo_section = str(repo)
|
||||
# fdroidserver.deploy.USER_RCLONE_CONF = str(rclone_file)
|
||||
fdroidserver.deploy.update_remote_storage_with_rclone(
|
||||
repo_section, is_index_only=True
|
||||
repo_section, awsbucket, is_index_only=True
|
||||
)
|
||||
self.assertFalse(dest_apk.is_file())
|
||||
self.assertTrue(dest_index.is_file())
|
||||
|
||||
@mock.patch.dict(os.environ, {'PATH': os.getenv('PATH')}, clear=True)
|
||||
@mock.patch('subprocess.check_output', _mock_rclone_config_file)
|
||||
def test_update_remote_storage_with_rclone_awsbucket_no_env_vars(self):
|
||||
with self.assertRaises(fdroidserver.exception.FDroidException):
|
||||
fdroidserver.deploy.update_remote_storage_with_rclone('repo', 'foobucket')
|
||||
|
||||
@mock.patch.dict(os.environ, {'PATH': os.getenv('PATH')}, clear=True)
|
||||
@mock.patch('subprocess.check_output', _mock_rclone_config_file)
|
||||
def test_update_remote_storage_with_rclone_awsbucket_no_AWS_SECRET_ACCESS_KEY(self):
|
||||
os.environ['AWS_ACCESS_KEY_ID'] = 'accesskey'
|
||||
with self.assertRaises(fdroidserver.exception.FDroidException):
|
||||
fdroidserver.deploy.update_remote_storage_with_rclone('repo', 'foobucket')
|
||||
|
||||
@mock.patch.dict(os.environ, {'PATH': os.getenv('PATH')}, clear=True)
|
||||
@mock.patch('subprocess.check_output', _mock_rclone_config_file)
|
||||
def test_update_remote_storage_with_rclone_awsbucket_no_AWS_ACCESS_KEY_ID(self):
|
||||
os.environ['AWS_SECRET_ACCESS_KEY'] = 'secrets' # nosec B105
|
||||
with self.assertRaises(fdroidserver.exception.FDroidException):
|
||||
fdroidserver.deploy.update_remote_storage_with_rclone('repo', 'foobucket')
|
||||
|
||||
@mock.patch.dict(os.environ, {'PATH': os.getenv('PATH')}, clear=True)
|
||||
@mock.patch('subprocess.check_output', _mock_rclone_config_file)
|
||||
@mock.patch('subprocess.call')
|
||||
@mock.patch('subprocess.check_output', lambda cmd, text: '/path/to/rclone.conf')
|
||||
def test_update_remote_storage_with_rclone_mock(self, mock_call):
|
||||
def test_update_remote_storage_with_rclone_awsbucket_env_vars(self, mock_call):
|
||||
awsbucket = 'test_bucket_folder'
|
||||
os.environ['AWS_ACCESS_KEY_ID'] = 'accesskey'
|
||||
os.environ['AWS_SECRET_ACCESS_KEY'] = 'secrets' # nosec B105
|
||||
|
||||
def _mock_subprocess_call(cmd):
|
||||
self.assertEqual(
|
||||
cmd,
|
||||
cmd[:5],
|
||||
[
|
||||
'rclone',
|
||||
'sync',
|
||||
'repo',
|
||||
'test_local_config:test_bucket_folder/fdroid/repo',
|
||||
'--delete-after',
|
||||
'--config',
|
||||
'.fdroid-deploy-rclone.conf',
|
||||
],
|
||||
)
|
||||
return 0
|
||||
|
||||
mock_call.side_effect = _mock_subprocess_call
|
||||
fdroidserver.deploy.config = {'awsbucket': awsbucket}
|
||||
fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket)
|
||||
mock_call.assert_called()
|
||||
|
||||
@mock.patch.dict(os.environ, {'PATH': os.getenv('PATH')}, clear=True)
|
||||
@mock.patch('subprocess.check_output', _mock_rclone_config_file)
|
||||
@mock.patch('subprocess.call')
|
||||
def test_update_remote_storage_with_rclone_mock_awsbucket(self, mock_call):
|
||||
awsbucket = 'test_bucket_folder'
|
||||
os.environ['AWS_ACCESS_KEY_ID'] = 'accesskey'
|
||||
os.environ['AWS_SECRET_ACCESS_KEY'] = 'secrets' # nosec B105
|
||||
self.last_cmd = None
|
||||
|
||||
def _mock_subprocess_call(cmd):
|
||||
self.last_cmd = cmd
|
||||
return 0
|
||||
|
||||
mock_call.side_effect = _mock_subprocess_call
|
||||
|
||||
fdroidserver.deploy.config = {'awsbucket': awsbucket}
|
||||
fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket)
|
||||
self.maxDiff = None
|
||||
self.assertEqual(
|
||||
self.last_cmd,
|
||||
[
|
||||
'rclone',
|
||||
'sync',
|
||||
'--delete-after',
|
||||
'--config',
|
||||
'.fdroid-deploy-rclone.conf',
|
||||
'repo',
|
||||
f'AWS-S3-US-East-1:{awsbucket}/fdroid/repo',
|
||||
],
|
||||
)
|
||||
|
||||
@mock.patch('subprocess.check_output', _mock_rclone_config_file)
|
||||
@mock.patch('subprocess.call')
|
||||
def test_update_remote_storage_with_rclone_mock_rclone_config(self, mock_call):
|
||||
awsbucket = 'test_bucket_folder'
|
||||
self.last_cmd = None
|
||||
|
||||
def _mock_subprocess_call(cmd):
|
||||
self.last_cmd = cmd
|
||||
return 0
|
||||
|
||||
mock_call.side_effect = _mock_subprocess_call
|
||||
|
||||
fdroidserver.deploy.config = {
|
||||
'awsbucket': 'test_bucket_folder',
|
||||
'rclone': True,
|
||||
'awsbucket': awsbucket,
|
||||
'rclone_config': 'test_local_config',
|
||||
}
|
||||
fdroidserver.deploy.update_remote_storage_with_rclone('repo')
|
||||
mock_call.assert_called_once()
|
||||
fdroidserver.deploy.update_remote_storage_with_rclone('repo', awsbucket)
|
||||
self.maxDiff = None
|
||||
self.assertEqual(
|
||||
self.last_cmd,
|
||||
[
|
||||
'rclone',
|
||||
'sync',
|
||||
'--delete-after',
|
||||
'repo',
|
||||
'test_local_config:test_bucket_folder/fdroid/repo',
|
||||
],
|
||||
)
|
||||
|
||||
def test_update_serverwebroot(self):
|
||||
"""rsync works with file paths, so this test uses paths for the URLs"""
|
||||
|
|
@ -668,399 +752,6 @@ class DeployTest(unittest.TestCase):
|
|||
name, fdroidserver.deploy.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
|
||||
)
|
||||
|
||||
def test_update_awsbucket_s3cmd(self):
|
||||
# setup parameters for this test run
|
||||
fdroidserver.common.options = mock.Mock()
|
||||
fdroidserver.common.options.no_checksum = True
|
||||
fdroidserver.common.options.verbose = False
|
||||
fdroidserver.common.options.quiet = True
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
fdroidserver.deploy.config = config
|
||||
fdroidserver.deploy.config["awsbucket"] = "bucket"
|
||||
fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
|
||||
fdroidserver.deploy.config["awssecretkey"] = "secretkey"
|
||||
fdroidserver.deploy.config["s3cmd"] = "s3cmd"
|
||||
|
||||
repo_section = 'repo'
|
||||
|
||||
# setup function for asserting subprocess.call invocations
|
||||
call_iteration = 0
|
||||
|
||||
def update_awsbucket_s3cmd_call(cmd):
|
||||
nonlocal call_iteration
|
||||
if call_iteration == 0:
|
||||
self.assertListEqual(
|
||||
cmd,
|
||||
[
|
||||
's3cmd',
|
||||
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
|
||||
'info',
|
||||
f"s3://{fdroidserver.deploy.config['awsbucket']}",
|
||||
],
|
||||
)
|
||||
elif call_iteration == 1:
|
||||
self.assertListEqual(
|
||||
cmd,
|
||||
[
|
||||
's3cmd',
|
||||
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
|
||||
'sync',
|
||||
'--acl-public',
|
||||
'--quiet',
|
||||
'--exclude',
|
||||
'repo/altstore-index.json',
|
||||
'--exclude',
|
||||
'repo/altstore-index.json.asc',
|
||||
'--exclude',
|
||||
'repo/entry.jar',
|
||||
'--exclude',
|
||||
'repo/entry.json',
|
||||
'--exclude',
|
||||
'repo/entry.json.asc',
|
||||
'--exclude',
|
||||
'repo/index-v1.jar',
|
||||
'--exclude',
|
||||
'repo/index-v1.json',
|
||||
'--exclude',
|
||||
'repo/index-v1.json.asc',
|
||||
'--exclude',
|
||||
'repo/index-v2.json',
|
||||
'--exclude',
|
||||
'repo/index-v2.json.asc',
|
||||
'--exclude',
|
||||
'repo/index.css',
|
||||
'--exclude',
|
||||
'repo/index.html',
|
||||
'--exclude',
|
||||
'repo/index.jar',
|
||||
'--exclude',
|
||||
'repo/index.png',
|
||||
'--exclude',
|
||||
'repo/index.xml',
|
||||
'--exclude',
|
||||
'repo/signer-index.jar',
|
||||
'--exclude',
|
||||
'repo/signer-index.json',
|
||||
'--exclude',
|
||||
'repo/signer-index.json.asc',
|
||||
'--no-check-md5',
|
||||
'--skip-existing',
|
||||
repo_section,
|
||||
f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
|
||||
],
|
||||
)
|
||||
elif call_iteration == 2:
|
||||
self.assertListEqual(
|
||||
cmd,
|
||||
[
|
||||
's3cmd',
|
||||
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
|
||||
'sync',
|
||||
'--acl-public',
|
||||
'--quiet',
|
||||
'--exclude',
|
||||
'repo/altstore-index.json',
|
||||
'--exclude',
|
||||
'repo/altstore-index.json.asc',
|
||||
'--exclude',
|
||||
'repo/entry.jar',
|
||||
'--exclude',
|
||||
'repo/entry.json',
|
||||
'--exclude',
|
||||
'repo/entry.json.asc',
|
||||
'--exclude',
|
||||
'repo/index-v1.jar',
|
||||
'--exclude',
|
||||
'repo/index-v1.json',
|
||||
'--exclude',
|
||||
'repo/index-v1.json.asc',
|
||||
'--exclude',
|
||||
'repo/index-v2.json',
|
||||
'--exclude',
|
||||
'repo/index-v2.json.asc',
|
||||
'--exclude',
|
||||
'repo/index.css',
|
||||
'--exclude',
|
||||
'repo/index.html',
|
||||
'--exclude',
|
||||
'repo/index.jar',
|
||||
'--exclude',
|
||||
'repo/index.png',
|
||||
'--exclude',
|
||||
'repo/index.xml',
|
||||
'--exclude',
|
||||
'repo/signer-index.jar',
|
||||
'--exclude',
|
||||
'repo/signer-index.json',
|
||||
'--exclude',
|
||||
'repo/signer-index.json.asc',
|
||||
'--no-check-md5',
|
||||
repo_section,
|
||||
f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
|
||||
],
|
||||
)
|
||||
elif call_iteration == 3:
|
||||
self.assertListEqual(
|
||||
cmd,
|
||||
[
|
||||
's3cmd',
|
||||
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
|
||||
'sync',
|
||||
'--acl-public',
|
||||
'--quiet',
|
||||
'--delete-removed',
|
||||
'--delete-after',
|
||||
'--no-check-md5',
|
||||
repo_section,
|
||||
f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
|
||||
],
|
||||
)
|
||||
else:
|
||||
self.fail('unexpected subprocess.call invocation')
|
||||
call_iteration += 1
|
||||
return 0
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
||||
os.mkdir('repo')
|
||||
os.symlink('repo/com.example.sym.apk', 'Sym.apk')
|
||||
os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
|
||||
os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
|
||||
with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
|
||||
fdroidserver.deploy.update_awsbucket_s3cmd(repo_section)
|
||||
self.assertEqual(call_iteration, 4, 'expected 4 invocations of subprocess.call')
|
||||
|
||||
def test_update_awsbucket_s3cmd_in_index_only_mode(self):
|
||||
# setup parameters for this test run
|
||||
fdroidserver.common.options = mock.Mock()
|
||||
fdroidserver.common.options.no_checksum = True
|
||||
fdroidserver.common.options.verbose = False
|
||||
fdroidserver.common.options.quiet = True
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
fdroidserver.deploy.config = config
|
||||
fdroidserver.deploy.config["awsbucket"] = "bucket"
|
||||
fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
|
||||
fdroidserver.deploy.config["awssecretkey"] = "secretkey"
|
||||
fdroidserver.deploy.config["s3cmd"] = "s3cmd"
|
||||
|
||||
repo_section = 'repo'
|
||||
|
||||
# setup function for asserting subprocess.call invocations
|
||||
call_iteration = 0
|
||||
|
||||
def update_awsbucket_s3cmd_call(cmd):
|
||||
nonlocal call_iteration
|
||||
if call_iteration == 0:
|
||||
self.assertListEqual(
|
||||
cmd,
|
||||
[
|
||||
's3cmd',
|
||||
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
|
||||
'info',
|
||||
f"s3://{fdroidserver.deploy.config['awsbucket']}",
|
||||
],
|
||||
)
|
||||
elif call_iteration == 1:
|
||||
self.assertListEqual(
|
||||
cmd,
|
||||
[
|
||||
's3cmd',
|
||||
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
|
||||
'sync',
|
||||
'--acl-public',
|
||||
'--quiet',
|
||||
'--include',
|
||||
'repo/altstore-index.json',
|
||||
'--include',
|
||||
'repo/altstore-index.json.asc',
|
||||
'--include',
|
||||
'repo/entry.jar',
|
||||
'--include',
|
||||
'repo/entry.json',
|
||||
'--include',
|
||||
'repo/entry.json.asc',
|
||||
'--include',
|
||||
'repo/index-v1.jar',
|
||||
'--include',
|
||||
'repo/index-v1.json',
|
||||
'--include',
|
||||
'repo/index-v1.json.asc',
|
||||
'--include',
|
||||
'repo/index-v2.json',
|
||||
'--include',
|
||||
'repo/index-v2.json.asc',
|
||||
'--include',
|
||||
'repo/index.css',
|
||||
'--include',
|
||||
'repo/index.html',
|
||||
'--include',
|
||||
'repo/index.jar',
|
||||
'--include',
|
||||
'repo/index.png',
|
||||
'--include',
|
||||
'repo/index.xml',
|
||||
'--include',
|
||||
'repo/signer-index.jar',
|
||||
'--include',
|
||||
'repo/signer-index.json',
|
||||
'--include',
|
||||
'repo/signer-index.json.asc',
|
||||
'--delete-removed',
|
||||
'--delete-after',
|
||||
'--no-check-md5',
|
||||
repo_section,
|
||||
f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
|
||||
],
|
||||
)
|
||||
else:
|
||||
self.fail('unexpected subprocess.call invocation')
|
||||
call_iteration += 1
|
||||
return 0
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
||||
os.mkdir('repo')
|
||||
os.symlink('repo/com.example.sym.apk', 'Sym.apk')
|
||||
os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
|
||||
os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
|
||||
with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
|
||||
fdroidserver.deploy.update_awsbucket_s3cmd(
|
||||
repo_section, is_index_only=True
|
||||
)
|
||||
self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
|
||||
|
||||
def test_update_awsbucket_libcloud(self):
|
||||
from libcloud.storage.base import Container
|
||||
|
||||
# setup parameters for this test run
|
||||
fdroidserver.common.options = mock.Mock()
|
||||
fdroidserver.common.options.no_checksum = True
|
||||
fdroidserver.common.options.verbose = False
|
||||
fdroidserver.common.options.quiet = True
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
fdroidserver.deploy.config = config
|
||||
fdroidserver.deploy.config["awsbucket"] = "bucket"
|
||||
fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
|
||||
fdroidserver.deploy.config["awssecretkey"] = "secretkey"
|
||||
fdroidserver.deploy.config["s3cmd"] = "s3cmd"
|
||||
|
||||
repo_section = 'repo'
|
||||
|
||||
os.chdir(self.testdir)
|
||||
repo = Path('repo')
|
||||
repo.mkdir(parents=True)
|
||||
fake_apk = repo / 'Sym.apk'
|
||||
with fake_apk.open('w') as fp:
|
||||
fp.write('not an APK, but has the right filename')
|
||||
fake_index = repo / fdroidserver.common.INDEX_FILES[0]
|
||||
with fake_index.open('w') as fp:
|
||||
fp.write('not an index, but has the right filename')
|
||||
|
||||
with mock.patch(
|
||||
'libcloud.storage.drivers.s3.S3StorageDriver'
|
||||
) as mock_driver_class:
|
||||
mock_driver = mock_driver_class.return_value
|
||||
mock_container = mock.MagicMock(spec=Container)
|
||||
mock_container.list_objects.return_value = [
|
||||
mock.MagicMock(name='Sym.apk'),
|
||||
mock.MagicMock(name=fdroidserver.common.INDEX_FILES[0]),
|
||||
]
|
||||
|
||||
mock_driver.get_container.return_value = mock_container
|
||||
mock_driver.upload_object_via_stream.return_value = None
|
||||
|
||||
fdroidserver.deploy.update_awsbucket_libcloud(repo_section)
|
||||
|
||||
mock_driver.get_container.assert_called_once_with(
|
||||
container_name=fdroidserver.deploy.config["awsbucket"]
|
||||
)
|
||||
mock_container.list_objects.assert_called_once_with()
|
||||
files_to_upload = [
|
||||
'fdroid/repo/Sym.apk',
|
||||
f"fdroid/repo/{fdroidserver.common.INDEX_FILES[0]}",
|
||||
]
|
||||
calls = [
|
||||
mock.call(
|
||||
iterator=mock.ANY,
|
||||
container=mock_container,
|
||||
object_name=file,
|
||||
extra={'acl': 'public-read'},
|
||||
)
|
||||
for file in files_to_upload
|
||||
]
|
||||
mock_driver.upload_object_via_stream.assert_has_calls(calls, any_order=True)
|
||||
self.assertEqual(mock_driver.upload_object_via_stream.call_count, 2)
|
||||
|
||||
def test_update_awsbucket_libcloud_in_index_only_mode(self):
|
||||
from libcloud.storage.base import Container
|
||||
|
||||
# setup parameters for this test run
|
||||
fdroidserver.common.options = mock.Mock()
|
||||
fdroidserver.common.options.no_checksum = True
|
||||
fdroidserver.common.options.verbose = False
|
||||
fdroidserver.common.options.quiet = True
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
fdroidserver.deploy.config = config
|
||||
fdroidserver.deploy.config["awsbucket"] = "bucket"
|
||||
fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
|
||||
fdroidserver.deploy.config["awssecretkey"] = "secretkey"
|
||||
fdroidserver.deploy.config["s3cmd"] = "s3cmd"
|
||||
|
||||
repo_section = 'repo'
|
||||
|
||||
os.chdir(self.testdir)
|
||||
repo = Path('repo')
|
||||
repo.mkdir(parents=True)
|
||||
fake_apk = repo / 'Sym.apk'
|
||||
with fake_apk.open('w') as fp:
|
||||
fp.write('not an APK, but has the right filename')
|
||||
fake_index = repo / fdroidserver.common.INDEX_FILES[0]
|
||||
with fake_index.open('w') as fp:
|
||||
fp.write('not an index, but has the right filename')
|
||||
|
||||
with mock.patch(
|
||||
'libcloud.storage.drivers.s3.S3StorageDriver'
|
||||
) as mock_driver_class:
|
||||
mock_driver = mock_driver_class.return_value
|
||||
mock_container = mock.MagicMock(spec=Container)
|
||||
mock_container.list_objects.return_value = [
|
||||
mock.MagicMock(name='Sym.apk'),
|
||||
mock.MagicMock(name=fdroidserver.common.INDEX_FILES[0]),
|
||||
]
|
||||
|
||||
mock_driver.get_container.return_value = mock_container
|
||||
mock_driver.upload_object_via_stream.return_value = None
|
||||
|
||||
fdroidserver.deploy.update_awsbucket_libcloud(
|
||||
repo_section, is_index_only=True
|
||||
)
|
||||
|
||||
mock_driver.get_container.assert_called_once_with(
|
||||
container_name=fdroidserver.deploy.config["awsbucket"]
|
||||
)
|
||||
mock_container.list_objects.assert_called_once_with()
|
||||
files_to_upload = [f"fdroid/repo/{fdroidserver.common.INDEX_FILES[0]}"]
|
||||
calls = [
|
||||
mock.call(
|
||||
iterator=mock.ANY,
|
||||
container=mock_container,
|
||||
object_name=file,
|
||||
extra={'acl': 'public-read'},
|
||||
)
|
||||
for file in files_to_upload
|
||||
]
|
||||
mock_driver.upload_object_via_stream.assert_has_calls(
|
||||
calls,
|
||||
any_order=False,
|
||||
)
|
||||
self.assertEqual(mock_driver.upload_object_via_stream.call_count, 1)
|
||||
|
||||
def test_update_servergitmirrors(self):
|
||||
# setup parameters for this test run
|
||||
fdroidserver.common.options = mock.Mock()
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
import configparser
|
||||
import itertools
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
|
|
@ -19,7 +21,7 @@ except ModuleNotFoundError:
|
|||
|
||||
from fdroidserver._yaml import yaml, yaml_dumper
|
||||
|
||||
from .shared_test_code import mkdir_testfiles
|
||||
from .shared_test_code import mkdir_testfiles, VerboseFalseOptions
|
||||
|
||||
# TODO: port generic tests that use index.xml to index-v2 (test that
|
||||
# explicitly test index-v0 should still use index.xml)
|
||||
|
|
@ -34,12 +36,17 @@ except KeyError:
|
|||
WORKSPACE = basedir.parent
|
||||
|
||||
from fdroidserver import common
|
||||
from fdroidserver import deploy
|
||||
|
||||
conf = {"sdk_path": os.getenv("ANDROID_HOME", "")}
|
||||
common.find_apksigner(conf)
|
||||
USE_APKSIGNER = "apksigner" in conf
|
||||
|
||||
|
||||
def docker_socket_exists(path="/var/run/docker.sock"):
|
||||
return os.path.exists(path) and stat.S_ISSOCK(os.stat(path).st_mode)
|
||||
|
||||
|
||||
@unittest.skipIf(sys.byteorder == 'big', 'androguard is not ported to big-endian')
|
||||
class IntegrationTest(unittest.TestCase):
|
||||
@classmethod
|
||||
|
|
@ -64,6 +71,7 @@ class IntegrationTest(unittest.TestCase):
|
|||
self.testdir = mkdir_testfiles(WORKSPACE, self)
|
||||
self.tmp_repo_root = self.testdir / "fdroid"
|
||||
self.tmp_repo_root.mkdir(parents=True)
|
||||
deploy.config = {}
|
||||
os.chdir(self.tmp_repo_root)
|
||||
|
||||
def tearDown(self):
|
||||
|
|
@ -1556,3 +1564,114 @@ class IntegrationTest(unittest.TestCase):
|
|||
self.fdroid_cmd + ["checkupdates", "--allow-dirty", "--auto", "-v"]
|
||||
)
|
||||
self.assertIn("CurrentVersionCode: 1", Path("metadata/fake.yml").read_text())
|
||||
|
||||
@unittest.skipUnless(docker_socket_exists(), "Docker is not available")
|
||||
def test_update_remote_storage_with_rclone_and_minio(self):
|
||||
try:
|
||||
from testcontainers.minio import MinioContainer
|
||||
except ImportError:
|
||||
self.skipTest('Requires testcontainers.minio to run')
|
||||
with MinioContainer(image="quay.io/minio/minio:latest") as minio:
|
||||
# Set up minio bukcet
|
||||
client = minio.get_client()
|
||||
client.make_bucket('test-bucket')
|
||||
host_ip = minio.get_config()['endpoint']
|
||||
|
||||
# Set up Repo dir
|
||||
os.chdir(self.testdir)
|
||||
repo_section = 'repo'
|
||||
repo = Path(repo_section)
|
||||
repo.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy(basedir / 'SpeedoMeterApp.main_1.apk', repo)
|
||||
shutil.copy(basedir / 'repo/index-v2.json', repo)
|
||||
|
||||
# write out config for test use
|
||||
rclone_config = configparser.ConfigParser()
|
||||
rclone_config.add_section("test-minio-config")
|
||||
rclone_config.set("test-minio-config", "type", "s3")
|
||||
rclone_config.set("test-minio-config", "provider", "Minio")
|
||||
rclone_config.set("test-minio-config", "endpoint", "http://" + host_ip)
|
||||
rclone_config.set("test-minio-config", "acl", "public-read")
|
||||
rclone_config.set("test-minio-config", "env_auth", "true")
|
||||
rclone_config.set("test-minio-config", "region", "us-east-1")
|
||||
rclone_config.set("test-minio-config", "access_key_id", "minioadmin")
|
||||
rclone_config.set("test-minio-config", "secret_access_key", "minioadmin")
|
||||
|
||||
rclone_config_path = Path('rclone_config_path')
|
||||
rclone_config_path.mkdir(parents=True, exist_ok=True)
|
||||
rclone_file = rclone_config_path / 'rclone-minio.conf'
|
||||
with open(rclone_file, "w", encoding="utf-8") as configfile:
|
||||
rclone_config.write(configfile)
|
||||
|
||||
# set up config for run
|
||||
awsbucket = "test-bucket"
|
||||
deploy.config['awsbucket'] = awsbucket
|
||||
deploy.config['rclone_config'] = "test-minio-config"
|
||||
deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
|
||||
common.options = VerboseFalseOptions
|
||||
|
||||
# call function
|
||||
deploy.update_remote_storage_with_rclone(repo_section, awsbucket)
|
||||
|
||||
# check if apk and index file are available
|
||||
bucket_content = client.list_objects('test-bucket', recursive=True)
|
||||
files_in_bucket = {obj.object_name for obj in bucket_content}
|
||||
self.assertEqual(
|
||||
files_in_bucket,
|
||||
{'fdroid/repo/SpeedoMeterApp.main_1.apk', 'fdroid/repo/index-v2.json'},
|
||||
)
|
||||
|
||||
@unittest.skipUnless(docker_socket_exists(), "Docker is not available")
|
||||
def test_update_remote_storage_with_rclone_and_minio_in_index_only_mode(self):
|
||||
try:
|
||||
from testcontainers.minio import MinioContainer
|
||||
except ImportError:
|
||||
self.skipTest('Requires testcontainers.minio to run')
|
||||
with MinioContainer(image="quay.io/minio/minio:latest") as minio:
|
||||
# Set up minio bukcet
|
||||
client = minio.get_client()
|
||||
client.make_bucket('test-bucket')
|
||||
host_ip = minio.get_config()['endpoint']
|
||||
|
||||
# Set up Repo dir
|
||||
os.chdir(self.testdir)
|
||||
repo_section = 'repo'
|
||||
repo = Path(repo_section)
|
||||
repo.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy(basedir / 'SpeedoMeterApp.main_1.apk', repo)
|
||||
shutil.copy(basedir / 'repo/index-v2.json', repo)
|
||||
|
||||
# write out config for test use
|
||||
rclone_config = configparser.ConfigParser()
|
||||
rclone_config.add_section("test-minio-config")
|
||||
rclone_config.set("test-minio-config", "type", "s3")
|
||||
rclone_config.set("test-minio-config", "provider", "Minio")
|
||||
rclone_config.set("test-minio-config", "endpoint", "http://" + host_ip)
|
||||
rclone_config.set("test-minio-config", "acl", "public-read")
|
||||
rclone_config.set("test-minio-config", "env_auth", "true")
|
||||
rclone_config.set("test-minio-config", "region", "us-east-1")
|
||||
rclone_config.set("test-minio-config", "access_key_id", "minioadmin")
|
||||
rclone_config.set("test-minio-config", "secret_access_key", "minioadmin")
|
||||
|
||||
rclone_config_path = Path('rclone_config_path')
|
||||
rclone_config_path.mkdir(parents=True, exist_ok=True)
|
||||
rclone_file = rclone_config_path / 'rclone-minio.conf'
|
||||
with open(rclone_file, "w", encoding="utf-8") as configfile:
|
||||
rclone_config.write(configfile)
|
||||
|
||||
# set up config for run
|
||||
awsbucket = "test-bucket"
|
||||
deploy.config['awsbucket'] = awsbucket
|
||||
deploy.config['rclone_config'] = "test-minio-config"
|
||||
deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
|
||||
common.options = VerboseFalseOptions
|
||||
|
||||
# call function
|
||||
deploy.update_remote_storage_with_rclone(
|
||||
repo_section, awsbucket, is_index_only=True
|
||||
)
|
||||
|
||||
# check if apk and index file are available
|
||||
bucket_content = client.list_objects('test-bucket', recursive=True)
|
||||
files_in_bucket = {obj.object_name for obj in bucket_content}
|
||||
self.assertEqual(files_in_bucket, {'fdroid/repo/index-v2.json'})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue