mirrors/fdroidserver
1
0
Fork 0
mirror of https://github.com/f-droid/fdroidserver.git synced 2025-11-13 10:40:29 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'yaml-config' into 'master'

Browse source 
move config.py to config.yml

See merge request fdroid/fdroidserver!571
This commit is contained in:
Michael Pöhn 2020-11-05 23:00:32 +00:00
commit 196dc75481
18 changed files with 512 additions and 254 deletions
Download patch file Download diff file Expand all files Collapse all files

4
fdroidserver/build.py
View file

@ -141,8 +141,8 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
ftp.chdir(homedir)
ftp.put(os.path.join(serverpath, '..', 'buildserver',
'config.buildserver.py'), 'config.py')
ftp.chmod('config.py', 0o600)
'config.buildserver.yml'), 'config.yml')
ftp.chmod('config.yml', 0o600)
# Copy over the ID (head commit hash) of the fdroidserver in use...
with open(os.path.join(os.getcwd(), 'tmp', 'fdroidserverid'), 'wb') as fp:

97
fdroidserver/common.py
View file

@ -39,6 +39,7 @@ import socket
import base64
import urllib.parse
import urllib.request
import yaml
import zipfile
import tempfile
import json
@ -284,14 +285,18 @@ def regsub_file(pattern, repl, path):
f.write(text)
def read_config(opts, config_file='config.py'):
def read_config(opts):
"""Read the repository config
The config is read from config_file, which is in the current
directory when any of the repo management commands are used. If
there is a local metadata file in the git repo, then config.py is
there is a local metadata file in the git repo, then the config is
not required, just use defaults.
config.yml is the preferred form because no code is executed when
reading it. config.py is deprecated and supported for backwards
compatibility.
"""
global config, options
@ -301,14 +306,25 @@ def read_config(opts, config_file='config.py'):
options = opts
config = {}
config_file = 'config.yml'
old_config_file = 'config.py'
if os.path.isfile(config_file):
if os.path.exists(config_file) and os.path.exists(old_config_file):
logging.error(_("""Conflicting config files! Using {newfile}, ignoring {oldfile}!""")
.format(oldfile=old_config_file, newfile=config_file))
if os.path.exists(config_file):
logging.debug(_("Reading '{config_file}'").format(config_file=config_file))
with io.open(config_file, "rb") as f:
code = compile(f.read(), config_file, 'exec')
exec(code, None, config) # nosec TODO switch to YAML file
with open(config_file) as fp:
config = yaml.safe_load(fp)
elif os.path.exists(old_config_file):
logging.warning(_("""{oldfile} is deprecated, use {newfile}""")
.format(oldfile=old_config_file, newfile=config_file))
with io.open(old_config_file, "rb") as fp:
code = compile(fp.read(), old_config_file, 'exec')
exec(code, None, config) # nosec TODO automatically migrate
else:
logging.warning(_("No 'config.py' found, using defaults."))
logging.warning(_("No config.yml found, using defaults."))
for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
if k in config:
@ -329,10 +345,14 @@ def read_config(opts, config_file='config.py'):
'-providerArg', 'opensc-fdroid.cfg']
if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
st = os.stat(config_file)
if os.path.exists(config_file):
f = config_file
elif os.path.exists(old_config_file):
f = old_config_file
st = os.stat(f)
if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
logging.warning(_("unsafe permissions on '{config_file}' (should be 0600)!")
.format(config_file=config_file))
.format(config_file=f))
fill_config_defaults(config)
@ -368,6 +388,29 @@ def read_config(opts, config_file='config.py'):
limit = config['git_mirror_size_limit']
config['git_mirror_size_limit'] = parse_human_readable_size(limit)
for configname, dictvalue in config.items():
if configname == 'java_paths':
new = dict()
for k, v in dictvalue.items():
new[str(k)] = v
config[configname] = new
elif configname in ('ndk_paths', 'java_paths', 'char_limits', 'keyaliases'):
continue
elif isinstance(dictvalue, dict):
for k, v in dictvalue.items():
if k == 'env':
env = os.getenv(v)
if env:
config[configname] = env
else:
del(config[configname])
logging.error(_('Environment variable {var} from {configname} is not set!')
.format(var=k, configname=configname))
else:
del(config[configname])
logging.error(_('Unknown entry {key} in {configname}')
.format(key=k, configname=configname))
return config
@ -396,10 +439,10 @@ def assert_config_keystore(config):
nosigningkey = False
if 'repo_keyalias' not in config:
nosigningkey = True
logging.critical(_("'repo_keyalias' not found in config.py!"))
logging.critical(_("'repo_keyalias' not found in config.yml!"))
if 'keystore' not in config:
nosigningkey = True
logging.critical(_("'keystore' not found in config.py!"))
logging.critical(_("'keystore' not found in config.yml!"))
elif config['keystore'] == 'NONE':
if not config.get('smartcardoptions'):
nosigningkey = True
@ -409,10 +452,10 @@ def assert_config_keystore(config):
logging.critical("'" + config['keystore'] + "' does not exist!")
if 'keystorepass' not in config:
nosigningkey = True
logging.critical(_("'keystorepass' not found in config.py!"))
logging.critical(_("'keystorepass' not found in config.yml!"))
if 'keypass' not in config and config.get('keystore') != 'NONE':
nosigningkey = True
logging.critical(_("'keypass' not found in config.py!"))
logging.critical(_("'keypass' not found in config.yml!"))
if nosigningkey:
raise FDroidException("This command requires a signing key, "
+ "you can create one using: fdroid update --create-key")
@ -513,7 +556,7 @@ def test_sdk_exists(thisconfig):
test_aapt_version(thisconfig['aapt'])
return True
else:
logging.error(_("'sdk_path' not set in 'config.py'!"))
logging.error(_("'sdk_path' not set in config.yml!"))
return False
if thisconfig['sdk_path'] == default_config['sdk_path']:
logging.error(_('No Android SDK found!'))
@ -3518,19 +3561,24 @@ def load_stats_fdroid_signing_key_fingerprints():
def write_to_config(thisconfig, key, value=None, config_file=None):
'''write a key/value to the local config.py
'''write a key/value to the local config.yml or config.py
NOTE: only supports writing string variables.
:param thisconfig: config dictionary
:param key: variable name in config.py to be overwritten/added
:param key: variable name in config to be overwritten/added
:param value: optional value to be written, instead of fetched
from 'thisconfig' dictionary.
'''
if value is None:
origkey = key + '_orig'
value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
cfg = config_file if config_file else 'config.py'
if config_file:
cfg = config_file
elif os.path.exists('config.py') and not os.path.exists('config.yml'):
cfg = 'config.py'
else:
cfg = 'config.yml'
# load config file, create one if it doesn't exist
if not os.path.exists(cfg):
@ -3546,10 +3594,17 @@ def write_to_config(thisconfig, key, value=None, config_file=None):
# regex for finding and replacing python string variable
# definitions/initializations
pattern = re.compile(r'^[\s#]*' + key + r'\s*=\s*"[^"]*"')
repl = key + ' = "' + value + '"'
pattern2 = re.compile(r'^[\s#]*' + key + r"\s*=\s*'[^']*'")
repl2 = key + " = '" + value + "'"
if cfg.endswith('.py'):
pattern = re.compile(r'^[\s#]*' + key + r'\s*=\s*"[^"]*"')
repl = key + ' = "' + value + '"'
pattern2 = re.compile(r'^[\s#]*' + key + r"\s*=\s*'[^']*'")
repl2 = key + " = '" + value + "'"
else:
# assume .yml as default
pattern = re.compile(r'^[\s#]*' + key + r':.*')
repl = yaml.dump({key: value}, default_flow_style=False)
pattern2 = pattern
repl2 = repl
# If we replaced this line once, we make sure won't be a
# second instance of this line for this key in the document.

4
fdroidserver/deploy.py
View file

@ -52,7 +52,7 @@ def update_awsbucket(repo_section):
subdirectories) to the AWS S3 "bucket". The contents of that subdir of the
bucket will first be deleted.
Requires AWS credentials set in config.py: awsaccesskeyid, awssecretkey
Requires AWS credentials set in config.yml: awsaccesskeyid, awssecretkey
'''
logging.debug('Syncing "' + repo_section + '" to Amazon S3 bucket "'
@ -160,7 +160,7 @@ def update_awsbucket_libcloud(repo_section):
if not config.get('awsaccesskeyid') or not config.get('awssecretkey'):
raise FDroidException(
_('To use awsbucket, awssecretkey and awsaccesskeyid must also be set in config.py!'))
_('To use awsbucket, awssecretkey and awsaccesskeyid must also be set in config.yml!'))
awsbucket = config['awsbucket']
if os.path.exists(USER_S3CFG):

27
fdroidserver/init.py
View file

@ -36,13 +36,14 @@ options = None
def disable_in_config(key, value):
'''write a key/value to the local config.py, then comment it out'''
with open('config.py', 'r') as f:
'''write a key/value to the local config.yml, then comment it out'''
import yaml
with open('config.yml') as f:
data = f.read()
pattern = r'\n[\s#]*' + key + r'\s*=\s*"[^"]*"'
repl = '\n#' + key + ' = "' + value + '"'
pattern = r'\n[\s#]*' + key + r':.*'
repl = '\n#' + yaml.dump({key: value}, default_flow_style=False)
data = re.sub(pattern, repl, data)
with open('config.py', 'w') as f:
with open('config.yml', 'w') as f:
f.writelines(data)
@ -114,13 +115,13 @@ def main():
raise FDroidException(_("Android SDK not found at {path}!")
.format(path=test_config['sdk_path']))
if not os.path.exists('config.py'):
if not os.path.exists('config.yml') and not os.path.exists('config.py'):
# 'metadata' and 'tmp' are created in fdroid
if not os.path.exists('repo'):
os.mkdir('repo')
shutil.copy(os.path.join(examplesdir, 'fdroid-icon.png'), fdroiddir)
shutil.copyfile(os.path.join(examplesdir, 'config.py'), 'config.py')
os.chmod('config.py', 0o0600)
shutil.copyfile(os.path.join(examplesdir, 'config.yml'), 'config.yml')
os.chmod('config.yml', 0o0600)
# If android_home is None, test_config['sdk_path'] will be used and
# "$ANDROID_HOME" may be used if the env var is set up correctly.
# If android_home is not None, the path given from the command line
@ -132,7 +133,7 @@ def main():
logging.info('Try running `fdroid init` in an empty directory.')
raise FDroidException('Repository already exists.')
# now that we have a local config.py, read configuration...
# now that we have a local config.yml, read configuration...
config = common.read_config(options)
# enable apksigner by default so v2/v3 APK signatures validate
@ -143,7 +144,7 @@ def main():
# left for the user to configure
# find or generate the keystore for the repo signing key. First try the
# path written in the default config.py. Then check if the user has
# path written in the default config.yml. Then check if the user has
# specified a path from the command line, which will trump all others.
# Otherwise, create ~/.local/share/fdroidserver and stick it in there. If
# keystore is set to NONE, that means that Java will look for keys in a
@ -192,7 +193,7 @@ def main():
f.write('name = OpenSC\nlibrary = ')
f.write(opensc_so)
f.write('\n')
logging.info("Repo setup using a smartcard HSM. Please edit keystorepass and repo_keyalias in config.py.")
logging.info("Repo setup using a smartcard HSM. Please edit keystorepass and repo_keyalias in config.yml.")
logging.info("If you want to generate a new repo signing key in the HSM you can do that with 'fdroid update "
"--create-key'.")
elif os.path.exists(keystore):
@ -202,7 +203,7 @@ def main():
if keydname:
to_set.remove('keydname')
logging.warning('\n' + _('Using existing keystore "{path}"').format(path=keystore)
+ '\n' + _('Now set these in config.py:') + ' '
+ '\n' + _('Now set these in config.yml:') + ' '
+ ', '.join(to_set) + '\n')
else:
password = common.genpassword()
@ -228,7 +229,7 @@ def main():
msg += '\n Alias for key in store:\t' + repo_keyalias
msg += '\n\n' + '''To complete the setup, add your APKs to "%s"
then run "fdroid update -c; fdroid update". You might also want to edit
"config.py" to set the URL, repo name, and more. You should also set up
"config.yml" to set the URL, repo name, and more. You should also set up
a signing key (a temporary one might have been automatically generated).
For more info: https://f-droid.org/docs/Setup_an_F-Droid_App_Repo