split out options from read_config()

There is no longer any reason for these to be intertwined.

This deliberately avoids touching some files as much as possible because
they are super tangled and due to be replaced.  Those files are:

* fdroidserver/build.py
* fdroidserver/update.py

# Conflicts:
#	tests/testcommon.py

# Conflicts:
#	fdroidserver/btlog.py
#	fdroidserver/import_subcommand.py
This commit is contained in:
Hans-Christoph Steiner 2024-05-08 16:26:46 +02:00
parent 685efa23d4
commit 18f3acc32e
53 changed files with 317 additions and 265 deletions

View file

@ -47,9 +47,6 @@ from . import deploy
from .exception import FDroidException
options = None
def make_binary_transparency_log(
repodirs: collections.abc.Iterable,
btrepo: str = 'binary_transparency',
@ -175,9 +172,8 @@ def main():
------
:exc:`~fdroidserver.exception.FDroidException`
If the specified or default Git repository does not exist.
"""
global options
"""
parser = ArgumentParser()
common.setup_global_opts(parser)
parser.add_argument(
@ -196,7 +192,7 @@ def main():
default=None,
help=_("Push the log to this git remote repository"),
)
options = parser.parse_args()
options = common.parse_args(parser)
if options.verbose:
logging.getLogger("requests").setLevel(logging.INFO)

View file

@ -1014,7 +1014,7 @@ def parse_commandline():
parser.add_argument("-w", "--wiki", default=False, action="store_true",
help=argparse.SUPPRESS)
metadata.add_metadata_arguments(parser)
options = parser.parse_args()
options = common.parse_args(parser)
metadata.warnings_action = options.W
# Force --stop with --on-server to get correct exit code
@ -1076,7 +1076,7 @@ def main():
if not options.appid and not options.all:
parser.error("option %s: If you really want to build all the apps, use --all" % "all")
config = common.read_config(opts=options)
config = common.read_config()
if config['build_server_always']:
options.server = True

View file

@ -506,7 +506,7 @@ def operate_vercode(operation: str, vercode: int) -> int:
return vercode
def checkupdates_app(app: metadata.App) -> None:
def checkupdates_app(app: metadata.App, auto: bool, commit: bool = False) -> None:
"""Check for new versions and updated name of a single app.
Also write back changes to the metadata file and create a Git commit if
@ -582,7 +582,7 @@ def checkupdates_app(app: metadata.App) -> None:
logging.info('...updating to version %s' % ver)
commitmsg = 'Update CurrentVersion of %s to %s' % (name, ver)
if options.auto:
if auto:
mode = app.AutoUpdateMode
if not app.CurrentVersionCode:
raise MetaDataException(
@ -665,7 +665,7 @@ def checkupdates_app(app: metadata.App) -> None:
if commitmsg:
metadata.write_metadata(app.metadatapath, app)
if options.commit:
if commit:
logging.info("Commiting update for " + app.metadatapath)
gitcmd = ["git", "commit", "-m", commitmsg]
if 'auto_author' in config:
@ -695,7 +695,6 @@ def status_update_json(processed: list, failed: dict) -> None:
config = None
options = None
start_timestamp = time.gmtime()
@ -705,7 +704,7 @@ def main():
The behaviour of this function is influenced by the configuration file as
well as command line parameters.
"""
global config, options
global config
# Parse command line...
parser = ArgumentParser()
@ -720,10 +719,10 @@ def main():
parser.add_argument("--allow-dirty", action="store_true", default=False,
help=_("Run on git repo that has uncommitted changes"))
metadata.add_metadata_arguments(parser)
options = parser.parse_args()
options = common.parse_args(parser)
metadata.warnings_action = options.W
config = common.read_config(options)
config = common.read_config()
if not options.allow_dirty:
status = subprocess.check_output(['git', 'status', '--porcelain'])
@ -749,7 +748,7 @@ def main():
logging.info(msg)
try:
checkupdates_app(app)
checkupdates_app(app, options.auto, options.commit)
processed.append(appid)
except Exception as e:
msg = _("...checkupdate failed for {appid} : {error}").format(appid=appid, error=e)

View file

@ -191,6 +191,34 @@ default_config = {
}
def get_options():
"""Return options as set up by parse_args().
This provides an easy way to get the global instance without
having to think about very confusing import and submodule
visibility. The code should be probably refactored so it does not
need this. If each individual option value was always passed to
functions as args, for example.
https://docs.python.org/3/reference/import.html#submodules
"""
return fdroidserver.common.options
def parse_args(parser):
"""Call parser.parse_args(), store result in module-level variable and return it.
This is needed to set up the copy of the options instance in the
fdroidserver.common module. A subcommand only needs to call this
if it uses functions from fdroidserver.common that expect the
"options" variable to be initialized.
"""
fdroidserver.common.options = parser.parse_args()
return fdroidserver.common.options
def setup_global_opts(parser):
try: # the buildserver VM might not have PIL installed
from PIL import PngImagePlugin
@ -373,7 +401,7 @@ def fill_config_defaults(thisconfig):
thisconfig['gradle_version_dir'] = str(Path(thisconfig['cachedir']) / 'gradle')
def get_config(opts=None):
def get_config():
"""Get the initalized, singleton config instance.
config and options are intertwined in read_config(), so they have
@ -395,18 +423,16 @@ def get_config(opts=None):
instance in fdroidserver.common.
"""
global config, options
global config
if config is not None:
return config
read_config(opts=opts)
read_config()
# make sure these values are available in common.py even if they didn't
# declare global in a scope
common.config = config
if opts is not None:
common.options = opts
return config
@ -439,7 +465,7 @@ def config_type_check(path, data):
)
def read_config(opts=None):
def read_config():
"""Read the repository config.
The config is read from config_file, which is in the current
@ -458,13 +484,11 @@ def read_config(opts=None):
in git, it makes sense to use a globally standard encoding.
"""
global config, options
global config
if config is not None:
return config
options = opts
config = {}
config_file = 'config.yml'
old_config_file = 'config.py'

View file

@ -36,7 +36,6 @@ from . import index
from .exception import FDroidException
config = None
options = None
start_timestamp = time.gmtime()
GIT_BRANCH = 'master'
@ -148,9 +147,10 @@ def update_awsbucket_s3cmd(repo_section):
raise FDroidException()
s3cmd_sync = s3cmd + ['sync', '--acl-public']
if options.verbose:
options = common.get_options()
if options and options.verbose:
s3cmd_sync += ['--verbose']
if options.quiet:
if options and options.quiet:
s3cmd_sync += ['--quiet']
s3url = s3bucketurl + '/fdroid/'
@ -312,6 +312,7 @@ def update_serverwebroot(serverwebroot, repo_section):
_('rsync is missing or broken: {error}').format(error=e)
) from e
rsyncargs = ['rsync', '--archive', '--delete-after', '--safe-links']
options = common.get_options()
if not options or not options.no_checksum:
rsyncargs.append('--checksum')
if options and options.verbose:
@ -387,7 +388,7 @@ def sync_from_localcopy(repo_section, local_copy_dir):
# trailing slashes have a meaning in rsync which is not needed here, so
# make sure both paths have exactly one trailing slash
common.local_rsync(
options,
common.get_options(),
os.path.join(local_copy_dir, repo_section).rstrip('/') + '/',
repo_section.rstrip('/') + '/',
)
@ -407,7 +408,7 @@ def update_localcopy(repo_section, local_copy_dir):
"""
# local_copy_dir is guaranteed to have a trailing slash in main() below
common.local_rsync(options, repo_section, local_copy_dir)
common.local_rsync(common.get_options(), repo_section, local_copy_dir)
offline_copy = os.path.join(os.getcwd(), BINARY_TRANSPARENCY_DIR)
if os.path.isdir(os.path.join(offline_copy, '.git')):
@ -446,6 +447,8 @@ def update_servergitmirrors(servergitmirrors, repo_section):
)
return
options = common.get_options()
# right now we support only 'repo' git-mirroring
if repo_section == 'repo':
git_mirror_path = 'git-mirror'
@ -595,7 +598,7 @@ def upload_to_android_observatory(repo_section):
requests # stop unused import warning
if options.verbose:
if common.get_options().verbose:
logging.getLogger("requests").setLevel(logging.INFO)
logging.getLogger("urllib3").setLevel(logging.INFO)
else:
@ -849,7 +852,7 @@ def push_binary_transparency(git_repo_path, git_remote):
def main():
global config, options
global config
parser = ArgumentParser()
common.setup_global_opts(parser)
@ -876,8 +879,8 @@ def main():
default=False,
help=_("If a git mirror gets to big, allow the archive to be deleted"),
)
options = parser.parse_args()
config = common.read_config(options)
options = common.parse_args(parser)
config = common.read_config()
if config.get('nonstandardwebroot') is True:
standardwebroot = False

View file

@ -28,7 +28,6 @@ from .common import FDroidPopen
from .exception import FDroidException
config = None
options = None
start_timestamp = time.gmtime()
@ -42,14 +41,14 @@ def status_update_json(signed):
def main():
global config, options
global config
# Parse command line...
parser = ArgumentParser()
common.setup_global_opts(parser)
options = parser.parse_args()
common.parse_args(parser)
config = common.read_config(options)
config = common.read_config()
repodirs = ['repo']
if config['archive_older'] != 0:

View file

@ -45,7 +45,6 @@ from .exception import FDroidException
config = None
options = None
def handle_retree_error_on_windows(function, path, excinfo):
@ -55,7 +54,7 @@ def handle_retree_error_on_windows(function, path, excinfo):
function(path)
def clone_to_tmp_dir(app: metadata.App) -> Path:
def clone_to_tmp_dir(app: metadata.App, rev=None) -> Path:
"""Clone the source repository of an app to a temporary directory for further processing.
Parameters
@ -67,6 +66,7 @@ def clone_to_tmp_dir(app: metadata.App) -> Path:
-------
tmp_dir
The (temporary) directory the apps source has been cloned into.
"""
tmp_dir = Path('tmp')
tmp_dir.mkdir(exist_ok=True)
@ -76,7 +76,7 @@ def clone_to_tmp_dir(app: metadata.App) -> Path:
if tmp_dir.exists():
shutil.rmtree(str(tmp_dir), onerror=handle_retree_error_on_windows)
vcs = common.getvcs(app.RepoType, app.Repo, tmp_dir)
vcs.gotorevision(options.rev)
vcs.gotorevision(rev)
return tmp_dir
@ -236,8 +236,9 @@ def main():
the current directory is not a Git repository, no application ID could
be found, no Gradle project could be found or there is already metadata
for the found application ID.
"""
global config, options
global config
# Parse command line...
parser = ArgumentParser()
@ -255,10 +256,10 @@ def main():
parser.add_argument("--rev", default=None,
help=_("Allows a different revision (or git branch) to be specified for the initial import"))
metadata.add_metadata_arguments(parser)
options = parser.parse_args()
options = common.parse_args(parser)
metadata.warnings_action = options.W
config = common.read_config(options)
config = common.read_config()
apps = metadata.read_metadata()
app = None
@ -289,7 +290,7 @@ def main():
write_local_file = True
elif options.url:
app = get_app_from_url(options.url)
tmp_importer_dir = clone_to_tmp_dir(app)
tmp_importer_dir = clone_to_tmp_dir(app, options.rev)
git_repo = git.Repo(tmp_importer_dir)
if not options.omit_disable:

View file

@ -32,7 +32,6 @@ from . import common
from .exception import FDroidException
config = {}
options = None
def disable_in_config(key, value):
@ -49,7 +48,7 @@ def disable_in_config(key, value):
def main():
global options, config
global config
# Parse command line...
parser = ArgumentParser()
@ -81,7 +80,7 @@ def main():
default=False,
help=_("Do not prompt for Android SDK path, just fail"),
)
options = parser.parse_args()
options = common.parse_args(parser)
common.set_console_logging(options.verbose)
@ -171,7 +170,7 @@ def main():
raise FDroidException('Repository already exists.')
# now that we have a local config.yml, read configuration...
config = common.read_config(options)
config = common.read_config()
# the NDK is optional and there may be multiple versions of it, so it's
# left for the user to configure

View file

@ -28,7 +28,6 @@ from . import common
from .common import SdkToolsPopen
from .exception import FDroidException
options = None
config = None
@ -44,7 +43,7 @@ def devices():
def main():
global options, config
global config
# Parse command line...
parser = ArgumentParser(
@ -63,7 +62,7 @@ def main():
default=False,
help=_("Install all signed applications available"),
)
options = parser.parse_args()
options = common.parse_args(parser)
common.set_console_logging(options.verbose)
@ -73,7 +72,7 @@ def main():
% "all"
)
config = common.read_config(options)
config = common.read_config()
output_dir = 'repo'
if not os.path.isdir(output_dir):

View file

@ -31,7 +31,6 @@ from . import metadata
from . import rewritemeta
config = None
options = None
def enforce_https(domain):
@ -503,7 +502,7 @@ def check_files_dir(app):
def check_format(app):
if options.format and not rewritemeta.proper_format(app):
if common.options.format and not rewritemeta.proper_format(app):
yield _("Run rewritemeta to fix formatting")
@ -787,7 +786,7 @@ def lint_config(arg):
def main():
global config, options
global config
# Parse command line...
parser = ArgumentParser()
@ -812,10 +811,10 @@ def main():
"appid", nargs='*', help=_("application ID of file to operate on")
)
metadata.add_metadata_arguments(parser)
options = parser.parse_args()
options = common.parse_args(parser)
metadata.warnings_action = options.W
config = common.read_config(options)
config = common.read_config()
load_antiFeatures_config()
load_categories_config()

View file

@ -15,11 +15,9 @@ from . import common
from . import index
from . import update
options = None
def _run_wget(path, urls):
if options.verbose:
def _run_wget(path, urls, verbose=False):
if verbose:
verbose = '--verbose'
else:
verbose = '--no-verbose'
@ -48,8 +46,6 @@ def _run_wget(path, urls):
def main():
global options
parser = ArgumentParser()
common.setup_global_opts(parser)
parser.add_argument(
@ -93,7 +89,7 @@ def main():
parser.add_argument(
"--output-dir", default=None, help=_("The directory to write the mirror to")
)
options = parser.parse_args()
options = common.parse_args(parser)
common.set_console_logging(options.verbose)
@ -119,7 +115,7 @@ def main():
)
if fingerprint:
config = common.read_config(options)
config = common.read_config()
if not ('jarsigner' in config or 'apksigner' in config):
logging.error(
_('Java JDK not found! Install in standard location or set java_paths!')
@ -229,7 +225,7 @@ def main():
_append_to_url_path(section, f[:-4] + '.log.gz')
)
_run_wget(sectiondir, urls)
_run_wget(sectiondir, urls, options.verbose)
for app in data['apps']:
localized = app.get('localized')
@ -242,7 +238,7 @@ def main():
if f:
filepath_tuple = components + (f,)
urls.append(_append_to_url_path(*filepath_tuple))
_run_wget(os.path.join(basedir, *components), urls)
_run_wget(os.path.join(basedir, *components), urls, options.verbose)
for k in update.SCREENSHOT_DIRS:
urls = []
filelist = d.get(k)
@ -251,7 +247,11 @@ def main():
for f in filelist:
filepath_tuple = components + (f,)
urls.append(_append_to_url_path(*filepath_tuple))
_run_wget(os.path.join(basedir, *components), urls)
_run_wget(
os.path.join(basedir, *components),
urls,
options.verbose,
)
urls = dict()
for app in data['apps']:
@ -269,7 +269,11 @@ def main():
for icondir in icondirs:
if icondir in urls:
_run_wget(os.path.join(basedir, section, icondir), urls[icondir])
_run_wget(
os.path.join(basedir, section, icondir),
urls[icondir],
options.verbose,
)
if __name__ == "__main__":

View file

@ -259,8 +259,7 @@ def main():
help=_("Set maximum releases in repo before older ones are archived"),
)
# TODO add --with-btlog
options = parser.parse_args()
common.options = options
options = common.parse_args(parser)
# force a tighter umask since this writes private key material
umask = os.umask(0o077)
@ -428,7 +427,7 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
with open('config.yml', 'w') as fp:
yaml.dump(config, fp, default_flow_style=False)
os.chmod('config.yml', 0o600)
config = common.read_config(options)
config = common.read_config()
common.assert_config_keystore(config)
for root, dirs, files in os.walk(cibase):

View file

@ -49,7 +49,6 @@ from .common import FDroidPopen
from .exception import BuildException, FDroidException
config = None
options = None
start_timestamp = time.gmtime()
@ -269,7 +268,7 @@ def create_key_if_not_existing(keyalias):
def main():
global config, options
global config
# Parse command line...
parser = ArgumentParser(
@ -289,10 +288,10 @@ def main():
help=_("application ID with optional versionCode in the form APPID[:VERCODE]"),
)
metadata.add_metadata_arguments(parser)
options = parser.parse_args()
options = common.parse_args(parser)
metadata.warnings_action = options.W
config = common.read_config(options)
config = common.read_config()
if not ('jarsigner' in config and 'keytool' in config):
logging.critical(

View file

@ -20,8 +20,6 @@ from argparse import ArgumentParser
from . import common
from . import metadata
options = None
def main():
parser = ArgumentParser()
@ -29,7 +27,7 @@ def main():
metadata.add_metadata_arguments(parser)
options = parser.parse_args()
metadata.warnings_action = options.W
common.read_config(None)
common.read_config()
metadata.read_metadata()

View file

@ -29,7 +29,6 @@ from . import common
from . import metadata
config = None
options = None
def proper_format(app):
@ -62,7 +61,7 @@ def remove_blank_flags_from_builds(builds):
def main():
global config, options
global config
parser = ArgumentParser()
common.setup_global_opts(parser)
@ -77,10 +76,10 @@ def main():
"appid", nargs='*', help=_("application ID of file to operate on")
)
metadata.add_metadata_arguments(parser)
options = parser.parse_args()
options = common.parse_args(parser)
metadata.warnings_action = options.W
config = common.read_config(options)
config = common.read_config()
# Get all apps...
allapps = metadata.read_metadata(options.appid)

View file

@ -40,8 +40,6 @@ from . import metadata
from .exception import BuildException, VCSException, ConfigurationException
from . import scanner
options = None
@dataclass
class MessageStore:
@ -332,8 +330,9 @@ class ScannerTool:
self.scanner_data_lookup()
config = common.get_config()
if (options and options.refresh_scanner) or config.get('refresh_scanner'):
options = common.get_options()
options_refresh_scanner = options and options.refresh_scanner
if options_refresh_scanner or common.get_config().get('refresh_scanner'):
self.refresh()
self.load()
@ -589,6 +588,7 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
-------
0 if the problem was ignored/deleted/is only a warning, 1 otherwise
"""
options = common.get_options()
if toignore(path_in_build_dir):
return ignoreproblem(what, path_in_build_dir, json_per_build)
if todelete(path_in_build_dir):
@ -776,9 +776,6 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
def main():
global options
# Parse command line...
parser = ArgumentParser(
usage="%(prog)s [options] [(APPID[:VERCODE] | path/to.apk) ...]"
)
@ -793,7 +790,7 @@ def main():
parser.add_argument("-e", "--exit-code", action="store_true", default=False,
help=_("Exit with a non-zero code if problems were found"))
metadata.add_metadata_arguments(parser)
options = parser.parse_args()
options = common.parse_args(parser)
metadata.warnings_action = options.W
json_output = dict()
@ -804,7 +801,7 @@ def main():
logging.getLogger().setLevel(logging.ERROR)
# initialize/load configuration values
common.get_config(opts=options)
common.get_config()
probcount = 0

View file

@ -103,11 +103,8 @@ def main():
"APK", nargs='*', help=_("signed APK, either a file-path or HTTPS URL.")
)
parser.add_argument("--no-check-https", action="store_true", default=False)
options = parser.parse_args()
options = common.parse_args(parser)
common.set_console_logging(options.verbose)
# Read config.py...
common.read_config(options)
common.read_config()
extract(options)

View file

@ -29,7 +29,6 @@ from . import metadata
from .exception import FDroidException
config = None
options = None
start_timestamp = time.gmtime()
@ -175,13 +174,13 @@ def status_update_json(signed):
def main():
global config, options
global config
parser = ArgumentParser()
common.setup_global_opts(parser)
options = parser.parse_args()
common.parse_args(parser)
config = common.read_config(options)
config = common.read_config()
if 'jarsigner' not in config:
raise FDroidException(

View file

@ -2574,10 +2574,10 @@ def main():
parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
help=_("Include APKs that are signed with disabled algorithms like MD5"))
metadata.add_metadata_arguments(parser)
options = parser.parse_args()
options = common.parse_args(parser)
metadata.warnings_action = options.W
config = common.read_config(options)
config = common.read_config()
common.setup_status_output(start_timestamp)
if not (('jarsigner' in config or 'apksigner' in config)

View file

@ -30,7 +30,6 @@ from . import common
from . import net
from .exception import FDroidException
options = None
config = None
@ -146,7 +145,7 @@ def write_json_report(url, remote_apk, unsigned_apk, compare_result):
def main():
global options, config
global config
# Parse command line...
parser = ArgumentParser(
@ -170,9 +169,9 @@ def main():
default=False,
help=_("Output JSON report to file named after APK."),
)
options = parser.parse_args()
options = common.parse_args(parser)
config = common.read_config(options)
config = common.read_config()
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):