mirror of
https://github.com/f-droid/fdroidserver.git
synced 2025-11-04 06:30:27 +03:00
8b11e098db
```
FAIL: test_get_upstream_main_branch (tests.test_checkupdates.CheckupdatesTest.test_get_upstream_main_branch)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/builds/fdroid/fdroidserver/tests/test_checkupdates.py", line 509, in test_get_upstream_main_branch
self.assertEqual(
~~~~~~~~~~~~~~~~^
f'upstream/{testvalue}',
^^^^^^^^^^^^^^^^^^^^^^^^
branch,
^^^^^^^
f'The default branch should be called {testvalue}!',
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
AssertionError: 'upstream/foo' != 'upstream/main'
- upstream/foo
+ upstream/main
: The default branch should be called foo!
```
967 lines
34 KiB
Python
967 lines
34 KiB
Python
#!/usr/bin/env python3
|
|
"""Check for updates to applications."""
|
|
#
|
|
# checkupdates.py - part of the FDroid server tools
|
|
# Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
|
|
# Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import configparser
|
|
import copy
|
|
import logging
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from argparse import ArgumentParser
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
import git
|
|
|
|
from . import _, common, metadata, net
|
|
from .exception import (
|
|
FDroidException,
|
|
MetaDataException,
|
|
NoSubmodulesException,
|
|
VCSException,
|
|
)
|
|
|
|
# https://gitlab.com/fdroid/checkupdates-runner/-/blob/1861899262a62a4ed08fa24e5449c0368dfb7617/.gitlab-ci.yml#L36
|
|
BOT_EMAIL = 'fdroidci@bubu1.eu'
|
|
|
|
|
|
def check_http(app: metadata.App) -> tuple[Optional[str], Optional[int]]:
|
|
"""Check for a new version by looking at a document retrieved via HTTP.
|
|
|
|
The app's UpdateCheckData field is used to provide the information
|
|
required.
|
|
|
|
Parameters
|
|
----------
|
|
app
|
|
The App instance to check for updates for.
|
|
|
|
Returns
|
|
-------
|
|
version
|
|
The found versionName or None if the versionName should be ignored
|
|
according to UpdateCheckIgnore.
|
|
vercode
|
|
The found versionCode or None if the versionCode should be ignored
|
|
according to UpdateCheckIgnore.
|
|
|
|
Raises
|
|
------
|
|
:exc:`~fdroidserver.exception.FDroidException`
|
|
If UpdateCheckData is missing or is an invalid URL or if there is no
|
|
match for the provided versionName or versionCode regex.
|
|
"""
|
|
if not app.UpdateCheckData:
|
|
raise FDroidException('Missing Update Check Data')
|
|
|
|
urlcode, codeex, urlver, verex = app.UpdateCheckData.split('|')
|
|
parsed = urllib.parse.urlparse(urlcode)
|
|
if not parsed.netloc or not parsed.scheme or parsed.scheme != 'https':
|
|
raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode))
|
|
if urlver != '.':
|
|
parsed = urllib.parse.urlparse(urlver)
|
|
if not parsed.netloc or not parsed.scheme or parsed.scheme != 'https':
|
|
raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlver))
|
|
|
|
logging.debug("...requesting {0}".format(urlcode))
|
|
req = urllib.request.Request(urlcode, None, headers=net.HEADERS)
|
|
resp = urllib.request.urlopen(req, None, 20) # nosec B310 scheme is filtered above
|
|
page = resp.read().decode('utf-8')
|
|
|
|
m = re.search(codeex, page)
|
|
if not m:
|
|
raise FDroidException("No RE match for versionCode")
|
|
vercode = common.version_code_string_to_int(m.group(1).strip())
|
|
|
|
if urlver != '.':
|
|
logging.debug("...requesting {0}".format(urlver))
|
|
req = urllib.request.Request(urlver, None)
|
|
resp = urllib.request.urlopen(req, None, 20) # nosec B310 scheme is filtered above
|
|
page = resp.read().decode('utf-8')
|
|
|
|
m = re.search(verex, page)
|
|
if not m:
|
|
raise FDroidException("No RE match for version")
|
|
version = m.group(1)
|
|
|
|
if app.UpdateCheckIgnore and re.search(app.UpdateCheckIgnore, version):
|
|
logging.info("Version {version} for {appid} is ignored".format(version=version, appid=app.id))
|
|
return (None, None)
|
|
|
|
return (version, vercode)
|
|
|
|
|
|
def check_tags(app: metadata.App, pattern: str) -> tuple[str, int, str]:
|
|
"""Check for a new version by looking at the tags in the source repo.
|
|
|
|
Whether this can be used reliably or not depends on
|
|
the development procedures used by the project's developers. Use it with
|
|
caution, because it's inappropriate for many projects.
|
|
|
|
Parameters
|
|
----------
|
|
app
|
|
The App instance to check for updates for.
|
|
pattern
|
|
The pattern a tag needs to match to be considered.
|
|
|
|
Returns
|
|
-------
|
|
versionName
|
|
The highest found versionName.
|
|
versionCode
|
|
The highest found versionCode.
|
|
ref
|
|
The Git reference, commit hash or tag name, of the highest found
|
|
versionName, versionCode.
|
|
|
|
Raises
|
|
------
|
|
:exc:`~fdroidserver.exception.MetaDataException`
|
|
If this function is not suitable for the RepoType of the app or
|
|
information is missing to perform this type of check.
|
|
:exc:`~fdroidserver.exception.FDroidException`
|
|
If no matching tags or no information whatsoever could be found.
|
|
"""
|
|
if app.RepoType == 'srclib':
|
|
build_dir = Path('build/srclib') / app.Repo
|
|
repotype = common.getsrclibvcs(app.Repo)
|
|
else:
|
|
build_dir = Path('build') / app.id
|
|
repotype = app.RepoType
|
|
|
|
if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
|
|
raise MetaDataException(_('Tags update mode only works for git, hg, bzr and git-svn repositories currently'))
|
|
|
|
if repotype == 'git-svn' and ';' not in app.Repo:
|
|
raise MetaDataException(_('Tags update mode used in git-svn, but the repo was not set up with tags'))
|
|
|
|
# Set up vcs interface and make sure we have the latest code...
|
|
vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
|
|
|
|
vcs.gotorevision(None)
|
|
|
|
last_build = get_last_build_from_app(app)
|
|
|
|
try_init_submodules(app, last_build, vcs)
|
|
|
|
htag = None
|
|
hver = None
|
|
hcode = 0
|
|
|
|
tags = []
|
|
if repotype == 'git':
|
|
tags = vcs.latesttags()
|
|
else:
|
|
tags = vcs.gettags()
|
|
if not tags:
|
|
raise FDroidException(_('No tags found'))
|
|
|
|
logging.debug("All tags: " + ','.join(tags))
|
|
if pattern:
|
|
pat = re.compile(pattern)
|
|
tags = [tag for tag in tags if pat.match(tag)]
|
|
if not tags:
|
|
raise FDroidException(_('No matching tags found'))
|
|
logging.debug("Matching tags: " + ','.join(tags))
|
|
|
|
if len(tags) > 5 and repotype == 'git':
|
|
tags = tags[:5]
|
|
logging.debug("Latest tags: " + ','.join(tags))
|
|
|
|
for tag in tags:
|
|
logging.debug("Check tag: '{0}'".format(tag))
|
|
vcs.gotorevision(tag)
|
|
try_init_submodules(app, last_build, vcs)
|
|
|
|
if app.UpdateCheckData:
|
|
filecode, codeex, filever, verex = app.UpdateCheckData.split('|')
|
|
|
|
if filecode:
|
|
filecode = build_dir / filecode
|
|
if not filecode.is_file():
|
|
logging.debug("UpdateCheckData file {0} not found in tag {1}".format(filecode, tag))
|
|
continue
|
|
filecontent = filecode.read_text()
|
|
else:
|
|
filecontent = tag
|
|
|
|
vercode = tag
|
|
if codeex:
|
|
m = re.search(codeex, filecontent)
|
|
if not m:
|
|
logging.debug(f"UpdateCheckData regex {codeex} for versionCode"
|
|
f" has no match in tag {tag}")
|
|
continue
|
|
|
|
vercode = m.group(1).strip()
|
|
|
|
if filever:
|
|
if filever != '.':
|
|
filever = build_dir / filever
|
|
if filever.is_file():
|
|
filecontent = filever.read_text()
|
|
else:
|
|
logging.debug("UpdateCheckData file {0} not found in tag {1}".format(filever, tag))
|
|
else:
|
|
filecontent = tag
|
|
|
|
version = tag
|
|
if verex:
|
|
m = re.search(verex, filecontent)
|
|
if not m:
|
|
logging.debug(f"UpdateCheckData regex {verex} for versionName"
|
|
f" has no match in tag {tag}")
|
|
continue
|
|
|
|
version = m.group(1)
|
|
|
|
logging.debug("UpdateCheckData found version {0} ({1})"
|
|
.format(version, vercode))
|
|
vercode = common.version_code_string_to_int(vercode)
|
|
if vercode > hcode:
|
|
htag = tag
|
|
hcode = vercode
|
|
hver = version
|
|
else:
|
|
for subdir in possible_subdirs(app):
|
|
root_dir = build_dir / subdir
|
|
paths = common.manifest_paths(root_dir, last_build.gradle)
|
|
version, vercode, _package = common.parse_androidmanifests(paths, app)
|
|
if version in ('Unknown', 'Ignore'):
|
|
version = tag
|
|
if vercode:
|
|
logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
|
|
.format(subdir, version, vercode))
|
|
if vercode > hcode:
|
|
htag = tag
|
|
hcode = vercode
|
|
hver = version
|
|
|
|
if hver:
|
|
if htag != tags[0]:
|
|
logging.warning(
|
|
"{appid}: latest tag {tag} does not contain highest version {version}".format(
|
|
appid=app.id, tag=tags[0], version=hver
|
|
)
|
|
)
|
|
try:
|
|
commit = vcs.getref(htag)
|
|
if commit:
|
|
return (hver, hcode, commit)
|
|
except VCSException:
|
|
pass
|
|
return (hver, hcode, htag)
|
|
raise FDroidException(_("Couldn't find any version information"))
|
|
|
|
|
|
def check_repomanifest(app: metadata.App, branch: Optional[str] = None) -> tuple[str, int]:
|
|
"""Check for a new version by looking at the AndroidManifest.xml at the HEAD of the source repo.
|
|
|
|
Whether this can be used reliably or not depends on
|
|
the development procedures used by the project's developers. Use it with
|
|
caution, because it's inappropriate for many projects.
|
|
|
|
Parameters
|
|
----------
|
|
app
|
|
The App instance to check for updates for.
|
|
branch
|
|
The VCS branch where to search for versionCode, versionName.
|
|
|
|
Returns
|
|
-------
|
|
versionName
|
|
The highest found versionName.
|
|
versionCode
|
|
The highest found versionCode.
|
|
|
|
Raises
|
|
------
|
|
:exc:`~fdroidserver.exception.FDroidException`
|
|
If no package id or no version information could be found.
|
|
"""
|
|
if app.RepoType == 'srclib':
|
|
build_dir = Path('build/srclib') / app.Repo
|
|
repotype = common.getsrclibvcs(app.Repo)
|
|
else:
|
|
build_dir = Path('build') / app.id
|
|
repotype = app.RepoType
|
|
|
|
# Set up vcs interface and make sure we have the latest code...
|
|
vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
|
|
|
|
if repotype == 'git':
|
|
if branch:
|
|
branch = 'origin/' + branch
|
|
vcs.gotorevision(branch)
|
|
elif repotype == 'git-svn':
|
|
vcs.gotorevision(branch)
|
|
elif repotype == 'hg':
|
|
vcs.gotorevision(branch)
|
|
elif repotype == 'bzr':
|
|
vcs.gotorevision(None)
|
|
|
|
last_build = get_last_build_from_app(app)
|
|
try_init_submodules(app, last_build, vcs)
|
|
|
|
hpak = None
|
|
hver = None
|
|
hcode = 0
|
|
for subdir in possible_subdirs(app):
|
|
root_dir = build_dir / subdir
|
|
paths = common.manifest_paths(root_dir, last_build.gradle)
|
|
version, vercode, package = common.parse_androidmanifests(paths, app)
|
|
if vercode:
|
|
logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
|
|
.format(subdir, version, vercode))
|
|
if vercode > hcode:
|
|
hpak = package
|
|
hcode = vercode
|
|
hver = version
|
|
|
|
if not hpak:
|
|
raise FDroidException(_("Couldn't find package ID"))
|
|
if hver:
|
|
return (hver, hcode)
|
|
raise FDroidException(_("Couldn't find any version information"))
|
|
|
|
|
|
def try_init_submodules(app: metadata.App, last_build: metadata.Build, vcs: common.vcs):
|
|
"""Try to init submodules if the last build entry uses them.
|
|
|
|
They might have been removed from the app's repo in the meantime,
|
|
so if we can't find any submodules we continue with the updates check.
|
|
If there is any other error in initializing them then we stop the check.
|
|
"""
|
|
if last_build.submodules:
|
|
try:
|
|
vcs.initsubmodules()
|
|
except NoSubmodulesException:
|
|
logging.info("No submodules present for {}".format(_getappname(app)))
|
|
except VCSException:
|
|
logging.info("submodule broken for {}".format(_getappname(app)))
|
|
|
|
|
|
def dirs_with_manifest(startdir: str):
|
|
"""Find directories containing a manifest file.
|
|
|
|
Yield all directories under startdir that contain any of the manifest
|
|
files, and thus are probably an Android project.
|
|
|
|
Parameters
|
|
----------
|
|
startdir
|
|
Directory to be walked down for search
|
|
|
|
Yields
|
|
------
|
|
path : :class:`pathlib.Path` or None
|
|
A directory that contains a manifest file of an Android project, None if
|
|
no directory could be found
|
|
"""
|
|
for root, dirs, files in os.walk(startdir):
|
|
dirs.sort()
|
|
if any(m in files for m in [
|
|
'AndroidManifest.xml', 'pom.xml', 'build.gradle', 'build.gradle.kts']):
|
|
yield Path(root)
|
|
|
|
|
|
def possible_subdirs(app: metadata.App):
|
|
"""Try to find a new subdir starting from the root build_dir.
|
|
|
|
Yields said subdir relative to the build dir if found, None otherwise.
|
|
|
|
Parameters
|
|
----------
|
|
app
|
|
The app to check for subdirs
|
|
|
|
Yields
|
|
------
|
|
subdir : :class:`pathlib.Path` or None
|
|
A possible subdir, None if no subdir could be found
|
|
"""
|
|
if app.RepoType == 'srclib':
|
|
build_dir = Path('build/srclib') / app.Repo
|
|
else:
|
|
build_dir = Path('build') / app.id
|
|
|
|
last_build = get_last_build_from_app(app)
|
|
|
|
for d in dirs_with_manifest(build_dir):
|
|
m_paths = common.manifest_paths(d, last_build.gradle)
|
|
package = common.parse_androidmanifests(m_paths, app)[2]
|
|
if package is not None or app.UpdateCheckName == "Ignore":
|
|
subdir = d.relative_to(build_dir)
|
|
logging.debug("Adding possible subdir %s" % subdir)
|
|
yield subdir
|
|
|
|
|
|
def _getappname(app: metadata.App) -> str:
|
|
return common.get_app_display_name(app)
|
|
|
|
|
|
def _getcvname(app: metadata.App) -> str:
|
|
return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
|
|
|
|
|
|
def fetch_autoname(app: metadata.App, tag: str) -> Optional[str]:
|
|
"""Fetch AutoName.
|
|
|
|
Get the to be displayed name of an app from the source code and adjust the
|
|
App instance in case it is different name has been found.
|
|
|
|
Parameters
|
|
----------
|
|
app
|
|
The App instance to get the AutoName for.
|
|
tag
|
|
Tag to fetch AutoName at.
|
|
|
|
Returns
|
|
-------
|
|
commitmsg
|
|
Commit message about the name change. None in case checking for the
|
|
name is disabled, a VCSException occured or no name could be found.
|
|
"""
|
|
if not app.RepoType or app.UpdateCheckMode in ('None', 'Static') \
|
|
or app.UpdateCheckName == "Ignore":
|
|
return None
|
|
|
|
if app.RepoType == 'srclib':
|
|
build_dir = Path('build/srclib') / app.Repo
|
|
else:
|
|
build_dir = Path('build') / app.id
|
|
|
|
try:
|
|
vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
|
|
vcs.gotorevision(tag)
|
|
except VCSException:
|
|
return None
|
|
|
|
last_build = get_last_build_from_app(app)
|
|
|
|
logging.debug("...fetch auto name from " + str(build_dir))
|
|
new_name = None
|
|
for subdir in possible_subdirs(app):
|
|
root_dir = build_dir / subdir
|
|
new_name = common.fetch_real_name(root_dir, last_build.gradle)
|
|
if new_name is not None:
|
|
break
|
|
commitmsg = None
|
|
if new_name:
|
|
logging.debug("...got autoname '" + new_name + "'")
|
|
if new_name != app.AutoName:
|
|
app.AutoName = new_name
|
|
if not commitmsg:
|
|
commitmsg = "Set autoname of {0}".format(_getappname(app))
|
|
else:
|
|
logging.debug("...couldn't get autoname")
|
|
|
|
return commitmsg
|
|
|
|
|
|
def operate_vercode(operation: str, vercode: int) -> int:
|
|
"""Calculate a new versionCode from a mathematical operation.
|
|
|
|
Parameters
|
|
----------
|
|
operation
|
|
The operation to execute to get the new versionCode.
|
|
vercode
|
|
The versionCode for replacing "%c" in the operation.
|
|
|
|
Returns
|
|
-------
|
|
vercode
|
|
The new versionCode obtained by executing the operation.
|
|
|
|
Raises
|
|
------
|
|
:exc:`~fdroidserver.exception.MetaDataException`
|
|
If the operation is invalid.
|
|
"""
|
|
if not common.VERCODE_OPERATION_RE.match(operation):
|
|
raise MetaDataException(_('Invalid VercodeOperation: {field}')
|
|
.format(field=operation))
|
|
oldvercode = vercode
|
|
op = operation.replace("%c", str(oldvercode))
|
|
vercode = common.calculate_math_string(op)
|
|
logging.debug("Applied vercode operation: %d -> %d" % (oldvercode, vercode))
|
|
return vercode
|
|
|
|
|
|
def checkupdates_app(app: metadata.App, auto: bool, commit: bool = False) -> None:
|
|
"""Check for new versions and updated name of a single app.
|
|
|
|
Also write back changes to the metadata file and create a Git commit if
|
|
requested.
|
|
|
|
Parameters
|
|
----------
|
|
app
|
|
The app to check for updates for.
|
|
|
|
Raises
|
|
------
|
|
:exc:`~fdroidserver.exception.MetaDataException`
|
|
If the app has an invalid UpdateCheckMode or AutoUpdateMode.
|
|
:exc:`~fdroidserver.exception.FDroidException`
|
|
If no version information could be found, the current version is newer
|
|
than the found version, auto-update was requested but an app has no
|
|
CurrentVersionCode or (Git) commiting the changes failed.
|
|
"""
|
|
# If a change is made, commitmsg should be set to a description of it.
|
|
# Only if this is set, changes will be written back to the metadata.
|
|
commitmsg = None
|
|
|
|
tag = None
|
|
mode = app.UpdateCheckMode
|
|
if mode.startswith('Tags'):
|
|
pattern = mode[5:] if len(mode) > 4 else None
|
|
(version, vercode, tag) = check_tags(app, pattern)
|
|
elif mode == 'RepoManifest':
|
|
(version, vercode) = check_repomanifest(app)
|
|
elif mode.startswith('RepoManifest/'):
|
|
tag = mode[13:]
|
|
(version, vercode) = check_repomanifest(app, tag)
|
|
elif mode == 'HTTP':
|
|
(version, vercode) = check_http(app)
|
|
elif mode in ('None', 'Static'):
|
|
logging.debug('Checking disabled')
|
|
return
|
|
else:
|
|
raise MetaDataException(_('Invalid UpdateCheckMode: {mode}').format(mode=mode))
|
|
|
|
if not version or not vercode:
|
|
raise FDroidException(_('no version information found'))
|
|
|
|
if app.VercodeOperation:
|
|
vercodes = sorted([
|
|
operate_vercode(operation, vercode) for operation in app.VercodeOperation
|
|
])
|
|
else:
|
|
vercodes = [vercode]
|
|
|
|
updating = False
|
|
if vercodes[-1] == app.CurrentVersionCode:
|
|
logging.debug("...up to date")
|
|
elif vercodes[-1] > app.CurrentVersionCode:
|
|
logging.debug("...updating - old vercode={0}, new vercode={1}".format(
|
|
app.CurrentVersionCode, vercodes[-1]))
|
|
app.CurrentVersion = version
|
|
app.CurrentVersionCode = vercodes[-1]
|
|
updating = True
|
|
else:
|
|
raise FDroidException(
|
|
_('current version is newer: old vercode={old}, new vercode={new}').format(
|
|
old=app.CurrentVersionCode, new=vercodes[-1]
|
|
)
|
|
)
|
|
|
|
commitmsg = fetch_autoname(app, tag)
|
|
|
|
if updating:
|
|
name = _getappname(app)
|
|
ver = _getcvname(app)
|
|
logging.info('...updating to version %s' % ver)
|
|
commitmsg = 'Update CurrentVersion of %s to %s' % (name, ver)
|
|
|
|
if auto:
|
|
mode = app.AutoUpdateMode
|
|
if not app.CurrentVersionCode:
|
|
raise MetaDataException(
|
|
_("Can't auto-update app with no CurrentVersionCode")
|
|
)
|
|
elif mode in ('None', 'Static'):
|
|
pass
|
|
elif mode.startswith('Version'):
|
|
pattern = mode[8:]
|
|
suffix = ''
|
|
if pattern.startswith('+'):
|
|
try:
|
|
suffix, pattern = pattern[1:].split(' ', 1)
|
|
except ValueError as exc:
|
|
raise MetaDataException("Invalid AutoUpdateMode: " + mode) from exc
|
|
|
|
gotcur = False
|
|
latest = None
|
|
builds = app.get('Builds', [])
|
|
|
|
if builds:
|
|
latest = builds[-1]
|
|
if latest.versionCode == app.CurrentVersionCode:
|
|
gotcur = True
|
|
elif latest.versionCode > app.CurrentVersionCode:
|
|
raise FDroidException(
|
|
_(
|
|
'latest build recipe is newer: '
|
|
'old vercode={old}, new vercode={new}'
|
|
).format(old=latest.versionCode, new=app.CurrentVersionCode)
|
|
)
|
|
|
|
if not gotcur:
|
|
newbuilds = copy.deepcopy(builds[-len(vercodes):])
|
|
|
|
# These are either built-in or invalid in newer system versions
|
|
bookworm_blocklist = [
|
|
'apt-get install -y openjdk-11-jdk',
|
|
'apt-get install openjdk-11-jdk-headless',
|
|
'apt-get install -y openjdk-11-jdk-headless',
|
|
'apt-get install -t stretch-backports openjdk-11-jdk-headless openjdk-11-jre-headless',
|
|
'apt-get install -y -t stretch-backports openjdk-11-jdk-headless openjdk-11-jre-headless',
|
|
'apt-get install -y openjdk-17-jdk',
|
|
'apt-get install openjdk-17-jdk-headless',
|
|
'apt-get install -y openjdk-17-jdk-headless',
|
|
'update-alternatives --auto java',
|
|
'update-java-alternatives -a',
|
|
]
|
|
|
|
for build in newbuilds:
|
|
if "sudo" in build:
|
|
if any("openjdk-11" in line for line in build["sudo"]) or any("openjdk-17" in line for line in build["sudo"]):
|
|
build["sudo"] = [line for line in build["sudo"] if line not in bookworm_blocklist]
|
|
if build["sudo"] == ['apt-get update']:
|
|
build["sudo"] = ''
|
|
|
|
for b, v in zip(newbuilds, vercodes):
|
|
b.disable = False
|
|
b.versionCode = v
|
|
b.versionName = app.CurrentVersion + suffix.replace(
|
|
'%c', str(v)
|
|
)
|
|
logging.info("...auto-generating build for " + b.versionName)
|
|
if tag:
|
|
b.commit = tag
|
|
else:
|
|
commit = pattern.replace('%v', app.CurrentVersion)
|
|
commit = commit.replace('%c', str(v))
|
|
b.commit = commit
|
|
|
|
app['Builds'].extend(newbuilds)
|
|
|
|
name = _getappname(app)
|
|
ver = _getcvname(app)
|
|
commitmsg = "Update %s to %s" % (name, ver)
|
|
else:
|
|
raise MetaDataException(
|
|
_('Invalid AutoUpdateMode: {mode}').format(mode=mode)
|
|
)
|
|
|
|
if commitmsg:
|
|
metadata.write_metadata(app.metadatapath, app)
|
|
if commit:
|
|
logging.info("Commiting update for " + app.metadatapath)
|
|
gitcmd = ["git", "commit", "-m", commitmsg]
|
|
gitcmd.extend(["--", app.metadatapath])
|
|
if subprocess.call(gitcmd) != 0:
|
|
raise FDroidException("Git commit failed")
|
|
|
|
|
|
def get_last_build_from_app(app: metadata.App) -> metadata.Build:
|
|
"""Get the last build entry of an app."""
|
|
if app.get('Builds'):
|
|
return app['Builds'][-1]
|
|
else:
|
|
return metadata.Build()
|
|
|
|
|
|
def get_upstream_main_branch(git_repo):
|
|
refs = list()
|
|
for ref in git_repo.remotes.upstream.refs:
|
|
if ref.name != 'upstream/HEAD':
|
|
refs.append(ref.name)
|
|
if len(refs) == 1:
|
|
return refs[0]
|
|
for name in ('upstream/main', 'upstream/master'):
|
|
if name in refs:
|
|
return name
|
|
try:
|
|
with git_repo.config_reader() as reader:
|
|
return 'upstream/%s' % reader.get_value('init', 'defaultBranch')
|
|
except configparser.NoSectionError:
|
|
return 'upstream/main'
|
|
|
|
|
|
def checkout_appid_branch(appid):
|
|
"""Prepare the working branch named after the appid.
|
|
|
|
This sets up everything for checkupdates_app() to run and add
|
|
commits. If there is an existing branch named after the appid,
|
|
and it has commits from users other than the checkupdates-bot,
|
|
then this will return False. Otherwise, it returns True.
|
|
|
|
The checkupdates-runner must set the committer email address in
|
|
the git config. Then any commit with a committer or author that
|
|
does not match that will be considered to have human edits. That
|
|
email address is currently set in:
|
|
https://gitlab.com/fdroid/checkupdates-runner/-/blob/1861899262a62a4ed08fa24e5449c0368dfb7617/.gitlab-ci.yml#L36
|
|
|
|
"""
|
|
logging.debug(f'Creating merge request branch for {appid}')
|
|
git_repo = git.Repo.init('.')
|
|
upstream_main = get_upstream_main_branch(git_repo)
|
|
for remote in git_repo.remotes:
|
|
remote.fetch()
|
|
try:
|
|
git_repo.remotes.origin.fetch(f'{appid}:refs/remotes/origin/{appid}')
|
|
except Exception as e:
|
|
logging.debug('"%s" branch not found on origin remote:\n\t%s', appid, e)
|
|
if appid in git_repo.remotes.origin.refs:
|
|
start_point = f"origin/{appid}"
|
|
for commit in git_repo.iter_commits(
|
|
f'{upstream_main}...{start_point}', right_only=True
|
|
):
|
|
if commit.committer.email != BOT_EMAIL or commit.author.email != BOT_EMAIL:
|
|
return False
|
|
else:
|
|
start_point = upstream_main
|
|
git_repo.git.checkout('-B', appid, start_point)
|
|
git_repo.git.rebase(upstream_main, strategy_option='ours', kill_after_timeout=120)
|
|
return True
|
|
|
|
|
|
def get_changes_versus_ref(git_repo, ref, f):
|
|
changes = []
|
|
for m in re.findall(
|
|
r"^[+-].*", git_repo.git.diff(f"{ref}", '--', f), flags=re.MULTILINE
|
|
):
|
|
if not re.match(r"^(\+\+\+|---) ", m):
|
|
changes.append(m)
|
|
return changes
|
|
|
|
|
|
def push_commits(branch_name='checkupdates'):
|
|
"""Make git branch then push commits as merge request.
|
|
|
|
The appid is parsed from the actual file that was changed so that
|
|
only the right branch is ever updated.
|
|
|
|
This uses the appid as the standard branch name so that there is
|
|
only ever one open merge request per-app. If multiple apps are
|
|
included in the branch, then 'checkupdates' is used as branch
|
|
name. This is to support the old way operating, e.g. in batches.
|
|
|
|
This uses GitLab "Push Options" to create a merge request. Git
|
|
Push Options are config data that can be sent via `git push
|
|
--push-option=... origin foo`.
|
|
|
|
References
|
|
----------
|
|
* https://docs.gitlab.com/ee/user/project/push_options.html
|
|
|
|
"""
|
|
if branch_name != "checkupdates":
|
|
if callable(getattr(git.SymbolicReference, "_check_ref_name_valid", None)):
|
|
git.SymbolicReference._check_ref_name_valid(branch_name)
|
|
|
|
git_repo = git.Repo.init('.')
|
|
upstream_main = get_upstream_main_branch(git_repo)
|
|
files = set()
|
|
for commit in git_repo.iter_commits(f'{upstream_main}...HEAD', right_only=True):
|
|
files.update(commit.stats.files.keys())
|
|
|
|
files = list(files)
|
|
if len(files) == 1:
|
|
m = re.match(r'metadata/(\S+)\.yml', files[0])
|
|
if m:
|
|
branch_name = m.group(1) # appid
|
|
if not files:
|
|
return
|
|
|
|
# https://git-scm.com/docs/git-check-ref-format Git refname can't end with .lock
|
|
if branch_name.endswith(".lock"):
|
|
branch_name = f"{branch_name}_"
|
|
|
|
remote = git_repo.remotes.origin
|
|
if branch_name in remote.refs:
|
|
if not get_changes_versus_ref(git_repo, f'origin/{branch_name}', files[0]):
|
|
return
|
|
|
|
git_repo.create_head(branch_name, force=True)
|
|
push_options = [
|
|
'merge_request.create',
|
|
'merge_request.remove_source_branch',
|
|
'merge_request.title=bot: ' + git_repo.branches[branch_name].commit.summary,
|
|
'merge_request.description='
|
|
+ '~%s checkupdates-bot run %s' % (branch_name, os.getenv('CI_JOB_URL')),
|
|
]
|
|
|
|
# mark as draft if there are only changes to CurrentVersion:
|
|
current_version_only = True
|
|
for m in get_changes_versus_ref(git_repo, upstream_main, files[0]):
|
|
if not re.match(r"^[-+]CurrentVersion", m):
|
|
current_version_only = False
|
|
break
|
|
if current_version_only:
|
|
push_options.append('merge_request.draft')
|
|
|
|
progress = git.RemoteProgress()
|
|
|
|
pushinfos = remote.push(
|
|
f"HEAD:refs/heads/{branch_name}",
|
|
progress=progress,
|
|
force=True,
|
|
set_upstream=True,
|
|
push_option=push_options,
|
|
)
|
|
|
|
for pushinfo in pushinfos:
|
|
logging.info(pushinfo.summary)
|
|
# Show potentially useful messages from git remote
|
|
if progress:
|
|
for line in progress.other_lines:
|
|
logging.info(line)
|
|
if pushinfo.flags & (
|
|
git.remote.PushInfo.ERROR
|
|
| git.remote.PushInfo.REJECTED
|
|
| git.remote.PushInfo.REMOTE_FAILURE
|
|
| git.remote.PushInfo.REMOTE_REJECTED
|
|
):
|
|
raise FDroidException(
|
|
f'{remote.url} push failed: {pushinfo.flags} {pushinfo.summary}'
|
|
)
|
|
else:
|
|
logging.info(remote.url + ': ' + pushinfo.summary)
|
|
|
|
|
|
def prune_empty_appid_branches(git_repo=None, main_branch='main'):
|
|
"""Remove empty branches from checkupdates-bot git remote."""
|
|
if git_repo is None:
|
|
git_repo = git.Repo.init('.')
|
|
upstream_main = get_upstream_main_branch(git_repo)
|
|
main_branch = upstream_main.split('/')[1]
|
|
|
|
remote = git_repo.remotes.origin
|
|
remote.update(prune=True)
|
|
merged_branches = git_repo.git().branch(remotes=True, merged=upstream_main).split()
|
|
for remote_branch in merged_branches:
|
|
if not remote_branch or '/' not in remote_branch:
|
|
continue
|
|
if remote_branch.split('/')[1] not in (main_branch, 'HEAD'):
|
|
for ref in git_repo.remotes.origin.refs:
|
|
if remote_branch == ref.name:
|
|
remote.push(':%s' % ref.remote_head, force=True) # rm remote branch
|
|
|
|
|
|
def status_update_json(processed: list, failed: dict) -> None:
|
|
"""Output a JSON file with metadata about this run."""
|
|
logging.debug(_('Outputting JSON'))
|
|
output = common.setup_status_output(start_timestamp)
|
|
if processed:
|
|
output['processed'] = processed
|
|
if failed:
|
|
output['failed'] = failed
|
|
common.write_status_json(output)
|
|
|
|
|
|
config = None
|
|
start_timestamp = time.gmtime()
|
|
|
|
|
|
def main():
|
|
"""Check for updates for one or more apps.
|
|
|
|
The behaviour of this function is influenced by the configuration file as
|
|
well as command line parameters.
|
|
"""
|
|
global config
|
|
|
|
# Parse command line...
|
|
parser = ArgumentParser()
|
|
common.setup_global_opts(parser)
|
|
parser.add_argument("appid", nargs='*', help=_("application ID of file to operate on"))
|
|
parser.add_argument("--auto", action="store_true", default=False,
|
|
help=_("Process auto-updates"))
|
|
parser.add_argument("--autoonly", action="store_true", default=False,
|
|
help=_("Only process apps with auto-updates"))
|
|
parser.add_argument("--commit", action="store_true", default=False,
|
|
help=_("Commit changes"))
|
|
parser.add_argument("--merge-request", action="store_true", default=False,
|
|
help=_("Commit changes, push, then make a merge request"))
|
|
parser.add_argument("--allow-dirty", action="store_true", default=False,
|
|
help=_("Run on git repo that has uncommitted changes"))
|
|
metadata.add_metadata_arguments(parser)
|
|
options = common.parse_args(parser)
|
|
metadata.warnings_action = options.W
|
|
|
|
config = common.read_config()
|
|
|
|
if not options.allow_dirty:
|
|
status = subprocess.check_output(['git', 'status', '--porcelain'])
|
|
if status:
|
|
logging.error(_('Build metadata git repo has uncommited changes!'))
|
|
sys.exit(1)
|
|
|
|
if options.merge_request and not (options.appid and len(options.appid) == 1):
|
|
logging.error(_('--merge-request only runs on a single appid!'))
|
|
sys.exit(1)
|
|
|
|
apps = common.read_app_args(options.appid)
|
|
|
|
processed = []
|
|
failed = dict()
|
|
exit_code = 0
|
|
for appid, app in apps.items():
|
|
|
|
if options.autoonly and app.AutoUpdateMode in ('None', 'Static'):
|
|
logging.debug(_("Nothing to do for {appid}.").format(appid=appid))
|
|
continue
|
|
|
|
msg = _("Processing {appid}").format(appid=appid)
|
|
logging.info(msg)
|
|
|
|
try:
|
|
if options.merge_request:
|
|
if not checkout_appid_branch(appid):
|
|
msg = _("...checkupdate failed for {appid} : {error}").format(
|
|
appid=appid,
|
|
error='Open merge request with human edits, skipped.',
|
|
)
|
|
logging.warning(msg)
|
|
failed[appid] = msg
|
|
continue
|
|
|
|
checkupdates_app(app, options.auto, options.commit or options.merge_request)
|
|
processed.append(appid)
|
|
except Exception as e:
|
|
msg = _("...checkupdate failed for {appid} : {error}").format(appid=appid, error=e)
|
|
logging.error(msg)
|
|
logging.debug(traceback.format_exc())
|
|
failed[appid] = str(e)
|
|
exit_code = 1
|
|
|
|
if options.appid and options.merge_request:
|
|
push_commits()
|
|
prune_empty_appid_branches()
|
|
|
|
status_update_json(processed, failed)
|
|
sys.exit(exit_code)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|