mirrors/fdroidserver
1
0
Fork 0
mirror of https://github.com/f-droid/fdroidserver.git synced 2025-11-04 06:30:27 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

🛰️ deploy: github releases

Browse source 
Implemented basic support for using `fdroid delpoy` to upload APKs from
the repo to GitHub releases.
This commit is contained in:
Michael Pöhn 2024-04-05 17:34:16 +02:00
parent aeb8a7a3e5
commit 1b19293ab0
No known key found for this signature in database
GPG key ID: 725F386C05529A5A
3 changed files with 255 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

22
examples/config.yml
View file

@ -211,6 +211,28 @@
# - url: https://gitlab.com/user/repo
# index_only: true
# These settings allows using `fdroid deploy` for publishing APK files from
# your repository to GitHub Releases. (You should also run `fdroid update`
# every time before deploying to GitHub releases to update index files.) Here's
# an example for this deployment automation:
# https://github.com/f-droid/fdroidclient/releases/
#
# It is highly recommended to use a "Fine-grained personal access token", which
# is restriced to the minimum required permissions, which are:
# * Metadata - read
# * Contents - read/write
# Also make sure to limit access only to the GitHub repository you're deploying
# to. (https://github.com/settings/personal-access-tokens/new)
#
# github_releases:
# - repo: f-droid/fdroidclient
# token: {env: GITHUB_TOKEN}
# packages:
# - org.fdroid.basic
# - org.fdroid.fdroid
# Most git hosting services have hard size limits for each git repo.
# `fdroid deploy` will delete the git history when the git mirror repo
# approaches this limit to ensure that the repo will still fit when

78
fdroidserver/deploy.py
View file

@ -31,9 +31,10 @@ import yaml
from argparse import ArgumentParser
import logging
from shlex import split
import pathlib
import shutil
import git
from pathlib import Path
import fdroidserver.github
from . import _
from . import common
@ -663,7 +664,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
return
options = common.get_options()
workspace_dir = Path(os.getcwd())
workspace_dir = pathlib.Path(os.getcwd())
# right now we support only 'repo' git-mirroring
if repo_section == 'repo':
@ -1115,6 +1116,76 @@ def push_binary_transparency(git_repo_path, git_remote):
raise FDroidException(_("Pushing to remote server failed!"))
def upload_to_github_releases(repo_section, gh_config):
repo_dir = pathlib.Path(repo_section)
idx_path = repo_dir / 'index-v2.json'
if not idx_path.is_file():
logging.waring(
_(
"Error deploying 'github_releases', {} not present. (You might "
"need to run `fdroid update` first.)"
).format(idx_path)
)
return
known_packages = {}
with open(idx_path, 'r') as f:
idx = json.load(f)
for repo_conf in gh_config:
for package_name in repo_conf.get('packages', []):
package = idx.get('packages', {}).get(package_name, {})
for version in package.get('versions', {}).values():
if package_name not in known_packages:
known_packages[package_name] = {}
ver_name = version['manifest']['versionName']
apk_path = repo_dir / version['file']['name'][1:]
files = [apk_path]
asc_path = pathlib.Path(str(apk_path) + '.asc')
if asc_path.is_file():
files.append(asc_path)
idsig_path = pathlib.Path(str(apk_path) + '.idsig')
if idsig_path.is_file():
files.append(idsig_path)
known_packages[package_name][ver_name] = files
for repo_conf in gh_config:
upload_to_github_releases_repo(repo_conf, known_packages)
def upload_to_github_releases_repo(repo_conf, known_packages):
repo = repo_conf.get('repo')
if not repo:
logging.warning(_("One of the 'github_releases' config items is missing the 'repo' value. skipping ..."))
return
token = repo_conf.get('token')
if not token:
logging.warning(_("One of the 'github_releases' config itmes is missing the 'token' value. skipping ..."))
return
packages = repo_conf.get('packages', [])
if not packages:
logging.warning(_("One of the 'github_releases' config itmes is missing the 'packages' value. skipping ..."))
return
# lookup all versionNames (git tags) for all packages available in the
# local fdroid repo
all_local_versions = set()
for package_name in repo_conf['packages']:
for version in known_packages.get(package_name, {}).keys():
all_local_versions.add(version)
gh = fdroidserver.github.GithubApi(token, repo)
unreleased_tags = gh.list_unreleased_tags()
for version in all_local_versions:
if version in unreleased_tags:
# collect files associated with this github release
files = []
for package in packages:
files.extend(known_packages.get(package, {}).get(version, []))
# create new release on github and upload all associated files
gh.create_release(version, files)
def main():
global config
@ -1194,6 +1265,7 @@ def main():
and not config.get('androidobservatory')
and not config.get('binary_transparency_remote')
and not config.get('virustotal_apikey')
and not config.get('github_releases')
and local_copy_dir is None
):
logging.warning(
@ -1236,6 +1308,8 @@ def main():
upload_to_android_observatory(repo_section)
if config.get('virustotal_apikey'):
upload_to_virustotal(repo_section, config.get('virustotal_apikey'))
if config.get('github_releases'):
upload_to_github_releases(repo_section, config.get('github_releases'))
binary_transparency_remote = config.get('binary_transparency_remote')
if binary_transparency_remote:

157
fdroidserver/github.py Normal file
View file

@ -0,0 +1,157 @@
#!/usr/bin/env python3
#
# github.py - part of the FDroid server tools
# Copyright (C) 2024, Michael Pöhn, michael@poehn.at
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import json
import pathlib
import urllib.request
import urllib.parse
class GithubApi:
"""
Warpper for some select calls to GitHub Json/REST API.
This class wraps some calls to api.github.com. This is not intended to be a
general API wrapper. Instead it's purpose is to return pre-filtered and
transformed data that's playing well with other fdroidserver functions.
"""
def __init__(self, api_token, repo_path):
self._api_token = api_token
self._repo_path = repo_path
def _req(self, url, data=None):
h = {
"Accept": "application/vnd.github+json",
"Authorization": f"Bearer {self._api_token}",
"X-GitHub-Api-Version": "2022-11-28",
}
return urllib.request.Request(
url,
headers=h,
data=data,
)
def list_released_tags(self):
"""List of all tags that are associated with a release for this repo on GitHub."""
names = []
req = self._req(f"https://api.github.com/repos/{self._repo_path}/releases")
with urllib.request.urlopen(req) as resp:
releases = json.load(resp)
for release in releases:
names.append(release['tag_name'])
return names
def list_unreleased_tags(self):
all_tags = self.list_all_tags()
released_tags = self.list_released_tags()
return [x for x in all_tags if x not in released_tags]
def tag_exists(self, tag):
"""
Check if git tag is present on github.
https://docs.github.com/en/rest/git/refs?apiVersion=2022-11-28#list-matching-references--fine-grained-access-tokens
"""
req = self._req(
f"https://api.github.com/repos/{self._repo_path}/git/matching-refs/tags/{tag}"
)
with urllib.request.urlopen(req) as resp:
rd = json.load(resp)
return len(rd) == 1 and rd[0].get("ref", False) == f"refs/tags/{tag}"
return False
def list_all_tags(self):
"""Get list of all tags for this repo on GitHub."""
tags = []
req = self._req(
f"https://api.github.com/repos/{self._repo_path}/git/matching-refs/tags/"
)
with urllib.request.urlopen(req) as resp:
refs = json.load(resp)
for ref in refs:
r = ref['ref']
if r.startswith('refs/tags/'):
tags.append(r[10:])
return tags
def create_release(self, tag, files):
"""
Create a new release on github.
also see: https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#create-a-release
:returns: True if release was created, False if release already exists
:raises: urllib exceptions in case of network or api errors, also
raises an exception when the tag doesn't exists.
"""
# Querying github to create a new release for a non-existent tag, will
# also create that tag on github. So we need an additional check to
# prevent this behavior.
if not self.tag_exists(tag):
raise Exception(
f"can't create github release for {self._repo_path} {tag}, tag doesn't exists"
)
# create the relase on github
req = self._req(
f"https://api.github.com/repos/{self._repo_path}/releases",
data=json.dumps(
{
"tag_name": tag,
}
).encode("utf-8"),
)
try:
with urllib.request.urlopen(req) as resp:
release_id = json.load(resp)['id']
except urllib.error.HTTPError as e:
if e.status == 422:
codes = [x['code'] for x in json.load(e).get('errors', [])]
if "already_exists" in codes:
return False
raise e
# attach / upload all files for the relase
for file in files:
self._create_release_asset(release_id, file)
return True
def _create_release_asset(self, release_id, file):
"""
Attach a file to a release on GitHub.
This uploads a file to github relases, it will be attached to the supplied release
also see: https://docs.github.com/en/rest/releases/assets?apiVersion=2022-11-28#upload-a-release-asset
"""
file = pathlib.Path(file)
with open(file, 'rb') as f:
req = urllib.request.Request(
f"https://uploads.github.com/repos/{self._repo_path}/releases/{release_id}/assets?name={file.name}",
headers={
"Accept": "application/vnd.github+json",
"Authorization": f"Bearer {self._api_token}",
"X-GitHub-Api-Version": "2022-11-28",
"Content-Type": "application/octet-stream",
},
data=f.read(),
)
with urllib.request.urlopen(req):
return True
return False