From 1b19293ab0787cdcf55c0669e20b1245ff84af4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Fri, 5 Apr 2024 17:34:16 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=9B=B0=EF=B8=8F=20=20deploy:=20github=20r?= =?UTF-8?q?eleases?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implemented basic support for using `fdroid delpoy` to upload APKs from the repo to GitHub releases. --- examples/config.yml | 22 ++++++ fdroidserver/deploy.py | 78 +++++++++++++++++++- fdroidserver/github.py | 157 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 255 insertions(+), 2 deletions(-) create mode 100644 fdroidserver/github.py diff --git a/examples/config.yml b/examples/config.yml index 0337e6f0..8efa41ea 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -211,6 +211,28 @@ # - url: https://gitlab.com/user/repo # index_only: true + +# These settings allows using `fdroid deploy` for publishing APK files from +# your repository to GitHub Releases. (You should also run `fdroid update` +# every time before deploying to GitHub releases to update index files.) Here's +# an example for this deployment automation: +# https://github.com/f-droid/fdroidclient/releases/ +# +# It is highly recommended to use a "Fine-grained personal access token", which +# is restriced to the minimum required permissions, which are: +# * Metadata - read +# * Contents - read/write +# Also make sure to limit access only to the GitHub repository you're deploying +# to. (https://github.com/settings/personal-access-tokens/new) +# +# github_releases: +# - repo: f-droid/fdroidclient +# token: {env: GITHUB_TOKEN} +# packages: +# - org.fdroid.basic +# - org.fdroid.fdroid + + # Most git hosting services have hard size limits for each git repo. # `fdroid deploy` will delete the git history when the git mirror repo # approaches this limit to ensure that the repo will still fit when diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index e87703c5..4393d1d7 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -31,9 +31,10 @@ import yaml from argparse import ArgumentParser import logging from shlex import split +import pathlib import shutil import git -from pathlib import Path +import fdroidserver.github from . import _ from . import common @@ -663,7 +664,7 @@ def update_servergitmirrors(servergitmirrors, repo_section): return options = common.get_options() - workspace_dir = Path(os.getcwd()) + workspace_dir = pathlib.Path(os.getcwd()) # right now we support only 'repo' git-mirroring if repo_section == 'repo': @@ -1115,6 +1116,76 @@ def push_binary_transparency(git_repo_path, git_remote): raise FDroidException(_("Pushing to remote server failed!")) +def upload_to_github_releases(repo_section, gh_config): + repo_dir = pathlib.Path(repo_section) + idx_path = repo_dir / 'index-v2.json' + if not idx_path.is_file(): + logging.waring( + _( + "Error deploying 'github_releases', {} not present. (You might " + "need to run `fdroid update` first.)" + ).format(idx_path) + ) + return + + known_packages = {} + with open(idx_path, 'r') as f: + idx = json.load(f) + for repo_conf in gh_config: + for package_name in repo_conf.get('packages', []): + package = idx.get('packages', {}).get(package_name, {}) + for version in package.get('versions', {}).values(): + if package_name not in known_packages: + known_packages[package_name] = {} + ver_name = version['manifest']['versionName'] + apk_path = repo_dir / version['file']['name'][1:] + files = [apk_path] + asc_path = pathlib.Path(str(apk_path) + '.asc') + if asc_path.is_file(): + files.append(asc_path) + idsig_path = pathlib.Path(str(apk_path) + '.idsig') + if idsig_path.is_file(): + files.append(idsig_path) + known_packages[package_name][ver_name] = files + + for repo_conf in gh_config: + upload_to_github_releases_repo(repo_conf, known_packages) + + +def upload_to_github_releases_repo(repo_conf, known_packages): + repo = repo_conf.get('repo') + if not repo: + logging.warning(_("One of the 'github_releases' config items is missing the 'repo' value. skipping ...")) + return + token = repo_conf.get('token') + if not token: + logging.warning(_("One of the 'github_releases' config itmes is missing the 'token' value. skipping ...")) + return + packages = repo_conf.get('packages', []) + if not packages: + logging.warning(_("One of the 'github_releases' config itmes is missing the 'packages' value. skipping ...")) + return + + # lookup all versionNames (git tags) for all packages available in the + # local fdroid repo + all_local_versions = set() + for package_name in repo_conf['packages']: + for version in known_packages.get(package_name, {}).keys(): + all_local_versions.add(version) + + gh = fdroidserver.github.GithubApi(token, repo) + unreleased_tags = gh.list_unreleased_tags() + + for version in all_local_versions: + if version in unreleased_tags: + # collect files associated with this github release + files = [] + for package in packages: + files.extend(known_packages.get(package, {}).get(version, [])) + # create new release on github and upload all associated files + gh.create_release(version, files) + + def main(): global config @@ -1194,6 +1265,7 @@ def main(): and not config.get('androidobservatory') and not config.get('binary_transparency_remote') and not config.get('virustotal_apikey') + and not config.get('github_releases') and local_copy_dir is None ): logging.warning( @@ -1236,6 +1308,8 @@ def main(): upload_to_android_observatory(repo_section) if config.get('virustotal_apikey'): upload_to_virustotal(repo_section, config.get('virustotal_apikey')) + if config.get('github_releases'): + upload_to_github_releases(repo_section, config.get('github_releases')) binary_transparency_remote = config.get('binary_transparency_remote') if binary_transparency_remote: diff --git a/fdroidserver/github.py b/fdroidserver/github.py new file mode 100644 index 00000000..6356c4ae --- /dev/null +++ b/fdroidserver/github.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# +# github.py - part of the FDroid server tools +# Copyright (C) 2024, Michael Pöhn, michael@poehn.at +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import json +import pathlib +import urllib.request +import urllib.parse + + +class GithubApi: + """ + Warpper for some select calls to GitHub Json/REST API. + + This class wraps some calls to api.github.com. This is not intended to be a + general API wrapper. Instead it's purpose is to return pre-filtered and + transformed data that's playing well with other fdroidserver functions. + """ + + def __init__(self, api_token, repo_path): + self._api_token = api_token + self._repo_path = repo_path + + def _req(self, url, data=None): + h = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self._api_token}", + "X-GitHub-Api-Version": "2022-11-28", + } + return urllib.request.Request( + url, + headers=h, + data=data, + ) + + def list_released_tags(self): + """List of all tags that are associated with a release for this repo on GitHub.""" + names = [] + req = self._req(f"https://api.github.com/repos/{self._repo_path}/releases") + with urllib.request.urlopen(req) as resp: + releases = json.load(resp) + for release in releases: + names.append(release['tag_name']) + return names + + def list_unreleased_tags(self): + all_tags = self.list_all_tags() + released_tags = self.list_released_tags() + return [x for x in all_tags if x not in released_tags] + + def tag_exists(self, tag): + """ + Check if git tag is present on github. + + https://docs.github.com/en/rest/git/refs?apiVersion=2022-11-28#list-matching-references--fine-grained-access-tokens + """ + req = self._req( + f"https://api.github.com/repos/{self._repo_path}/git/matching-refs/tags/{tag}" + ) + with urllib.request.urlopen(req) as resp: + rd = json.load(resp) + return len(rd) == 1 and rd[0].get("ref", False) == f"refs/tags/{tag}" + return False + + def list_all_tags(self): + """Get list of all tags for this repo on GitHub.""" + tags = [] + req = self._req( + f"https://api.github.com/repos/{self._repo_path}/git/matching-refs/tags/" + ) + with urllib.request.urlopen(req) as resp: + refs = json.load(resp) + for ref in refs: + r = ref['ref'] + if r.startswith('refs/tags/'): + tags.append(r[10:]) + return tags + + def create_release(self, tag, files): + """ + Create a new release on github. + + also see: https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#create-a-release + + :returns: True if release was created, False if release already exists + :raises: urllib exceptions in case of network or api errors, also + raises an exception when the tag doesn't exists. + """ + # Querying github to create a new release for a non-existent tag, will + # also create that tag on github. So we need an additional check to + # prevent this behavior. + if not self.tag_exists(tag): + raise Exception( + f"can't create github release for {self._repo_path} {tag}, tag doesn't exists" + ) + # create the relase on github + req = self._req( + f"https://api.github.com/repos/{self._repo_path}/releases", + data=json.dumps( + { + "tag_name": tag, + } + ).encode("utf-8"), + ) + try: + with urllib.request.urlopen(req) as resp: + release_id = json.load(resp)['id'] + except urllib.error.HTTPError as e: + if e.status == 422: + codes = [x['code'] for x in json.load(e).get('errors', [])] + if "already_exists" in codes: + return False + raise e + + # attach / upload all files for the relase + for file in files: + self._create_release_asset(release_id, file) + + return True + + def _create_release_asset(self, release_id, file): + """ + Attach a file to a release on GitHub. + + This uploads a file to github relases, it will be attached to the supplied release + + also see: https://docs.github.com/en/rest/releases/assets?apiVersion=2022-11-28#upload-a-release-asset + """ + file = pathlib.Path(file) + with open(file, 'rb') as f: + req = urllib.request.Request( + f"https://uploads.github.com/repos/{self._repo_path}/releases/{release_id}/assets?name={file.name}", + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self._api_token}", + "X-GitHub-Api-Version": "2022-11-28", + "Content-Type": "application/octet-stream", + }, + data=f.read(), + ) + with urllib.request.urlopen(req): + return True + return False