🛰️ deploy: github releases

Implemented basic support for using `fdroid delpoy` to upload APKs from
the repo to GitHub releases.

examples/config.yml

@@ -211,6 +211,28 @@
 #   - url: https://gitlab.com/user/repo
 #     index_only: true
 
+
+# These settings allows using `fdroid deploy` for publishing APK files from
+# your repository to GitHub Releases. (You should also run `fdroid update`
+# every time before deploying to GitHub releases to update index files.) Here's
+# an example for this deployment automation:
+# https://github.com/f-droid/fdroidclient/releases/
+#
+# It is highly recommended to use a "Fine-grained personal access token", which
+# is restriced to the minimum required permissions, which are:
+#  * Metadata - read
+#  * Contents - read/write
+# Also make sure to limit access only to the GitHub repository you're deploying
+# to. (https://github.com/settings/personal-access-tokens/new)
+#
+# github_releases:
+#   - repo: f-droid/fdroidclient
+#     token: {env: GITHUB_TOKEN}
+#     packages:
+#       - org.fdroid.basic
+#       - org.fdroid.fdroid
+
+
 # Most git hosting services have hard size limits for each git repo.
 # `fdroid deploy` will delete the git history when the git mirror repo
 # approaches this limit to ensure that the repo will still fit when

fdroidserver/deploy.py

@@ -31,9 +31,10 @@ import yaml
 from argparse import ArgumentParser
 import logging
 from shlex import split
+import pathlib
 import shutil
 import git
-from pathlib import Path
+import fdroidserver.github
 
 from . import _
 from . import common
@@ -663,7 +664,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
         return
 
     options = common.get_options()
-    workspace_dir = Path(os.getcwd())
+    workspace_dir = pathlib.Path(os.getcwd())
 
     # right now we support only 'repo' git-mirroring
     if repo_section == 'repo':
@@ -1115,6 +1116,76 @@ def push_binary_transparency(git_repo_path, git_remote):
             raise FDroidException(_("Pushing to remote server failed!"))
 
 
+def upload_to_github_releases(repo_section, gh_config):
+    repo_dir = pathlib.Path(repo_section)
+    idx_path = repo_dir / 'index-v2.json'
+    if not idx_path.is_file():
+        logging.waring(
+            _(
+                "Error deploying 'github_releases', {} not present. (You might "
+                "need to run `fdroid update` first.)"
+            ).format(idx_path)
+        )
+        return
+
+    known_packages = {}
+    with open(idx_path, 'r') as f:
+        idx = json.load(f)
+        for repo_conf in gh_config:
+            for package_name in repo_conf.get('packages', []):
+                package = idx.get('packages', {}).get(package_name, {})
+                for version in package.get('versions', {}).values():
+                    if package_name not in known_packages:
+                        known_packages[package_name] = {}
+                    ver_name = version['manifest']['versionName']
+                    apk_path = repo_dir / version['file']['name'][1:]
+                    files = [apk_path]
+                    asc_path = pathlib.Path(str(apk_path) + '.asc')
+                    if asc_path.is_file():
+                        files.append(asc_path)
+                    idsig_path = pathlib.Path(str(apk_path) + '.idsig')
+                    if idsig_path.is_file():
+                        files.append(idsig_path)
+                    known_packages[package_name][ver_name] = files
+
+    for repo_conf in gh_config:
+        upload_to_github_releases_repo(repo_conf, known_packages)
+
+
+def upload_to_github_releases_repo(repo_conf, known_packages):
+    repo = repo_conf.get('repo')
+    if not repo:
+        logging.warning(_("One of the 'github_releases' config items is missing the 'repo' value. skipping ..."))
+        return
+    token = repo_conf.get('token')
+    if not token:
+        logging.warning(_("One of the 'github_releases' config itmes is missing the 'token' value. skipping ..."))
+        return
+    packages = repo_conf.get('packages', [])
+    if not packages:
+        logging.warning(_("One of the 'github_releases' config itmes is missing the 'packages' value. skipping ..."))
+        return
+
+    # lookup all versionNames (git tags) for all packages available in the
+    # local fdroid repo
+    all_local_versions = set()
+    for package_name in repo_conf['packages']:
+        for version in known_packages.get(package_name, {}).keys():
+            all_local_versions.add(version)
+
+    gh = fdroidserver.github.GithubApi(token, repo)
+    unreleased_tags = gh.list_unreleased_tags()
+
+    for version in all_local_versions:
+        if version in unreleased_tags:
+            # collect files associated with this github release
+            files = []
+            for package in packages:
+                files.extend(known_packages.get(package, {}).get(version, []))
+            # create new release on github and upload all associated files
+            gh.create_release(version, files)
+
+
 def main():
     global config
 
@@ -1194,6 +1265,7 @@ def main():
         and not config.get('androidobservatory')
         and not config.get('binary_transparency_remote')
         and not config.get('virustotal_apikey')
+        and not config.get('github_releases')
         and local_copy_dir is None
     ):
         logging.warning(
@@ -1236,6 +1308,8 @@ def main():
             upload_to_android_observatory(repo_section)
         if config.get('virustotal_apikey'):
             upload_to_virustotal(repo_section, config.get('virustotal_apikey'))
+        if config.get('github_releases'):
+            upload_to_github_releases(repo_section, config.get('github_releases'))
 
     binary_transparency_remote = config.get('binary_transparency_remote')
     if binary_transparency_remote:

fdroidserver/github.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+#
+# github.py - part of the FDroid server tools
+# Copyright (C) 2024, Michael Pöhn, michael@poehn.at
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import pathlib
+import urllib.request
+import urllib.parse
+
+
+class GithubApi:
+    """
+    Warpper for some select calls to GitHub Json/REST API.
+
+    This class wraps some calls to api.github.com. This is not intended to be a
+    general API wrapper. Instead it's purpose is to return pre-filtered and
+    transformed data that's playing well with other fdroidserver functions.
+    """
+
+    def __init__(self, api_token, repo_path):
+        self._api_token = api_token
+        self._repo_path = repo_path
+
+    def _req(self, url, data=None):
+        h = {
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"Bearer {self._api_token}",
+            "X-GitHub-Api-Version": "2022-11-28",
+        }
+        return urllib.request.Request(
+            url,
+            headers=h,
+            data=data,
+        )
+
+    def list_released_tags(self):
+        """List of all tags that are associated with a release for this repo on GitHub."""
+        names = []
+        req = self._req(f"https://api.github.com/repos/{self._repo_path}/releases")
+        with urllib.request.urlopen(req) as resp:
+            releases = json.load(resp)
+            for release in releases:
+                names.append(release['tag_name'])
+        return names
+
+    def list_unreleased_tags(self):
+        all_tags = self.list_all_tags()
+        released_tags = self.list_released_tags()
+        return [x for x in all_tags if x not in released_tags]
+
+    def tag_exists(self, tag):
+        """
+        Check if git tag is present on github.
+
+        https://docs.github.com/en/rest/git/refs?apiVersion=2022-11-28#list-matching-references--fine-grained-access-tokens
+        """
+        req = self._req(
+            f"https://api.github.com/repos/{self._repo_path}/git/matching-refs/tags/{tag}"
+        )
+        with urllib.request.urlopen(req) as resp:
+            rd = json.load(resp)
+            return len(rd) == 1 and rd[0].get("ref", False) == f"refs/tags/{tag}"
+        return False
+
+    def list_all_tags(self):
+        """Get list of all tags for this repo on GitHub."""
+        tags = []
+        req = self._req(
+            f"https://api.github.com/repos/{self._repo_path}/git/matching-refs/tags/"
+        )
+        with urllib.request.urlopen(req) as resp:
+            refs = json.load(resp)
+            for ref in refs:
+                r = ref['ref']
+                if r.startswith('refs/tags/'):
+                    tags.append(r[10:])
+        return tags
+
+    def create_release(self, tag, files):
+        """
+        Create a new release on github.
+
+        also see: https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#create-a-release
+
+        :returns: True if release was created, False if release already exists
+        :raises: urllib exceptions in case of network or api errors, also
+                 raises an exception when the tag doesn't exists.
+        """
+        # Querying github to create a new release for a non-existent tag, will
+        # also create that tag on github. So we need an additional check to
+        # prevent this behavior.
+        if not self.tag_exists(tag):
+            raise Exception(
+                f"can't create github release for {self._repo_path} {tag}, tag doesn't exists"
+            )
+        # create the relase on github
+        req = self._req(
+            f"https://api.github.com/repos/{self._repo_path}/releases",
+            data=json.dumps(
+                {
+                    "tag_name": tag,
+                }
+            ).encode("utf-8"),
+        )
+        try:
+            with urllib.request.urlopen(req) as resp:
+                release_id = json.load(resp)['id']
+        except urllib.error.HTTPError as e:
+            if e.status == 422:
+                codes = [x['code'] for x in json.load(e).get('errors', [])]
+                if "already_exists" in codes:
+                    return False
+            raise e
+
+        # attach / upload all files for the relase
+        for file in files:
+            self._create_release_asset(release_id, file)
+
+        return True
+
+    def _create_release_asset(self, release_id, file):
+        """
+        Attach a file to a release on GitHub.
+
+        This uploads a file to github relases, it will be attached to the supplied release
+
+        also see: https://docs.github.com/en/rest/releases/assets?apiVersion=2022-11-28#upload-a-release-asset
+        """
+        file = pathlib.Path(file)
+        with open(file, 'rb') as f:
+            req = urllib.request.Request(
+                f"https://uploads.github.com/repos/{self._repo_path}/releases/{release_id}/assets?name={file.name}",
+                headers={
+                    "Accept": "application/vnd.github+json",
+                    "Authorization": f"Bearer {self._api_token}",
+                    "X-GitHub-Api-Version": "2022-11-28",
+                    "Content-Type": "application/octet-stream",
+                },
+                data=f.read(),
+            )
+            with urllib.request.urlopen(req):
+                return True
+            return False