Merge branch 'parse-donation-links-from-funding.yml' into 'master'

update: insert donation links based on FUNDING.yml

See merge request fdroid/fdroidserver!754

.gitlab-ci.yml

@@ -36,7 +36,7 @@ metadata_v0:
     - cd fdroiddata
     - ../tests/dump_internal_metadata_format.py
     - sed -i
-          -e '/kivy:\sfalse/d'
+          -e '/Liberapay:/d'
           -e '/OpenCollective/d'
           metadata/dump_*/*.yaml
     - diff -uw metadata/dump_*

fdroidserver/lint.py

@@ -141,9 +141,11 @@ regex_checks = {
     ],
     'Donate': http_checks + [
         (re.compile(r'.*flattr\.com'),
-         _("Flattr donation methods belong in the FlattrID flag")),
+         _("Flattr donation methods belong in the FlattrID: field")),
         (re.compile(r'.*liberapay\.com'),
-         _("Liberapay donation methods belong in the LiberapayID flag")),
+         _("Liberapay donation methods belong in the Liberapay: field")),
+        (re.compile(r'.*opencollective\.com'),
+         _("OpenCollective donation methods belong in the OpenCollective: field")),
     ],
     'Changelog': http_checks,
     'Author Name': [

fdroidserver/metadata.py

@@ -41,6 +41,10 @@ from fdroidserver.exception import MetaDataException, FDroidException
 srclibs = None
 warnings_action = None
 
+# validates usernames based on a loose collection of rules from GitHub, GitLab,
+# Liberapay and issuehunt.  This is mostly to block abuse.
+VALID_USERNAME_REGEX = re.compile(r'^[a-z\d](?:[a-z\d/._-]){0,38}$', re.IGNORECASE)
+
 
 def warn_or_exception(value, cause=None):
     '''output warning or Exception depending on -W'''
@@ -73,6 +77,7 @@ app_fields = set([
     'Changelog',
     'Donate',
     'FlattrID',
+    'Liberapay',
     'LiberapayID',
     'OpenCollective',
     'Bitcoin',
@@ -117,6 +122,7 @@ yaml_app_field_order = [
     'Changelog',
     'Donate',
     'FlattrID',
+    'Liberapay',
     'LiberapayID',
     'OpenCollective',
     'Bitcoin',
@@ -177,6 +183,7 @@ class App(dict):
         self.Changelog = ''
         self.Donate = None
         self.FlattrID = None
+        self.Liberapay = None
         self.LiberapayID = None
         self.OpenCollective = None
         self.Bitcoin = None
@@ -450,12 +457,16 @@ valuetypes = {
                    r'^[0-9a-z]+$',
                    ['FlattrID']),
 
+    FieldValidator("Liberapay",
+                   VALID_USERNAME_REGEX,
+                   ['Liberapay']),
+
     FieldValidator("Liberapay ID",
                    r'^[0-9]+$',
                    ['LiberapayID']),
 
     FieldValidator("Open Collective",
-                   r'^[0-9a-zA-Z_-]+$',
+                   VALID_USERNAME_REGEX,
                    ['OpenCollective']),
 
     FieldValidator("HTTP link",

fdroidserver/update.py

@@ -31,10 +31,15 @@ import zipfile
 import hashlib
 import json
 import time
+import yaml
 import copy
 from datetime import datetime
 from argparse import ArgumentParser
 from base64 import urlsafe_b64encode
+try:
+    from yaml import CSafeLoader as SafeLoader
+except ImportError:
+    from yaml import SafeLoader
 
 import collections
 from binascii import hexlify
@@ -854,6 +859,122 @@ def _get_base_hash_extension(f):
     return base, None, extension
 
 
+def sanitize_funding_yml_entry(entry):
+    """FUNDING.yml comes from upstream repos, entries must be sanitized"""
+    if type(entry) not in (bytes, int, float, list, str):
+        return
+    if isinstance(entry, bytes):
+        entry = entry.decode()
+    elif isinstance(entry, list):
+        if entry:
+            entry = entry[0]
+        else:
+            return
+    try:
+        entry = str(entry)
+    except (TypeError, ValueError):
+        return
+    if len(entry) > 2048:
+        logging.warning(_('Ignoring FUNDING.yml entry longer than 2048: %s') % entry[:2048])
+        return
+    if '\n' in entry:
+        return
+    return entry.strip()
+
+
+def sanitize_funding_yml_name(name):
+    """Sanitize usernames that come from FUNDING.yml"""
+    entry = sanitize_funding_yml_entry(name)
+    if entry:
+        m = metadata.VALID_USERNAME_REGEX.match(entry)
+        if m:
+            return m.group()
+    return
+
+
+def insert_funding_yml_donation_links(apps):
+    """include donation links from FUNDING.yml in app's source repo
+
+    GitHub made a standard file format for declaring donation
+    links. This parses that format from upstream repos to include in
+    metadata here.  GitHub supports mostly proprietary services, so
+    this logic adds proprietary services only as Donate: links.
+
+    FUNDING.yml can be either in the root of the project, or in the
+    ".github" subdir.
+
+    https://help.github.com/en/articles/displaying-a-sponsor-button-in-your-repository#about-funding-files
+
+    """
+
+    if not os.path.isdir('build'):
+        return  # nothing to do
+    for packageName, app in apps.items():
+        sourcedir = os.path.join('build', packageName)
+        if not os.path.isdir(sourcedir):
+            continue
+        for f in ([os.path.join(sourcedir, 'FUNDING.yml'), ]
+                  + glob.glob(os.path.join(sourcedir, '.github', 'FUNDING.yml'))):
+            if not os.path.isfile(f):
+                continue
+            data = None
+            try:
+                with open(f) as fp:
+                    data = yaml.load(fp, Loader=SafeLoader)
+            except yaml.YAMLError as e:
+                logging.error(_('Found bad funding file "{path}" for "{name}":')
+                              .format(path=f, name=packageName))
+                logging.error(e)
+            if not data or type(data) != dict:
+                continue
+            if not app.get('Liberapay') and 'liberapay' in data:
+                s = sanitize_funding_yml_name(data['liberapay'])
+                if s:
+                    app['Liberapay'] = s
+            if not app.get('OpenCollective') and 'open_collective' in data:
+                s = sanitize_funding_yml_name(data['open_collective'])
+                if s:
+                    app['OpenCollective'] = s
+            if not app.get('Donate'):
+                del(data['liberapay'])
+                del(data['open_collective'])
+                # this tuple provides a preference ordering
+                for k in ('custom', 'github', 'patreon', 'community_bridge', 'ko_fi', 'issuehunt'):
+                    v = data.get(k)
+                    if not v:
+                        continue
+                    if k == 'custom':
+                        s = sanitize_funding_yml_entry(v)
+                        if s:
+                            app['Donate'] = s
+                            break
+                    elif k == 'community_bridge':
+                        s = sanitize_funding_yml_name(v)
+                        if s:
+                            app['Donate'] = 'https://funding.communitybridge.org/projects/' + s
+                            break
+                    elif k == 'github':
+                        s = sanitize_funding_yml_name(v)
+                        if s:
+                            app['Donate'] = 'https://github.com/sponsors/' + s
+                            break
+                    elif k == 'issuehunt':
+                        s = sanitize_funding_yml_name(v)
+                        if s:
+                            app['Donate'] = 'https://issuehunt.io/r/' + s
+                            break
+                    elif k == 'ko_fi':
+                        s = sanitize_funding_yml_name(v)
+                        if s:
+                            app['Donate'] = 'https://ko-fi.com/' + s
+                            break
+                    elif k == 'patreon':
+                        s = sanitize_funding_yml_name(v)
+                        if s:
+                            app['Donate'] = 'https://patreon.com/' + s
+                            break
+
+
 def copy_triple_t_store_metadata(apps):
     """Include store metadata from the app's source repo
 
@@ -2179,6 +2300,7 @@ def main():
                 else:
                     logging.warning(msg + '\n\t' + _('Use `fdroid update -c` to create it.'))
 
+    insert_funding_yml_donation_links(apps)
     copy_triple_t_store_metadata(apps)
     insert_obbs(repodirs[0], apps, apks)
     insert_localized_app_metadata(apps)

tests/funding-usernames.yaml

@@ -0,0 +1,197 @@
+bad:
+  - "Robert'); DROP TABLE Students; --"
+  - ''
+  - -a-b
+  - '1234567890123456789012345678901234567890'
+  - ~derp@darp---++asdf
+  - foo@bar.com
+  - me++
+  - --me
+bitcoin:
+  - 3Lbz4vdt15Fsa4wVD3Yk8uGf6ugKKY4zSc
+community_bridge: []
+custom:
+  - bc1qvll2mp5ndwd4sgycu4ad2ken4clhjac7mdlcaj
+  - http://www.roguetemple.com/z/donate.php
+  - https://donate.openfoodfacts.org
+  - https://email.faircode.eu/donate/
+  - https://etchdroid.depau.eu/donate/
+  - https://f-droid.org/about/
+  - https://flattr.com/github/bk138
+  - https://gultsch.de/donate.html
+  - https://jahir.dev/donate
+  - https://kodi.tv/contribute/donate
+  - https://link.xbrowsersync.org/cryptos
+  - https://manyver.se/donate
+  - https://paypal.me/DanielQuahShaoHian
+  - https://paypal.me/deletescape
+  - https://paypal.me/freaktechnik
+  - https://paypal.me/hpoul
+  - https://paypal.me/imkosh
+  - https://paypal.me/paphonb
+  - https://paypal.me/vocabletrainer
+  - https://pendulums.io/donation.html
+  - https://play.google.com/store/apps/details?id=de.dennisguse.opentracks.playstore
+  - https://play.google.com/store/apps/details?id=eu.faircode.email
+  - https://raw.githubusercontent.com/Blankj/AndroidUtilCode/master/art/donate.png
+  - https://raw.githubusercontent.com/CarGuo/GSYGithubAppFlutter/master/thanks.jpg
+  - https://raw.githubusercontent.com/GanZhiXiong/GZXTaoBaoAppFlutter/blob/master/preview_images/thanks.png
+  - https://seriesgui.de/whypay
+  - https://transportr.app/donate/
+  - https://www.bountysource.com/teams/nextcloud/issues?tracker_ids=38838206
+  - https://www.donationalerts.com/r/blidingmage835
+  - https://www.hellotux.com/f-droid
+  - https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=8UH5MBVYM3J36
+  - https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=E2FCXCT6837GL
+  - https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=FMLNN8GXZKJEE
+  - https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=K7HVLE6J7SXXA
+  - https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=ZD39ZE7MGEGBL&source=url
+  - https://www.paypal.me/SimpleMobileTools
+  - https://www.paypal.me/TheAlphamerc/
+  - https://www.paypal.me/avirias
+  - https://www.paypal.me/btimofeev
+  - https://www.paypal.me/enricocid
+  - https://www.paypal.me/gsnathan
+  - https://www.paypal.me/nikita36078
+  - https://www.paypal.me/sahdeep
+  - https://www.paypal.me/saulhenriquez
+  - https://www.simplemobiletools.com/donate
+  - https://www.youtube.com/watch?v=ZmrNc1ZhBkQ
+  - paypal.me/amangautam1
+  - paypal.me/pools/c/8lCZfNnU0u
+  - paypal.me/psoffritti
+github:
+  - 00-Evan
+  - adrcotfas
+  - afollestad
+  - ar-
+  - BarnabyShearer
+  - CarGuo
+  - cketti
+  - eighthave
+  - emansih
+  - GanZhiXiong
+  - gpeal
+  - hpoul
+  - i--
+  - inorichi
+  - inputmice
+  - jahirfiquitiva
+  - johnjohndoe
+  - kaloudis
+  - kiwix
+  - ligi
+  - M66B
+  - mikepenz
+  - Mygod
+  - paroj
+  - PerfectSlayer
+  - sschueller
+  - tateisu
+  - tibbi
+  - westnordost
+  - x1unix
+  - xn--nding-jua
+  - zenorogue
+issuehunt:
+  - bk138/multivnc
+ko_fi:
+  - afollestad
+  - fennifith
+  - inorichi
+  - mastalab
+  - psoffritti
+liberapay:
+  - ActivityDiary
+  - AndStatus
+  - BM835
+  - Briar
+  - DAVx5
+  - F-Droid-Data
+  - Feeel
+  - Fruit-Radar-Development
+  - Gadgetbridge
+  - GuardianProject
+  - Hocuri
+  - KOReader
+  - Kanedias
+  - Kunzisoft
+  - MaxK
+  - NovaVideoPlayer
+  - Phie
+  - Rudloff
+  - Schoumi
+  - Syncthing-Fork
+  - TeamNewPipe
+  - Telegram-FOSS
+  - Transportr
+  - Varlorg
+  - Wesnoth
+  - ZiiS
+  - ar-
+  - bk138
+  - btimofeev
+  - bubblineyuri
+  - dennis.guse
+  - developerfromjokela
+  - devgianlu
+  - eneiluj
+  - experiment322
+  - fdossena
+  - fennifith
+  - freaktechnik
+  - gsantner
+  - hisname
+  - hsn6
+  - iNPUTmice
+  - inputmice
+  - k9mail
+  - matrixdotorg
+  - mmarif
+  - moezbhatti
+  - proninyaroslav
+  - quite
+  - renyuneyun
+  - rocketnine.space
+  - sanskritbscs
+  - sschueller
+  - sschueller/donate
+  - stefan-niedermann
+  - tasks
+  - teamkodi
+  - thermatk
+  - tom79
+  - wallabag
+  - westnordost
+  - whyorean
+  - wilko
+  - xbrowsersync
+  - yeriomin
+  - zeh
+open_collective:
+  - avirias
+  - curl
+  - libsodium
+  - manyverse
+  - mastalab
+  - tusky
+otechie: []
+patreon:
+  - BaldPhone
+  - Bm835
+  - FastHub
+  - Teamkodi
+  - andrestaltz
+  - bk138
+  - depau
+  - iamSahdeep
+  - ligi
+  - ogre1
+  - orhunp
+  - tiborkaputa
+  - tom79
+  - westnordost
+  - xbrowsersync
+  - yairm210
+  - zenorogue
+tidelift: []

tests/metadata.TestCase

@@ -100,6 +100,21 @@ class MetadataTest(unittest.TestCase):
         self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                           'tb1qw5r8drrejxrrg4y5rrrrrraryrrrrwrkxrjrsx', 'fake.app.id')
 
+    def test_valid_funding_yml_regex(self):
+        """Check the regex can find all the cases"""
+        with open(os.path.join(self.basedir, 'funding-usernames.yaml')) as fp:
+            data = yaml.safe_load(fp)
+
+        for k, entries in data.items():
+            for entry in entries:
+                m = fdroidserver.metadata.VALID_USERNAME_REGEX.match(entry)
+                if k == 'custom':
+                    pass
+                elif k == 'bad':
+                    self.assertIsNone(m, 'this is an invalid %s username: {%s}' % (k, entry))
+                else:
+                    self.assertIsNotNone(m, 'this is a valid %s username: {%s}' % (k, entry))
+
     def test_read_metadata(self):
 
         def _build_yaml_representer(dumper, data):

tests/metadata/dump/com.politedroid.yaml

@@ -17,6 +17,7 @@ Disabled: null
 Donate: null
 FlattrID: null
 IssueTracker: https://github.com/miguelvps/PoliteDroid/issues
+Liberapay: null
 LiberapayID: null
 License: GPL-3.0-only
 Litecoin: null

tests/metadata/dump/org.adaway.yaml

@@ -40,7 +40,8 @@ Disabled: null
 Donate: http://sufficientlysecure.org/index.php/adaway
 FlattrID: '369138'
 IssueTracker: https://github.com/dschuermann/ad-away/issues
-LiberapayID: null
+Liberapay: null
+LiberapayID: '1234567890'
 License: GPL-3.0-only
 Litecoin: null
 MaintainerNotes: ''

tests/metadata/dump/org.smssecure.smssecure.yaml

@@ -37,6 +37,7 @@ Disabled: null
 Donate: null
 FlattrID: null
 IssueTracker: https://github.com/SMSSecure/SMSSecure/issues
+Liberapay: null
 LiberapayID: null
 License: GPL-3.0-only
 Litecoin: null

tests/metadata/dump/org.videolan.vlc.yaml

@@ -24,6 +24,7 @@ Disabled: null
 Donate: http://www.videolan.org/contribute.html#money
 FlattrID: null
 IssueTracker: http://www.videolan.org/support/index.html#bugs
+Liberapay: null
 LiberapayID: null
 License: GPL-3.0-only
 Litecoin: null

tests/metadata/info.guardianproject.checkey.yml

@@ -7,6 +7,7 @@ SourceCode: https://github.com/guardianproject/checkey
 IssueTracker: https://dev.guardianproject.info/projects/checkey/issues
 Translation: https://www.transifex.com/otf/checkey
 Bitcoin: 1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk
+Liberapay: GuardianProject
 
 AutoName: Checkey
 

tests/metadata/obb.main.oldversion.yml

@@ -3,6 +3,7 @@ Categories:
 License: GPL-3.0-only
 SourceCode: https://github.com/eighthave/urzip
 Bitcoin: 1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk
+Liberapay: 12334
 
 AutoName: OBB Main Old Version
 

tests/metadata/org.adaway.yml

@@ -8,6 +8,7 @@ IssueTracker: https://github.com/dschuermann/ad-away/issues
 Translation: https://www.transifex.com/dominikschuermann/adaway
 Donate: http://sufficientlysecure.org/index.php/adaway
 FlattrID: '369138'
+LiberapayID: '1234567890'
 
 AutoName: AdAway
 Summary: Block advertisements

tests/repo/index-v1.json

@@ -104,6 +104,7 @@
         "Development"
       ],
       "suggestedVersionCode": "99999999",
+      "liberapay": "12334",
       "license": "GPL-3.0-only",
       "name": "OBB Main Old Version",
       "sourceCode": "https://github.com/eighthave/urzip",

tests/update.TestCase

@@ -8,6 +8,7 @@ import inspect
 import logging
 import optparse
 import os
+import random
 import shutil
 import subprocess
 import sys
@@ -33,6 +34,13 @@ import fdroidserver.update
 from fdroidserver.common import FDroidPopen
 
 
+DONATION_FIELDS = (
+    'Donate',
+    'Liberapay',
+    'OpenCollective',
+)
+
+
 class UpdateTest(unittest.TestCase):
     '''fdroid update'''
 
@@ -972,6 +980,86 @@ class UpdateTest(unittest.TestCase):
                                       'VercodeOperation': '',
                                       'WebSite': ''})
 
+    def test_insert_funding_yml_donation_links(self):
+        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+        os.chdir(testdir)
+        os.mkdir('build')
+        content = textwrap.dedent("""
+            community_bridge: ''
+            custom: [LINK1, LINK2]
+            github: USERNAME
+            issuehunt: USERNAME
+            ko_fi: USERNAME
+            liberapay: USERNAME
+            open_collective: USERNAME
+            otechie: USERNAME
+            patreon: USERNAME
+        """)
+        app = fdroidserver.metadata.App()
+        app.id = 'fake.app.id'
+        apps = {app.id: app}
+        os.mkdir(os.path.join('build', app.id))
+        fdroidserver.update.insert_funding_yml_donation_links(apps)
+        for field in DONATION_FIELDS:
+            self.assertFalse(app.get(field))
+        with open(os.path.join('build', app.id, 'FUNDING.yml'), 'w') as fp:
+            fp.write(content)
+
+        fdroidserver.update.insert_funding_yml_donation_links(apps)
+        for field in DONATION_FIELDS:
+            self.assertIsNotNone(app.get(field), field)
+        self.assertEqual('LINK1', app.get('Donate'))
+        self.assertEqual('USERNAME', app.get('Liberapay'))
+        self.assertEqual('USERNAME', app.get('OpenCollective'))
+
+        app['Donate'] = 'keepme'
+        app['Liberapay'] = 'keepme'
+        app['OpenCollective'] = 'keepme'
+        fdroidserver.update.insert_funding_yml_donation_links(apps)
+        for field in DONATION_FIELDS:
+            self.assertEqual('keepme', app.get(field))
+
+    def test_insert_funding_yml_donation_links_with_corrupt_file(self):
+        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+        os.chdir(testdir)
+        os.mkdir('build')
+        app = fdroidserver.metadata.App()
+        app.id = 'fake.app.id'
+        apps = {app.id: app}
+        os.mkdir(os.path.join('build', app.id))
+        with open(os.path.join('build', app.id, 'FUNDING.yml'), 'w') as fp:
+            fp.write(textwrap.dedent("""
+                opencollective: foo
+                custom: []
+                liberapay: :
+            """))
+        fdroidserver.update.insert_funding_yml_donation_links(apps)
+        for field in DONATION_FIELDS:
+            self.assertIsNone(app.get(field))
+
+    def test_sanitize_funding_yml(self):
+        with open(os.path.join(self.basedir, 'funding-usernames.yaml')) as fp:
+            data = yaml.safe_load(fp)
+        for k, entries in data.items():
+            for entry in entries:
+                if k in 'custom':
+                    m = fdroidserver.update.sanitize_funding_yml_entry(entry)
+                else:
+                    m = fdroidserver.update.sanitize_funding_yml_name(entry)
+                if k == 'bad':
+                    self.assertIsNone(m)
+                else:
+                    self.assertIsNotNone(m)
+        self.assertIsNone(fdroidserver.update.sanitize_funding_yml_entry('foo\nbar'))
+        self.assertIsNone(fdroidserver.update.sanitize_funding_yml_entry(
+            ''.join(chr(random.randint(65, 90)) for _ in range(2049))))
+
+        # not recommended but valid entries
+        self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(12345))
+        self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(5.0))
+        self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(' WhyIncludeWhitespace '))
+        self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(['first', 'second']))
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))