Merge branch 'parse-donation-links-from-funding.yml' into 'master'

update: insert donation links based on FUNDING.yml

See merge request fdroid/fdroidserver!754

tests/funding-usernames.yaml

@@ -0,0 +1,197 @@
+bad:
+  - "Robert'); DROP TABLE Students; --"
+  - ''
+  - -a-b
+  - '1234567890123456789012345678901234567890'
+  - ~derp@darp---++asdf
+  - foo@bar.com
+  - me++
+  - --me
+bitcoin:
+  - 3Lbz4vdt15Fsa4wVD3Yk8uGf6ugKKY4zSc
+community_bridge: []
+custom:
+  - bc1qvll2mp5ndwd4sgycu4ad2ken4clhjac7mdlcaj
+  - http://www.roguetemple.com/z/donate.php
+  - https://donate.openfoodfacts.org
+  - https://email.faircode.eu/donate/
+  - https://etchdroid.depau.eu/donate/
+  - https://f-droid.org/about/
+  - https://flattr.com/github/bk138
+  - https://gultsch.de/donate.html
+  - https://jahir.dev/donate
+  - https://kodi.tv/contribute/donate
+  - https://link.xbrowsersync.org/cryptos
+  - https://manyver.se/donate
+  - https://paypal.me/DanielQuahShaoHian
+  - https://paypal.me/deletescape
+  - https://paypal.me/freaktechnik
+  - https://paypal.me/hpoul
+  - https://paypal.me/imkosh
+  - https://paypal.me/paphonb
+  - https://paypal.me/vocabletrainer
+  - https://pendulums.io/donation.html
+  - https://play.google.com/store/apps/details?id=de.dennisguse.opentracks.playstore
+  - https://play.google.com/store/apps/details?id=eu.faircode.email
+  - https://raw.githubusercontent.com/Blankj/AndroidUtilCode/master/art/donate.png
+  - https://raw.githubusercontent.com/CarGuo/GSYGithubAppFlutter/master/thanks.jpg
+  - https://raw.githubusercontent.com/GanZhiXiong/GZXTaoBaoAppFlutter/blob/master/preview_images/thanks.png
+  - https://seriesgui.de/whypay
+  - https://transportr.app/donate/
+  - https://www.bountysource.com/teams/nextcloud/issues?tracker_ids=38838206
+  - https://www.donationalerts.com/r/blidingmage835
+  - https://www.hellotux.com/f-droid
+  - https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=8UH5MBVYM3J36
+  - https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=E2FCXCT6837GL
+  - https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=FMLNN8GXZKJEE
+  - https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=K7HVLE6J7SXXA
+  - https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=ZD39ZE7MGEGBL&source=url
+  - https://www.paypal.me/SimpleMobileTools
+  - https://www.paypal.me/TheAlphamerc/
+  - https://www.paypal.me/avirias
+  - https://www.paypal.me/btimofeev
+  - https://www.paypal.me/enricocid
+  - https://www.paypal.me/gsnathan
+  - https://www.paypal.me/nikita36078
+  - https://www.paypal.me/sahdeep
+  - https://www.paypal.me/saulhenriquez
+  - https://www.simplemobiletools.com/donate
+  - https://www.youtube.com/watch?v=ZmrNc1ZhBkQ
+  - paypal.me/amangautam1
+  - paypal.me/pools/c/8lCZfNnU0u
+  - paypal.me/psoffritti
+github:
+  - 00-Evan
+  - adrcotfas
+  - afollestad
+  - ar-
+  - BarnabyShearer
+  - CarGuo
+  - cketti
+  - eighthave
+  - emansih
+  - GanZhiXiong
+  - gpeal
+  - hpoul
+  - i--
+  - inorichi
+  - inputmice
+  - jahirfiquitiva
+  - johnjohndoe
+  - kaloudis
+  - kiwix
+  - ligi
+  - M66B
+  - mikepenz
+  - Mygod
+  - paroj
+  - PerfectSlayer
+  - sschueller
+  - tateisu
+  - tibbi
+  - westnordost
+  - x1unix
+  - xn--nding-jua
+  - zenorogue
+issuehunt:
+  - bk138/multivnc
+ko_fi:
+  - afollestad
+  - fennifith
+  - inorichi
+  - mastalab
+  - psoffritti
+liberapay:
+  - ActivityDiary
+  - AndStatus
+  - BM835
+  - Briar
+  - DAVx5
+  - F-Droid-Data
+  - Feeel
+  - Fruit-Radar-Development
+  - Gadgetbridge
+  - GuardianProject
+  - Hocuri
+  - KOReader
+  - Kanedias
+  - Kunzisoft
+  - MaxK
+  - NovaVideoPlayer
+  - Phie
+  - Rudloff
+  - Schoumi
+  - Syncthing-Fork
+  - TeamNewPipe
+  - Telegram-FOSS
+  - Transportr
+  - Varlorg
+  - Wesnoth
+  - ZiiS
+  - ar-
+  - bk138
+  - btimofeev
+  - bubblineyuri
+  - dennis.guse
+  - developerfromjokela
+  - devgianlu
+  - eneiluj
+  - experiment322
+  - fdossena
+  - fennifith
+  - freaktechnik
+  - gsantner
+  - hisname
+  - hsn6
+  - iNPUTmice
+  - inputmice
+  - k9mail
+  - matrixdotorg
+  - mmarif
+  - moezbhatti
+  - proninyaroslav
+  - quite
+  - renyuneyun
+  - rocketnine.space
+  - sanskritbscs
+  - sschueller
+  - sschueller/donate
+  - stefan-niedermann
+  - tasks
+  - teamkodi
+  - thermatk
+  - tom79
+  - wallabag
+  - westnordost
+  - whyorean
+  - wilko
+  - xbrowsersync
+  - yeriomin
+  - zeh
+open_collective:
+  - avirias
+  - curl
+  - libsodium
+  - manyverse
+  - mastalab
+  - tusky
+otechie: []
+patreon:
+  - BaldPhone
+  - Bm835
+  - FastHub
+  - Teamkodi
+  - andrestaltz
+  - bk138
+  - depau
+  - iamSahdeep
+  - ligi
+  - ogre1
+  - orhunp
+  - tiborkaputa
+  - tom79
+  - westnordost
+  - xbrowsersync
+  - yairm210
+  - zenorogue
+tidelift: []

tests/metadata.TestCase

@@ -100,6 +100,21 @@ class MetadataTest(unittest.TestCase):
         self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                           'tb1qw5r8drrejxrrg4y5rrrrrraryrrrrwrkxrjrsx', 'fake.app.id')
 
+    def test_valid_funding_yml_regex(self):
+        """Check the regex can find all the cases"""
+        with open(os.path.join(self.basedir, 'funding-usernames.yaml')) as fp:
+            data = yaml.safe_load(fp)
+
+        for k, entries in data.items():
+            for entry in entries:
+                m = fdroidserver.metadata.VALID_USERNAME_REGEX.match(entry)
+                if k == 'custom':
+                    pass
+                elif k == 'bad':
+                    self.assertIsNone(m, 'this is an invalid %s username: {%s}' % (k, entry))
+                else:
+                    self.assertIsNotNone(m, 'this is a valid %s username: {%s}' % (k, entry))
+
     def test_read_metadata(self):
 
         def _build_yaml_representer(dumper, data):

tests/metadata/dump/com.politedroid.yaml

@@ -17,6 +17,7 @@ Disabled: null
 Donate: null
 FlattrID: null
 IssueTracker: https://github.com/miguelvps/PoliteDroid/issues
+Liberapay: null
 LiberapayID: null
 License: GPL-3.0-only
 Litecoin: null

tests/metadata/dump/org.adaway.yaml

@@ -40,7 +40,8 @@ Disabled: null
 Donate: http://sufficientlysecure.org/index.php/adaway
 FlattrID: '369138'
 IssueTracker: https://github.com/dschuermann/ad-away/issues
-LiberapayID: null
+Liberapay: null
+LiberapayID: '1234567890'
 License: GPL-3.0-only
 Litecoin: null
 MaintainerNotes: ''

tests/metadata/dump/org.smssecure.smssecure.yaml

@@ -37,6 +37,7 @@ Disabled: null
 Donate: null
 FlattrID: null
 IssueTracker: https://github.com/SMSSecure/SMSSecure/issues
+Liberapay: null
 LiberapayID: null
 License: GPL-3.0-only
 Litecoin: null

tests/metadata/dump/org.videolan.vlc.yaml

@@ -24,6 +24,7 @@ Disabled: null
 Donate: http://www.videolan.org/contribute.html#money
 FlattrID: null
 IssueTracker: http://www.videolan.org/support/index.html#bugs
+Liberapay: null
 LiberapayID: null
 License: GPL-3.0-only
 Litecoin: null

tests/metadata/info.guardianproject.checkey.yml

@@ -7,6 +7,7 @@ SourceCode: https://github.com/guardianproject/checkey
 IssueTracker: https://dev.guardianproject.info/projects/checkey/issues
 Translation: https://www.transifex.com/otf/checkey
 Bitcoin: 1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk
+Liberapay: GuardianProject
 
 AutoName: Checkey
 

tests/metadata/obb.main.oldversion.yml

@@ -3,6 +3,7 @@ Categories:
 License: GPL-3.0-only
 SourceCode: https://github.com/eighthave/urzip
 Bitcoin: 1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk
+Liberapay: 12334
 
 AutoName: OBB Main Old Version
 

tests/metadata/org.adaway.yml

@@ -8,6 +8,7 @@ IssueTracker: https://github.com/dschuermann/ad-away/issues
 Translation: https://www.transifex.com/dominikschuermann/adaway
 Donate: http://sufficientlysecure.org/index.php/adaway
 FlattrID: '369138'
+LiberapayID: '1234567890'
 
 AutoName: AdAway
 Summary: Block advertisements

tests/repo/index-v1.json

@@ -104,6 +104,7 @@
         "Development"
       ],
       "suggestedVersionCode": "99999999",
+      "liberapay": "12334",
       "license": "GPL-3.0-only",
       "name": "OBB Main Old Version",
       "sourceCode": "https://github.com/eighthave/urzip",

tests/update.TestCase

@@ -8,6 +8,7 @@ import inspect
 import logging
 import optparse
 import os
+import random
 import shutil
 import subprocess
 import sys
@@ -33,6 +34,13 @@ import fdroidserver.update
 from fdroidserver.common import FDroidPopen
 
 
+DONATION_FIELDS = (
+    'Donate',
+    'Liberapay',
+    'OpenCollective',
+)
+
+
 class UpdateTest(unittest.TestCase):
     '''fdroid update'''
 
@@ -972,6 +980,86 @@ class UpdateTest(unittest.TestCase):
                                       'VercodeOperation': '',
                                       'WebSite': ''})
 
+    def test_insert_funding_yml_donation_links(self):
+        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+        os.chdir(testdir)
+        os.mkdir('build')
+        content = textwrap.dedent("""
+            community_bridge: ''
+            custom: [LINK1, LINK2]
+            github: USERNAME
+            issuehunt: USERNAME
+            ko_fi: USERNAME
+            liberapay: USERNAME
+            open_collective: USERNAME
+            otechie: USERNAME
+            patreon: USERNAME
+        """)
+        app = fdroidserver.metadata.App()
+        app.id = 'fake.app.id'
+        apps = {app.id: app}
+        os.mkdir(os.path.join('build', app.id))
+        fdroidserver.update.insert_funding_yml_donation_links(apps)
+        for field in DONATION_FIELDS:
+            self.assertFalse(app.get(field))
+        with open(os.path.join('build', app.id, 'FUNDING.yml'), 'w') as fp:
+            fp.write(content)
+
+        fdroidserver.update.insert_funding_yml_donation_links(apps)
+        for field in DONATION_FIELDS:
+            self.assertIsNotNone(app.get(field), field)
+        self.assertEqual('LINK1', app.get('Donate'))
+        self.assertEqual('USERNAME', app.get('Liberapay'))
+        self.assertEqual('USERNAME', app.get('OpenCollective'))
+
+        app['Donate'] = 'keepme'
+        app['Liberapay'] = 'keepme'
+        app['OpenCollective'] = 'keepme'
+        fdroidserver.update.insert_funding_yml_donation_links(apps)
+        for field in DONATION_FIELDS:
+            self.assertEqual('keepme', app.get(field))
+
+    def test_insert_funding_yml_donation_links_with_corrupt_file(self):
+        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+        os.chdir(testdir)
+        os.mkdir('build')
+        app = fdroidserver.metadata.App()
+        app.id = 'fake.app.id'
+        apps = {app.id: app}
+        os.mkdir(os.path.join('build', app.id))
+        with open(os.path.join('build', app.id, 'FUNDING.yml'), 'w') as fp:
+            fp.write(textwrap.dedent("""
+                opencollective: foo
+                custom: []
+                liberapay: :
+            """))
+        fdroidserver.update.insert_funding_yml_donation_links(apps)
+        for field in DONATION_FIELDS:
+            self.assertIsNone(app.get(field))
+
+    def test_sanitize_funding_yml(self):
+        with open(os.path.join(self.basedir, 'funding-usernames.yaml')) as fp:
+            data = yaml.safe_load(fp)
+        for k, entries in data.items():
+            for entry in entries:
+                if k in 'custom':
+                    m = fdroidserver.update.sanitize_funding_yml_entry(entry)
+                else:
+                    m = fdroidserver.update.sanitize_funding_yml_name(entry)
+                if k == 'bad':
+                    self.assertIsNone(m)
+                else:
+                    self.assertIsNotNone(m)
+        self.assertIsNone(fdroidserver.update.sanitize_funding_yml_entry('foo\nbar'))
+        self.assertIsNone(fdroidserver.update.sanitize_funding_yml_entry(
+            ''.join(chr(random.randint(65, 90)) for _ in range(2049))))
+
+        # not recommended but valid entries
+        self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(12345))
+        self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(5.0))
+        self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(' WhyIncludeWhitespace '))
+        self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(['first', 'second']))
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))