From 56bed02a29b51152556c80bc8fd31f4557f6a47f Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 21 Feb 2024 18:01:56 +0100 Subject: [PATCH 01/15] install: download_apk() fetchs APKs by appid based on the index --- fdroidserver/common.py | 93 +++++++++++++++++++++++++++++++++++++++++ fdroidserver/install.py | 45 +++++++++++++++++++- tests/common.TestCase | 16 +++++++ 3 files changed, 152 insertions(+), 2 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 7553f74b..5d54ac09 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -28,6 +28,7 @@ # common.py is imported by all modules, so do not import third-party # libraries here as they will become a requirement for all commands. +import copy import difflib from typing import List import git @@ -60,6 +61,7 @@ from base64 import urlsafe_b64encode from binascii import hexlify from datetime import datetime, timedelta, timezone from queue import Queue +from urllib.parse import urlparse, urlunparse from zipfile import ZipFile import fdroidserver.metadata @@ -435,6 +437,14 @@ def get_config(): return config +def get_cachedir(): + cachedir = config and config.get('cachedir') + if cachedir and os.path.exists(cachedir): + return Path(cachedir) + else: + return Path(tempfile.mkdtemp()) + + def regsub_file(pattern, repl, path): with open(path, 'rb') as f: text = f.read() @@ -609,6 +619,17 @@ def parse_mirrors_config(mirrors): raise TypeError(_('only accepts strings, lists, and tuples')) +def append_filename_to_mirrors(filename, mirrors): + """Append the filename to all "url" entries in the mirrors dict.""" + appended = copy.deepcopy(mirrors) + for mirror in appended: + parsed = urlparse(mirror['url']) + mirror['url'] = urlunparse( + parsed._replace(path=os.path.join(parsed.path, filename)) + ) + return appended + + def file_entry(filename, hash_value=None): meta = {} meta["name"] = "/" + Path(filename).as_posix().split("/", 1)[1] @@ -4620,3 +4641,75 @@ def _install_ndk(ndk): logging.info( _('Set NDK {release} ({version}) up').format(release=ndk, version=version) ) + + +FDROIDORG_MIRRORS = [ + { + 'isPrimary': True, + 'url': 'https://f-droid.org/repo', + 'dnsA': ['65.21.79.229', '136.243.44.143'], + 'dnsAAAA': ['2a01:4f8:212:c98::2', '2a01:4f9:3b:546d::2'], + 'worksWithoutSNI': True, + }, + { + 'url': 'http://fdroidorg6cooksyluodepej4erfctzk7rrjpjbbr6wx24jh3lqyfwyd.onion/fdroid/repo' + }, + { + 'url': 'http://dotsrccccbidkzg7oc7oj4ugxrlfbt64qebyunxbrgqhxiwj3nl6vcad.onion/fdroid/repo' + }, + { + 'url': 'http://ftpfaudev4triw2vxiwzf4334e3mynz7osqgtozhbc77fixncqzbyoyd.onion/fdroid/repo' + }, + { + 'url': 'http://lysator7eknrfl47rlyxvgeamrv7ucefgrrlhk7rouv3sna25asetwid.onion/pub/fdroid/repo' + }, + { + 'url': 'http://mirror.ossplanetnyou5xifr6liw5vhzwc2g2fmmlohza25wwgnnaw65ytfsad.onion/fdroid/repo' + }, + {'url': 'https://fdroid.tetaneutral.net/fdroid/repo', 'countryCode': 'FR'}, + { + 'url': 'https://ftp.agdsn.de/fdroid/repo', + 'countryCode': 'DE', + "dnsA": ["141.30.235.39"], + "dnsAAAA": ["2a13:dd85:b00:12::1"], + "worksWithoutSNI": True, + }, + { + 'url': 'https://ftp.fau.de/fdroid/repo', + 'countryCode': 'DE', + "dnsA": ["131.188.12.211"], + "dnsAAAA": ["2001:638:a000:1021:21::1"], + "worksWithoutSNI": True, + }, + {'url': 'https://ftp.gwdg.de/pub/android/fdroid/repo', 'countryCode': 'DE'}, + { + 'url': 'https://ftp.lysator.liu.se/pub/fdroid/repo', + 'countryCode': 'SE', + "dnsA": ["130.236.254.251", "130.236.254.253"], + "dnsAAAA": ["2001:6b0:17:f0a0::fb", "2001:6b0:17:f0a0::fd"], + "worksWithoutSNI": True, + }, + {'url': 'https://mirror.cyberbits.eu/fdroid/repo', 'countryCode': 'FR'}, + { + 'url': 'https://mirror.fcix.net/fdroid/repo', + 'countryCode': 'US', + "dnsA": ["23.152.160.16"], + "dnsAAAA": ["2620:13b:0:1000::16"], + "worksWithoutSNI": True, + }, + {'url': 'https://mirror.kumi.systems/fdroid/repo', 'countryCode': 'AT'}, + {'url': 'https://mirror.level66.network/fdroid/repo', 'countryCode': 'DE'}, + {'url': 'https://mirror.ossplanet.net/fdroid/repo', 'countryCode': 'TW'}, + {'url': 'https://mirrors.dotsrc.org/fdroid/repo', 'countryCode': 'DK'}, + {'url': 'https://opencolo.mm.fcix.net/fdroid/repo', 'countryCode': 'US'}, + { + 'url': 'https://plug-mirror.rcac.purdue.edu/fdroid/repo', + 'countryCode': 'US', + "dnsA": ["128.211.151.252"], + "dnsAAAA": ["2001:18e8:804:35::1337"], + "worksWithoutSNI": True, + }, +] +FDROIDORG_FINGERPRINT = ( + '43238D512C1E5EB2D6569F4A3AFBF5523418B82E0A3ED1552770ABB9A9C9CCAB' +) diff --git a/fdroidserver/install.py b/fdroidserver/install.py index b9370ee5..362d5aed 100644 --- a/fdroidserver/install.py +++ b/fdroidserver/install.py @@ -20,17 +20,58 @@ import sys import os import glob -from argparse import ArgumentParser import logging +from argparse import ArgumentParser +from pathlib import Path +from urllib.parse import urlencode, urlparse, urlunparse + from . import _ -from . import common +from . import common, index, net from .common import SdkToolsPopen from .exception import FDroidException config = None +DEFAULT_IPFS_GATEWAYS = ("https://gateway.ipfs.io/ipfs/",) + + +def download_apk(appid='org.fdroid.fdroid'): + """Download an APK from F-Droid via the first mirror that works.""" + url = urlunparse( + urlparse(common.FDROIDORG_MIRRORS[0]['url'])._replace( + query=urlencode({'fingerprint': common.FDROIDORG_FINGERPRINT}) + ) + ) + + data, _ignored = index.download_repo_index_v2(url) + app = data.get('packages', dict()).get(appid) + preferred_version = None + for version in app['versions'].values(): + if not preferred_version: + # if all else fails, use the first one + preferred_version = version + if not version.get('releaseChannels'): + # prefer APK in default release channel + preferred_version = version + break + print('skipping', version) + + mirrors = common.append_filename_to_mirrors( + preferred_version['file']['name'][1:], common.FDROIDORG_MIRRORS + ) + ipfsCIDv1 = preferred_version['file'].get('ipfsCIDv1') + if ipfsCIDv1: + for gateway in DEFAULT_IPFS_GATEWAYS: + mirrors.append({'url': os.path.join(gateway, ipfsCIDv1)}) + f = net.download_using_mirrors(mirrors) + if f and os.path.exists(f): + versionCode = preferred_version['manifest']['versionCode'] + f = Path(f) + return str(f.rename(f.with_stem(f'{appid}_{versionCode}')).resolve()) + + def devices(): p = SdkToolsPopen(['adb', "devices"]) if p.returncode != 0: diff --git a/tests/common.TestCase b/tests/common.TestCase index f04df702..c4959cb5 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -36,6 +36,7 @@ if localmodule not in sys.path: sys.path.insert(0, localmodule) import fdroidserver.index +import fdroidserver.install import fdroidserver.signindex import fdroidserver.common import fdroidserver.metadata @@ -2967,6 +2968,21 @@ class CommonTest(unittest.TestCase): knownapks.recordapk(fake_apk, default_date=datetime.now(timezone.utc)) self.assertEqual(knownapks.apks[fake_apk], now) + def test_append_filename_to_mirrors(self): + filename = 'test.apk' + url = 'https://example.com/fdroid/repo' + mirrors = [{'url': url}] + self.assertEqual( + [{'url': url + '/' + filename}], + fdroidserver.common.append_filename_to_mirrors(filename, mirrors), + ) + + def test_append_filename_to_mirrors_full(self): + filename = 'test.apk' + mirrors = fdroidserver.common.FDROIDORG_MIRRORS + for mirror in fdroidserver.common.append_filename_to_mirrors(filename, mirrors): + self.assertTrue(mirror['url'].endswith('/' + filename)) + APKS_WITH_JAR_SIGNATURES = ( ( From 49dcc53076f673e04f3f0cf98df1f02d5365a9c8 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 27 Feb 2024 17:07:57 +0100 Subject: [PATCH 02/15] install: download_fdroid_apk() to fetch the recommended initial APK --- fdroidserver/install.py | 12 ++++++++++++ tests/install.TestCase | 7 +++++++ 2 files changed, 19 insertions(+) diff --git a/fdroidserver/install.py b/fdroidserver/install.py index 362d5aed..3c9316e9 100644 --- a/fdroidserver/install.py +++ b/fdroidserver/install.py @@ -72,6 +72,18 @@ def download_apk(appid='org.fdroid.fdroid'): return str(f.rename(f.with_stem(f'{appid}_{versionCode}')).resolve()) +def download_fdroid_apk(): + """Directly download the current F-Droid APK and verify it. + + This downloads the "download button" link, which is the version + that is best tested for new installs. + + """ + mirror = common.FDROIDORG_MIRRORS[0] + mirror['url'] = urlunparse(urlparse(mirror['url'])._replace(path='F-Droid.apk')) + return net.download_using_mirrors([mirror]) + + def devices(): p = SdkToolsPopen(['adb', "devices"]) if p.returncode != 0: diff --git a/tests/install.TestCase b/tests/install.TestCase index cef5c022..540a2bf0 100755 --- a/tests/install.TestCase +++ b/tests/install.TestCase @@ -7,6 +7,8 @@ import os import sys import unittest +from pathlib import Path + localmodule = os.path.realpath( os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..') ) @@ -33,6 +35,11 @@ class InstallTest(unittest.TestCase): for device in devices: self.assertIsInstance(device, str) + @unittest.skipUnless(os.getenv('test_download_fdroid_apk'), 'requires net access') + def test_download_fdroid_apk(self): + f = fdroidserver.install.download_fdroid_apk() + self.assertTrue(Path(f).exists()) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From 2e3f6d273a608077a599d7a017137a878891a67e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 26 Feb 2024 18:13:53 +0100 Subject: [PATCH 03/15] net: download_using_mirrors() to download like fdroidclient does --- fdroidserver/net.py | 87 +++++++++++++++++++++++++++++++++++++++++++-- tests/net.TestCase | 22 ++++++++++++ 2 files changed, 106 insertions(+), 3 deletions(-) diff --git a/fdroidserver/net.py b/fdroidserver/net.py index 49d67f2c..5c6e0144 100644 --- a/fdroidserver/net.py +++ b/fdroidserver/net.py @@ -17,13 +17,20 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import copy import logging import os +import random import requests +import tempfile import time import urllib +import urllib3 from requests.adapters import HTTPAdapter, Retry -from requests.exceptions import ChunkedEncodingError + +from . import _, common + +logger = logging.getLogger(__name__) HEADERS = {'User-Agent': 'F-Droid'} @@ -64,14 +71,88 @@ def download_file(url, local_filename=None, dldir='tmp', retries=3, backoff_fact f.write(chunk) f.flush() return local_filename - except ChunkedEncodingError as err: + except requests.exceptions.ChunkedEncodingError as err: if i == retries: raise err - logging.warning('Download interrupted, retrying...') + logger.warning('Download interrupted, retrying...') time.sleep(backoff_factor * 2**i) raise ValueError("retries must be >= 0") +def download_using_mirrors(mirrors, local_filename=None): + """Try to download the file from any working mirror. + + Download the file that all URLs in the mirrors list point to, + trying all the tricks, starting with the most private methods + first. The list of mirrors is converted into a list of mirror + configurations to try, in order that the should be attempted. + + This builds mirror_configs_to_try using all possible combos to + try. If a mirror is marked with worksWithoutSNI: True, then this + logic will try it twice: first without SNI, then again with SNI. + + """ + mirrors = common.parse_mirrors_config(mirrors) + mirror_configs_to_try = [] + for mirror in mirrors: + mirror_configs_to_try.append(mirror) + if mirror.get('worksWithoutSNI'): + m = copy.deepcopy(mirror) + del m['worksWithoutSNI'] + mirror_configs_to_try.append(m) + + if not local_filename: + for mirror in mirrors: + filename = urllib.parse.urlparse(mirror['url']).path.split('/')[-1] + if filename: + break + if filename: + local_filename = os.path.join(common.get_cachedir(), filename) + else: + local_filename = tempfile.mkstemp(prefix='fdroid-') + + timeouts = (2, 10, 100) + last_exception = None + for timeout in timeouts: + for mirror in mirror_configs_to_try: + last_exception = None + urllib3.util.ssl_.HAS_SNI = not mirror.get('worksWithoutSNI') + try: + # the stream=True parameter keeps memory usage low + r = requests.get( + mirror['url'], + stream=True, + allow_redirects=False, + headers=HEADERS, + # add jitter to the timeout to be less predictable + timeout=timeout + random.randint(0, timeout), # nosec B311 + ) + if r.status_code != 200: + raise requests.exceptions.HTTPError(r.status_code, response=r) + with open(local_filename, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + f.flush() + return local_filename + except ( + ConnectionError, + requests.exceptions.ChunkedEncodingError, + requests.exceptions.ConnectionError, + requests.exceptions.ContentDecodingError, + requests.exceptions.HTTPError, + requests.exceptions.SSLError, + requests.exceptions.StreamConsumedError, + requests.exceptions.Timeout, + requests.exceptions.UnrewindableBodyError, + ) as e: + last_exception = e + logger.debug(_('Retrying failed download: %s') % str(e)) + # if it hasn't succeeded by now, then give up and raise last exception + if last_exception: + raise last_exception + + def http_get(url, etag=None, timeout=600): """Download the content from the given URL by making a GET request. diff --git a/tests/net.TestCase b/tests/net.TestCase index a50f5925..725bbbfd 100755 --- a/tests/net.TestCase +++ b/tests/net.TestCase @@ -121,6 +121,28 @@ class NetTest(unittest.TestCase): net.download_file('http://localhost:%d/f.txt' % server.port) server.stop() + def test_download_using_mirrors_retries(self): + server = RetryServer() + f = net.download_using_mirrors( + [ + 'https://fake.com/f.txt', # 404 or 301 Redirect + 'https://httpbin.org/status/403', + 'https://httpbin.org/status/500', + 'http://localhost:1/f.txt', # ConnectionError + 'http://localhost:%d/' % server.port, + ], + ) + # strip the HTTP headers and compare the reply + self.assertEqual(server.reply.split(b'\n\n')[1], Path(f).read_bytes()) + server.stop() + + def test_download_using_mirrors_retries_not_forever(self): + """The retry logic should eventually exit with an error.""" + server = RetryServer(failures=5) + with self.assertRaises(requests.exceptions.ConnectionError): + net.download_using_mirrors(['http://localhost:%d/' % server.port]) + server.stop() + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From 59fcfa5dec7240b0ca7bf21a3a1ed95a53a9cab3 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 4 Mar 2024 12:44:20 +0100 Subject: [PATCH 04/15] index: download_repo_index_v2() uses mirrors test_download_repo_index_v2_url_parsing is no longer needed, since all the things it tested are now handled in test_download_repo_index_v2 --- fdroidserver/common.py | 19 ++++++++++- fdroidserver/index.py | 32 +++++++++--------- tests/api.TestCase | 53 ++++++++++++++++++++--------- tests/common.TestCase | 17 ++++++++++ tests/index.TestCase | 76 ++++++++++++++++-------------------------- tests/testcommon.py | 3 ++ 6 files changed, 119 insertions(+), 81 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 5d54ac09..4bc70a2e 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -61,7 +61,7 @@ from base64 import urlsafe_b64encode from binascii import hexlify from datetime import datetime, timedelta, timezone from queue import Queue -from urllib.parse import urlparse, urlunparse +from urllib.parse import urlparse, urlsplit, urlunparse from zipfile import ZipFile import fdroidserver.metadata @@ -619,6 +619,23 @@ def parse_mirrors_config(mirrors): raise TypeError(_('only accepts strings, lists, and tuples')) +def get_mirrors(url, filename=None): + """Get list of dict entries for mirrors, appending filename if provided.""" + # TODO use cached index if it exists + if isinstance(url, str): + url = urlsplit(url) + + if url.netloc == 'f-droid.org': + mirrors = FDROIDORG_MIRRORS + else: + mirrors = parse_mirrors_config(url.geturl()) + + if filename: + return append_filename_to_mirrors(filename, mirrors) + else: + return mirrors + + def append_filename_to_mirrors(filename, mirrors): """Append the filename to all "url" entries in the mirrors dict.""" appended = copy.deepcopy(mirrors) diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 5ca59662..d3f9d44e 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -1633,7 +1633,7 @@ def download_repo_index_v1(url_str, etag=None, verify_fingerprint=True, timeout= return index, new_etag -def download_repo_index_v2(url_str, etag=None, verify_fingerprint=True, timeout=600): +def download_repo_index_v2(url_str, etag=None, verify_fingerprint=True, timeout=None): """Download and verifies index v2 file, then returns its data. Downloads the repository index from the given :param url_str and @@ -1652,8 +1652,13 @@ def download_repo_index_v2(url_str, etag=None, verify_fingerprint=True, timeout= - The new eTag as returned by the HTTP request """ + etag # etag is unused but needs to be there to keep the same API as the earlier functions. + url = urllib.parse.urlsplit(url_str) + if timeout is not None: + logging.warning('"timeout" argument of download_repo_index_v2() is deprecated!') + fingerprint = None if verify_fingerprint: query = urllib.parse.parse_qs(url.query) @@ -1665,29 +1670,22 @@ def download_repo_index_v2(url_str, etag=None, verify_fingerprint=True, timeout= path = url.path.rsplit('/', 1)[0] else: path = url.path.rstrip('/') + url = urllib.parse.SplitResult(url.scheme, url.netloc, path, '', '') - url = urllib.parse.SplitResult(url.scheme, url.netloc, path + '/entry.jar', '', '') - download, new_etag = net.http_get(url.geturl(), etag, timeout) + mirrors = common.get_mirrors(url, 'entry.jar') + f = net.download_using_mirrors(mirrors) + entry, public_key, fingerprint = get_index_from_jar(f, fingerprint) - if download is None: - return None, new_etag - - # jarsigner is used to verify the JAR, it requires a file for input - with tempfile.TemporaryDirectory() as dirname: - with (Path(dirname) / 'entry.jar').open('wb') as fp: - fp.write(download) - fp.flush() - entry, public_key, fingerprint = get_index_from_jar(fp.name, fingerprint) - - name = entry['index']['name'] sha256 = entry['index']['sha256'] - url = urllib.parse.SplitResult(url.scheme, url.netloc, path + name, '', '') - index, _ignored = net.http_get(url.geturl(), None, timeout) + mirrors = common.get_mirrors(url, entry['index']['name'][1:]) + f = net.download_using_mirrors(mirrors) + with open(f, 'rb') as fp: + index = fp.read() if sha256 != hashlib.sha256(index).hexdigest(): raise VerificationException( _("SHA-256 of {url} does not match entry!").format(url=url) ) - return json.loads(index), new_etag + return json.loads(index), None def get_index_from_jar(jarfile, fingerprint=None, allow_deprecated=False): diff --git a/tests/api.TestCase b/tests/api.TestCase index 0dbaefd8..e3e66765 100755 --- a/tests/api.TestCase +++ b/tests/api.TestCase @@ -2,6 +2,7 @@ import inspect import os +import shutil import sys import unittest from unittest import mock @@ -14,6 +15,8 @@ if localmodule not in sys.path: sys.path.insert(0, localmodule) import fdroidserver +from fdroidserver import common, signindex +from testcommon import GP_FINGERPRINT, mkdtemp class ApiTest(unittest.TestCase): @@ -29,6 +32,18 @@ class ApiTest(unittest.TestCase): self.basedir = os.path.join(localmodule, 'tests') os.chdir(self.basedir) + self._td = mkdtemp() + self.testdir = self._td.name + + common.config = None + config = common.read_config() + config['jarsigner'] = common.find_sdk_tools_cmd('jarsigner') + common.config = config + signindex.config = config + + def tearDown(self): + self._td.cleanup() + def test_download_repo_index_no_fingerprint(self): with self.assertRaises(fdroidserver.VerificationException): fdroidserver.download_repo_index("http://example.org") @@ -67,23 +82,31 @@ class ApiTest(unittest.TestCase): ) self.assertEqual(index_url, etag_set_to_url) - @mock.patch('fdroidserver.net.http_get') - def test_download_repo_index_v2_url_parsing(self, mock_http_get): - """Test whether it is trying to download the right file - - This passes the URL back via the etag return value just as a - hack to check which URL was actually attempted. - - """ - mock_http_get.side_effect = lambda url, etag, timeout: (None, url) - repo_url = 'https://example.org/fdroid/repo' - entry_url = 'https://example.org/fdroid/repo/entry.jar' - index_url = 'https://example.org/fdroid/repo/index-v2.json' - for url in (repo_url, entry_url, index_url): - _ignored, etag_set_to_url = fdroidserver.download_repo_index_v2( + @mock.patch('fdroidserver.net.download_using_mirrors') + def test_download_repo_index_v2(self, mock_download_using_mirrors): + """Basically a copy of IndexTest.test_download_repo_index_v2""" + mock_download_using_mirrors.side_effect = lambda mirrors: os.path.join( + self.testdir, 'repo', os.path.basename(mirrors[0]['url']) + ) + os.chdir(self.testdir) + signindex.config['keystore'] = os.path.join(self.basedir, 'keystore.jks') + os.mkdir('repo') + shutil.copy(os.path.join(self.basedir, 'repo', 'entry.json'), 'repo') + shutil.copy(os.path.join(self.basedir, 'repo', 'index-v2.json'), 'repo') + signindex.sign_index('repo', 'entry.json') + repo_url = 'https://fake.url/fdroid/repo' + entry_url = 'https://fake.url/fdroid/repo/entry.jar' + index_url = 'https://fake.url/fdroid/repo/index-v2.json' + fingerprint_url = 'https://fake.url/fdroid/repo?fingerprint=' + GP_FINGERPRINT + slash_url = 'https://fake.url/fdroid/repo//?fingerprint=' + GP_FINGERPRINT + for url in (repo_url, entry_url, index_url, fingerprint_url, slash_url): + data, _ignored = fdroidserver.download_repo_index_v2( url, verify_fingerprint=False ) - self.assertEqual(entry_url, etag_set_to_url) + self.assertEqual(['repo', 'packages'], list(data)) + self.assertEqual( + 'My First F-Droid Repo Demo', data['repo']['name']['en-US'] + ) if __name__ == "__main__": diff --git a/tests/common.TestCase b/tests/common.TestCase index c4959cb5..c2e03243 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -2968,6 +2968,23 @@ class CommonTest(unittest.TestCase): knownapks.recordapk(fake_apk, default_date=datetime.now(timezone.utc)) self.assertEqual(knownapks.apks[fake_apk], now) + def test_get_mirrors_fdroidorg(self): + mirrors = fdroidserver.common.get_mirrors( + 'https://f-droid.org/repo', 'entry.jar' + ) + self.assertEqual( + 'https://f-droid.org/repo/entry.jar', + mirrors[0]['url'], + ) + + def test_get_mirrors_other(self): + self.assertEqual( + [{'url': 'https://example.com/fdroid/repo/index-v2.json'}], + fdroidserver.common.get_mirrors( + 'https://example.com/fdroid/repo', 'index-v2.json' + ), + ) + def test_append_filename_to_mirrors(self): filename = 'test.apk' url = 'https://example.com/fdroid/repo' diff --git a/tests/index.TestCase b/tests/index.TestCase index 0e23c71b..facc9e77 100755 --- a/tests/index.TestCase +++ b/tests/index.TestCase @@ -25,13 +25,10 @@ if localmodule not in sys.path: import fdroidserver from fdroidserver import common, index, publish, signindex, update -from testcommon import TmpCwd, mkdtemp, parse_args_for_test +from testcommon import GP_FINGERPRINT, TmpCwd, mkdtemp, parse_args_for_test from pathlib import Path -GP_FINGERPRINT = 'B7C2EEFD8DAC7806AF67DFCD92EB18126BC08312A7F2D6F3862E46013C7A6135' - - class Options: nosign = True pretty = False @@ -183,32 +180,11 @@ class IndexTest(unittest.TestCase): ilist = index.download_repo_index(url, verify_fingerprint=False) self.assertEqual(index_url, ilist[1]) # etag item used to return URL - @patch('fdroidserver.net.http_get') - def test_download_repo_index_v2_url_parsing(self, mock_http_get): - """Test whether it is trying to download the right file - - This passes the URL back via the etag return value just as a - hack to check which URL was actually attempted. - - """ - mock_http_get.side_effect = lambda url, etag, timeout: (None, url) - repo_url = 'https://fake.url/fdroid/repo' - entry_url = 'https://fake.url/fdroid/repo/entry.jar' - index_url = 'https://fake.url/fdroid/repo/index-v2.json' - fingerprint_url = 'https://fake.url/fdroid/repo?fingerprint=' + GP_FINGERPRINT - slash_url = 'https://fake.url/fdroid/repo//?fingerprint=' + GP_FINGERPRINT - for url in (repo_url, entry_url, index_url, fingerprint_url, slash_url): - ilist = index.download_repo_index_v2(url, verify_fingerprint=False) - self.assertEqual(entry_url, ilist[1]) # etag item used to return URL - - @patch('fdroidserver.net.http_get') - def test_download_repo_index_v2(self, mock_http_get): - def http_get_def(url, etag, timeout): # pylint: disable=unused-argument - f = os.path.basename(url) - with open(os.path.join(self.testdir, 'repo', f), 'rb') as fp: - return (fp.read(), 'fakeetag') - - mock_http_get.side_effect = http_get_def + @patch('fdroidserver.net.download_using_mirrors') + def test_download_repo_index_v2(self, mock_download_using_mirrors): + mock_download_using_mirrors.side_effect = lambda mirrors: os.path.join( + self.testdir, 'repo', os.path.basename(mirrors[0]['url']) + ) os.chdir(self.testdir) signindex.config['keystore'] = os.path.join(self.basedir, 'keystore.jks') os.mkdir('repo') @@ -223,15 +199,15 @@ class IndexTest(unittest.TestCase): for url in (repo_url, entry_url, index_url, fingerprint_url, slash_url): data, _ignored = index.download_repo_index_v2(url, verify_fingerprint=False) self.assertEqual(['repo', 'packages'], list(data.keys())) + self.assertEqual( + 'My First F-Droid Repo Demo', data['repo']['name']['en-US'] + ) - @patch('fdroidserver.net.http_get') - def test_download_repo_index_v2_bad_fingerprint(self, mock_http_get): - def http_get_def(url, etag, timeout): # pylint: disable=unused-argument - f = os.path.basename(url) - with open(os.path.join(self.testdir, 'repo', f), 'rb') as fp: - return (fp.read(), 'fakeetag') - - mock_http_get.side_effect = http_get_def + @patch('fdroidserver.net.download_using_mirrors') + def test_download_repo_index_v2_bad_fingerprint(self, mock_download_using_mirrors): + mock_download_using_mirrors.side_effect = lambda mirrors: os.path.join( + self.testdir, 'repo', os.path.basename(mirrors[0]['url']) + ) os.chdir(self.testdir) signindex.config['keystore'] = os.path.join(self.basedir, 'keystore.jks') os.mkdir('repo') @@ -243,22 +219,26 @@ class IndexTest(unittest.TestCase): with self.assertRaises(fdroidserver.exception.VerificationException): data, _ignored = index.download_repo_index_v2(bad_fp_url) - @patch('fdroidserver.net.http_get') - def test_download_repo_index_v2_entry_verify(self, mock_http_get): - def http_get_def(url, etag, timeout): # pylint: disable=unused-argument - return (b'not the entry.jar file contents', 'fakeetag') + @patch('fdroidserver.net.download_using_mirrors') + def test_download_repo_index_v2_entry_verify(self, mock_download_using_mirrors): + def download_using_mirrors_def(mirrors): + f = os.path.join(tempfile.mkdtemp(), os.path.basename(mirrors[0]['url'])) + Path(f).write_text('not the entry.jar file contents') + return f - mock_http_get.side_effect = http_get_def + mock_download_using_mirrors.side_effect = download_using_mirrors_def url = 'https://fake.url/fdroid/repo?fingerprint=' + GP_FINGERPRINT with self.assertRaises(fdroidserver.exception.VerificationException): data, _ignored = index.download_repo_index_v2(url) - @patch('fdroidserver.net.http_get') - def test_download_repo_index_v2_index_verify(self, mock_http_get): - def http_get_def(url, etag, timeout): # pylint: disable=unused-argument - return (b'not the index-v2.json file contents', 'fakeetag') + @patch('fdroidserver.net.download_using_mirrors') + def test_download_repo_index_v2_index_verify(self, mock_download_using_mirrors): + def download_using_mirrors_def(mirrors): + f = os.path.join(tempfile.mkdtemp(), os.path.basename(mirrors[0]['url'])) + Path(f).write_text('not the index-v2.json file contents') + return f - mock_http_get.side_effect = http_get_def + mock_download_using_mirrors.side_effect = download_using_mirrors_def os.chdir(self.testdir) signindex.config['keystore'] = os.path.join(self.basedir, 'keystore.jks') os.mkdir('repo') diff --git a/tests/testcommon.py b/tests/testcommon.py index f0fd11bd..edb54fb0 100644 --- a/tests/testcommon.py +++ b/tests/testcommon.py @@ -24,6 +24,9 @@ import unittest.mock from pathlib import Path +GP_FINGERPRINT = 'B7C2EEFD8DAC7806AF67DFCD92EB18126BC08312A7F2D6F3862E46013C7A6135' + + class TmpCwd: """Context-manager for temporarily changing the current working directory.""" From f1b110942afe7bd61b272db7a0fb3c556b4689aa Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 13 Mar 2024 21:44:49 +0100 Subject: [PATCH 05/15] net: let localhost RetryServer tests run with an HTTP proxy active --- tests/net.TestCase | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/net.TestCase b/tests/net.TestCase index 725bbbfd..1c3d5e88 100755 --- a/tests/net.TestCase +++ b/tests/net.TestCase @@ -107,6 +107,7 @@ class NetTest(unittest.TestCase): self.assertTrue(os.path.exists(f)) self.assertEqual('tmp/com.downloader.aegis-3175421.apk', f) + @patch.dict(os.environ, clear=True) def test_download_file_retries(self): server = RetryServer() f = net.download_file('http://localhost:%d/f.txt' % server.port) @@ -114,6 +115,7 @@ class NetTest(unittest.TestCase): self.assertEqual(server.reply.split(b'\n\n')[1], Path(f).read_bytes()) server.stop() + @patch.dict(os.environ, clear=True) def test_download_file_retries_not_forever(self): """The retry logic should eventually exit with an error.""" server = RetryServer(failures=5) From 681d705da06d8497af743939cae49d0bdab9d3ae Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 5 Mar 2024 11:47:58 +0100 Subject: [PATCH 06/15] install: reliable algorithm for picking devices from adb output Versions of this algorithm are used elsewhere: * https://github.com/openatx/adbutils/blob/master/adbutils/_adb.py --- fdroidserver/install.py | 91 ++++++++++++++----------- tests/install.TestCase | 144 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 193 insertions(+), 42 deletions(-) diff --git a/fdroidserver/install.py b/fdroidserver/install.py index 3c9316e9..4f65e050 100644 --- a/fdroidserver/install.py +++ b/fdroidserver/install.py @@ -28,7 +28,6 @@ from urllib.parse import urlencode, urlparse, urlunparse from . import _ from . import common, index, net -from .common import SdkToolsPopen from .exception import FDroidException config = None @@ -85,14 +84,57 @@ def download_fdroid_apk(): def devices(): - p = SdkToolsPopen(['adb', "devices"]) + """Get the list of device serials for use with adb commands.""" + p = common.SdkToolsPopen(['adb', "devices"]) if p.returncode != 0: raise FDroidException("An error occured when finding devices: %s" % p.output) - lines = [line for line in p.output.splitlines() if not line.startswith('* ')] - if len(lines) < 3: - return [] - lines = lines[1:-1] - return [line.split()[0] for line in lines] + serials = list() + for line in p.output.splitlines(): + columns = line.strip().split("\t", maxsplit=1) + if len(columns) == 2: + serial, status = columns + if status == 'device': + serials.append(serial) + else: + d = {'serial': serial, 'status': status} + logging.warning(_('adb reports {serial} is "{status}"!'.format(**d))) + return serials + + +def install_apks_to_devices(apks): + """Install the list of APKs to all Android devices reported by `adb devices`.""" + for apk in apks: + # Get device list each time to avoid device not found errors + devs = devices() + if not devs: + raise FDroidException(_("No attached devices found")) + logging.info(_("Installing %s...") % apk) + for dev in devs: + logging.info( + _("Installing '{apkfilename}' on {dev}...").format( + apkfilename=apk, dev=dev + ) + ) + p = common.SdkToolsPopen(['adb', "-s", dev, "install", apk]) + fail = "" + for line in p.output.splitlines(): + if line.startswith("Failure"): + fail = line[9:-1] + if not fail: + continue + + if fail == "INSTALL_FAILED_ALREADY_EXISTS": + logging.warning( + _('"{apkfilename}" is already installed on {dev}.').format( + apkfilename=apk, dev=dev + ) + ) + else: + raise FDroidException( + _("Failed to install '{apkfilename}' on {dev}: {error}").format( + apkfilename=apk, dev=dev, error=fail + ) + ) def main(): @@ -152,45 +194,14 @@ def main(): for appid, apk in apks.items(): if not apk: raise FDroidException(_("No signed APK available for %s") % appid) + install_apks_to_devices(apks.values()) else: apks = { common.publishednameinfo(apkfile)[0]: apkfile for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))) } - - for appid, apk in apks.items(): - # Get device list each time to avoid device not found errors - devs = devices() - if not devs: - raise FDroidException(_("No attached devices found")) - logging.info(_("Installing %s...") % apk) - for dev in devs: - logging.info( - _("Installing '{apkfilename}' on {dev}...").format( - apkfilename=apk, dev=dev - ) - ) - p = SdkToolsPopen(['adb', "-s", dev, "install", apk]) - fail = "" - for line in p.output.splitlines(): - if line.startswith("Failure"): - fail = line[9:-1] - if not fail: - continue - - if fail == "INSTALL_FAILED_ALREADY_EXISTS": - logging.warning( - _('"{apkfilename}" is already installed on {dev}.').format( - apkfilename=apk, dev=dev - ) - ) - else: - raise FDroidException( - _("Failed to install '{apkfilename}' on {dev}: {error}").format( - apkfilename=apk, dev=dev, error=fail - ) - ) + install_apks_to_devices(apks.values()) logging.info('\n' + _('Finished')) diff --git a/tests/install.TestCase b/tests/install.TestCase index 540a2bf0..6e701469 100755 --- a/tests/install.TestCase +++ b/tests/install.TestCase @@ -5,9 +5,11 @@ import inspect import os import sys +import textwrap import unittest from pathlib import Path +from unittest.mock import Mock, patch localmodule = os.path.realpath( os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..') @@ -16,13 +18,17 @@ print('localmodule: ' + localmodule) if localmodule not in sys.path: sys.path.insert(0, localmodule) -import fdroidserver.common -import fdroidserver.install +import fdroidserver +from fdroidserver import common, install +from fdroidserver.exception import BuildException, FDroidException class InstallTest(unittest.TestCase): '''fdroidserver/install.py''' + def tearDown(self): + common.config = None + def test_devices(self): config = dict() fdroidserver.common.fill_config_defaults(config) @@ -35,6 +41,140 @@ class InstallTest(unittest.TestCase): for device in devices: self.assertIsInstance(device, str) + def test_devices_fail(self): + common.config = dict() + common.fill_config_defaults(common.config) + common.config['adb'] = '/bin/false' + with self.assertRaises(FDroidException): + fdroidserver.install.devices() + + def test_devices_fail_nonexistent(self): + """This is mostly just to document this strange difference in behavior""" + common.config = dict() + common.fill_config_defaults(common.config) + common.config['adb'] = '/nonexistent' + with self.assertRaises(BuildException): + fdroidserver.install.devices() + + @patch('fdroidserver.common.SdkToolsPopen') + def test_devices_with_mock_none(self, mock_SdkToolsPopen): + p = Mock() + mock_SdkToolsPopen.return_value = p + p.output = 'List of devices attached\n\n' + p.returncode = 0 + common.config = dict() + common.fill_config_defaults(common.config) + self.assertEqual([], fdroidserver.install.devices()) + + @patch('fdroidserver.common.SdkToolsPopen') + def test_devices_with_mock_one(self, mock_SdkToolsPopen): + p = Mock() + mock_SdkToolsPopen.return_value = p + p.output = 'List of devices attached\n05995813\tdevice\n\n' + p.returncode = 0 + common.config = dict() + common.fill_config_defaults(common.config) + self.assertEqual(['05995813'], fdroidserver.install.devices()) + + @patch('fdroidserver.common.SdkToolsPopen') + def test_devices_with_mock_many(self, mock_SdkToolsPopen): + p = Mock() + mock_SdkToolsPopen.return_value = p + p.output = textwrap.dedent( + """* daemon not running; starting now at tcp:5037 + * daemon started successfully + List of devices attached + RZCT809FTQM device + 05995813 device + emulator-5556 device + emulator-5554 unauthorized + 0a388e93 no permissions (missing udev rules? user is in the plugdev group); see [http://developer.android.com/tools/device.html] + 986AY133QL device + 09301JEC215064 device + 015d165c3010200e device + 4DCESKVGUC85VOTO device + + """ + ) + p.returncode = 0 + common.config = dict() + common.fill_config_defaults(common.config) + self.assertEqual( + [ + 'RZCT809FTQM', + '05995813', + 'emulator-5556', + '986AY133QL', + '09301JEC215064', + '015d165c3010200e', + '4DCESKVGUC85VOTO', + ], + fdroidserver.install.devices(), + ) + + @patch('fdroidserver.common.SdkToolsPopen') + def test_devices_with_mock_error(self, mock_SdkToolsPopen): + p = Mock() + mock_SdkToolsPopen.return_value = p + p.output = textwrap.dedent( + """* daemon not running. starting it now on port 5037 * + * daemon started successfully * + ** daemon still not running + error: cannot connect to daemon + """ + ) + p.returncode = 0 + common.config = dict() + common.fill_config_defaults(common.config) + self.assertEqual([], fdroidserver.install.devices()) + + @patch('fdroidserver.common.SdkToolsPopen') + def test_devices_with_mock_no_permissions(self, mock_SdkToolsPopen): + p = Mock() + mock_SdkToolsPopen.return_value = p + p.output = textwrap.dedent( + """List of devices attached + ???????????????? no permissions + """ + ) + p.returncode = 0 + common.config = dict() + common.fill_config_defaults(common.config) + self.assertEqual([], fdroidserver.install.devices()) + + @patch('fdroidserver.common.SdkToolsPopen') + def test_devices_with_mock_unauthorized(self, mock_SdkToolsPopen): + p = Mock() + mock_SdkToolsPopen.return_value = p + p.output = textwrap.dedent( + """List of devices attached + aeef5e4e unauthorized + """ + ) + p.returncode = 0 + common.config = dict() + common.fill_config_defaults(common.config) + self.assertEqual([], fdroidserver.install.devices()) + + @patch('fdroidserver.common.SdkToolsPopen') + def test_devices_with_mock_no_permissions_with_serial(self, mock_SdkToolsPopen): + p = Mock() + mock_SdkToolsPopen.return_value = p + p.output = textwrap.dedent( + """List of devices attached + 4DCESKVGUC85VOTO no permissions (missing udev rules? user is in the plugdev group); see [http://developer.android.com/tools/device.html] + + """ + ) + p.returncode = 0 + common.config = dict() + common.fill_config_defaults(common.config) + self.assertEqual([], fdroidserver.install.devices()) + + @patch('fdroidserver.net.download_using_mirrors', lambda m: 'testvalue') + def test_download_fdroid_apk_smokecheck(self): + self.assertEqual('testvalue', install.download_fdroid_apk()) + @unittest.skipUnless(os.getenv('test_download_fdroid_apk'), 'requires net access') def test_download_fdroid_apk(self): f = fdroidserver.install.download_fdroid_apk() From c7bc8d0fea4874d0093aae21560056a01283ca89 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 27 Feb 2024 20:02:51 +0100 Subject: [PATCH 07/15] install: function to fetch, verify and install the F-Droid.apk --- fdroidserver/install.py | 73 ++++++++++++++++++++++++++++++++++++----- tests/install.TestCase | 43 +++++++++++++++++++++++- 2 files changed, 107 insertions(+), 9 deletions(-) diff --git a/fdroidserver/install.py b/fdroidserver/install.py index 4f65e050..2bb8293a 100644 --- a/fdroidserver/install.py +++ b/fdroidserver/install.py @@ -20,6 +20,7 @@ import sys import os import glob +import locale import logging from argparse import ArgumentParser @@ -83,6 +84,60 @@ def download_fdroid_apk(): return net.download_using_mirrors([mirror]) +def install_fdroid_apk(privacy_mode=False): + """Download and install F-Droid.apk using all tricks we can muster. + + By default, this first tries to fetch the official install APK + which is offered when someone clicks the "download" button on + https://f-droid.org/. Then it will try all the mirrors and + methods until it gets something successful, or runs out of + options. + + There is privacy_mode which tries to download from mirrors first, + so that this downloads from a mirror that has many different kinds + of files available, thereby breaking the clear link to F-Droid. + + Returns + ------- + None for success or the error message. + + """ + if locale.getlocale()[0].split('_')[-1] in ('CN', 'HK', 'IR', 'TM'): + logging.warning(_('Privacy mode was enabled based on your locale.')) + privacy_mode = True + + if privacy_mode or not (config and config.get('jarsigner')): + download_methods = [download_fdroid_apk] + else: + download_methods = [download_apk, download_fdroid_apk] + for method in download_methods: + try: + f = method() + break + except Exception as e: + logging.info(e) + else: + return _('F-Droid.apk could not be downloaded from any known source!') + + if config and config['apksigner']: + # TODO this should always verify, but that requires APK sig verification in Python #94 + logging.info(_('Verifying package {path} with apksigner.').format(path=f)) + common.verify_apk_signature(f) + fingerprint = common.apk_signer_fingerprint(f) + if fingerprint.upper() != common.FDROIDORG_FINGERPRINT: + return _('{path} has the wrong fingerprint ({fingerprint})!').format( + path=f, fingerprint=fingerprint + ) + + if config and config.get('adb'): + if devices(): + install_apks_to_devices([f]) + os.remove(f) + else: + os.remove(f) + return _('No devices found for `adb install`! Please plug one in.') + + def devices(): """Get the list of device serials for use with adb commands.""" p = common.SdkToolsPopen(['adb', "devices"]) @@ -162,17 +217,16 @@ def main(): common.set_console_logging(options.verbose) if not options.appid and not options.all: - parser.error( - _("option %s: If you really want to install all the signed apps, use --all") - % "all" - ) + # TODO implement me, including a -y/--yes flag + print('TODO prompt the user if they want to download and install F-Droid.apk') config = common.read_config() output_dir = 'repo' - if not os.path.isdir(output_dir): - logging.info(_("No signed output directory - nothing to do")) - sys.exit(0) + if (options.appid or options.all) and not os.path.isdir(output_dir): + logging.error(_("No signed output directory - nothing to do")) + # TODO prompt user if they want to download from f-droid.org + sys.exit(1) if options.appid: vercodes = common.read_pkg_args(options.appid, True) @@ -196,13 +250,16 @@ def main(): raise FDroidException(_("No signed APK available for %s") % appid) install_apks_to_devices(apks.values()) - else: + elif options.all: apks = { common.publishednameinfo(apkfile)[0]: apkfile for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))) } install_apks_to_devices(apks.values()) + else: + sys.exit(install_fdroid_apk()) + logging.info('\n' + _('Finished')) diff --git a/tests/install.TestCase b/tests/install.TestCase index 6e701469..3124719b 100755 --- a/tests/install.TestCase +++ b/tests/install.TestCase @@ -171,13 +171,54 @@ class InstallTest(unittest.TestCase): common.fill_config_defaults(common.config) self.assertEqual([], fdroidserver.install.devices()) + @staticmethod + def _download_raise(privacy_mode): + raise Exception('fake failed download') + + @patch('fdroidserver.install.download_apk') + @patch('fdroidserver.install.download_fdroid_apk') + def test_install_fdroid_apk_privacy_mode_true( + self, download_fdroid_apk, download_apk + ): + download_apk.side_effect = self._download_raise + download_fdroid_apk.side_effect = self._download_raise + fdroidserver.common.config = {'jarsigner': 'fakepath'} + install.install_fdroid_apk(privacy_mode=True) + download_apk.assert_not_called() + download_fdroid_apk.assert_called_once() + + @patch('fdroidserver.install.download_apk') + @patch('fdroidserver.install.download_fdroid_apk') + def test_install_fdroid_apk_privacy_mode_false( + self, download_fdroid_apk, download_apk + ): + download_apk.side_effect = self._download_raise + download_fdroid_apk.side_effect = self._download_raise + fdroidserver.common.config = {'jarsigner': 'fakepath'} + install.install_fdroid_apk(privacy_mode=False) + download_apk.assert_not_called() + download_fdroid_apk.assert_called_once() + + @patch('fdroidserver.install.download_apk') + @patch('fdroidserver.install.download_fdroid_apk') + @patch('locale.getlocale', lambda: ('zh_CN', 'UTF-8')) + def test_install_fdroid_apk_privacy_mode_locale_auto( + self, download_fdroid_apk, download_apk + ): + download_apk.side_effect = self._download_raise + download_fdroid_apk.side_effect = self._download_raise + fdroidserver.common.config = {'jarsigner': 'fakepath'} + install.install_fdroid_apk(privacy_mode=None) + download_apk.assert_not_called() + download_fdroid_apk.assert_called_once() + @patch('fdroidserver.net.download_using_mirrors', lambda m: 'testvalue') def test_download_fdroid_apk_smokecheck(self): self.assertEqual('testvalue', install.download_fdroid_apk()) @unittest.skipUnless(os.getenv('test_download_fdroid_apk'), 'requires net access') def test_download_fdroid_apk(self): - f = fdroidserver.install.download_fdroid_apk() + f = install.download_fdroid_apk() self.assertTrue(Path(f).exists()) From 3a3417f6f880536c4eb8450a2014245afaa4add2 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 21 Oct 2024 20:04:02 +0200 Subject: [PATCH 08/15] install: add --privacy-mode flag --- completion/bash-completion | 4 ++-- fdroidserver/install.py | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/completion/bash-completion b/completion/bash-completion index 810355a0..71f1b447 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -109,8 +109,8 @@ __complete_gpgsign() { } __complete_install() { - opts="-v -q" - lopts="--verbose --quiet --all" + opts="-v -q -a -p" + lopts="--verbose --quiet --all --privacy-mode" case "${cur}" in -*) __complete_options diff --git a/fdroidserver/install.py b/fdroidserver/install.py index 2bb8293a..f9f8dd62 100644 --- a/fdroidserver/install.py +++ b/fdroidserver/install.py @@ -37,7 +37,8 @@ config = None DEFAULT_IPFS_GATEWAYS = ("https://gateway.ipfs.io/ipfs/",) -def download_apk(appid='org.fdroid.fdroid'): +# pylint: disable=unused-argument +def download_apk(appid='org.fdroid.fdroid', privacy_mode=False): """Download an APK from F-Droid via the first mirror that works.""" url = urlunparse( urlparse(common.FDROIDORG_MIRRORS[0]['url'])._replace( @@ -72,7 +73,7 @@ def download_apk(appid='org.fdroid.fdroid'): return str(f.rename(f.with_stem(f'{appid}_{versionCode}')).resolve()) -def download_fdroid_apk(): +def download_fdroid_apk(privacy_mode=False): # pylint: disable=unused-argument """Directly download the current F-Droid APK and verify it. This downloads the "download button" link, which is the version @@ -112,7 +113,7 @@ def install_fdroid_apk(privacy_mode=False): download_methods = [download_apk, download_fdroid_apk] for method in download_methods: try: - f = method() + f = method(privacy_mode=privacy_mode) break except Exception as e: logging.info(e) @@ -212,6 +213,13 @@ def main(): default=False, help=_("Install all signed applications available"), ) + parser.add_argument( + "-p", + "--privacy-mode", + action="store_true", + default=False, + help=_("Download F-Droid.apk using mirrors that leak less to the network"), + ) options = common.parse_args(parser) common.set_console_logging(options.verbose) @@ -258,7 +266,7 @@ def main(): install_apks_to_devices(apks.values()) else: - sys.exit(install_fdroid_apk()) + sys.exit(install_fdroid_apk(options.privacy_mode)) logging.info('\n' + _('Finished')) From b77eba824be8f8cb0ec062f4e0209a833b13639b Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 28 Oct 2024 22:56:35 +0100 Subject: [PATCH 09/15] install: convert to common.get_config() --- fdroidserver/install.py | 13 ++++--------- tests/install.TestCase | 2 +- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/fdroidserver/install.py b/fdroidserver/install.py index f9f8dd62..9606dbe5 100644 --- a/fdroidserver/install.py +++ b/fdroidserver/install.py @@ -31,8 +31,6 @@ from . import _ from . import common, index, net from .exception import FDroidException -config = None - DEFAULT_IPFS_GATEWAYS = ("https://gateway.ipfs.io/ipfs/",) @@ -107,7 +105,7 @@ def install_fdroid_apk(privacy_mode=False): logging.warning(_('Privacy mode was enabled based on your locale.')) privacy_mode = True - if privacy_mode or not (config and config.get('jarsigner')): + if privacy_mode or not (common.config and common.config.get('jarsigner')): download_methods = [download_fdroid_apk] else: download_methods = [download_apk, download_fdroid_apk] @@ -120,7 +118,7 @@ def install_fdroid_apk(privacy_mode=False): else: return _('F-Droid.apk could not be downloaded from any known source!') - if config and config['apksigner']: + if common.config and common.config.get('apksigner'): # TODO this should always verify, but that requires APK sig verification in Python #94 logging.info(_('Verifying package {path} with apksigner.').format(path=f)) common.verify_apk_signature(f) @@ -130,7 +128,7 @@ def install_fdroid_apk(privacy_mode=False): path=f, fingerprint=fingerprint ) - if config and config.get('adb'): + if common.config and common.config.get('adb'): if devices(): install_apks_to_devices([f]) os.remove(f) @@ -194,9 +192,6 @@ def install_apks_to_devices(apks): def main(): - global config - - # Parse command line... parser = ArgumentParser( usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]" ) @@ -228,7 +223,7 @@ def main(): # TODO implement me, including a -y/--yes flag print('TODO prompt the user if they want to download and install F-Droid.apk') - config = common.read_config() + common.get_config() output_dir = 'repo' if (options.appid or options.all) and not os.path.isdir(output_dir): diff --git a/tests/install.TestCase b/tests/install.TestCase index 3124719b..70f58b95 100755 --- a/tests/install.TestCase +++ b/tests/install.TestCase @@ -196,7 +196,7 @@ class InstallTest(unittest.TestCase): download_fdroid_apk.side_effect = self._download_raise fdroidserver.common.config = {'jarsigner': 'fakepath'} install.install_fdroid_apk(privacy_mode=False) - download_apk.assert_not_called() + download_apk.assert_called_once() download_fdroid_apk.assert_called_once() @patch('fdroidserver.install.download_apk') From 3da48e64bcc69ca874ed70021e21b5feab2d30fd Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 31 Oct 2024 23:52:15 +0100 Subject: [PATCH 10/15] install: use Maven Central as additional source --- fdroidserver/install.py | 51 +++++++++++++++++++++++++++++++++++++++-- tests/install.TestCase | 24 +++++++++++++++---- 2 files changed, 68 insertions(+), 7 deletions(-) diff --git a/fdroidserver/install.py b/fdroidserver/install.py index 9606dbe5..c3a64ae8 100644 --- a/fdroidserver/install.py +++ b/fdroidserver/install.py @@ -23,6 +23,8 @@ import glob import locale import logging +import defusedxml.ElementTree as XMLElementTree + from argparse import ArgumentParser from pathlib import Path from urllib.parse import urlencode, urlparse, urlunparse @@ -33,6 +35,27 @@ from .exception import FDroidException DEFAULT_IPFS_GATEWAYS = ("https://gateway.ipfs.io/ipfs/",) +MAVEN_CENTRAL_MIRRORS = [ + { + "url": "https://repo1.maven.org/maven2/", + "dnsA": ["199.232.16.209"], + "worksWithoutSNI": True, + }, + { + "url": "https://repo.maven.apache.org/maven2/", + "dnsA": ["199.232.16.215"], + "worksWithoutSNI": True, + }, + { + "url": "https://maven-central-asia.storage-download.googleapis.com/maven2/", + }, + { + "url": "https://maven-central-eu.storage-download.googleapis.com/maven2/", + }, + { + "url": "https://maven-central.storage-download.googleapis.com/maven2/", + }, +] # pylint: disable=unused-argument @@ -83,6 +106,24 @@ def download_fdroid_apk(privacy_mode=False): # pylint: disable=unused-argument return net.download_using_mirrors([mirror]) +def download_fdroid_apk_from_maven(privacy_mode=False): + """Download F-Droid.apk from Maven Central and official mirrors.""" + path = 'org/fdroid/fdroid/F-Droid' + if privacy_mode: + mirrors = MAVEN_CENTRAL_MIRRORS[:2] # skip the Google servers + else: + mirrors = MAVEN_CENTRAL_MIRRORS + mirrors = common.append_filename_to_mirrors( + os.path.join(path, 'maven-metadata.xml'), mirrors + ) + metadata = net.download_using_mirrors(mirrors) + version = XMLElementTree.parse(metadata).getroot().findall('*.//latest')[0].text + mirrors = common.append_filename_to_mirrors( + os.path.join(path, version, f'F-Droid-{version}.apk'), mirrors + ) + return net.download_using_mirrors(mirrors) + + def install_fdroid_apk(privacy_mode=False): """Download and install F-Droid.apk using all tricks we can muster. @@ -106,9 +147,15 @@ def install_fdroid_apk(privacy_mode=False): privacy_mode = True if privacy_mode or not (common.config and common.config.get('jarsigner')): - download_methods = [download_fdroid_apk] + download_methods = [ + download_fdroid_apk_from_maven, + ] else: - download_methods = [download_apk, download_fdroid_apk] + download_methods = [ + download_apk, + download_fdroid_apk_from_maven, + download_fdroid_apk, + ] for method in download_methods: try: f = method(privacy_mode=privacy_mode) diff --git a/tests/install.TestCase b/tests/install.TestCase index 70f58b95..c09b05d8 100755 --- a/tests/install.TestCase +++ b/tests/install.TestCase @@ -177,40 +177,49 @@ class InstallTest(unittest.TestCase): @patch('fdroidserver.install.download_apk') @patch('fdroidserver.install.download_fdroid_apk') + @patch('fdroidserver.install.download_fdroid_apk_from_maven') def test_install_fdroid_apk_privacy_mode_true( - self, download_fdroid_apk, download_apk + self, maven, download_fdroid_apk, download_apk ): download_apk.side_effect = self._download_raise download_fdroid_apk.side_effect = self._download_raise + maven.side_effect = self._download_raise fdroidserver.common.config = {'jarsigner': 'fakepath'} install.install_fdroid_apk(privacy_mode=True) download_apk.assert_not_called() - download_fdroid_apk.assert_called_once() + download_fdroid_apk.assert_not_called() + maven.assert_called_once() @patch('fdroidserver.install.download_apk') @patch('fdroidserver.install.download_fdroid_apk') + @patch('fdroidserver.install.download_fdroid_apk_from_maven') def test_install_fdroid_apk_privacy_mode_false( - self, download_fdroid_apk, download_apk + self, maven, download_fdroid_apk, download_apk ): download_apk.side_effect = self._download_raise download_fdroid_apk.side_effect = self._download_raise + maven.side_effect = self._download_raise fdroidserver.common.config = {'jarsigner': 'fakepath'} install.install_fdroid_apk(privacy_mode=False) download_apk.assert_called_once() download_fdroid_apk.assert_called_once() + maven.assert_called_once() @patch('fdroidserver.install.download_apk') @patch('fdroidserver.install.download_fdroid_apk') + @patch('fdroidserver.install.download_fdroid_apk_from_maven') @patch('locale.getlocale', lambda: ('zh_CN', 'UTF-8')) def test_install_fdroid_apk_privacy_mode_locale_auto( - self, download_fdroid_apk, download_apk + self, maven, download_fdroid_apk, download_apk ): download_apk.side_effect = self._download_raise download_fdroid_apk.side_effect = self._download_raise + maven.side_effect = self._download_raise fdroidserver.common.config = {'jarsigner': 'fakepath'} install.install_fdroid_apk(privacy_mode=None) download_apk.assert_not_called() - download_fdroid_apk.assert_called_once() + download_fdroid_apk.assert_not_called() + maven.assert_called_once() @patch('fdroidserver.net.download_using_mirrors', lambda m: 'testvalue') def test_download_fdroid_apk_smokecheck(self): @@ -221,6 +230,11 @@ class InstallTest(unittest.TestCase): f = install.download_fdroid_apk() self.assertTrue(Path(f).exists()) + @unittest.skipUnless(os.getenv('test_download_fdroid_apk'), 'requires net access') + def test_download_fdroid_apk_from_maven(self): + f = install.download_fdroid_apk_from_maven() + self.assertTrue(Path(f).exists()) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From 1eb6516f16bd2d63e2118819fd33494c50bf61e7 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 21 Oct 2024 23:20:11 +0200 Subject: [PATCH 11/15] install: add IPNS download method --- fdroidserver/install.py | 13 +++++++++++++ tests/install.TestCase | 20 +++++++++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/fdroidserver/install.py b/fdroidserver/install.py index c3a64ae8..9645f89d 100644 --- a/fdroidserver/install.py +++ b/fdroidserver/install.py @@ -106,6 +106,17 @@ def download_fdroid_apk(privacy_mode=False): # pylint: disable=unused-argument return net.download_using_mirrors([mirror]) +def download_fdroid_apk_from_ipns(privacy_mode=False): + """Download the F-Droid APK from an IPNS repo.""" + cid = 'k51qzi5uqu5dl4hbcksbdmplanu9n4hivnqsupqe6vzve1pdbeh418ssptldd3' + mirrors = [ + {"url": f"https://ipfs.io/ipns/{cid}/F-Droid.apk"}, + ] + if not privacy_mode: + mirrors.append({"url": f"https://{cid}.ipns.dweb.link/F-Droid.apk"}) + return net.download_using_mirrors(mirrors) + + def download_fdroid_apk_from_maven(privacy_mode=False): """Download F-Droid.apk from Maven Central and official mirrors.""" path = 'org/fdroid/fdroid/F-Droid' @@ -149,11 +160,13 @@ def install_fdroid_apk(privacy_mode=False): if privacy_mode or not (common.config and common.config.get('jarsigner')): download_methods = [ download_fdroid_apk_from_maven, + download_fdroid_apk_from_ipns, ] else: download_methods = [ download_apk, download_fdroid_apk_from_maven, + download_fdroid_apk_from_ipns, download_fdroid_apk, ] for method in download_methods: diff --git a/tests/install.TestCase b/tests/install.TestCase index c09b05d8..535832d2 100755 --- a/tests/install.TestCase +++ b/tests/install.TestCase @@ -177,48 +177,57 @@ class InstallTest(unittest.TestCase): @patch('fdroidserver.install.download_apk') @patch('fdroidserver.install.download_fdroid_apk') + @patch('fdroidserver.install.download_fdroid_apk_from_ipns') @patch('fdroidserver.install.download_fdroid_apk_from_maven') def test_install_fdroid_apk_privacy_mode_true( - self, maven, download_fdroid_apk, download_apk + self, maven, ipns, download_fdroid_apk, download_apk ): download_apk.side_effect = self._download_raise download_fdroid_apk.side_effect = self._download_raise + ipns.side_effect = self._download_raise maven.side_effect = self._download_raise fdroidserver.common.config = {'jarsigner': 'fakepath'} install.install_fdroid_apk(privacy_mode=True) download_apk.assert_not_called() download_fdroid_apk.assert_not_called() + ipns.assert_called_once() maven.assert_called_once() @patch('fdroidserver.install.download_apk') @patch('fdroidserver.install.download_fdroid_apk') + @patch('fdroidserver.install.download_fdroid_apk_from_ipns') @patch('fdroidserver.install.download_fdroid_apk_from_maven') def test_install_fdroid_apk_privacy_mode_false( - self, maven, download_fdroid_apk, download_apk + self, maven, ipns, download_fdroid_apk, download_apk ): download_apk.side_effect = self._download_raise download_fdroid_apk.side_effect = self._download_raise + ipns.side_effect = self._download_raise maven.side_effect = self._download_raise fdroidserver.common.config = {'jarsigner': 'fakepath'} install.install_fdroid_apk(privacy_mode=False) download_apk.assert_called_once() download_fdroid_apk.assert_called_once() + ipns.assert_called_once() maven.assert_called_once() @patch('fdroidserver.install.download_apk') @patch('fdroidserver.install.download_fdroid_apk') + @patch('fdroidserver.install.download_fdroid_apk_from_ipns') @patch('fdroidserver.install.download_fdroid_apk_from_maven') @patch('locale.getlocale', lambda: ('zh_CN', 'UTF-8')) def test_install_fdroid_apk_privacy_mode_locale_auto( - self, maven, download_fdroid_apk, download_apk + self, maven, ipns, download_fdroid_apk, download_apk ): download_apk.side_effect = self._download_raise download_fdroid_apk.side_effect = self._download_raise + ipns.side_effect = self._download_raise maven.side_effect = self._download_raise fdroidserver.common.config = {'jarsigner': 'fakepath'} install.install_fdroid_apk(privacy_mode=None) download_apk.assert_not_called() download_fdroid_apk.assert_not_called() + ipns.assert_called_once() maven.assert_called_once() @patch('fdroidserver.net.download_using_mirrors', lambda m: 'testvalue') @@ -235,6 +244,11 @@ class InstallTest(unittest.TestCase): f = install.download_fdroid_apk_from_maven() self.assertTrue(Path(f).exists()) + @unittest.skipUnless(os.getenv('test_download_fdroid_apk'), 'requires net access') + def test_download_fdroid_apk_from_ipns(self): + f = install.download_fdroid_apk_from_ipns() + self.assertTrue(Path(f).exists()) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From 560472e4e5585bdf53e251219b99658a6383041a Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 22 Oct 2024 00:12:43 +0200 Subject: [PATCH 12/15] install: download from GitHub Releases --- fdroidserver/github.py | 21 ++++++++++++++++++--- fdroidserver/install.py | 15 ++++++++++++++- tests/install.TestCase | 20 +++++++++++++++++--- 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/fdroidserver/github.py b/fdroidserver/github.py index c522f522..0a6844d9 100644 --- a/fdroidserver/github.py +++ b/fdroidserver/github.py @@ -23,12 +23,15 @@ import urllib.parse class GithubApi: - """ - Warpper for some select calls to GitHub Json/REST API. + """Wrapper for some select calls to GitHub Json/REST API. This class wraps some calls to api.github.com. This is not intended to be a general API wrapper. Instead it's purpose is to return pre-filtered and transformed data that's playing well with other fdroidserver functions. + + With the GitHub API, the token is optional, but it has pretty + severe rate limiting. + """ def __init__(self, api_token, repo_path): @@ -41,9 +44,10 @@ class GithubApi: def _req(self, url, data=None): h = { "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {self._api_token}", "X-GitHub-Api-Version": "2022-11-28", } + if self._api_token: + h["Authorization"] = f"Bearer {self._api_token}" return urllib.request.Request( url, headers=h, @@ -65,6 +69,17 @@ class GithubApi: released_tags = self.list_released_tags() return [x for x in all_tags if x not in released_tags] + def get_latest_apk(self): + req = self._req( + f"https://api.github.com/repos/{self._repo_path}/releases/latest" + ) + with urllib.request.urlopen(req) as resp: # nosec CWE-22 disable bandit warning + assets = json.load(resp)['assets'] + for asset in assets: + url = asset.get('browser_download_url') + if url and url.endswith('.apk'): + return url + def tag_exists(self, tag): """ Check if git tag is present on github. diff --git a/fdroidserver/install.py b/fdroidserver/install.py index 9645f89d..7deb1939 100644 --- a/fdroidserver/install.py +++ b/fdroidserver/install.py @@ -30,7 +30,7 @@ from pathlib import Path from urllib.parse import urlencode, urlparse, urlunparse from . import _ -from . import common, index, net +from . import common, github, index, net from .exception import FDroidException @@ -106,6 +106,17 @@ def download_fdroid_apk(privacy_mode=False): # pylint: disable=unused-argument return net.download_using_mirrors([mirror]) +def download_fdroid_apk_from_github(privacy_mode=False): + """Download F-Droid.apk from F-Droid's GitHub Releases.""" + if common.config and not privacy_mode: + token = common.config.get('github_token') + else: + token = None + gh = github.GithubApi(token, 'https://github.com/f-droid/fdroidclient') + latest_apk = gh.get_latest_apk() + return net.download_file(latest_apk) + + def download_fdroid_apk_from_ipns(privacy_mode=False): """Download the F-Droid APK from an IPNS repo.""" cid = 'k51qzi5uqu5dl4hbcksbdmplanu9n4hivnqsupqe6vzve1pdbeh418ssptldd3' @@ -161,11 +172,13 @@ def install_fdroid_apk(privacy_mode=False): download_methods = [ download_fdroid_apk_from_maven, download_fdroid_apk_from_ipns, + download_fdroid_apk_from_github, ] else: download_methods = [ download_apk, download_fdroid_apk_from_maven, + download_fdroid_apk_from_github, download_fdroid_apk_from_ipns, download_fdroid_apk, ] diff --git a/tests/install.TestCase b/tests/install.TestCase index 535832d2..ea53f6a3 100755 --- a/tests/install.TestCase +++ b/tests/install.TestCase @@ -177,56 +177,65 @@ class InstallTest(unittest.TestCase): @patch('fdroidserver.install.download_apk') @patch('fdroidserver.install.download_fdroid_apk') + @patch('fdroidserver.install.download_fdroid_apk_from_github') @patch('fdroidserver.install.download_fdroid_apk_from_ipns') @patch('fdroidserver.install.download_fdroid_apk_from_maven') def test_install_fdroid_apk_privacy_mode_true( - self, maven, ipns, download_fdroid_apk, download_apk + self, maven, ipns, github, download_fdroid_apk, download_apk ): download_apk.side_effect = self._download_raise download_fdroid_apk.side_effect = self._download_raise + github.side_effect = self._download_raise ipns.side_effect = self._download_raise maven.side_effect = self._download_raise fdroidserver.common.config = {'jarsigner': 'fakepath'} install.install_fdroid_apk(privacy_mode=True) download_apk.assert_not_called() download_fdroid_apk.assert_not_called() + github.assert_called_once() ipns.assert_called_once() maven.assert_called_once() @patch('fdroidserver.install.download_apk') @patch('fdroidserver.install.download_fdroid_apk') + @patch('fdroidserver.install.download_fdroid_apk_from_github') @patch('fdroidserver.install.download_fdroid_apk_from_ipns') @patch('fdroidserver.install.download_fdroid_apk_from_maven') def test_install_fdroid_apk_privacy_mode_false( - self, maven, ipns, download_fdroid_apk, download_apk + self, maven, ipns, github, download_fdroid_apk, download_apk ): download_apk.side_effect = self._download_raise download_fdroid_apk.side_effect = self._download_raise + github.side_effect = self._download_raise ipns.side_effect = self._download_raise maven.side_effect = self._download_raise fdroidserver.common.config = {'jarsigner': 'fakepath'} install.install_fdroid_apk(privacy_mode=False) download_apk.assert_called_once() download_fdroid_apk.assert_called_once() + github.assert_called_once() ipns.assert_called_once() maven.assert_called_once() @patch('fdroidserver.install.download_apk') @patch('fdroidserver.install.download_fdroid_apk') + @patch('fdroidserver.install.download_fdroid_apk_from_github') @patch('fdroidserver.install.download_fdroid_apk_from_ipns') @patch('fdroidserver.install.download_fdroid_apk_from_maven') @patch('locale.getlocale', lambda: ('zh_CN', 'UTF-8')) def test_install_fdroid_apk_privacy_mode_locale_auto( - self, maven, ipns, download_fdroid_apk, download_apk + self, maven, ipns, github, download_fdroid_apk, download_apk ): download_apk.side_effect = self._download_raise download_fdroid_apk.side_effect = self._download_raise + github.side_effect = self._download_raise ipns.side_effect = self._download_raise maven.side_effect = self._download_raise fdroidserver.common.config = {'jarsigner': 'fakepath'} install.install_fdroid_apk(privacy_mode=None) download_apk.assert_not_called() download_fdroid_apk.assert_not_called() + github.assert_called_once() ipns.assert_called_once() maven.assert_called_once() @@ -249,6 +258,11 @@ class InstallTest(unittest.TestCase): f = install.download_fdroid_apk_from_ipns() self.assertTrue(Path(f).exists()) + @unittest.skipUnless(os.getenv('test_download_fdroid_apk'), 'requires net access') + def test_download_fdroid_apk_from_github(self): + f = install.download_fdroid_apk_from_github() + self.assertTrue(Path(f).exists()) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From 4d22a7f67f6f38199b9d3454569ebce167fa7afd Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 22 Oct 2024 18:58:49 +0200 Subject: [PATCH 13/15] install: Y/n prompt, --yes, --no controls downloading F-Droid.apk --- completion/bash-completion | 4 +-- fdroidserver/install.py | 63 +++++++++++++++++++++++++++++++++----- tests/run-tests | 2 +- 3 files changed, 59 insertions(+), 10 deletions(-) diff --git a/completion/bash-completion b/completion/bash-completion index 71f1b447..5f6e6b23 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -109,8 +109,8 @@ __complete_gpgsign() { } __complete_install() { - opts="-v -q -a -p" - lopts="--verbose --quiet --all --privacy-mode" + opts="-v -q -a -p -n -y" + lopts="--verbose --quiet --all --privacy-mode --no --yes" case "${cur}" in -*) __complete_options diff --git a/fdroidserver/install.py b/fdroidserver/install.py index 7deb1939..f814feff 100644 --- a/fdroidserver/install.py +++ b/fdroidserver/install.py @@ -22,6 +22,8 @@ import os import glob import locale import logging +import termios +import tty import defusedxml.ElementTree as XMLElementTree @@ -264,6 +266,23 @@ def install_apks_to_devices(apks): ) +def read_char(): + """Read input from the terminal prompt one char at a time.""" + fd = sys.stdin.fileno() + old_settings = termios.tcgetattr(fd) + try: + tty.setraw(fd) + ch = sys.stdin.read(1) + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) + return ch + + +def strtobool(val): + """Convert a localized string representation of truth to True or False.""" + return val.lower() in ('', 'y', 'yes', _('yes'), _('true')) # '' is pressing Enter + + def main(): parser = ArgumentParser( usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]" @@ -288,16 +307,49 @@ def main(): default=False, help=_("Download F-Droid.apk using mirrors that leak less to the network"), ) + parser.add_argument( + "-y", + "--yes", + action="store_true", + default=None, + help=_("Automatic yes to all prompts."), + ) + parser.add_argument( + "-n", + "--no", + action="store_false", + dest='yes', + help=_("Automatic no to all prompts."), + ) options = common.parse_args(parser) common.set_console_logging(options.verbose) - - if not options.appid and not options.all: - # TODO implement me, including a -y/--yes flag - print('TODO prompt the user if they want to download and install F-Droid.apk') + logging.captureWarnings(True) # for SNIMissingWarning common.get_config() + if not options.appid and not options.all: + run_install = options.yes + if options.yes is None and sys.stdout.isatty(): + print( + _( + 'Would you like to download and install F-Droid.apk via adb? (YES/no)' + ), + flush=True, + ) + answer = '' + while True: + in_char = read_char() + if in_char == '\r': # Enter key + break + if not in_char.isprintable(): + sys.exit(1) + answer += in_char + run_install = strtobool(answer) + if run_install: + sys.exit(install_fdroid_apk(options.privacy_mode)) + sys.exit(1) + output_dir = 'repo' if (options.appid or options.all) and not os.path.isdir(output_dir): logging.error(_("No signed output directory - nothing to do")) @@ -333,9 +385,6 @@ def main(): } install_apks_to_devices(apks.values()) - else: - sys.exit(install_fdroid_apk(options.privacy_mode)) - logging.info('\n' + _('Finished')) diff --git a/tests/run-tests b/tests/run-tests index 6ba54e03..9a0dafbc 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -730,7 +730,7 @@ $fdroid scanner # run these to get their output, but the are not setup, so don't fail $fdroid build || true $fdroid import || true -$fdroid install || true +$fdroid install --no || true #------------------------------------------------------------------------------# From 27e3b5066ac191e44de2324edf2b604bd13d5723 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 22 Oct 2024 19:21:11 +0200 Subject: [PATCH 14/15] install: add --no-privacy-mode as an argument --- fdroidserver/install.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/fdroidserver/install.py b/fdroidserver/install.py index f814feff..8103a02e 100644 --- a/fdroidserver/install.py +++ b/fdroidserver/install.py @@ -27,7 +27,7 @@ import tty import defusedxml.ElementTree as XMLElementTree -from argparse import ArgumentParser +from argparse import ArgumentParser, BooleanOptionalAction from pathlib import Path from urllib.parse import urlencode, urlparse, urlunparse @@ -166,8 +166,13 @@ def install_fdroid_apk(privacy_mode=False): None for success or the error message. """ - if locale.getlocale()[0].split('_')[-1] in ('CN', 'HK', 'IR', 'TM'): - logging.warning(_('Privacy mode was enabled based on your locale.')) + country_code = locale.getlocale()[0].split('_')[-1] + if privacy_mode is None and country_code in ('CN', 'HK', 'IR', 'TM'): + logging.warning( + _('Privacy mode was enabled based on your locale ({country_code}).').format( + country_code=country_code + ) + ) privacy_mode = True if privacy_mode or not (common.config and common.config.get('jarsigner')): @@ -303,8 +308,8 @@ def main(): parser.add_argument( "-p", "--privacy-mode", - action="store_true", - default=False, + action=BooleanOptionalAction, + default=None, help=_("Download F-Droid.apk using mirrors that leak less to the network"), ) parser.add_argument( From acae5f1ac9afe62808b82992131526ab3f4590ca Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 23 Oct 2024 08:47:46 +0200 Subject: [PATCH 15/15] tests/run-tests: stop skipping install.TestCase --- tests/install.TestCase | 5 ++++- tests/run-tests | 4 ---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/install.TestCase b/tests/install.TestCase index ea53f6a3..351cc420 100755 --- a/tests/install.TestCase +++ b/tests/install.TestCase @@ -33,7 +33,10 @@ class InstallTest(unittest.TestCase): config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config - config['adb'] = fdroidserver.common.find_sdk_tools_cmd('adb') + try: + config['adb'] = fdroidserver.common.find_sdk_tools_cmd('adb') + except FDroidException as e: + self.skipTest(f'Skipping test because: {e}') self.assertTrue(os.path.exists(config['adb'])) self.assertTrue(os.path.isfile(config['adb'])) devices = fdroidserver.install.devices() diff --git a/tests/run-tests b/tests/run-tests index 9a0dafbc..303327a5 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -172,10 +172,6 @@ echo_header "run unit tests" cd $WORKSPACE/tests for testcase in $WORKSPACE/tests/*.TestCase; do - if [ $testcase == $WORKSPACE/tests/install.TestCase ]; then - echo "skipping install.TestCase, its too troublesome in CI builds" - continue - fi if [ $(uname) != "Linux" ] && [ $testcase == $WORKSPACE/tests/nightly.TestCase ]; then echo "skipping nightly.TestCase, it currently only works GNU/Linux" continue