From cf0100cf1155b7544a87613d8ba2f335489d264e Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Mon, 5 Sep 2022 17:14:51 +0200 Subject: [PATCH 1/3] Merge comparisons with 'in' --- fdroidserver/checkupdates.py | 2 +- fdroidserver/common.py | 2 +- fdroidserver/import_subcommand.py | 2 +- fdroidserver/init.py | 2 +- fdroidserver/update.py | 26 ++++++++++++++------------ makebuildserver | 2 +- 6 files changed, 19 insertions(+), 17 deletions(-) diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py index 35466193..aeda879f 100644 --- a/fdroidserver/checkupdates.py +++ b/fdroidserver/checkupdates.py @@ -191,7 +191,7 @@ def check_tags(app, pattern): root_dir = build_dir / subdir paths = common.manifest_paths(root_dir, last_build.gradle) version, vercode, _package = common.parse_androidmanifests(paths, app) - if version == 'Unknown' or version == 'Ignore': + if version in ('Unknown', 'Ignore'): version = tag if vercode: logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})" diff --git a/fdroidserver/common.py b/fdroidserver/common.py index e9056634..17b939d3 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -2616,7 +2616,7 @@ def get_apk_id_androguard(apkfile): if axml.getName() == 'manifest': break - elif _type == END_TAG or _type == TEXT or _type == END_DOCUMENT: + elif _type in (END_TAG, TEXT, END_DOCUMENT): raise RuntimeError('{path}: must be the first element in AndroidManifest.xml' .format(path=apkfile)) diff --git a/fdroidserver/import_subcommand.py b/fdroidserver/import_subcommand.py index e32a9f90..ec3b0036 100644 --- a/fdroidserver/import_subcommand.py +++ b/fdroidserver/import_subcommand.py @@ -142,7 +142,7 @@ def get_app_from_url(url): app.RepoType = 'git' app.SourceCode = url app.IssueTracker = url + '/issues' - elif parsed.netloc == 'gitlab.com' or parsed.netloc == 'framagit.org': + elif parsed.netloc in ('gitlab.com', 'framagit.org'): # git can be fussy with gitlab URLs unless they end in .git if url.endswith('.git'): url = url[:-4] diff --git a/fdroidserver/init.py b/fdroidserver/init.py index c429f096..1fbba382 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -98,7 +98,7 @@ def main(): # exist, prompt the user using platform-specific default # and if the user leaves it blank, ignore and move on. default_sdk_path = '' - if sys.platform == 'win32' or sys.platform == 'cygwin': + if sys.platform in ('win32', 'cygwin'): p = os.path.join( os.getenv('USERPROFILE'), 'AppData', 'Local', 'Android', 'android-sdk' ) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index fe3a0874..11aa801e 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -108,7 +108,7 @@ def px_to_dpi(px): def get_icon_dir(repodir, density): - if density == '0' or density == '65534': + if density in ('0', '65534'): return os.path.join(repodir, "icons") else: return os.path.join(repodir, "icons-%s" % density) @@ -647,7 +647,7 @@ def _strip_and_copy_image(in_file, outpath): except Exception as e: logging.error(_("Failed copying {path}: {error}".format(path=in_file, error=e))) return - elif extension == 'jpg' or extension == 'jpeg': + elif extension in ('jpg', 'jpeg'): try: with open(in_file, 'rb') as fp: in_image = Image.open(fp) @@ -866,16 +866,16 @@ def copy_triple_t_store_metadata(apps): locale = segments[-2] for f in files: - if f == 'fulldescription' or f == 'full-description.txt': + if f in ('fulldescription', 'full-description.txt'): _set_localized_text_entry(app, locale, 'description', os.path.join(root, f)) - elif f == 'shortdescription' or f == 'short-description.txt': + elif f in ('shortdescription', 'short-description.txt'): _set_localized_text_entry(app, locale, 'summary', os.path.join(root, f)) - elif f == 'title' or f == 'title.txt': + elif f in ('title', 'title.txt'): _set_localized_text_entry(app, locale, 'name', os.path.join(root, f)) - elif f == 'video' or f == 'video-url.txt': + elif f in ('video', 'video-url.txt'): _set_localized_text_entry(app, locale, 'video', os.path.join(root, f)) elif f == 'whatsnew': @@ -884,11 +884,11 @@ def copy_triple_t_store_metadata(apps): elif f == 'default.txt' and segments[-2] == 'release-notes': _set_localized_text_entry(app, locale, 'whatsNew', os.path.join(root, f)) - elif f == 'contactEmail' or f == 'contact-email.txt': + elif f in ('contactEmail', 'contact-email.txt'): _set_author_entry(app, 'authorEmail', os.path.join(root, f)) - elif f == 'contactPhone' or f == 'contact-phone.txt': + elif f in ('contactPhone', 'contact-phone.txt'): _set_author_entry(app, 'authorPhone', os.path.join(root, f)) - elif f == 'contactWebsite' or f == 'contact-website.txt': + elif f in ('contactWebsite', 'contact-website.txt'): _set_author_entry(app, 'authorWebSite', os.path.join(root, f)) else: base, extension = common.get_extension(f) @@ -1113,7 +1113,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): repodir = repodir.encode() for name in os.listdir(repodir): file_extension = common.get_file_extension(name) - if file_extension == 'apk' or file_extension == 'obb': + if file_extension in ('apk', 'obb'): continue filename = os.path.join(repodir, name) name_utf8 = name.decode() @@ -1405,8 +1405,10 @@ def scan_apk_androguard(apk, apkfile): if key not in item.attrib: continue feature = str(item.attrib[key]) - if feature != "android.hardware.screen.portrait" \ - and feature != "android.hardware.screen.landscape": + if feature not in ( + 'android.hardware.screen.portrait', + 'android.hardware.screen.landscape', + ): if feature.startswith("android.feature."): feature = feature[16:] required = item.attrib.get(xmlns + 'required') diff --git a/makebuildserver b/makebuildserver index 428dec50..cb4d1b92 100755 --- a/makebuildserver +++ b/makebuildserver @@ -616,7 +616,7 @@ if __name__ == '__main__': virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8') except subprocess.CalledProcessError: virt = 'none' - if virt == 'qemu' or virt == 'kvm' or virt == 'bochs': + if virt in ('qemu', 'kvm', 'bochs'): logging.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt') config['vm_provider'] = 'libvirt' elif virt != 'none': From c89a9f0e8b4def0f8e77cbcba88ea515ff98a4fd Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Mon, 5 Sep 2022 17:15:15 +0200 Subject: [PATCH 2/3] Add timeout argument to requests.(get,post) --- fdroidserver/common.py | 2 +- fdroidserver/deploy.py | 10 +++++----- fdroidserver/net.py | 4 +++- fdroidserver/scanner.py | 2 +- tests/gradle-release-checksums.py | 5 ++++- tests/import_subcommand.TestCase | 2 +- tests/ndk-release-checksums.py | 3 ++- tests/net.TestCase | 2 +- tests/nightly.TestCase | 2 +- tests/openssl-version-check-test.py | 2 +- tests/scanner.TestCase | 4 +++- 11 files changed, 23 insertions(+), 15 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 17b939d3..035ee506 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1331,7 +1331,7 @@ class vcs_gitsvn(vcs): # git-svn sucks at certificate validation, this throws useful errors: try: import requests - r = requests.head(remote) + r = requests.head(remote, timeout=300) r.raise_for_status() except Exception as e: raise VCSException('SVN certificate pre-validation failed: ' + str(e)) from e diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 953c6f71..cecf8c65 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -506,7 +506,7 @@ def upload_apk_to_android_observatory(path): apkfilename = os.path.basename(path) r = requests.post('https://androidobservatory.org/', data={'q': update.sha256sum(path), 'searchby': 'hash'}, - headers=net.HEADERS) + headers=net.HEADERS, timeout=300) if r.status_code == 200: # from now on XPath will be used to retrieve the message in the HTML # androidobservatory doesn't have a nice API to talk with @@ -534,7 +534,7 @@ def upload_apk_to_android_observatory(path): r = requests.post('https://androidobservatory.org/upload', files={'apk': (apkfilename, open(path, 'rb'))}, headers=net.HEADERS, - allow_redirects=False) + allow_redirects=False, timeout=300) def upload_to_virustotal(repo_section, virustotal_apikey): @@ -586,7 +586,7 @@ def upload_apk_to_virustotal(virustotal_apikey, packageName, apkName, hash, needs_file_upload = False while True: r = requests.get('https://www.virustotal.com/vtapi/v2/file/report?' - + urllib.parse.urlencode(data), headers=headers) + + urllib.parse.urlencode(data), headers=headers, timeout=300) if r.status_code == 200: response = r.json() if response['response_code'] == 0: @@ -620,7 +620,7 @@ def upload_apk_to_virustotal(virustotal_apikey, packageName, apkName, hash, elif size > 32000000: # VirusTotal API requires fetching a URL to upload bigger files r = requests.get('https://www.virustotal.com/vtapi/v2/file/scan/upload_url?' - + urllib.parse.urlencode(data), headers=headers) + + urllib.parse.urlencode(data), headers=headers, timeout=300) if r.status_code == 200: upload_url = r.json().get('upload_url') elif r.status_code == 403: @@ -638,7 +638,7 @@ def upload_apk_to_virustotal(virustotal_apikey, packageName, apkName, hash, files = { 'file': (apkName, open(repofilename, 'rb')) } - r = requests.post(upload_url, data=data, headers=headers, files=files) + r = requests.post(upload_url, data=data, headers=headers, files=files, timeout=300) logging.debug(_('If this upload fails, try manually uploading to {url}') .format(url=manual_url)) r.raise_for_status() diff --git a/fdroidserver/net.py b/fdroidserver/net.py index da88df29..688eda68 100644 --- a/fdroidserver/net.py +++ b/fdroidserver/net.py @@ -28,7 +28,9 @@ def download_file(url, local_filename=None, dldir='tmp'): if local_filename is None: local_filename = os.path.join(dldir, filename) # the stream=True parameter keeps memory usage low - r = requests.get(url, stream=True, allow_redirects=True, headers=HEADERS) + r = requests.get( + url, stream=True, allow_redirects=True, headers=HEADERS, timeout=300 + ) r.raise_for_status() with open(local_filename, 'wb') as f: for chunk in r.iter_content(chunk_size=1024): diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index c6cdf0cf..d4f14c05 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -174,7 +174,7 @@ def load_exodus_trackers_signatures(): """ signatures = [] exodus_url = "https://reports.exodus-privacy.eu.org/api/trackers" - r = requests.get(exodus_url) + r = requests.get(exodus_url, timeout=300) data = r.json() for e in data['trackers']: signatures.append( diff --git a/tests/gradle-release-checksums.py b/tests/gradle-release-checksums.py index 1fca803c..38784052 100755 --- a/tests/gradle-release-checksums.py +++ b/tests/gradle-release-checksums.py @@ -14,7 +14,10 @@ checksums = None versions = dict() while not checksums: - r = requests.get('https://gitlab.com/fdroid/gradle-transparency-log/-/raw/master/checksums.json') + r = requests.get( + 'https://gitlab.com/fdroid/gradle-transparency-log/-/raw/master/checksums.json', + timeout=300, + ) if r.status_code == 200: checksums = r.json() diff --git a/tests/import_subcommand.TestCase b/tests/import_subcommand.TestCase index 8c2d7792..8b871ee3 100755 --- a/tests/import_subcommand.TestCase +++ b/tests/import_subcommand.TestCase @@ -45,7 +45,7 @@ class ImportTest(unittest.TestCase): fdroidserver.common.config = config url = 'https://gitlab.com/fdroid/ci-test-app' - r = requests.head(url) + r = requests.head(url, timeout=300) if r.status_code != 200: print("ERROR", url, 'unreachable (', r.status_code, ')') print('Skipping ImportTest!') diff --git a/tests/ndk-release-checksums.py b/tests/ndk-release-checksums.py index b948abf0..2f39bc68 100755 --- a/tests/ndk-release-checksums.py +++ b/tests/ndk-release-checksums.py @@ -15,7 +15,8 @@ versions = dict() while not checksums: r = requests.get( - 'https://gitlab.com/fdroid/android-sdk-transparency-log/-/raw/master/checksums.json' + 'https://gitlab.com/fdroid/android-sdk-transparency-log/-/raw/master/checksums.json', + timeout=300, ) if r.status_code == 200: checksums = r.json() diff --git a/tests/net.TestCase b/tests/net.TestCase index 3f2d51a1..e5ea6b24 100755 --- a/tests/net.TestCase +++ b/tests/net.TestCase @@ -35,7 +35,7 @@ class NetTest(unittest.TestCase): @patch('requests.get') def test_download_file_url_parsing(self, requests_get): - def _get(url, stream, allow_redirects, headers): # pylint: disable=W0613 + def _get(url, stream, allow_redirects, headers, timeout): # pylint: disable=W0613 return MagicMock() requests_get.side_effect = _get diff --git a/tests/nightly.TestCase b/tests/nightly.TestCase index afe34752..cf3ccf8b 100755 --- a/tests/nightly.TestCase +++ b/tests/nightly.TestCase @@ -33,7 +33,7 @@ class NightlyTest(unittest.TestCase): ]: url = nightly.get_repo_base_url(clone_url, repo_git_base) self.assertEqual(result, url) - r = requests.head(os.path.join(url, 'repo/index-v1.jar')) + r = requests.head(os.path.join(url, 'repo/index-v1.jar'), timeout=300) # gitlab.com often returns 403 Forbidden from their cloudflare restrictions self.assertTrue(r.status_code in (200, 403), 'should not be a redirect') diff --git a/tests/openssl-version-check-test.py b/tests/openssl-version-check-test.py index 69d4d1ce..d4022126 100755 --- a/tests/openssl-version-check-test.py +++ b/tests/openssl-version-check-test.py @@ -13,7 +13,7 @@ import requests versions = [ ] -r = requests.get('https://www.openssl.org/news/changelog.html') +r = requests.get('https://www.openssl.org/news/changelog.html', timeout=300) safe = set() bad = set() diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index ac27d0b3..9455f6cd 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -428,7 +428,9 @@ class Test_load_exodus_trackers_signatures(unittest.TestCase): "fdroidserver.scanner._exodus_compile_signatures", self.compilesig_func ): result_sigs, result_regex = fdroidserver.scanner.load_exodus_trackers_signatures() - self.requests_func.assert_called_once_with("https://reports.exodus-privacy.eu.org/api/trackers") + self.requests_func.assert_called_once_with( + "https://reports.exodus-privacy.eu.org/api/trackers", timeout=300 + ) self.assertEqual(len(result_sigs), 2) self.assertListEqual([1, 2], sorted([x.id for x in result_sigs])) From 5b79e7aea8b10e28bcaa9d72a4fa2be2dc7d6dba Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Mon, 5 Sep 2022 17:15:42 +0200 Subject: [PATCH 3/3] Fix trailing whitespace --- tests/extra/manual-vmtools-test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/extra/manual-vmtools-test.py b/tests/extra/manual-vmtools-test.py index 6b6c6964..03ef22f5 100755 --- a/tests/extra/manual-vmtools-test.py +++ b/tests/extra/manual-vmtools-test.py @@ -112,7 +112,7 @@ def main(args): if __name__ == '__main__': logging.basicConfig(format='%(message)s', level=logging.DEBUG) - + parser = ArgumentParser(description="""\ This is intended for manually testing vmtools.py