From ce54dbfc11969af7bdf51f9b28687a123db475fb Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 15 Aug 2019 12:19:29 +0200 Subject: [PATCH 1/4] setup.py: use officially documented way of including README https://packaging.python.org/tutorials/packaging-projects/ --- setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1f51e157..c255f64e 100755 --- a/setup.py +++ b/setup.py @@ -49,10 +49,14 @@ def get_data_files(): return data_files +with open("README.md", "r") as fh: + long_description = fh.read() + + setup(name='fdroidserver', version='1.2a', description='F-Droid Server Tools', - long_description='README.md', + long_description=long_description, long_description_content_type='text/markdown', author='The F-Droid Project', author_email='team@f-droid.org', From fbdecbceb7caf50fe3335eb599180b2df1e690ec Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 21 Aug 2019 14:40:41 +0200 Subject: [PATCH 2/4] setup.py: ruamel.yaml >= 0.15 required for yml rewrites --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c255f64e..f38759dc 100755 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ setup(name='fdroidserver', 'python-vagrant', 'PyYAML', 'qrcode', - 'ruamel.yaml >= 0.13', + 'ruamel.yaml >= 0.15', 'requests >= 2.5.2, != 2.11.0, != 2.12.2, != 2.18.0', 'docker-py >= 1.9, < 2.0', ], From 508af00e84e3d48aa9f0f018451867eba2336726 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 28 Aug 2019 12:25:53 +0200 Subject: [PATCH 3/4] update: only copy graphics and screenshots if mtime/size has changed Instead of copying every time, trust the filesystem to tell us when the file has changed. --- .gitignore | 1 + fdroidserver/update.py | 38 +++++++++++++++++++++++++++----------- tests/update.TestCase | 7 +++++++ 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 461fe410..73a13d4c 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,7 @@ makebuildserver.config.py /tests/archive/index.xml /tests/archive/index-v1.jar /tests/archive/index-v1.json +/tests/metadata/org.videolan.vlc/en-US/icon*.png /tests/repo/index.jar /tests/repo/index_unsigned.jar /tests/repo/index-v1.jar diff --git a/fdroidserver/update.py b/fdroidserver/update.py index f7440786..ce8ab0f4 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -707,33 +707,52 @@ def _set_author_entry(app, key, f): app[key] = text -def _strip_and_copy_image(inpath, outpath): +def _strip_and_copy_image(in_file, outpath): """Remove any metadata from image and copy it to new path Sadly, image metadata like EXIF can be used to exploit devices. It is not used at all in the F-Droid ecosystem, so its much safer just to remove it entirely. - """ + This uses size+mtime to check for a new file since this process + actually modifies the resulting file to strip out the EXIF. + + outpath can be path to either a file or dir. The dir that outpath + refers to must exist before calling this. + + """ + logging.debug('copying ' + in_file + ' ' + outpath) - extension = common.get_extension(inpath)[1] if os.path.isdir(outpath): - outpath = os.path.join(outpath, os.path.basename(inpath)) + out_file = os.path.join(outpath, os.path.basename(in_file)) + else: + out_file = outpath + + if os.path.exists(out_file): + in_stat = os.stat(in_file) + out_stat = os.stat(out_file) + if in_stat.st_size == out_stat.st_size \ + and in_stat.st_mtime == out_stat.st_mtime: + return + + extension = common.get_extension(in_file)[1] if extension == 'png': - with open(inpath, 'rb') as fp: + with open(in_file, 'rb') as fp: in_image = Image.open(fp) - in_image.save(outpath, "PNG", optimize=True, + in_image.save(out_file, "PNG", optimize=True, pnginfo=BLANK_PNG_INFO, icc_profile=None) elif extension == 'jpg' or extension == 'jpeg': - with open(inpath, 'rb') as fp: + with open(in_file, 'rb') as fp: in_image = Image.open(fp) data = list(in_image.getdata()) out_image = Image.new(in_image.mode, in_image.size) out_image.putdata(data) - out_image.save(outpath, "JPEG", optimize=True) + out_image.save(out_file, "JPEG", optimize=True) else: raise FDroidException(_('Unsupported file type "{extension}" for repo graphic') .format(extension=extension)) + stat_result = os.stat(in_file) + os.utime(out_file, times=(stat_result.st_atime, stat_result.st_mtime)) def copy_triple_t_store_metadata(apps): @@ -845,7 +864,6 @@ def copy_triple_t_store_metadata(apps): os.makedirs(destdir, mode=0o755, exist_ok=True) sourcefile = os.path.join(root, f) destfile = os.path.join(destdir, repofilename) - logging.debug('copying ' + sourcefile + ' ' + destfile) _strip_and_copy_image(sourcefile, destfile) @@ -934,7 +952,6 @@ def insert_localized_app_metadata(apps): destdir = os.path.join('repo', packageName, locale) if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS: os.makedirs(destdir, mode=0o755, exist_ok=True) - logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir) _strip_and_copy_image(os.path.join(root, f), destdir) for d in dirs: if d in SCREENSHOT_DIRS: @@ -946,7 +963,6 @@ def insert_localized_app_metadata(apps): if extension in ALLOWED_EXTENSIONS: screenshotdestdir = os.path.join(destdir, d) os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True) - logging.debug('copying ' + f + ' ' + screenshotdestdir) _strip_and_copy_image(f, screenshotdestdir) repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z]*'))) diff --git a/tests/update.TestCase b/tests/update.TestCase index 1baa141e..454780c8 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -66,6 +66,13 @@ class UpdateTest(unittest.TestCase): shutil.copytree(os.path.join('source-files', 'eu.siacs.conversations'), os.path.join('build', 'eu.siacs.conversations')) + testfile = os.path.join('repo', 'org.videolan.vlc', 'en-US', 'icon.png') + cpdir = os.path.join('metadata', 'org.videolan.vlc', 'en-US') + cpfile = os.path.join(cpdir, 'icon.png') + os.makedirs(cpdir, exist_ok=True) + shutil.copy(testfile, cpfile) + shutil.copystat(testfile, cpfile) + apps = dict() for packageName in ('info.guardianproject.urzip', 'org.videolan.vlc', 'obb.mainpatch.current', 'com.nextcloud.client', 'com.nextcloud.client.dev', From 70e7e720b9c2f3cbcf3ae44105abf613fd96767d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 28 Aug 2019 13:42:40 +0200 Subject: [PATCH 4/4] update: use graphics filename with hash in index to support caching Using a filename based on the hash of the contents means that the caching algorithms for fdroidclient and browsers can safely cache the file forever using the filename, since this guarantees that the contents will never change for a given filename. This does not cover screenshots, only icon.png, featureGraphic.png, tvBanner.png, and promoGraphic.png. fdroidserver#689 fdroid-website!453 --- .gitignore | 3 +++ CHANGELOG.md | 2 ++ fdroidserver/update.py | 42 ++++++++++++++++++++++++++++++++++++---- tests/repo/index-v1.json | 8 ++++---- tests/update.TestCase | 25 ++++++++++++++++-------- 5 files changed, 64 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 73a13d4c..25aab3b3 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,9 @@ makebuildserver.config.py /tests/repo/info.guardianproject.urzip/ /tests/repo/info.guardianproject.checkey/en-US/phoneScreenshots/checkey-phone.png /tests/repo/info.guardianproject.checkey/en-US/phoneScreenshots/checkey.png +/tests/repo/obb.mainpatch.current/en-US/featureGraphic_ffhLaojxbGAfu9ROe1MJgK5ux8d0OVc6b65nmvOBaTk=.png +/tests/repo/obb.mainpatch.current/en-US/icon_WI0pkO3LsklrsTAnRr-OQSxkkoMY41lYe2-fAvXLiLg=.png +/tests/repo/org.videolan.vlc/en-US/icon_yAfSvPRJukZzMMfUzvbYqwaD1XmHXNtiPBtuPVHW-6s=.png /tests/urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk /unsigned/ diff --git a/CHANGELOG.md b/CHANGELOG.md index a486c5f8..20f0e850 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ([!663](https://gitlab.com/fdroid/fdroidserver/merge_requests/663)) * added support for gradle 5.5.1 ([!656](https://gitlab.com/fdroid/fdroidserver/merge_requests/656)) +* add SHA256 to filename of repo graphics + ([!669](https://gitlab.com/fdroid/fdroidserver/merge_requests/669)) ### Fixed * checkupdates: UpdateCheckIngore gets properly observed now diff --git a/fdroidserver/update.py b/fdroidserver/update.py index ce8ab0f4..d1dd11bf 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -34,6 +34,7 @@ import time import copy from datetime import datetime from argparse import ArgumentParser +from base64 import urlsafe_b64encode import collections from binascii import hexlify @@ -522,6 +523,18 @@ def sha256sum(filename): return sha.hexdigest() +def sha256base64(filename): + '''Calculate the sha256 of the given file as URL-safe base64''' + hasher = hashlib.sha256() + with open(filename, 'rb') as f: + while True: + t = f.read(16384) + if len(t) == 0: + break + hasher.update(t) + return urlsafe_b64encode(hasher.digest()).decode() + + def has_known_vulnerability(filename): """checks for known vulnerabilities in the APK @@ -755,6 +768,16 @@ def _strip_and_copy_image(in_file, outpath): os.utime(out_file, times=(stat_result.st_atime, stat_result.st_mtime)) +def _get_base_hash_extension(f): + '''split a graphic/screenshot filename into base, sha256, and extension + ''' + base, extension = common.get_extension(f) + sha256_index = base.find('_') + if sha256_index > 0: + return base[:sha256_index], base[sha256_index + 1:], extension + return base, None, extension + + def copy_triple_t_store_metadata(apps): """Include store metadata from the app's source repo @@ -965,8 +988,8 @@ def insert_localized_app_metadata(apps): os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True) _strip_and_copy_image(f, screenshotdestdir) - repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z]*'))) - for d in repofiles: + repodirs = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z]*'))) + for d in repodirs: if not os.path.isdir(d): continue for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))): @@ -977,7 +1000,7 @@ def insert_localized_app_metadata(apps): locale = segments[2] screenshotdir = segments[3] filename = os.path.basename(f) - base, extension = common.get_extension(filename) + base, sha256, extension = _get_base_hash_extension(filename) if packageName not in apps: logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!') @@ -989,7 +1012,18 @@ def insert_localized_app_metadata(apps): logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f)) elif base in GRAPHIC_NAMES: # there can only be zero or one of these per locale - graphics[base] = filename + basename = base + '.' + extension + basepath = os.path.join(os.path.dirname(f), basename) + if sha256: + if not os.path.samefile(f, basepath): + os.unlink(f) + else: + sha256 = sha256base64(f) + filename = base + '_' + sha256 + '.' + extension + index_file = os.path.join(os.path.dirname(f), filename) + if not os.path.exists(index_file): + os.link(f, index_file, follow_symlinks=False) + graphics[base] = filename elif screenshotdir in SCREENSHOT_DIRS: # there can any number of these per locale logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f)) diff --git a/tests/repo/index-v1.json b/tests/repo/index-v1.json index a5a1d1e9..492b6051 100644 --- a/tests/repo/index-v1.json +++ b/tests/repo/index-v1.json @@ -141,8 +141,8 @@ "lastUpdated": 1496275200000, "localized": { "en-US": { - "featureGraphic": "featureGraphic.png", - "icon": "icon.png", + "featureGraphic": "featureGraphic_ffhLaojxbGAfu9ROe1MJgK5ux8d0OVc6b65nmvOBaTk=.png", + "icon": "icon_WI0pkO3LsklrsTAnRr-OQSxkkoMY41lYe2-fAvXLiLg=.png", "phoneScreenshots": [ "screenshot-main.png" ], @@ -196,8 +196,8 @@ "localized": { "en-US": { "description": "full description\n", - "featureGraphic": "featureGraphic.png", - "icon": "icon.png", + "featureGraphic": "featureGraphic_GFRT5BovZsENGpJq1HqPODGWBRPWQsx25B95Ol5w_wU=.png", + "icon": "icon_NJXNzMcyf-v9i5a1ElJi0j9X1LvllibCa48xXYPlOqQ=.png", "name": "title\n", "summary": "short description\n", "video": "video\n" diff --git a/tests/update.TestCase b/tests/update.TestCase index 454780c8..fc5f69bb 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -66,9 +66,10 @@ class UpdateTest(unittest.TestCase): shutil.copytree(os.path.join('source-files', 'eu.siacs.conversations'), os.path.join('build', 'eu.siacs.conversations')) + testfilename = 'icon_yAfSvPRJukZzMMfUzvbYqwaD1XmHXNtiPBtuPVHW-6s=.png' testfile = os.path.join('repo', 'org.videolan.vlc', 'en-US', 'icon.png') cpdir = os.path.join('metadata', 'org.videolan.vlc', 'en-US') - cpfile = os.path.join(cpdir, 'icon.png') + cpfile = os.path.join(cpdir, testfilename) os.makedirs(cpdir, exist_ok=True) shutil.copy(testfile, cpfile) shutil.copystat(testfile, cpfile) @@ -98,8 +99,12 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.insert_localized_app_metadata(apps) appdir = os.path.join('repo', 'info.guardianproject.urzip', 'en-US') - self.assertTrue(os.path.isfile(os.path.join(appdir, 'icon.png'))) - self.assertTrue(os.path.isfile(os.path.join(appdir, 'featureGraphic.png'))) + self.assertTrue(os.path.isfile(os.path.join( + appdir, + 'icon_NJXNzMcyf-v9i5a1ElJi0j9X1LvllibCa48xXYPlOqQ=.png'))) + self.assertTrue(os.path.isfile(os.path.join( + appdir, + 'featureGraphic_GFRT5BovZsENGpJq1HqPODGWBRPWQsx25B95Ol5w_wU=.png'))) self.assertEqual(6, len(apps)) for packageName, app in apps.items(): @@ -112,16 +117,20 @@ class UpdateTest(unittest.TestCase): self.assertEqual('title\n', app['localized']['en-US']['name']) self.assertEqual('short description\n', app['localized']['en-US']['summary']) self.assertEqual('video\n', app['localized']['en-US']['video']) - self.assertEqual('icon.png', app['localized']['en-US']['icon']) - self.assertEqual('featureGraphic.png', app['localized']['en-US']['featureGraphic']) + self.assertEqual('icon_NJXNzMcyf-v9i5a1ElJi0j9X1LvllibCa48xXYPlOqQ=.png', + app['localized']['en-US']['icon']) + self.assertEqual('featureGraphic_GFRT5BovZsENGpJq1HqPODGWBRPWQsx25B95Ol5w_wU=.png', + app['localized']['en-US']['featureGraphic']) self.assertEqual('100\n', app['localized']['en-US']['whatsNew']) elif packageName == 'org.videolan.vlc': - self.assertEqual('icon.png', app['localized']['en-US']['icon']) + self.assertEqual(testfilename, app['localized']['en-US']['icon']) self.assertEqual(9, len(app['localized']['en-US']['phoneScreenshots'])) self.assertEqual(15, len(app['localized']['en-US']['sevenInchScreenshots'])) elif packageName == 'obb.mainpatch.current': - self.assertEqual('icon.png', app['localized']['en-US']['icon']) - self.assertEqual('featureGraphic.png', app['localized']['en-US']['featureGraphic']) + self.assertEqual('icon_WI0pkO3LsklrsTAnRr-OQSxkkoMY41lYe2-fAvXLiLg=.png', + app['localized']['en-US']['icon']) + self.assertEqual('featureGraphic_ffhLaojxbGAfu9ROe1MJgK5ux8d0OVc6b65nmvOBaTk=.png', + app['localized']['en-US']['featureGraphic']) self.assertEqual(1, len(app['localized']['en-US']['phoneScreenshots'])) self.assertEqual(1, len(app['localized']['en-US']['sevenInchScreenshots'])) elif packageName == 'com.nextcloud.client':