From e8269387340a1e8e08a5537a2eb58061d3c0a3b1 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 10 Oct 2014 20:47:21 -0400 Subject: [PATCH 1/5] static URLs to "Current Version" of each app MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I wrote up the feature to automatically generate symlinks with a constant name that points to the current release version. I have it on by default, with a *config.py* option to turn it off. There is also an option to set where the symlink name comes from which defaults to app['Name'] i.e. F-Droid.apk, but can easily be set to app['id'], i.e. _org.fdroid.fdroid.apk_. I think the best place for the symlinks is in the root of the repo, so like https://f-droid.org/F-Droid.apk or https://guardianproject.info/fdroid/ChatSecure.apk For the case of the current FDroid static link https://f-droid.org/FDroid.apk it can just be a symlink to the generated one (https://f-droid.org/F-Droid.apk or https://f-droid.org/org.fdroid.fdroid.apk). Right now, this feature is all or nothing, meaning it generates symlinks for all apps in the repo, or none. I can’t think of any problems that this might cause since its only symlinks, so the amount of disk space is tiny. Also, I think it would be useful for having an easy “Download this app” button on each app’s page on the “Browse” view. As long as this button is less prominent than the “Download F-Droid” button, and it is clear that it is better to use the FDroid app than doing direct downloads. For the f-droid.org repo, the symlinks should probably be based on app['id'] to prevent name conflicts. more info here: https://f-droid.org/forums/topic/static-urls-to-current-version-of-each-app/ --- examples/config.py | 9 +++++++++ fdroidserver/common.py | 2 ++ fdroidserver/update.py | 16 ++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/examples/config.py b/examples/config.py index e1272649..974a4d5b 100644 --- a/examples/config.py +++ b/examples/config.py @@ -52,6 +52,15 @@ archive_description = """ The repository of older versions of applications from the main demo repository. """ +# `fdroid update` will create a link to the current version of a given app. +# This provides a static path to the current APK. To disable the creation of +# this link, uncomment this: +# make_current_version_link = False + +# By default, the "current version" link will be based on the "Name" of the +# app from the metadata. You can change it to use a different field from the +# metadata here: +# current_version_name_source = 'id' # The ID of a GPG key for making detached signatures for apks. Optional. # gpgkey = '1DBA2E89' diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 033cba8b..f6e9538f 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -47,6 +47,8 @@ default_config = { 'mvn3': "mvn", 'gradle': 'gradle', 'sync_from_local_copy_dir': False, + 'make_current_version_link': True, + 'current_version_name_source': 'Name', 'update_stats': False, 'stats_ignore': [], 'stats_server': None, diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 225f594d..cd59a656 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -824,7 +824,15 @@ def make_index(apps, sortedids, apks, repodir, archive, categories): apklist[i]['apkname'], apklist[i + 1]['apkname'])) sys.exit(1) + current_version_code = 0 + current_version_file = None for apk in apklist: + # find the APK for the "Current Version" + if current_version_code < apk['versioncode']: + current_version_code = apk['versioncode'] + if current_version_code < int(app['Current Version Code']): + current_version_file = apk['apkname'] + apkel = doc.createElement("package") apel.appendChild(apkel) addElement('version', apk['version'], doc, apkel) @@ -857,6 +865,14 @@ def make_index(apps, sortedids, apks, repodir, archive, categories): if len(apk['features']) > 0: addElement('features', ','.join(apk['features']), doc, apkel) + if current_version_file is not None \ + and config['make_current_version_link'] \ + and repodir == 'repo': # only create these + apklinkname = app[config['current_version_name_source']] + '.apk' + if os.path.exists(apklinkname): + os.remove(apklinkname) + os.symlink(os.path.join(repodir, current_version_file), apklinkname) + of = open(os.path.join(repodir, 'index.xml'), 'wb') if options.pretty: output = doc.toprettyxml() From 803ec39e23eb357b9d376f20fae56b49c76f895b Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 10 Oct 2014 20:54:32 -0400 Subject: [PATCH 2/5] when symlinking current version of app, include gpg sig if it exists Let's make it easy for people to also find the GPG signature, and promote easy verification! --- fdroidserver/update.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index cd59a656..b96d860e 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -869,9 +869,18 @@ def make_index(apps, sortedids, apks, repodir, archive, categories): and config['make_current_version_link'] \ and repodir == 'repo': # only create these apklinkname = app[config['current_version_name_source']] + '.apk' + current_version_path = os.path.join(repodir, current_version_file) if os.path.exists(apklinkname): os.remove(apklinkname) - os.symlink(os.path.join(repodir, current_version_file), apklinkname) + os.symlink(current_version_path, apklinkname) + # also symlink gpg signature, if it exists + for extension in ('.asc', '.sig'): + sigfile_path = current_version_path + extension + if os.path.exists(sigfile_path): + siglinkname = apklinkname + extension + if os.path.exists(siglinkname): + os.remove(siglinkname) + os.symlink(sigfile_path, siglinkname) of = open(os.path.join(repodir, 'index.xml'), 'wb') if options.pretty: From b33cae375a1edbf879be13c9412b2cbe61fab040 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 10 Oct 2014 21:12:48 -0400 Subject: [PATCH 3/5] remove url-unsafe characters from "current version" symlink names This prevents the URL from having ugly %20 stuff in the app name. --- fdroidserver/update.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index b96d860e..0339f460 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -868,7 +868,9 @@ def make_index(apps, sortedids, apks, repodir, archive, categories): if current_version_file is not None \ and config['make_current_version_link'] \ and repodir == 'repo': # only create these - apklinkname = app[config['current_version_name_source']] + '.apk' + sanitized_name = re.sub('''[ '"&%?+=]''', '', + app[config['current_version_name_source']]) + apklinkname = sanitized_name + '.apk' current_version_path = os.path.join(repodir, current_version_file) if os.path.exists(apklinkname): os.remove(apklinkname) From 8e9e17892d2da0ecc0ed712b6ef8bcf46ba793e3 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 10 Oct 2014 23:50:27 -0400 Subject: [PATCH 4/5] server: upload "current version" symlinks if requested If `fdroid update` generates the "current version" symlinks, then `fdroid server update` should upload them to the server. --- fdroidserver/server.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/fdroidserver/server.py b/fdroidserver/server.py index e0217836..a9a04ecd 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -18,6 +18,7 @@ # along with this program. If not, see . import sys +import glob import hashlib import os import paramiko @@ -143,6 +144,16 @@ def update_serverwebroot(serverwebroot, repo_section): sys.exit(1) if subprocess.call(rsyncargs + [indexjar, sectionpath]) != 0: sys.exit(1) + # upload "current version" symlinks if requested + if config['make_current_version_link'] and repo_section == 'repo': + links_to_upload = [] + for f in glob.glob('*.apk') \ + + glob.glob('*.apk.asc') + glob.glob('*.apk.sig'): + if os.path.islink(f): + links_to_upload.append(f) + if len(links_to_upload) > 0: + if subprocess.call(rsyncargs + links_to_upload + [serverwebroot]) != 0: + sys.exit(1) def _local_sync(fromdir, todir): From 7a07f5973154ea0626e4cb4b5fa35dd74b2e168a Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 10 Oct 2014 23:47:16 -0400 Subject: [PATCH 5/5] server: always use same rsync destination path to allow for strict setups for more info on how: http://positon.org/rsync-command-restriction-over-ssh http://ramblings.narrabilis.com/using-rsync-with-ssh --- fdroidserver/server.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/fdroidserver/server.py b/fdroidserver/server.py index a9a04ecd..b5da4be2 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -121,7 +121,9 @@ def update_awsbucket(repo_section): def update_serverwebroot(serverwebroot, repo_section): - rsyncargs = ['rsync', '--archive', '--delete'] + # use a checksum comparison for accurate comparisons on different + # filesystems, for example, FAT has a low resolution timestamp + rsyncargs = ['rsync', '--archive', '--delete', '--checksum'] if options.verbose: rsyncargs += ['--verbose'] if options.quiet: @@ -132,17 +134,18 @@ def update_serverwebroot(serverwebroot, repo_section): rsyncargs += ['-e', 'ssh -i ' + config['identity_file']] indexxml = os.path.join(repo_section, 'index.xml') indexjar = os.path.join(repo_section, 'index.jar') - # serverwebroot is guaranteed to have a trailing slash in common.py + # upload the first time without the index so that the repo stays working + # while this update is running. Then once it is complete, rerun the + # command again to upload the index. Always using the same target with + # rsync allows for very strict settings on the receiving server, you can + # literally specify the one rsync command that is allowed to run in + # ~/.ssh/authorized_keys. (serverwebroot is guaranteed to have a trailing + # slash in common.py) if subprocess.call(rsyncargs + ['--exclude', indexxml, '--exclude', indexjar, repo_section, serverwebroot]) != 0: sys.exit(1) - # use stricter checking on the indexes since they provide the signature - rsyncargs += ['--checksum'] - sectionpath = serverwebroot + repo_section - if subprocess.call(rsyncargs + [indexxml, sectionpath]) != 0: - sys.exit(1) - if subprocess.call(rsyncargs + [indexjar, sectionpath]) != 0: + if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0: sys.exit(1) # upload "current version" symlinks if requested if config['make_current_version_link'] and repo_section == 'repo':