From 6f6c074a6bd2e4f5a57e399f174ad05b681c786e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 14 Apr 2014 16:12:17 -0400 Subject: [PATCH 01/12] when running `pip install` tests, cache downloads from pypi Since this runs frequently without much change, it makes sense to cache the downloaded source packages as much as possible. There are probably better ways to do this, but this is really easy. https://stackoverflow.com/questions/4806448/how-do-i-install-from-a-local-cache-with-pip --- jenkins-build | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/jenkins-build b/jenkins-build index 7cc2b4be..dbdea777 100755 --- a/jenkins-build +++ b/jenkins-build @@ -25,6 +25,14 @@ if [ -z $ANDROID_HOME ]; then fi fi + +#------------------------------------------------------------------------------# +# cache pypi downloads +if [ -z $PIP_DOWNLOAD_CACHE ]; then + export PIP_DOWNLOAD_CACHE=$HOME/.pip_download_cache +fi + + #------------------------------------------------------------------------------# # required Java 7 keytool/jarsigner for :file support From e59e900a644f5a755a6b2199159ef62bef2d9e11 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 14 Apr 2014 22:11:36 -0400 Subject: [PATCH 02/12] fix warnings in python files manifest warning: no files found matching 'jenkins-build.sh' warning: no files found matching 'buildserver/cookbooks' --- MANIFEST.in | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index a323c531..47cd528e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,11 +2,10 @@ include README include COPYING include fd-commit include fdroid -include jenkins-build.sh +include jenkins-build include makebuildserver include updateplugin include buildserver/config.buildserver.py -include buildserver/cookbooks include buildserver/fixpaths.sh include buildserver/cookbooks/android-ndk/recipes/default.rb include buildserver/cookbooks/android-sdk/recipes/default.rb From 0cbe9690c91703c95d91a6cc82e6e0509767f7d9 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 14 Apr 2014 22:14:46 -0400 Subject: [PATCH 03/12] update manual to use public https:// URLs to git repos git@gitlab.com URLs require a gitlab login, https:// URLs do not --- docs/fdroid.texi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/fdroid.texi b/docs/fdroid.texi index d1be9223..6bdcb563 100644 --- a/docs/fdroid.texi +++ b/docs/fdroid.texi @@ -160,7 +160,7 @@ certainly want to work from a git clone of the tools at this stage. To get started: @example -git clone git@gitlab.com:fdroid/fdroidserver.git +git clone https://gitlab.com/fdroid/fdroidserver.git @end example You now have lots of stuff in the fdroidserver directory, but the most @@ -177,7 +177,7 @@ repository management tasks. You can either create a brand new one, or grab a copy of the data used by the main F-Droid repository: @example -git clone git@gitlab.com:fdroid/fdroiddata.git +git clone https://gitlab.com/fdroid/fdroiddata.git @end example Regardless of the intended usage of the tools, you will always need to set From a7d1d9a54d53a66a16a2724569126a1c91a2375d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 15 Apr 2014 23:48:48 -0400 Subject: [PATCH 04/12] overwrite password files if they exist The .fdroid.*.txt password files are only meant to be a conduit for the passwords, so blow them away everytime. The canonical password is stored in config.py. It might makes sense to replace these files with env vars using -storepass:env and -keypass:env. I figured that the passwords are already in a file, config.py, so adding more files in the same location with the same perms would not increase the risk at all. --- fdroidserver/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index ed567af8..df7c4905 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -123,7 +123,7 @@ def write_password_file(pwtype, password=None): command line argments ''' filename = '.fdroid.' + pwtype + '.txt' - fd = os.open(filename, os.O_CREAT | os.O_WRONLY, 0600) + fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600) if password == None: os.write(fd, config[pwtype]) else: From 1ca7949bb1237233f50a200eb0ab6959fe17769c Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 17 Apr 2014 14:39:47 -0400 Subject: [PATCH 05/12] if using crypto smartcard, remind user to plug it in --- fdroidserver/update.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index b2d490b2..53c999a2 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -645,7 +645,10 @@ def make_index(apps, apks, repodir, archive, categories): '-storepass:file', config['keystorepassfile']] + config['smartcardoptions']) if p.returncode != 0: - logging.critical("Failed to get repo pubkey") + msg = "Failed to get repo pubkey!" + if config['keystore'] == 'NONE': + msg += ' Is your crypto smartcard plugged in?' + logging.critical(msg) sys.exit(1) global repo_pubkey_fingerprint repo_pubkey_fingerprint = cert_fingerprint(p.stdout) From 6b65257516b324f94d81860874ab13fbe34b7916 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 14 Apr 2014 23:44:20 -0400 Subject: [PATCH 06/12] make `fdroid server` check whether serverwebroot is set Having serverwebroot optional in `fdroid server` means that it can support multiple methods of hosting, like cloud storage services. `fdroid server` can also then support multiple repo hosting options at the same time. --- examples/config.py | 4 ++-- fdroidserver/server.py | 54 +++++++++++++++++++++++------------------- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/examples/config.py b/examples/config.py index 4556233e..dcc283a0 100644 --- a/examples/config.py +++ b/examples/config.py @@ -100,10 +100,10 @@ keyaliases['com.example.another.plugin'] = '@com.example.another' # generated repo to the server that is it hosted on. It must end in the # standard public repo name of "/fdroid", but can be in up to three levels of # sub-directories (i.e. /var/www/packagerepos/fdroid). -serverwebroot = 'user@example:/var/www/fdroid' +#serverwebroot = 'user@example:/var/www/fdroid' # If you want to force 'fdroid server' to use a non-standard serverwebroot -#nonstandardwebroot = True +#nonstandardwebroot = False #Wiki details wiki_protocol = "http" diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 83cbce82..6346b778 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -50,19 +50,22 @@ def main(): logging.critical("The only commands currently supported are 'init' and 'update'") sys.exit(1) - serverwebroot = config['serverwebroot'].rstrip('/').replace('//', '/') - host, fdroiddir = serverwebroot.split(':') - serverrepobase = os.path.basename(fdroiddir) + if 'serverwebroot' in config: + serverwebroot = config['serverwebroot'].rstrip('/').replace('//', '/') + host, fdroiddir = serverwebroot.split(':') + serverrepobase = os.path.basename(fdroiddir) + if serverrepobase != 'fdroid' and standardwebroot: + logging.error('serverwebroot does not end with "fdroid", ' + + 'perhaps you meant one of these:\n\t' + + serverwebroot.rstrip('/') + '/fdroid\n\t' + + serverwebroot.rstrip('/').rstrip(serverrepobase) + 'fdroid') + sys.exit(1) + else: + serverwebroot = None if 'nonstandardwebroot' in config and config['nonstandardwebroot'] == True: standardwebroot = False else: standardwebroot = True - if serverrepobase != 'fdroid' and standardwebroot: - print('ERROR: serverwebroot does not end with "fdroid", ' - + 'perhaps you meant one of these:\n\t' - + serverwebroot.rstrip('/') + '/fdroid\n\t' - + serverwebroot.rstrip('/').rstrip(serverrepobase) + 'fdroid') - sys.exit(1) repodirs = ['repo'] if config['archive_older'] != 0: @@ -70,24 +73,27 @@ def main(): for repodir in repodirs: if args[0] == 'init': - if subprocess.call(['ssh', '-v', host, + if serverwebroot == None: + logging.warn('No serverwebroot set! Edit your config.py to set it.') + elif subprocess.call(['ssh', '-v', host, 'mkdir -p', fdroiddir + '/' + repodir]) != 0: sys.exit(1) elif args[0] == 'update': - index = os.path.join(repodir, 'index.xml') - indexjar = os.path.join(repodir, 'index.jar') - if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', - '--exclude', index, '--exclude', indexjar, - repodir, config['serverwebroot']]) != 0: - sys.exit(1) - if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', - index, - config['serverwebroot'] + '/' + repodir]) != 0: - sys.exit(1) - if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', - indexjar, - config['serverwebroot'] + '/' + repodir]) != 0: - sys.exit(1) + if serverwebroot != None: + index = os.path.join(repodir, 'index.xml') + indexjar = os.path.join(repodir, 'index.jar') + if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', + '--exclude', index, '--exclude', indexjar, + repodir, config['serverwebroot']]) != 0: + sys.exit(1) + if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', + index, + config['serverwebroot'] + '/' + repodir]) != 0: + sys.exit(1) + if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', + indexjar, + config['serverwebroot'] + '/' + repodir]) != 0: + sys.exit(1) sys.exit(0) From 5ce3b61a2a7b83b3480ac248a65862623a1014ed Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 15 Apr 2014 20:00:31 -0400 Subject: [PATCH 07/12] reorg `fdroid server` to allow for multiple server types Right now, ssh+rsync is the only supported server upload type. Things like cloud storage services are useful storage bins for fdroid repos since they are often not blocked while specific websites like Google Play are. --- fdroidserver/server.py | 65 +++++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 6346b778..b06866ab 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -28,8 +28,23 @@ config = None options = None -def main(): +def update_serverwebroot(repo_section): + index = os.path.join(repo_section, 'index.xml') + indexjar = os.path.join(repo_section, 'index.jar') + if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', + '--exclude', index, '--exclude', indexjar, + repo_section, config['serverwebroot']]) != 0: + sys.exit(1) + if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', + index, + config['serverwebroot'] + '/' + repo_section]) != 0: + sys.exit(1) + if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', + indexjar, + config['serverwebroot'] + '/' + repo_section]) != 0: + sys.exit(1) +def main(): global config, options # Parse command line... @@ -50,6 +65,11 @@ def main(): logging.critical("The only commands currently supported are 'init' and 'update'") sys.exit(1) + if 'nonstandardwebroot' in config and config['nonstandardwebroot'] == True: + standardwebroot = False + else: + standardwebroot = True + if 'serverwebroot' in config: serverwebroot = config['serverwebroot'].rstrip('/').replace('//', '/') host, fdroiddir = serverwebroot.split(':') @@ -62,38 +82,25 @@ def main(): sys.exit(1) else: serverwebroot = None - if 'nonstandardwebroot' in config and config['nonstandardwebroot'] == True: - standardwebroot = False - else: - standardwebroot = True - repodirs = ['repo'] + if serverwebroot == None: + logging.warn('No serverwebroot set! Edit your config.py to set one.') + sys.exit(1) + + repo_sections = ['repo'] if config['archive_older'] != 0: - repodirs.append('archive') + repo_sections.append('archive') - for repodir in repodirs: - if args[0] == 'init': - if serverwebroot == None: - logging.warn('No serverwebroot set! Edit your config.py to set it.') - elif subprocess.call(['ssh', '-v', host, - 'mkdir -p', fdroiddir + '/' + repodir]) != 0: - sys.exit(1) - elif args[0] == 'update': + if args[0] == 'init': + if serverwebroot != None: + for repo_section in repo_sections: + if subprocess.call(['ssh', '-v', host, + 'mkdir -p', fdroiddir + '/' + repo_section]) != 0: + sys.exit(1) + elif args[0] == 'update': + for repo_section in repo_sections: if serverwebroot != None: - index = os.path.join(repodir, 'index.xml') - indexjar = os.path.join(repodir, 'index.jar') - if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', - '--exclude', index, '--exclude', indexjar, - repodir, config['serverwebroot']]) != 0: - sys.exit(1) - if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', - index, - config['serverwebroot'] + '/' + repodir]) != 0: - sys.exit(1) - if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', - indexjar, - config['serverwebroot'] + '/' + repodir]) != 0: - sys.exit(1) + update_serverwebroot(repo_section) sys.exit(0) From d1cd817759633732c42d7b197bc681e6e088befa Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 15 Apr 2014 20:47:03 -0400 Subject: [PATCH 08/12] implement -q and -v for `fdroid server` The --quiet and --verbose options that are standard with the fdroid tools were not implemented yet with the `server` command. --- fdroidserver/common.py | 7 +++++++ fdroidserver/server.py | 35 +++++++++++++++++++++++------------ 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index df7c4905..f9db5506 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -115,6 +115,13 @@ def read_config(opts, config_file='config.py'): if k in config: write_password_file(k) + # since this is used with rsync, where trailing slashes have meaning, + # ensure there is always a trailing slash + if 'serverwebroot' in config: + if config['serverwebroot'][-1] != '/': + config['serverwebroot'] += '/' + config['serverwebroot'] = config['serverwebroot'].replace('//', '/') + return config def write_password_file(pwtype, password=None): diff --git a/fdroidserver/server.py b/fdroidserver/server.py index b06866ab..69a72f57 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -29,19 +29,23 @@ options = None def update_serverwebroot(repo_section): + rsyncargs = ['rsync', '-u', '-r', '--delete'] + if options.verbose: + rsyncargs += ['--verbose'] + if options.quiet: + rsyncargs += ['--quiet'] index = os.path.join(repo_section, 'index.xml') indexjar = os.path.join(repo_section, 'index.jar') - if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', - '--exclude', index, '--exclude', indexjar, + # serverwebroot is guaranteed to have a trailing slash in common.py + if subprocess.call(rsyncargs + + ['--exclude', index, '--exclude', indexjar, repo_section, config['serverwebroot']]) != 0: sys.exit(1) - if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', - index, - config['serverwebroot'] + '/' + repo_section]) != 0: + if subprocess.call(rsyncargs + + [index, config['serverwebroot'] + repo_section]) != 0: sys.exit(1) - if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', - indexjar, - config['serverwebroot'] + '/' + repo_section]) != 0: + if subprocess.call(rsyncargs + + [indexjar, config['serverwebroot'] + repo_section]) != 0: sys.exit(1) def main(): @@ -71,8 +75,8 @@ def main(): standardwebroot = True if 'serverwebroot' in config: - serverwebroot = config['serverwebroot'].rstrip('/').replace('//', '/') - host, fdroiddir = serverwebroot.split(':') + serverwebroot = config['serverwebroot'] + host, fdroiddir = serverwebroot.rstrip('/').split(':') serverrepobase = os.path.basename(fdroiddir) if serverrepobase != 'fdroid' and standardwebroot: logging.error('serverwebroot does not end with "fdroid", ' @@ -93,9 +97,16 @@ def main(): if args[0] == 'init': if serverwebroot != None: + sshargs = ['ssh'] + if options.quiet: + sshargs += ['-q'] for repo_section in repo_sections: - if subprocess.call(['ssh', '-v', host, - 'mkdir -p', fdroiddir + '/' + repo_section]) != 0: + cmd = sshargs + [host, 'mkdir -p', fdroiddir + '/' + repo_section] + if options.verbose: + # ssh -v produces different output than rsync -v, so this + # simulates rsync -v + logging.info(' '.join(cmd)) + if subprocess.call(cmd) != 0: sys.exit(1) elif args[0] == 'update': for repo_section in repo_sections: From f0def08adddfbbc6c05def6c86be5974329a307c Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 15 Apr 2014 22:41:28 -0400 Subject: [PATCH 09/12] support cloud storage with libcloud, starting with Amazon AWS S3 apache-libcloud enables uploading to basically any cloud storage service. This is the first implementation that allows `fdroid server` to push a repo up to a AWS S3 'bucket'. Supporting other cloud storage services should mostly be a matter of finding the libcloud "Provider" and setting the access creditials. fixes #3137 https://dev.guardianproject.info/issues/3137 --- examples/config.py | 9 +++++++ fdroidserver/server.py | 58 +++++++++++++++++++++++++++++++++++++----- setup.py | 1 + 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/examples/config.py b/examples/config.py index dcc283a0..5c489fc7 100644 --- a/examples/config.py +++ b/examples/config.py @@ -102,6 +102,15 @@ keyaliases['com.example.another.plugin'] = '@com.example.another' # sub-directories (i.e. /var/www/packagerepos/fdroid). #serverwebroot = 'user@example:/var/www/fdroid' +# To upload the repo to an Amazon S3 bucket using `fdroid server update`. +# Warning, this deletes and recreates the whole fdroid/ directory each +# time. This is based on apache-libcloud, which supports basically all cloud +# storage services, so it should be easy to port the fdroid server tools to +# any of them. +#awsbucket = 'myawsfdroid' +#awsaccesskeyid = 'SEE0CHAITHEIMAUR2USA' +#awssecretkey = 'yourverysecretkeywordpassphraserighthere' + # If you want to force 'fdroid server' to use a non-standard serverwebroot #nonstandardwebroot = False diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 69a72f57..c87df290 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -27,6 +27,53 @@ import common config = None options = None +def update_awsbucket(repo_section): + ''' + Upload the contents of the directory `repo_section` (including + subdirectories) to the AWS S3 "bucket". The contents of that subdir of the + bucket will first be deleted. + + Requires AWS credentials set in config.py: awsaccesskeyid, awssecretkey + ''' + + import libcloud.security + libcloud.security.VERIFY_SSL_CERT = True + from libcloud.storage.types import Provider + from libcloud.storage.providers import get_driver + + if 'awsaccesskeyid' not in config or 'awssecretkey' not in config: + logging.error('To use awsbucket, you must set awssecretkey and awsaccesskeyid in config.py!') + sys.exit(1) + awsbucket = config['awsbucket'] + + cls = get_driver(Provider.S3) + driver = cls(config['awsaccesskeyid'], config['awssecretkey']) + container = driver.get_container(container_name=awsbucket) + + upload_dir = 'fdroid/' + repo_section + if options.verbose: + logging.info('Deleting existing repo on Amazon S3 bucket: "' + awsbucket + + '/' + upload_dir + '"') + for obj in container.list_objects(): + if obj.name.startswith(upload_dir + '/'): + obj.delete() + if options.verbose: + logging.info(' deleted ' + obj.name) + + if options.verbose: + logging.info('Uploading to Amazon S3 bucket: "' + awsbucket + '/' + upload_dir + '"') + for root, _, files in os.walk(os.path.join(os.getcwd(), repo_section)): + for name in files: + file_to_upload = os.path.join(root, name) + object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd()) + + if options.verbose: + logging.info(' ' + file_to_upload + '...') + extra = { 'acl': 'public-read' } + driver.upload_object(file_path=file_to_upload, + container=container, + object_name=object_name, + extra=extra) def update_serverwebroot(repo_section): rsyncargs = ['rsync', '-u', '-r', '--delete'] @@ -84,11 +131,8 @@ def main(): + serverwebroot.rstrip('/') + '/fdroid\n\t' + serverwebroot.rstrip('/').rstrip(serverrepobase) + 'fdroid') sys.exit(1) - else: - serverwebroot = None - - if serverwebroot == None: - logging.warn('No serverwebroot set! Edit your config.py to set one.') + elif 'awsbucket' not in config: + logging.warn('No serverwebroot or awsbucket set! Edit your config.py to set one or both.') sys.exit(1) repo_sections = ['repo'] @@ -110,8 +154,10 @@ def main(): sys.exit(1) elif args[0] == 'update': for repo_section in repo_sections: - if serverwebroot != None: + if 'serverwebroot' in config: update_serverwebroot(repo_section) + if 'awsbucket' in config: + update_awsbucket(repo_section) sys.exit(0) diff --git a/setup.py b/setup.py index 35ecc8e5..66e8de14 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ setup(name='fdroidserver', 'paramiko', 'PIL', 'python-magic', + 'apache-libcloud >= 0.5', ], classifiers=[ 'Development Status :: 3 - Alpha', From 98033f3270f882d817ecbe9fffb93d8ff79eb33b Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 22 Apr 2014 23:11:58 -0400 Subject: [PATCH 10/12] if the AWS S3 bucket does not exist, create it This makes the AWS S3 setup dead simple: just put in a awsbucket name of your choosing, set the AWS credentials, and it'll do the rest, whether the bucket exists already or not. S3 buckets are trivial to delete too, in case of error: `s3cmd rb s3://mybadbucketname`. --- fdroidserver/server.py | 8 ++++++-- setup.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/fdroidserver/server.py b/fdroidserver/server.py index c87df290..1e78cbfc 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -38,7 +38,7 @@ def update_awsbucket(repo_section): import libcloud.security libcloud.security.VERIFY_SSL_CERT = True - from libcloud.storage.types import Provider + from libcloud.storage.types import Provider, ContainerDoesNotExistError from libcloud.storage.providers import get_driver if 'awsaccesskeyid' not in config or 'awssecretkey' not in config: @@ -48,7 +48,11 @@ def update_awsbucket(repo_section): cls = get_driver(Provider.S3) driver = cls(config['awsaccesskeyid'], config['awssecretkey']) - container = driver.get_container(container_name=awsbucket) + try: + container = driver.get_container(container_name=awsbucket) + except ContainerDoesNotExistError: + container = driver.create_container(container_name=awsbucket) + logging.info('Created new container "' + container.name + '"') upload_dir = 'fdroid/' + repo_section if options.verbose: diff --git a/setup.py b/setup.py index 66e8de14..b4dd7e5e 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup(name='fdroidserver', 'paramiko', 'PIL', 'python-magic', - 'apache-libcloud >= 0.5', + 'apache-libcloud >= 0.14.1', ], classifiers=[ 'Development Status :: 3 - Alpha', From 7248432d2d89f5c98c96ee3ec90d5162d21db652 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 17 Apr 2014 18:20:36 -0400 Subject: [PATCH 11/12] make awsbucket upload only new or changed files, ignore existing Since it is possible to check the file size and MD5 hash of the file up on the AWS S3 bucket, `fdroid server update` can check that a file needs to be updated before actually deleting and uploading the new file. fixes #3137 https://dev.guardianproject.info/issues/3137 --- fdroidserver/server.py | 64 ++++++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 1e78cbfc..72c75945 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -18,6 +18,7 @@ # along with this program. If not, see . import sys +import hashlib import os import subprocess from optparse import OptionParser @@ -55,29 +56,62 @@ def update_awsbucket(repo_section): logging.info('Created new container "' + container.name + '"') upload_dir = 'fdroid/' + repo_section - if options.verbose: - logging.info('Deleting existing repo on Amazon S3 bucket: "' + awsbucket - + '/' + upload_dir + '"') + objs = dict() for obj in container.list_objects(): if obj.name.startswith(upload_dir + '/'): - obj.delete() - if options.verbose: - logging.info(' deleted ' + obj.name) + objs[obj.name] = obj - if options.verbose: - logging.info('Uploading to Amazon S3 bucket: "' + awsbucket + '/' + upload_dir + '"') for root, _, files in os.walk(os.path.join(os.getcwd(), repo_section)): for name in files: + upload = False file_to_upload = os.path.join(root, name) object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd()) + if not object_name in objs: + upload = True + else: + obj = objs.pop(object_name) + if obj.size != os.path.getsize(file_to_upload): + upload = True + else: + # if the sizes match, then compare by MD5 + md5 = hashlib.md5() + with open(file_to_upload, 'rb') as f: + while True: + data = f.read(8192) + if not data: + break + md5.update(data) + if obj.hash != md5.hexdigest(): + s3url = 's3://' + awsbucket + '/' + obj.name + logging.info(' deleting ' + s3url) + if not driver.delete_object(obj): + logging.warn('Could not delete ' + s3url) + upload = True - if options.verbose: - logging.info(' ' + file_to_upload + '...') - extra = { 'acl': 'public-read' } - driver.upload_object(file_path=file_to_upload, - container=container, - object_name=object_name, - extra=extra) + if upload: + if options.verbose: + logging.info(' uploading "' + file_to_upload + '"...') + extra = { 'acl': 'public-read' } + if file_to_upload.endswith('.sig'): + extra['content_type'] = 'application/pgp-signature' + elif file_to_upload.endswith('.asc'): + extra['content_type'] = 'application/pgp-signature' + logging.info(' uploading ' + os.path.relpath(file_to_upload) + + ' to s3://' + awsbucket + '/' + obj.name) + obj = driver.upload_object(file_path=file_to_upload, + container=container, + object_name=object_name, + verify_hash=False, + extra=extra) + # delete the remnants in the bucket, they do not exist locally + while objs: + object_name, obj = objs.popitem() + s3url = 's3://' + awsbucket + '/' + object_name + if object_name.startswith(upload_dir): + logging.warn(' deleting ' + s3url) + driver.delete_object(obj) + else: + logging.info(' skipping ' + s3url) def update_serverwebroot(repo_section): rsyncargs = ['rsync', '-u', '-r', '--delete'] From c429751a1e2df9f16f8150a9b77ef9d10667d680 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 18 Apr 2014 13:12:29 -0400 Subject: [PATCH 12/12] add generic installation instructions to README --- README | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README b/README index 4a9ac075..218deff8 100644 --- a/README +++ b/README @@ -10,3 +10,21 @@ assist in creating, testing and submitting metadata to the main repository. For documentation, please see the docs directory. Alternatively, visit https://f-droid.org/manual/ + + +Installing +---------- + +The easiest way to install the fdroidserver tools is to use virtualenv and pip +(if you are Debian/Ubuntu/Mint/etc, you can first try installing using +`apt-get install fdroidserver`). First, make sure you have virtualenv +installed, it should be included in your OS's Python distribution or via other +mechanisms like dnf/yum/pacman/emerge/Fink/MacPorts/Brew. Then here's how to +install: + + git clone https://gitlab.com/fdroid/fdroidserver.git + cd fdroidserver + virtualenv env/ + . env/bin/activate + pip install -e . + python setup.py install