From c7962e7c6d3952bf809cb53399660349f975943f Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 2 Jul 2014 20:54:52 -0400 Subject: [PATCH 1/4] server init: replace ssh subprocess with paramiko It is easier to handle programming with python rather than subprocess calls so I replaced the subprocess call to 'ssh' with paramiko. This also makes fdroid more portable since it no longer relies on the local system having ssh installed. --- fdroidserver/server.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 4e902304..0911b5be 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -20,6 +20,8 @@ import sys import hashlib import os +import paramiko +import pwd import subprocess from optparse import OptionParser import logging @@ -255,17 +257,26 @@ def main(): if args[0] == 'init': if config.get('serverwebroot'): - sshargs = ['ssh'] - if options.quiet: - sshargs += ['-q'] + ssh = paramiko.SSHClient() + ssh.load_system_host_keys() + sshstr, remotepath = config['serverwebroot'].rstrip('/').split(':') + if sshstr.find('@') >= 0: + username, hostname = sshstr.split('@') + else: + username = pwd.getpwuid(os.getuid())[0] # get effective uid + hostname = sshstr + ssh.connect(hostname, username=username) + sftp = ssh.open_sftp() + if os.path.basename(remotepath) \ + not in sftp.listdir(os.path.dirname(remotepath)): + sftp.mkdir(remotepath, mode=0755) for repo_section in repo_sections: - cmd = sshargs + [host, 'mkdir -p', fdroiddir + '/' + repo_section] - if options.verbose: - # ssh -v produces different output than rsync -v, so this - # simulates rsync -v - logging.info(' '.join(cmd)) - if subprocess.call(cmd) != 0: - sys.exit(1) + repo_path = os.path.join(remotepath, repo_section) + if os.path.basename(repo_path) \ + not in sftp.listdir(remotepath): + sftp.mkdir(repo_path, mode=0755) + sftp.close() + ssh.close() elif args[0] == 'update': for repo_section in repo_sections: if local_copy_dir is not None: From f34c842f552b580c033424f9adfedb1a1fb46215 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 2 Jul 2014 20:57:47 -0400 Subject: [PATCH 2/4] auto-clean newlines and spaces in repo/archive descriptions This gives us flexibility in how the blocks of text can be formatted in config.py, but also provides a more useful format for displaying since the client can decide where to wrap the text. --- examples/config.py | 17 +++++++------ fdroidserver/common.py | 28 +++++++++++++++++---- tests/description-parsing.py | 48 ++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 13 deletions(-) create mode 100755 tests/description-parsing.py diff --git a/examples/config.py b/examples/config.py index ce9f0b87..3918774a 100644 --- a/examples/config.py +++ b/examples/config.py @@ -33,11 +33,12 @@ repo_maxage = 0 repo_url = "https://MyFirstFDroidRepo.org/fdroid/repo" repo_name = "My First FDroid Repo Demo" repo_icon = "fdroid-icon.png" -repo_description = ( - "This is a repository of apps to be used with FDroid. Applications in this " - + "repository are either official binaries built by the original application " - + "developers, or are binaries built from source by the admin of f-droid.org " - + "using the tools on https://gitlab.com/u/fdroid.") +repo_description = """ +This is a repository of apps to be used with FDroid. Applications in this +repository are either official binaries built by the original application +developers, or are binaries built from source by the admin of f-droid.org +using the tools on https://gitlab.com/u/fdroid. +""" # As above, but for the archive repo. # archive_older sets the number of versions kept in the main repo, with all @@ -47,9 +48,9 @@ archive_older = 3 archive_url = "https://f-droid.org/archive" archive_name = "My First FDroid Archive Demo" archive_icon = "fdroid-icon.png" -archive_description = ( - "The repository of older versions of applications from the main demo " - + "repository.") +archive_description = """ +The repository of older versions of applications from the main demo repository. +""" # The ID of a GPG key for making detached signatures for apks. Optional. diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 663af8b1..c04a5c69 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -62,11 +62,12 @@ def get_default_config(): 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo", 'repo_name': "My First FDroid Repo Demo", 'repo_icon': "fdroid-icon.png", - 'repo_description': ( - "This is a repository of apps to be used with FDroid. Applications in this " - + "repository are either official binaries built by the original application " - + "developers, or are binaries built from source by the admin of f-droid.org " - + "using the tools on https://gitlab.com/u/fdroid."), + 'repo_description': ''' + This is a repository of apps to be used with FDroid. Applications in this + repository are either official binaries built by the original application + developers, or are binaries built from source by the admin of f-droid.org + using the tools on https://gitlab.com/u/fdroid. + ''', 'archive_older': 0, } @@ -163,6 +164,10 @@ def read_config(opts, config_file='config.py'): if k in config: write_password_file(k) + for k in ["repo_description", "archive_description"]: + if k in config: + config[k] = clean_description(config[k]) + # since this is used with rsync, where trailing slashes have meaning, # ensure there is always a trailing slash if 'serverwebroot' in config: @@ -290,6 +295,19 @@ def has_extension(filename, extension): apk_regex = None +def clean_description(description): + 'Remove unneeded newlines and spaces from a block of description text' + returnstring = '' + # this is split up by paragraph to make removing the newlines easier + for paragraph in re.split(r'\n\n', description): + paragraph = re.sub('\r', '', paragraph) + paragraph = re.sub('\n', ' ', paragraph) + paragraph = re.sub(' {2,}', ' ', paragraph) + paragraph = re.sub('^\s*(\w)', r'\1', paragraph) + returnstring += paragraph + '\n\n' + return returnstring.rstrip('\n') + + def apknameinfo(filename): global apk_regex filename = os.path.basename(filename) diff --git a/tests/description-parsing.py b/tests/description-parsing.py new file mode 100755 index 00000000..05eba4b9 --- /dev/null +++ b/tests/description-parsing.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +import os +import sys + +sys.path.insert(1, os.path.join(os.getcwd(), '..', 'fdroidserver')) +import common + +config = common.get_default_config() + +testtext = ''' +This is a block of text that has been wrapped to fit nicely in PEP8 style: + +GnuPrivacyGuard extends the gpgcli command line tool to bring an integrated +privacy engine to your Android. It gives you command line access to the entire +GnuPG suite of encryption software. It also serves as the test bed for +complete Android integration for all of GnuPG's crypto services, including +OpenPGP, symmetric encryption, and more. + +GPG is GNU’s tool for end-to-end secure communication and encrypted data +storage. This trusted protocol is the free software alternative to PGP. This +app is built upon GnuPG 2.1, the new modularized version of GnuPG that now +supports S/MIME. + +GPG aims to provide an integrated experience, so clicking on PGP files should +"just work". You can also share files to GPG to encrypt them. GPG will also +respond when you click on a PGP fingerprint URL (one that starts with +openpgp4fpr:). + +Before using GPG, be sure to launch the app and let it finish its installation +process. Once it has completed, then you're ready to use it. The easiest way +to get started with GPG is to install [[jackpal.androidterm]]. GPG will +automatically configure Android Terminal Emulator as long as you have the +"Allow PATH extensions" settings enabled. +''' + +archive_description = """ +The repository of older versions of applications from the main demo repository. +""" + + +print('\n\n\n----------------------------------------------------') +print(common.clean_description(testtext)) +print('\n\n\n----------------------------------------------------') +print(common.clean_description(archive_description)) +print('\n\n\n----------------------------------------------------') +print(common.clean_description(config['repo_description'])) From 35ee4b1bc5d197b58f20cfe8172309bcf8ca74ba Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 2 Jul 2014 21:03:26 -0400 Subject: [PATCH 3/4] update local_copy_dir rsync to handle FAT and filesystems with perms With FAT filesystems, the user, group, and permissions will not be at all preserved. With file systems like ext4 that have perms, the umask might not be set to something that makes sense for the public repo files, which are meant to be published and therefore readible by all. If need be, it would be easy enough to add a config option for rsync's chmod string, to address setups that have specific permissions needs. fixes #23 https://gitlab.com/fdroid/fdroidserver/issues/23 --- fdroidserver/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 0911b5be..9cf5b919 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -143,7 +143,8 @@ def update_serverwebroot(repo_section): def _local_sync(fromdir, todir): - rsyncargs = ['rsync', '--archive', '--one-file-system', '--delete'] + rsyncargs = ['rsync', '--recursive', '--links', '--times', + '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w'] # use stricter rsync checking on all files since people using offline mode # are already prioritizing security above ease and speed rsyncargs += ['--checksum'] From 8c8fb8b156773c84b3ffcdca478c97d404895810 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 14 Jul 2014 15:03:58 -0400 Subject: [PATCH 4/4] support lists/tuples in 'serverwebroot' config item This allows the user to specify multiple servers to put the repo to, and `fdroid server update` will automatically push to them all. fixes #22 https://gitlab.com/fdroid/fdroidserver/issues/22 --- examples/config.py | 8 +++++++- fdroidserver/common.py | 19 ++++++++++++++----- fdroidserver/server.py | 23 +++++++++++------------ 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/examples/config.py b/examples/config.py index 3918774a..11c998a8 100644 --- a/examples/config.py +++ b/examples/config.py @@ -102,9 +102,15 @@ keyaliases['com.example.another.plugin'] = '@com.example.another' # rsync/ssh format for a remote host/path. This is used for syncing a locally # generated repo to the server that is it hosted on. It must end in the # standard public repo name of "/fdroid", but can be in up to three levels of -# sub-directories (i.e. /var/www/packagerepos/fdroid). +# sub-directories (i.e. /var/www/packagerepos/fdroid). You can include +# multiple servers to sync to by wrapping the whole thing in {} or [], and +# including the serverwebroot strings in a comma-separated list. # # serverwebroot = 'user@example:/var/www/fdroid' +# serverwebroot = { +# 'foo.com:/usr/share/nginx/www/fdroid', +# 'bar.info:/var/www/fdroid', +# } # optionally specific which identity file to use when using rsync over SSH diff --git a/fdroidserver/common.py b/fdroidserver/common.py index c04a5c69..3b1d99aa 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -168,12 +168,21 @@ def read_config(opts, config_file='config.py'): if k in config: config[k] = clean_description(config[k]) - # since this is used with rsync, where trailing slashes have meaning, - # ensure there is always a trailing slash if 'serverwebroot' in config: - if config['serverwebroot'][-1] != '/': - config['serverwebroot'] += '/' - config['serverwebroot'] = config['serverwebroot'].replace('//', '/') + if isinstance(config['serverwebroot'], basestring): + roots = [config['serverwebroot']] + elif all(isinstance(item, basestring) for item in config['serverwebroot']): + roots = config['serverwebroot'] + else: + raise TypeError('only accepts strings, lists, and tuples') + rootlist = [] + for rootstr in roots: + # since this is used with rsync, where trailing slashes have + # meaning, ensure there is always a trailing slash + if rootstr[-1] != '/': + rootstr += '/' + rootlist.append(rootstr.replace('//', '/')) + config['serverwebroot'] = rootlist return config diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 9cf5b919..473529db 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -116,7 +116,7 @@ def update_awsbucket(repo_section): logging.info(' skipping ' + s3url) -def update_serverwebroot(repo_section): +def update_serverwebroot(serverwebroot, repo_section): rsyncargs = ['rsync', '--archive', '--delete'] if options.verbose: rsyncargs += ['--verbose'] @@ -131,11 +131,11 @@ def update_serverwebroot(repo_section): # serverwebroot is guaranteed to have a trailing slash in common.py if subprocess.call(rsyncargs + ['--exclude', indexxml, '--exclude', indexjar, - repo_section, config['serverwebroot']]) != 0: + repo_section, serverwebroot]) != 0: sys.exit(1) # use stricter checking on the indexes since they provide the signature rsyncargs += ['--checksum'] - sectionpath = config['serverwebroot'] + repo_section + sectionpath = serverwebroot + repo_section if subprocess.call(rsyncargs + [indexxml, sectionpath]) != 0: sys.exit(1) if subprocess.call(rsyncargs + [indexjar, sectionpath]) != 0: @@ -202,12 +202,11 @@ def main(): else: standardwebroot = True - if config.get('serverwebroot'): - serverwebroot = config['serverwebroot'] + for serverwebroot in config.get('serverwebroot', []): host, fdroiddir = serverwebroot.rstrip('/').split(':') repobase = os.path.basename(fdroiddir) if standardwebroot and repobase != 'fdroid': - logging.error('serverwebroot does not end with "fdroid", ' + logging.error('serverwebroot path does not end with "fdroid", ' + 'perhaps you meant one of these:\n\t' + serverwebroot.rstrip('/') + '/fdroid\n\t' + serverwebroot.rstrip('/').rstrip(repobase) + 'fdroid') @@ -257,10 +256,10 @@ def main(): os.mkdir('archive') if args[0] == 'init': - if config.get('serverwebroot'): - ssh = paramiko.SSHClient() - ssh.load_system_host_keys() - sshstr, remotepath = config['serverwebroot'].rstrip('/').split(':') + ssh = paramiko.SSHClient() + ssh.load_system_host_keys() + for serverwebroot in config.get('serverwebroot', []): + sshstr, remotepath = serverwebroot.rstrip('/').split(':') if sshstr.find('@') >= 0: username, hostname = sshstr.split('@') else: @@ -285,8 +284,8 @@ def main(): sync_from_localcopy(repo_section, local_copy_dir) else: update_localcopy(repo_section, local_copy_dir) - if config.get('serverwebroot'): - update_serverwebroot(repo_section) + for serverwebroot in config.get('serverwebroot', []): + update_serverwebroot(serverwebroot, repo_section) if config.get('awsbucket'): update_awsbucket(repo_section)