diff --git a/examples/config.py b/examples/config.py index ce9f0b87..11c998a8 100644 --- a/examples/config.py +++ b/examples/config.py @@ -33,11 +33,12 @@ repo_maxage = 0 repo_url = "https://MyFirstFDroidRepo.org/fdroid/repo" repo_name = "My First FDroid Repo Demo" repo_icon = "fdroid-icon.png" -repo_description = ( - "This is a repository of apps to be used with FDroid. Applications in this " - + "repository are either official binaries built by the original application " - + "developers, or are binaries built from source by the admin of f-droid.org " - + "using the tools on https://gitlab.com/u/fdroid.") +repo_description = """ +This is a repository of apps to be used with FDroid. Applications in this +repository are either official binaries built by the original application +developers, or are binaries built from source by the admin of f-droid.org +using the tools on https://gitlab.com/u/fdroid. +""" # As above, but for the archive repo. # archive_older sets the number of versions kept in the main repo, with all @@ -47,9 +48,9 @@ archive_older = 3 archive_url = "https://f-droid.org/archive" archive_name = "My First FDroid Archive Demo" archive_icon = "fdroid-icon.png" -archive_description = ( - "The repository of older versions of applications from the main demo " - + "repository.") +archive_description = """ +The repository of older versions of applications from the main demo repository. +""" # The ID of a GPG key for making detached signatures for apks. Optional. @@ -101,9 +102,15 @@ keyaliases['com.example.another.plugin'] = '@com.example.another' # rsync/ssh format for a remote host/path. This is used for syncing a locally # generated repo to the server that is it hosted on. It must end in the # standard public repo name of "/fdroid", but can be in up to three levels of -# sub-directories (i.e. /var/www/packagerepos/fdroid). +# sub-directories (i.e. /var/www/packagerepos/fdroid). You can include +# multiple servers to sync to by wrapping the whole thing in {} or [], and +# including the serverwebroot strings in a comma-separated list. # # serverwebroot = 'user@example:/var/www/fdroid' +# serverwebroot = { +# 'foo.com:/usr/share/nginx/www/fdroid', +# 'bar.info:/var/www/fdroid', +# } # optionally specific which identity file to use when using rsync over SSH diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 663af8b1..3b1d99aa 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -62,11 +62,12 @@ def get_default_config(): 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo", 'repo_name': "My First FDroid Repo Demo", 'repo_icon': "fdroid-icon.png", - 'repo_description': ( - "This is a repository of apps to be used with FDroid. Applications in this " - + "repository are either official binaries built by the original application " - + "developers, or are binaries built from source by the admin of f-droid.org " - + "using the tools on https://gitlab.com/u/fdroid."), + 'repo_description': ''' + This is a repository of apps to be used with FDroid. Applications in this + repository are either official binaries built by the original application + developers, or are binaries built from source by the admin of f-droid.org + using the tools on https://gitlab.com/u/fdroid. + ''', 'archive_older': 0, } @@ -163,12 +164,25 @@ def read_config(opts, config_file='config.py'): if k in config: write_password_file(k) - # since this is used with rsync, where trailing slashes have meaning, - # ensure there is always a trailing slash + for k in ["repo_description", "archive_description"]: + if k in config: + config[k] = clean_description(config[k]) + if 'serverwebroot' in config: - if config['serverwebroot'][-1] != '/': - config['serverwebroot'] += '/' - config['serverwebroot'] = config['serverwebroot'].replace('//', '/') + if isinstance(config['serverwebroot'], basestring): + roots = [config['serverwebroot']] + elif all(isinstance(item, basestring) for item in config['serverwebroot']): + roots = config['serverwebroot'] + else: + raise TypeError('only accepts strings, lists, and tuples') + rootlist = [] + for rootstr in roots: + # since this is used with rsync, where trailing slashes have + # meaning, ensure there is always a trailing slash + if rootstr[-1] != '/': + rootstr += '/' + rootlist.append(rootstr.replace('//', '/')) + config['serverwebroot'] = rootlist return config @@ -290,6 +304,19 @@ def has_extension(filename, extension): apk_regex = None +def clean_description(description): + 'Remove unneeded newlines and spaces from a block of description text' + returnstring = '' + # this is split up by paragraph to make removing the newlines easier + for paragraph in re.split(r'\n\n', description): + paragraph = re.sub('\r', '', paragraph) + paragraph = re.sub('\n', ' ', paragraph) + paragraph = re.sub(' {2,}', ' ', paragraph) + paragraph = re.sub('^\s*(\w)', r'\1', paragraph) + returnstring += paragraph + '\n\n' + return returnstring.rstrip('\n') + + def apknameinfo(filename): global apk_regex filename = os.path.basename(filename) diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 4e902304..473529db 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -20,6 +20,8 @@ import sys import hashlib import os +import paramiko +import pwd import subprocess from optparse import OptionParser import logging @@ -114,7 +116,7 @@ def update_awsbucket(repo_section): logging.info(' skipping ' + s3url) -def update_serverwebroot(repo_section): +def update_serverwebroot(serverwebroot, repo_section): rsyncargs = ['rsync', '--archive', '--delete'] if options.verbose: rsyncargs += ['--verbose'] @@ -129,11 +131,11 @@ def update_serverwebroot(repo_section): # serverwebroot is guaranteed to have a trailing slash in common.py if subprocess.call(rsyncargs + ['--exclude', indexxml, '--exclude', indexjar, - repo_section, config['serverwebroot']]) != 0: + repo_section, serverwebroot]) != 0: sys.exit(1) # use stricter checking on the indexes since they provide the signature rsyncargs += ['--checksum'] - sectionpath = config['serverwebroot'] + repo_section + sectionpath = serverwebroot + repo_section if subprocess.call(rsyncargs + [indexxml, sectionpath]) != 0: sys.exit(1) if subprocess.call(rsyncargs + [indexjar, sectionpath]) != 0: @@ -141,7 +143,8 @@ def update_serverwebroot(repo_section): def _local_sync(fromdir, todir): - rsyncargs = ['rsync', '--archive', '--one-file-system', '--delete'] + rsyncargs = ['rsync', '--recursive', '--links', '--times', + '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w'] # use stricter rsync checking on all files since people using offline mode # are already prioritizing security above ease and speed rsyncargs += ['--checksum'] @@ -199,12 +202,11 @@ def main(): else: standardwebroot = True - if config.get('serverwebroot'): - serverwebroot = config['serverwebroot'] + for serverwebroot in config.get('serverwebroot', []): host, fdroiddir = serverwebroot.rstrip('/').split(':') repobase = os.path.basename(fdroiddir) if standardwebroot and repobase != 'fdroid': - logging.error('serverwebroot does not end with "fdroid", ' + logging.error('serverwebroot path does not end with "fdroid", ' + 'perhaps you meant one of these:\n\t' + serverwebroot.rstrip('/') + '/fdroid\n\t' + serverwebroot.rstrip('/').rstrip(repobase) + 'fdroid') @@ -254,18 +256,27 @@ def main(): os.mkdir('archive') if args[0] == 'init': - if config.get('serverwebroot'): - sshargs = ['ssh'] - if options.quiet: - sshargs += ['-q'] + ssh = paramiko.SSHClient() + ssh.load_system_host_keys() + for serverwebroot in config.get('serverwebroot', []): + sshstr, remotepath = serverwebroot.rstrip('/').split(':') + if sshstr.find('@') >= 0: + username, hostname = sshstr.split('@') + else: + username = pwd.getpwuid(os.getuid())[0] # get effective uid + hostname = sshstr + ssh.connect(hostname, username=username) + sftp = ssh.open_sftp() + if os.path.basename(remotepath) \ + not in sftp.listdir(os.path.dirname(remotepath)): + sftp.mkdir(remotepath, mode=0755) for repo_section in repo_sections: - cmd = sshargs + [host, 'mkdir -p', fdroiddir + '/' + repo_section] - if options.verbose: - # ssh -v produces different output than rsync -v, so this - # simulates rsync -v - logging.info(' '.join(cmd)) - if subprocess.call(cmd) != 0: - sys.exit(1) + repo_path = os.path.join(remotepath, repo_section) + if os.path.basename(repo_path) \ + not in sftp.listdir(remotepath): + sftp.mkdir(repo_path, mode=0755) + sftp.close() + ssh.close() elif args[0] == 'update': for repo_section in repo_sections: if local_copy_dir is not None: @@ -273,8 +284,8 @@ def main(): sync_from_localcopy(repo_section, local_copy_dir) else: update_localcopy(repo_section, local_copy_dir) - if config.get('serverwebroot'): - update_serverwebroot(repo_section) + for serverwebroot in config.get('serverwebroot', []): + update_serverwebroot(serverwebroot, repo_section) if config.get('awsbucket'): update_awsbucket(repo_section) diff --git a/tests/description-parsing.py b/tests/description-parsing.py new file mode 100755 index 00000000..05eba4b9 --- /dev/null +++ b/tests/description-parsing.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +import os +import sys + +sys.path.insert(1, os.path.join(os.getcwd(), '..', 'fdroidserver')) +import common + +config = common.get_default_config() + +testtext = ''' +This is a block of text that has been wrapped to fit nicely in PEP8 style: + +GnuPrivacyGuard extends the gpgcli command line tool to bring an integrated +privacy engine to your Android. It gives you command line access to the entire +GnuPG suite of encryption software. It also serves as the test bed for +complete Android integration for all of GnuPG's crypto services, including +OpenPGP, symmetric encryption, and more. + +GPG is GNU’s tool for end-to-end secure communication and encrypted data +storage. This trusted protocol is the free software alternative to PGP. This +app is built upon GnuPG 2.1, the new modularized version of GnuPG that now +supports S/MIME. + +GPG aims to provide an integrated experience, so clicking on PGP files should +"just work". You can also share files to GPG to encrypt them. GPG will also +respond when you click on a PGP fingerprint URL (one that starts with +openpgp4fpr:). + +Before using GPG, be sure to launch the app and let it finish its installation +process. Once it has completed, then you're ready to use it. The easiest way +to get started with GPG is to install [[jackpal.androidterm]]. GPG will +automatically configure Android Terminal Emulator as long as you have the +"Allow PATH extensions" settings enabled. +''' + +archive_description = """ +The repository of older versions of applications from the main demo repository. +""" + + +print('\n\n\n----------------------------------------------------') +print(common.clean_description(testtext)) +print('\n\n\n----------------------------------------------------') +print(common.clean_description(archive_description)) +print('\n\n\n----------------------------------------------------') +print(common.clean_description(config['repo_description']))