diff --git a/MANIFEST.in b/MANIFEST.in index a323c531..47cd528e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,11 +2,10 @@ include README include COPYING include fd-commit include fdroid -include jenkins-build.sh +include jenkins-build include makebuildserver include updateplugin include buildserver/config.buildserver.py -include buildserver/cookbooks include buildserver/fixpaths.sh include buildserver/cookbooks/android-ndk/recipes/default.rb include buildserver/cookbooks/android-sdk/recipes/default.rb diff --git a/README b/README index 4a9ac075..218deff8 100644 --- a/README +++ b/README @@ -10,3 +10,21 @@ assist in creating, testing and submitting metadata to the main repository. For documentation, please see the docs directory. Alternatively, visit https://f-droid.org/manual/ + + +Installing +---------- + +The easiest way to install the fdroidserver tools is to use virtualenv and pip +(if you are Debian/Ubuntu/Mint/etc, you can first try installing using +`apt-get install fdroidserver`). First, make sure you have virtualenv +installed, it should be included in your OS's Python distribution or via other +mechanisms like dnf/yum/pacman/emerge/Fink/MacPorts/Brew. Then here's how to +install: + + git clone https://gitlab.com/fdroid/fdroidserver.git + cd fdroidserver + virtualenv env/ + . env/bin/activate + pip install -e . + python setup.py install diff --git a/docs/fdroid.texi b/docs/fdroid.texi index d1be9223..6bdcb563 100644 --- a/docs/fdroid.texi +++ b/docs/fdroid.texi @@ -160,7 +160,7 @@ certainly want to work from a git clone of the tools at this stage. To get started: @example -git clone git@gitlab.com:fdroid/fdroidserver.git +git clone https://gitlab.com/fdroid/fdroidserver.git @end example You now have lots of stuff in the fdroidserver directory, but the most @@ -177,7 +177,7 @@ repository management tasks. You can either create a brand new one, or grab a copy of the data used by the main F-Droid repository: @example -git clone git@gitlab.com:fdroid/fdroiddata.git +git clone https://gitlab.com/fdroid/fdroiddata.git @end example Regardless of the intended usage of the tools, you will always need to set diff --git a/examples/config.py b/examples/config.py index 4556233e..5c489fc7 100644 --- a/examples/config.py +++ b/examples/config.py @@ -100,10 +100,19 @@ keyaliases['com.example.another.plugin'] = '@com.example.another' # generated repo to the server that is it hosted on. It must end in the # standard public repo name of "/fdroid", but can be in up to three levels of # sub-directories (i.e. /var/www/packagerepos/fdroid). -serverwebroot = 'user@example:/var/www/fdroid' +#serverwebroot = 'user@example:/var/www/fdroid' + +# To upload the repo to an Amazon S3 bucket using `fdroid server update`. +# Warning, this deletes and recreates the whole fdroid/ directory each +# time. This is based on apache-libcloud, which supports basically all cloud +# storage services, so it should be easy to port the fdroid server tools to +# any of them. +#awsbucket = 'myawsfdroid' +#awsaccesskeyid = 'SEE0CHAITHEIMAUR2USA' +#awssecretkey = 'yourverysecretkeywordpassphraserighthere' # If you want to force 'fdroid server' to use a non-standard serverwebroot -#nonstandardwebroot = True +#nonstandardwebroot = False #Wiki details wiki_protocol = "http" diff --git a/fdroidserver/common.py b/fdroidserver/common.py index ed567af8..f9db5506 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -115,6 +115,13 @@ def read_config(opts, config_file='config.py'): if k in config: write_password_file(k) + # since this is used with rsync, where trailing slashes have meaning, + # ensure there is always a trailing slash + if 'serverwebroot' in config: + if config['serverwebroot'][-1] != '/': + config['serverwebroot'] += '/' + config['serverwebroot'] = config['serverwebroot'].replace('//', '/') + return config def write_password_file(pwtype, password=None): @@ -123,7 +130,7 @@ def write_password_file(pwtype, password=None): command line argments ''' filename = '.fdroid.' + pwtype + '.txt' - fd = os.open(filename, os.O_CREAT | os.O_WRONLY, 0600) + fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600) if password == None: os.write(fd, config[pwtype]) else: diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 83cbce82..72c75945 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -18,6 +18,7 @@ # along with this program. If not, see . import sys +import hashlib import os import subprocess from optparse import OptionParser @@ -27,9 +28,112 @@ import common config = None options = None +def update_awsbucket(repo_section): + ''' + Upload the contents of the directory `repo_section` (including + subdirectories) to the AWS S3 "bucket". The contents of that subdir of the + bucket will first be deleted. + + Requires AWS credentials set in config.py: awsaccesskeyid, awssecretkey + ''' + + import libcloud.security + libcloud.security.VERIFY_SSL_CERT = True + from libcloud.storage.types import Provider, ContainerDoesNotExistError + from libcloud.storage.providers import get_driver + + if 'awsaccesskeyid' not in config or 'awssecretkey' not in config: + logging.error('To use awsbucket, you must set awssecretkey and awsaccesskeyid in config.py!') + sys.exit(1) + awsbucket = config['awsbucket'] + + cls = get_driver(Provider.S3) + driver = cls(config['awsaccesskeyid'], config['awssecretkey']) + try: + container = driver.get_container(container_name=awsbucket) + except ContainerDoesNotExistError: + container = driver.create_container(container_name=awsbucket) + logging.info('Created new container "' + container.name + '"') + + upload_dir = 'fdroid/' + repo_section + objs = dict() + for obj in container.list_objects(): + if obj.name.startswith(upload_dir + '/'): + objs[obj.name] = obj + + for root, _, files in os.walk(os.path.join(os.getcwd(), repo_section)): + for name in files: + upload = False + file_to_upload = os.path.join(root, name) + object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd()) + if not object_name in objs: + upload = True + else: + obj = objs.pop(object_name) + if obj.size != os.path.getsize(file_to_upload): + upload = True + else: + # if the sizes match, then compare by MD5 + md5 = hashlib.md5() + with open(file_to_upload, 'rb') as f: + while True: + data = f.read(8192) + if not data: + break + md5.update(data) + if obj.hash != md5.hexdigest(): + s3url = 's3://' + awsbucket + '/' + obj.name + logging.info(' deleting ' + s3url) + if not driver.delete_object(obj): + logging.warn('Could not delete ' + s3url) + upload = True + + if upload: + if options.verbose: + logging.info(' uploading "' + file_to_upload + '"...') + extra = { 'acl': 'public-read' } + if file_to_upload.endswith('.sig'): + extra['content_type'] = 'application/pgp-signature' + elif file_to_upload.endswith('.asc'): + extra['content_type'] = 'application/pgp-signature' + logging.info(' uploading ' + os.path.relpath(file_to_upload) + + ' to s3://' + awsbucket + '/' + obj.name) + obj = driver.upload_object(file_path=file_to_upload, + container=container, + object_name=object_name, + verify_hash=False, + extra=extra) + # delete the remnants in the bucket, they do not exist locally + while objs: + object_name, obj = objs.popitem() + s3url = 's3://' + awsbucket + '/' + object_name + if object_name.startswith(upload_dir): + logging.warn(' deleting ' + s3url) + driver.delete_object(obj) + else: + logging.info(' skipping ' + s3url) + +def update_serverwebroot(repo_section): + rsyncargs = ['rsync', '-u', '-r', '--delete'] + if options.verbose: + rsyncargs += ['--verbose'] + if options.quiet: + rsyncargs += ['--quiet'] + index = os.path.join(repo_section, 'index.xml') + indexjar = os.path.join(repo_section, 'index.jar') + # serverwebroot is guaranteed to have a trailing slash in common.py + if subprocess.call(rsyncargs + + ['--exclude', index, '--exclude', indexjar, + repo_section, config['serverwebroot']]) != 0: + sys.exit(1) + if subprocess.call(rsyncargs + + [index, config['serverwebroot'] + repo_section]) != 0: + sys.exit(1) + if subprocess.call(rsyncargs + + [indexjar, config['serverwebroot'] + repo_section]) != 0: + sys.exit(1) def main(): - global config, options # Parse command line... @@ -50,44 +154,48 @@ def main(): logging.critical("The only commands currently supported are 'init' and 'update'") sys.exit(1) - serverwebroot = config['serverwebroot'].rstrip('/').replace('//', '/') - host, fdroiddir = serverwebroot.split(':') - serverrepobase = os.path.basename(fdroiddir) if 'nonstandardwebroot' in config and config['nonstandardwebroot'] == True: standardwebroot = False else: standardwebroot = True - if serverrepobase != 'fdroid' and standardwebroot: - print('ERROR: serverwebroot does not end with "fdroid", ' - + 'perhaps you meant one of these:\n\t' - + serverwebroot.rstrip('/') + '/fdroid\n\t' - + serverwebroot.rstrip('/').rstrip(serverrepobase) + 'fdroid') + + if 'serverwebroot' in config: + serverwebroot = config['serverwebroot'] + host, fdroiddir = serverwebroot.rstrip('/').split(':') + serverrepobase = os.path.basename(fdroiddir) + if serverrepobase != 'fdroid' and standardwebroot: + logging.error('serverwebroot does not end with "fdroid", ' + + 'perhaps you meant one of these:\n\t' + + serverwebroot.rstrip('/') + '/fdroid\n\t' + + serverwebroot.rstrip('/').rstrip(serverrepobase) + 'fdroid') + sys.exit(1) + elif 'awsbucket' not in config: + logging.warn('No serverwebroot or awsbucket set! Edit your config.py to set one or both.') sys.exit(1) - repodirs = ['repo'] + repo_sections = ['repo'] if config['archive_older'] != 0: - repodirs.append('archive') + repo_sections.append('archive') - for repodir in repodirs: - if args[0] == 'init': - if subprocess.call(['ssh', '-v', host, - 'mkdir -p', fdroiddir + '/' + repodir]) != 0: - sys.exit(1) - elif args[0] == 'update': - index = os.path.join(repodir, 'index.xml') - indexjar = os.path.join(repodir, 'index.jar') - if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', - '--exclude', index, '--exclude', indexjar, - repodir, config['serverwebroot']]) != 0: - sys.exit(1) - if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', - index, - config['serverwebroot'] + '/' + repodir]) != 0: - sys.exit(1) - if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', - indexjar, - config['serverwebroot'] + '/' + repodir]) != 0: - sys.exit(1) + if args[0] == 'init': + if serverwebroot != None: + sshargs = ['ssh'] + if options.quiet: + sshargs += ['-q'] + for repo_section in repo_sections: + cmd = sshargs + [host, 'mkdir -p', fdroiddir + '/' + repo_section] + if options.verbose: + # ssh -v produces different output than rsync -v, so this + # simulates rsync -v + logging.info(' '.join(cmd)) + if subprocess.call(cmd) != 0: + sys.exit(1) + elif args[0] == 'update': + for repo_section in repo_sections: + if 'serverwebroot' in config: + update_serverwebroot(repo_section) + if 'awsbucket' in config: + update_awsbucket(repo_section) sys.exit(0) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index b2d490b2..53c999a2 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -645,7 +645,10 @@ def make_index(apps, apks, repodir, archive, categories): '-storepass:file', config['keystorepassfile']] + config['smartcardoptions']) if p.returncode != 0: - logging.critical("Failed to get repo pubkey") + msg = "Failed to get repo pubkey!" + if config['keystore'] == 'NONE': + msg += ' Is your crypto smartcard plugged in?' + logging.critical(msg) sys.exit(1) global repo_pubkey_fingerprint repo_pubkey_fingerprint = cert_fingerprint(p.stdout) diff --git a/jenkins-build b/jenkins-build index 7cc2b4be..dbdea777 100755 --- a/jenkins-build +++ b/jenkins-build @@ -25,6 +25,14 @@ if [ -z $ANDROID_HOME ]; then fi fi + +#------------------------------------------------------------------------------# +# cache pypi downloads +if [ -z $PIP_DOWNLOAD_CACHE ]; then + export PIP_DOWNLOAD_CACHE=$HOME/.pip_download_cache +fi + + #------------------------------------------------------------------------------# # required Java 7 keytool/jarsigner for :file support diff --git a/setup.py b/setup.py index 35ecc8e5..b4dd7e5e 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ setup(name='fdroidserver', 'paramiko', 'PIL', 'python-magic', + 'apache-libcloud >= 0.14.1', ], classifiers=[ 'Development Status :: 3 - Alpha',