From d73b43c5fc99a4aaf3444f0c2f7628f71e333915 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 26 Jun 2014 11:57:40 -0400 Subject: [PATCH 1/6] server: 'local_copy_dir' config/options to automate offline repo signing This allows a dir to be specified in config.py that `fdroid server update` will automatically rsync the repo to. The idea is that the path would point to an SD card on a fully offline machine that serves as the secure repo signing machine. --- examples/config.py | 20 ++++++++++++++ fdroidserver/server.py | 62 ++++++++++++++++++++++++++++++++++++++---- tests/run-tests | 35 ++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 5 deletions(-) diff --git a/examples/config.py b/examples/config.py index df1ec79b..faf704bf 100644 --- a/examples/config.py +++ b/examples/config.py @@ -98,28 +98,48 @@ keyaliases['com.example.app'] = 'example' # the @ prefix. keyaliases['com.example.another.plugin'] = '@com.example.another' + # The full path to the root of the repository. It must be specified in # rsync/ssh format for a remote host/path. This is used for syncing a locally # generated repo to the server that is it hosted on. It must end in the # standard public repo name of "/fdroid", but can be in up to three levels of # sub-directories (i.e. /var/www/packagerepos/fdroid). +# # serverwebroot = 'user@example:/var/www/fdroid' + # optionally specific which identity file to use when using rsync over SSH +# # identity_file = '~/.ssh/fdroid_id_rsa' + +# If you are running the repo signing process on a completely offline machine, +# which provides the best security, then you can specify a folder to sync the +# repo to when running `fdroid server update`. This is most likely going to +# be a USB thumb drive, SD Card, or some other kind of removable media. Make +# sure it is mounted before running `fdroid server update`. Using the +# standard folder called 'fdroid' as the specified folder is recommended, like +# with serverwebroot. +# +# local_copy_dir = '/media/MyUSBThumbDrive/fdroid' + + # To upload the repo to an Amazon S3 bucket using `fdroid server update`. # Warning, this deletes and recreates the whole fdroid/ directory each # time. This is based on apache-libcloud, which supports basically all cloud # storage services, so it should be easy to port the fdroid server tools to # any of them. +# # awsbucket = 'myawsfdroid' # awsaccesskeyid = 'SEE0CHAITHEIMAUR2USA' # awssecretkey = 'yourverysecretkeywordpassphraserighthere' + # If you want to force 'fdroid server' to use a non-standard serverwebroot +# # nonstandardwebroot = False + # Wiki details wiki_protocol = "http" wiki_server = "server" diff --git a/fdroidserver/server.py b/fdroidserver/server.py index bddc4b0f..3efe5603 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -140,6 +140,20 @@ def update_serverwebroot(repo_section): sys.exit(1) +def update_localcopy(repo_section, local_copy_dir): + rsyncargs = ['rsync', '--update', '--recursive', '--delete'] + # use stricter rsync checking on all files since people using offline mode + # are already prioritizing security above ease and speed + rsyncargs += ['--checksum'] + if options.verbose: + rsyncargs += ['--verbose'] + if options.quiet: + rsyncargs += ['--quiet'] + # local_copy_dir is guaranteed to have a trailing slash in main() below + if subprocess.call(rsyncargs + [repo_section, local_copy_dir]) != 0: + sys.exit(1) + + def main(): global config, options @@ -147,6 +161,8 @@ def main(): parser = OptionParser() parser.add_option("-i", "--identity-file", default=None, help="Specify an identity file to provide to SSH for rsyncing") + parser.add_option("--local-copy-dir", default=None, + help="Specify a local folder to sync the repo to") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-q", "--quiet", action="store_true", default=False, @@ -171,15 +187,49 @@ def main(): if config.get('serverwebroot'): serverwebroot = config['serverwebroot'] host, fdroiddir = serverwebroot.rstrip('/').split(':') - serverrepobase = os.path.basename(fdroiddir) - if serverrepobase != 'fdroid' and standardwebroot: + repobase = os.path.basename(fdroiddir) + if standardwebroot and repobase != 'fdroid': logging.error('serverwebroot does not end with "fdroid", ' + 'perhaps you meant one of these:\n\t' + serverwebroot.rstrip('/') + '/fdroid\n\t' - + serverwebroot.rstrip('/').rstrip(serverrepobase) + 'fdroid') + + serverwebroot.rstrip('/').rstrip(repobase) + 'fdroid') sys.exit(1) - elif not config.get('awsbucket'): - logging.warn('No serverwebroot or awsbucket set! Edit your config.py to set one or both.') + + if options.local_copy_dir is not None: + local_copy_dir = options.local_copy_dir + elif config.get('local_copy_dir'): + local_copy_dir = config['local_copy_dir'] + else: + local_copy_dir = None + if local_copy_dir is not None: + fdroiddir = local_copy_dir.rstrip('/') + if os.path.exists(fdroiddir) and not os.path.isdir(fdroiddir): + logging.error('local_copy_dir must be directory, not a file!') + sys.exit(1) + if not os.path.exists(os.path.dirname(fdroiddir)): + logging.error('The root dir for local_copy_dir "' + + os.path.dirname(fdroiddir) + + '" does not exist!') + sys.exit(1) + if not os.path.isabs(fdroiddir): + logging.error('local_copy_dir must be an absolute path!') + sys.exit(1) + repobase = os.path.basename(fdroiddir) + if standardwebroot and repobase != 'fdroid': + logging.error('local_copy_dir does not end with "fdroid", ' + + 'perhaps you meant: ' + fdroiddir + '/fdroid') + sys.exit(1) + if local_copy_dir[-1] != '/': + local_copy_dir += '/' + local_copy_dir = local_copy_dir.replace('//', '/') + if not os.path.exists(fdroiddir): + os.mkdir(fdroiddir) + + if not config.get('awsbucket') \ + and not config.get('serverwebroot') \ + and local_copy_dir is None: + logging.warn('No serverwebroot, local_copy_dir, or awsbucket set!' + + 'Edit your config.py to set at least one.') sys.exit(1) repo_sections = ['repo'] @@ -205,6 +255,8 @@ def main(): update_serverwebroot(repo_section) if config.get('awsbucket'): update_awsbucket(repo_section) + if local_copy_dir is not None: + update_localcopy(repo_section, local_copy_dir) sys.exit(0) diff --git a/tests/run-tests b/tests/run-tests index 72b8b3f3..8cccc044 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -70,6 +70,41 @@ if [ -z $aapt ]; then aapt=`ls -1 $ANDROID_HOME/build-tools/*/aapt | sort | tail -1` fi +#------------------------------------------------------------------------------# +echo_header "test config checks of local_copy_dir" + +REPOROOT=`create_test_dir` +cd $REPOROOT +$fdroid init +$fdroid update --create-metadata +$fdroid server update --local-copy-dir=/tmp/fdroid + +# now test the errors work +set +e +$fdroid server update --local-copy-dir=thisisnotanabsolutepath +if [ $? -eq 0 ]; then + echo "This should have failed because thisisnotanabsolutepath is not an absolute path!" + exit 1 +else + echo "testing absolute path checker passed" +fi +$fdroid server update --local-copy-dir=/tmp/IReallyDoubtThisPathExistsasdfasdf +if [ $? -eq 0 ]; then + echo "This should have failed because the path does not end with 'fdroid'!" + exit 1 +else + echo "testing dirname exists checker passed" +fi +$fdroid server update --local-copy-dir=/tmp/IReallyDoubtThisPathExistsasdfasdf/fdroid +if [ $? -eq 0 ]; then + echo "This should have failed because the dirname path does not exist!" + exit 1 +else + echo "testing dirname exists checker passed" +fi +set -e + + #------------------------------------------------------------------------------# echo_header "setup a new repo from scratch using ANDROID_HOME" From fd24416f4e8fa679929f278234d12841bb70a899 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 26 Jun 2014 13:05:24 -0400 Subject: [PATCH 2/6] switch serverwebroot rsync to --archive for guaranteed full sync In `fdroid server update`, the rsync command used --update, which `man rsync` says: "skip files that are newer on the receiver". That could cause issues of the public repo getting out of sync with the private, master repo. --archive is a better sync method since it aims to exactly reproduce the sending dir to the receiving dir. --- fdroidserver/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 3efe5603..100fff29 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -115,7 +115,7 @@ def update_awsbucket(repo_section): def update_serverwebroot(repo_section): - rsyncargs = ['rsync', '--update', '--recursive', '--delete'] + rsyncargs = ['rsync', '--archive', '--delete'] if options.verbose: rsyncargs += ['--verbose'] if options.quiet: From 25f6b0c246c43822a6aa38af958a804c3df611b4 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 26 Jun 2014 14:18:29 -0400 Subject: [PATCH 3/6] server: --sync-from-local-copy-dir for updating from offline signing repo To support a fully offline build/signing machine, there is the "local copy dir". The repo is generated on the offline machine and then copied to a local dir where a thumb drive or SD Card is mounted. Then on the online machine, using `fdroid server update --sync-from-local-copy-dir` allows the whole server update process to happen in a single command: 0. read config.py on online machine's repo 1. rsync from the local_copy_dir to the current dir 2. copy to serverwebroot, awsbucket, etc. --- examples/config.py | 8 ++++++++ fdroidserver/common.py | 1 + fdroidserver/server.py | 30 ++++++++++++++++++++++++------ tests/run-tests | 9 ++++++++- 4 files changed, 41 insertions(+), 7 deletions(-) diff --git a/examples/config.py b/examples/config.py index faf704bf..221b63bf 100644 --- a/examples/config.py +++ b/examples/config.py @@ -124,6 +124,14 @@ keyaliases['com.example.another.plugin'] = '@com.example.another' # local_copy_dir = '/media/MyUSBThumbDrive/fdroid' +# If you are using local_copy_dir on an offline build/signing server, once the +# thumb drive has been plugged into the online machine, it will need to be +# synced to the copy on the online machine. To make that happen +# automatically, set sync_from_local_copy_dir to True: +# +# sync_from_local_copy_dir = True + + # To upload the repo to an Amazon S3 bucket using `fdroid server update`. # Warning, this deletes and recreates the whole fdroid/ directory each # time. This is based on apache-libcloud, which supports basically all cloud diff --git a/fdroidserver/common.py b/fdroidserver/common.py index a4b3fb0f..afafffdf 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -47,6 +47,7 @@ def get_default_config(): 'mvn3': "mvn", 'gradle': 'gradle', 'archive_older': 0, + 'sync_from_local_copy_dir': False, 'update_stats': False, 'stats_to_carbon': False, 'repo_maxage': 0, diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 100fff29..6e686017 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -140,8 +140,8 @@ def update_serverwebroot(repo_section): sys.exit(1) -def update_localcopy(repo_section, local_copy_dir): - rsyncargs = ['rsync', '--update', '--recursive', '--delete'] +def _local_sync(fromdir, todir): + rsyncargs = ['rsync', '--archive', '--one-file-system', '--delete'] # use stricter rsync checking on all files since people using offline mode # are already prioritizing security above ease and speed rsyncargs += ['--checksum'] @@ -149,11 +149,24 @@ def update_localcopy(repo_section, local_copy_dir): rsyncargs += ['--verbose'] if options.quiet: rsyncargs += ['--quiet'] - # local_copy_dir is guaranteed to have a trailing slash in main() below - if subprocess.call(rsyncargs + [repo_section, local_copy_dir]) != 0: + logging.debug(' '.join(rsyncargs + [fromdir, todir])) + if subprocess.call(rsyncargs + [fromdir, todir]) != 0: sys.exit(1) +def sync_from_localcopy(repo_section, local_copy_dir): + logging.info('Syncing from local_copy_dir to this repo.') + # trailing slashes have a meaning in rsync which is not needed here, so + # remove them all + _local_sync(os.path.join(local_copy_dir, repo_section).rstrip('/'), + repo_section.rstrip('/')) + + +def update_localcopy(repo_section, local_copy_dir): + # local_copy_dir is guaranteed to have a trailing slash in main() below + _local_sync(repo_section, local_copy_dir) + + def main(): global config, options @@ -163,6 +176,8 @@ def main(): help="Specify an identity file to provide to SSH for rsyncing") parser.add_option("--local-copy-dir", default=None, help="Specify a local folder to sync the repo to") + parser.add_option("--sync-from-local-copy-dir", action="store_true", default=False, + help="Before uploading to servers, sync from local copy dir") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-q", "--quiet", action="store_true", default=False, @@ -251,12 +266,15 @@ def main(): sys.exit(1) elif args[0] == 'update': for repo_section in repo_sections: + if local_copy_dir is not None: + if config['sync_from_local_copy_dir'] and os.path.exists(repo_section): + sync_from_localcopy(repo_section, local_copy_dir) + else: + update_localcopy(repo_section, local_copy_dir) if config.get('serverwebroot'): update_serverwebroot(repo_section) if config.get('awsbucket'): update_awsbucket(repo_section) - if local_copy_dir is not None: - update_localcopy(repo_section, local_copy_dir) sys.exit(0) diff --git a/tests/run-tests b/tests/run-tests index 8cccc044..a5576765 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -106,7 +106,7 @@ set -e #------------------------------------------------------------------------------# -echo_header "setup a new repo from scratch using ANDROID_HOME" +echo_header "setup a new repo from scratch using ANDROID_HOME and do a local sync" REPOROOT=`create_test_dir` cd $REPOROOT @@ -115,6 +115,13 @@ copy_apks_into_repo $REPOROOT $fdroid update --create-metadata grep -F ' Date: Thu, 26 Jun 2014 19:52:11 -0400 Subject: [PATCH 5/6] include getsig.java and opensc-fdroid.cfg in the source tarball --- MANIFEST.in | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 47cd528e..93c086ab 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -23,6 +23,10 @@ include docs/update.sh include examples/config.py include examples/fdroid-icon.png include examples/makebs.config.py +include examples/opensc-fdroid.cfg +include fdroidserver/getsig/run.sh +include fdroidserver/getsig/make.sh +include fdroidserver/getsig/getsig.java include tests/run-tests include wp-fdroid/AndroidManifest.xml include wp-fdroid/android-permissions.php From 22473e20520c71a2679df3620dcdc094c3669c88 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 26 Jun 2014 19:52:19 -0400 Subject: [PATCH 6/6] set version to v0.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a3fcdcc8..d4566f07 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ if not os.path.exists('fdroidserver/getsig/getsig.class'): shell=True) setup(name='fdroidserver', - version='0.1', + version='0.2', description='F-Droid Server Tools', long_description=open('README').read(), author='The F-Droid Project',