From 25caa09420d6f3be3cc24273538a31c57ea629cf Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 16 Sep 2015 17:19:12 +0200 Subject: [PATCH 1/6] run-tests: verbose output for `fdroid update` to aid debugging I'm stumped by this stacktrace, hopefully debug logging will help: Unknown exception found! Traceback (most recent call last): File "./fdroidserver-0.4.0/fdroid", line 141, in main() File "./fdroidserver-0.4.0/fdroid", line 119, in main mod.main() File " line 1146, in main apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks) File " line 543, in scan_apks thisinfo['sig'] = getsig(os.path.join(os.getcwd(), apkfile)) File " line 361, in getsig with zipfile.ZipFile(apkpath, 'r') as apk: File "/usr/lib/python2.7/zipfile.py", line 770, in __init__ self._RealGetContents() File "/usr/lib/python2.7/zipfile.py", line 857, in _RealGetContents x._decodeExtra() File "/usr/lib/python2.7/zipfile.py", line 388, in _decodeExtra tp, ln = unpack('> config.py echo 'keystorepass = "foo"' >> config.py echo 'keypass = "foo"' >> config.py set +e -$fdroid update --create-metadata +$fdroid update --create-metadata --verbose if [ $? -eq 0 ]; then echo "This should have failed because this repo has a bad/fake keystore!" exit 1 @@ -462,7 +461,7 @@ cd $REPOROOT $fdroid init --keystore $KEYSTORE test -e $KEYSTORE cp $WORKSPACE/tests/urzip.apk $REPOROOT/repo/ -$fdroid update --create-metadata +$fdroid update --create-metadata --verbose $fdroid readmeta test -e repo/index.xml test -e repo/index.jar From b33add9ce7574b0dff069193a87aeea9d76c146d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 20 Jan 2016 20:38:18 +0100 Subject: [PATCH 2/6] name makebuildserver config the same as the script This keeps the numbers of names down to a minimum, and since the config is placed right next to the script, this keeps tab completion working nicely when the config file is in place. The old file name is still supported. --- MANIFEST.in | 2 +- .../{makebs.config.py => makebuildserver.config.py} | 0 makebuildserver | 10 +++++++++- setup.py | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) rename examples/{makebs.config.py => makebuildserver.config.py} (100%) diff --git a/MANIFEST.in b/MANIFEST.in index 92ddb75d..ca8eea6e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -21,7 +21,7 @@ include docs/index_versions.md include docs/update.sh include examples/config.py include examples/fdroid-icon.png -include examples/makebs.config.py +include examples/makebuildserver.config.py include examples/opensc-fdroid.cfg include tests/getsig/run.sh include tests/getsig/make.sh diff --git a/examples/makebs.config.py b/examples/makebuildserver.config.py similarity index 100% rename from examples/makebs.config.py rename to examples/makebuildserver.config.py diff --git a/makebuildserver b/makebuildserver index 38b776f1..9695ba2c 100755 --- a/makebuildserver +++ b/makebuildserver @@ -46,7 +46,13 @@ parser.add_option("--debian-mirror", default="http://ftp.uk.debian.org/debian/", options, args = parser.parse_args() config = {} -execfile('makebs.config.py', config) + +# load config file, if present +if os.path.exists('makebuildserver.config.py'): + execfile('makebuildserver.config.py', config) +elif os.path.exists('makebs.config.py'): + # this is the old name for the config file + execfile('makebs.config.py', config) if not os.path.exists('makebuildserver') or not os.path.exists(serverdir): print 'This must be run from the correct directory!' @@ -193,6 +199,8 @@ cachefiles = [ ('build-tools-23.0.2.zip', 'https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip', '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'), + # the binaries that Google uses are here: + # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/ ('gradle-1.4-bin.zip', 'https://services.gradle.org/distributions/gradle-1.4-bin.zip', 'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'), diff --git a/setup.py b/setup.py index 7834799b..4a266e7c 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ setup(name='fdroidserver', (data_prefix + '/share/doc/fdroidserver/examples', ['buildserver/config.buildserver.py', 'examples/config.py', - 'examples/makebs.config.py', + 'examples/makebuildserver.config.py', 'examples/opensc-fdroid.cfg', 'examples/fdroid-icon.png']), ], From 50713c7ce96203e7297e16848919ca2009a21d85 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 20 Jan 2016 20:40:25 +0100 Subject: [PATCH 3/6] makebuildserver: set default Debian mirror to http.debian.net This host automatically detects which is the closest mirror, then uses that one. It does so dynamically, so it'll work on machines that move too. Now that we are pushing more people to run F-Droid build servers, the defaults should take those use cases into account. --- examples/makebuildserver.config.py | 6 ++++++ makebuildserver | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/examples/makebuildserver.config.py b/examples/makebuildserver.config.py index 615029b3..f2c846d5 100644 --- a/examples/makebuildserver.config.py +++ b/examples/makebuildserver.config.py @@ -13,6 +13,12 @@ baseboxurl = "https://f-droid.org/jessie32.box" # or if you have a cached local copy, you can use that first: # baseboxurl = ["file:///home/fdroid/fdroidserver/cache/jessie32.box", "https://f-droid.org/jessie32.box"] +# To specify which Debian mirror the build server VM should use, by +# default it uses http.debian.net, which auto-detects which is the +# best mirror to use. +# +# debian_mirror = 'http://ftp.uk.debian.org/debian/' + # The amount of RAM the build server will have memory = 3584 diff --git a/makebuildserver b/makebuildserver index 9695ba2c..4fc9a28d 100755 --- a/makebuildserver +++ b/makebuildserver @@ -41,7 +41,7 @@ parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-c", "--clean", action="store_true", default=False, help="Build from scratch, rather than attempting to update the existing server") -parser.add_option("--debian-mirror", default="http://ftp.uk.debian.org/debian/", +parser.add_option("--debian-mirror", default="http://http.debian.net/debian/", help="Use the specified Debian mirror in the box's /etc/apt/sources.list.") options, args = parser.parse_args() From 3bd8ef2c8c49fd77108d8d3885fd6e66e4a38e73 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 20 Jan 2016 21:00:16 +0100 Subject: [PATCH 4/6] makebuildserver: set defaults in script so it can run without config This makes it so that ./makebuildserver will run without any config file, using the defaults that are embedded in the script itself. This is like how `fdroid` works. --- examples/makebuildserver.config.py | 15 ++++++++------- makebuildserver | 9 ++++++++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/examples/makebuildserver.config.py b/examples/makebuildserver.config.py index f2c846d5..9795bf76 100644 --- a/examples/makebuildserver.config.py +++ b/examples/makebuildserver.config.py @@ -3,13 +3,14 @@ # You may want to alter these before running ./makebuildserver # Name of the base box to use -basebox = "jessie32" +# basebox = "jessie32" # Location where testing32.box can be found, if you don't already have # it. For security reasons, it's recommended that you make your own # in a secure environment using trusted media (see the manual) but # you can use this default if you like... -baseboxurl = "https://f-droid.org/jessie32.box" +# baseboxurl = "https://f-droid.org/jessie32.box" +# # or if you have a cached local copy, you can use that first: # baseboxurl = ["file:///home/fdroid/fdroidserver/cache/jessie32.box", "https://f-droid.org/jessie32.box"] @@ -20,13 +21,13 @@ baseboxurl = "https://f-droid.org/jessie32.box" # debian_mirror = 'http://ftp.uk.debian.org/debian/' # The amount of RAM the build server will have -memory = 3584 +# memory = 3584 # The number of CPUs the build server will have -cpus = 1 +# cpus = 1 -# Debian package proxy server - if you have one, e.g. "http://192.168.0.19:8000" -aptproxy = None +# Debian package proxy server - if you have one +# aptproxy = "http://192.168.0.19:8000" # Set to True if your base box is 64 bit (e.g. testing32.box isn't) -arch64 = False +# arch64 = True diff --git a/makebuildserver b/makebuildserver index 4fc9a28d..ebd998b2 100755 --- a/makebuildserver +++ b/makebuildserver @@ -45,7 +45,14 @@ parser.add_option("--debian-mirror", default="http://http.debian.net/debian/", help="Use the specified Debian mirror in the box's /etc/apt/sources.list.") options, args = parser.parse_args() -config = {} +# set up default config +config = { + 'arch64': False, + 'basebox': 'jessie32', + 'baseboxurl': 'https://f-droid.org/jessie32.box', + 'cpus': 1, + 'memory': 3584, +} # load config file, if present if os.path.exists('makebuildserver.config.py'): From cda8fa00f56068bac49276bbecf9064337125189 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 21 Jan 2016 14:09:12 +0100 Subject: [PATCH 5/6] makebuildserver: cache defaults to ~/.cache/fdroidserver This also provides a config option to override that default. ~/.cache is a standard location on GNU/Linux machines for cached content. It is also good to have the cache outside of the git repo in case `git clean -fdx` is run, which would delete all files in the directory that are not part of the git repo, including buildserver/cache/ --- examples/makebuildserver.config.py | 7 +++++++ makebuildserver | 5 +++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/examples/makebuildserver.config.py b/examples/makebuildserver.config.py index 9795bf76..1d4bd0d3 100644 --- a/examples/makebuildserver.config.py +++ b/examples/makebuildserver.config.py @@ -14,6 +14,13 @@ # or if you have a cached local copy, you can use that first: # baseboxurl = ["file:///home/fdroid/fdroidserver/cache/jessie32.box", "https://f-droid.org/jessie32.box"] +# In the process of setting up the build server, many gigs of files +# are downloaded (Android SDK components, gradle, etc). These are +# cached so that they are not redownloaded each time. By default, +# these are stored in ~/.cache/fdroidserver +# +# cachedir = 'buildserver/cache' + # To specify which Debian mirror the build server VM should use, by # default it uses http.debian.net, which auto-detects which is the # best mirror to use. diff --git a/makebuildserver b/makebuildserver index ebd998b2..f745b0a3 100755 --- a/makebuildserver +++ b/makebuildserver @@ -50,6 +50,7 @@ config = { 'arch64': False, 'basebox': 'jessie32', 'baseboxurl': 'https://f-droid.org/jessie32.box', + 'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), 'cpus': 1, 'memory': 3584, } @@ -72,9 +73,9 @@ if options.clean: vagrant(['destroy', '-f'], cwd=serverdir, printout=options.verbose) # Update cached files. -cachedir = os.path.join('buildserver', 'cache') +cachedir = config['cachedir'] if not os.path.exists(cachedir): - os.mkdir(cachedir) + os.makedirs(cachedir, 0755) cachefiles = [ ('android-sdk_r24.4.1-linux.tgz', From 24a9f3b816ec8951bd43784e6bda98c2ea66984a Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 21 Jan 2016 15:41:26 +0100 Subject: [PATCH 6/6] makebuildserver: delete cached files with zero size If download fails to connect, it'll make a zero size file. Then the next run on ./makebuildserver will fail because the checksum will not be right. --- makebuildserver | 3 +++ 1 file changed, 3 insertions(+) diff --git a/makebuildserver b/makebuildserver index f745b0a3..4e575a09 100755 --- a/makebuildserver +++ b/makebuildserver @@ -305,6 +305,9 @@ def sha256_for_file(path): for f, src, shasum in cachefiles: relpath = os.path.join(cachedir, f) + # if download fails to connect, it'll make a zero size file + if os.path.exists(relpath) and os.stat(relpath).st_size == 0: + os.remove(relpath) if not os.path.exists(relpath): print "Downloading " + f + " to cache" if subprocess.call(['wget', src, '-O', f], cwd=cachedir) != 0: