Merge branch 'makebuildserver-for-debian-jenkins' into 'master'

Makebuildserver for debian jenkins

@mvdan and I worked out a plan with Holger, the Debian Developer running the reproducible builds jenkins build farm.  The idea is for that to just run `./makebuildserver` in the jenkins script.  Then we'll make sure that it works.  This also makes it easy for anyone to set up and run an FDroid build instance, which is important for things like the verification server and for people to be able to run reproducible builds on their own machine.

This does change some of the defaults, but they can all be overridden in the `makebs.config.py` for f-droid.org.

More comments in the commits.


See merge request !89

MANIFEST.in

@@ -21,7 +21,7 @@ include docs/index_versions.md
 include docs/update.sh
 include examples/config.py
 include examples/fdroid-icon.png
-include examples/makebs.config.py
+include examples/makebuildserver.config.py
 include examples/opensc-fdroid.cfg
 include tests/getsig/run.sh
 include tests/getsig/make.sh

examples/makebuildserver.config.py

@@ -3,24 +3,38 @@
 # You may want to alter these before running ./makebuildserver
 
 # Name of the base box to use
-basebox = "jessie32"
+# basebox = "jessie32"
 
 # Location where testing32.box can be found, if you don't already have
 # it. For security reasons, it's recommended that you make your own
 # in a secure environment using trusted media (see the manual) but
 # you can use this default if you like...
-baseboxurl = "https://f-droid.org/jessie32.box"
+# baseboxurl = "https://f-droid.org/jessie32.box"
+#
 # or if you have a cached local copy, you can use that first:
 # baseboxurl = ["file:///home/fdroid/fdroidserver/cache/jessie32.box", "https://f-droid.org/jessie32.box"]
 
+# In the process of setting up the build server, many gigs of files
+# are downloaded (Android SDK components, gradle, etc).  These are
+# cached so that they are not redownloaded each time. By default,
+# these are stored in ~/.cache/fdroidserver
+#
+# cachedir = 'buildserver/cache'
+
+# To specify which Debian mirror the build server VM should use, by
+# default it uses http.debian.net, which auto-detects which is the
+# best mirror to use.
+#
+# debian_mirror = 'http://ftp.uk.debian.org/debian/'
+
 # The amount of RAM the build server will have
-memory = 3584
+# memory = 3584
 
 # The number of CPUs the build server will have
-cpus = 1
+# cpus = 1
 
-# Debian package proxy server - if you have one, e.g. "http://192.168.0.19:8000"
-aptproxy = None
+# Debian package proxy server - if you have one
+# aptproxy = "http://192.168.0.19:8000"
 
 # Set to True if your base box is 64 bit (e.g. testing32.box isn't)
-arch64 = False
+# arch64 = True

makebuildserver

@@ -41,12 +41,26 @@ parser.add_option("-v", "--verbose", action="store_true", default=False,
                   help="Spew out even more information than normal")
 parser.add_option("-c", "--clean", action="store_true", default=False,
                   help="Build from scratch, rather than attempting to update the existing server")
-parser.add_option("--debian-mirror", default="http://ftp.uk.debian.org/debian/",
+parser.add_option("--debian-mirror", default="http://http.debian.net/debian/",
                   help="Use the specified Debian mirror in the box's /etc/apt/sources.list.")
 options, args = parser.parse_args()
 
-config = {}
-execfile('makebs.config.py', config)
+# set up default config
+config = {
+    'arch64': False,
+    'basebox': 'jessie32',
+    'baseboxurl': 'https://f-droid.org/jessie32.box',
+    'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'),
+    'cpus': 1,
+    'memory': 3584,
+}
+
+# load config file, if present
+if os.path.exists('makebuildserver.config.py'):
+    execfile('makebuildserver.config.py', config)
+elif os.path.exists('makebs.config.py'):
+    # this is the old name for the config file
+    execfile('makebs.config.py', config)
 
 if not os.path.exists('makebuildserver') or not os.path.exists(serverdir):
     print 'This must be run from the correct directory!'
@@ -59,9 +73,9 @@ if options.clean:
     vagrant(['destroy', '-f'], cwd=serverdir, printout=options.verbose)
 
 # Update cached files.
-cachedir = os.path.join('buildserver', 'cache')
+cachedir = config['cachedir']
 if not os.path.exists(cachedir):
-    os.mkdir(cachedir)
+    os.makedirs(cachedir, 0755)
 
 cachefiles = [
     ('android-sdk_r24.4.1-linux.tgz',
@@ -193,6 +207,8 @@ cachefiles = [
     ('build-tools-23.0.2.zip',
      'https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
+    # the binaries that Google uses are here:
+    # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
     ('gradle-1.4-bin.zip',
      'https://services.gradle.org/distributions/gradle-1.4-bin.zip',
      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
@@ -289,6 +305,9 @@ def sha256_for_file(path):
 
 for f, src, shasum in cachefiles:
     relpath = os.path.join(cachedir, f)
+    # if download fails to connect, it'll make a zero size file
+    if os.path.exists(relpath) and os.stat(relpath).st_size == 0:
+        os.remove(relpath)
     if not os.path.exists(relpath):
         print "Downloading " + f + " to cache"
         if subprocess.call(['wget', src, '-O', f], cwd=cachedir) != 0:

setup.py

@@ -22,7 +22,7 @@ setup(name='fdroidserver',
           (data_prefix + '/share/doc/fdroidserver/examples',
               ['buildserver/config.buildserver.py',
                   'examples/config.py',
-                  'examples/makebs.config.py',
+                  'examples/makebuildserver.config.py',
                   'examples/opensc-fdroid.cfg',
                   'examples/fdroid-icon.png']),
       ],

tests/run-tests

@@ -168,7 +168,7 @@ tar xzf `ls -1 $WORKSPACE/dist/fdroidserver-*.tar.gz | sort -n | tail -1`
 cd $REPOROOT
 ./fdroidserver-*/fdroid init
 copy_apks_into_repo $REPOROOT
-./fdroidserver-*/fdroid update --create-metadata
+./fdroidserver-*/fdroid update --create-metadata --verbose
 
 
 #------------------------------------------------------------------------------#
@@ -177,7 +177,7 @@ echo_header "test config checks of local_copy_dir"
 REPOROOT=`create_test_dir`
 cd $REPOROOT
 $fdroid init
-$fdroid update --create-metadata
+$fdroid update --create-metadata --verbose
 $fdroid readmeta
 $fdroid server update --local-copy-dir=/tmp/fdroid
 
@@ -214,7 +214,7 @@ REPOROOT=`create_test_dir`
 cd $REPOROOT
 $fdroid init
 copy_apks_into_repo $REPOROOT
-$fdroid update --create-metadata
+$fdroid update --create-metadata --verbose
 $fdroid readmeta
 grep -F '<application id=' repo/index.xml > /dev/null
 
@@ -318,7 +318,7 @@ cd $REPOROOT
 $fdroid init --keystore $KEYSTORE --android-home $STORED_ANDROID_HOME --no-prompt
 test -e $KEYSTORE
 copy_apks_into_repo $REPOROOT
-$fdroid update --create-metadata
+$fdroid update --create-metadata --verbose
 $fdroid readmeta
 grep -F '<application id=' repo/index.xml > /dev/null
 test -e repo/index.xml
@@ -334,7 +334,7 @@ cd $REPOROOT
 mkdir repo
 copy_apks_into_repo $REPOROOT
 $fdroid init
-$fdroid update --create-metadata
+$fdroid update --create-metadata --verbose
 $fdroid readmeta
 grep -F '<application id=' repo/index.xml > /dev/null
 
@@ -348,7 +348,7 @@ cd $REPOROOT
 $fdroid init --keystore $KEYSTORE
 test -e $KEYSTORE
 copy_apks_into_repo $REPOROOT
-$fdroid update --create-metadata
+$fdroid update --create-metadata --verbose
 $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
@@ -376,7 +376,7 @@ set -e
 $fdroid update --create-key
 test -e $KEYSTORE
 copy_apks_into_repo $REPOROOT
-$fdroid update --create-metadata
+$fdroid update --create-metadata --verbose
 $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
@@ -392,14 +392,14 @@ cd $REPOROOT
 $fdroid init --keystore $KEYSTORE
 test -e $KEYSTORE
 copy_apks_into_repo $REPOROOT
-$fdroid update --create-metadata
+$fdroid update --create-metadata --verbose
 $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
 grep -F '<application id=' repo/index.xml > /dev/null
 test -e $REPOROOT/repo/info.guardianproject.urzip_100.apk || \
     cp $WORKSPACE/tests/urzip.apk $REPOROOT/repo/
-$fdroid update --create-metadata
+$fdroid update --create-metadata --verbose
 $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
@@ -408,7 +408,6 @@ grep -F '<application id=' repo/index.xml > /dev/null
 
 #------------------------------------------------------------------------------#
 echo_header "setup a new repo from scratch with a HSM/smartcard"
-
 REPOROOT=`create_test_dir`
 cd $REPOROOT
 $fdroid init --keystore NONE
@@ -427,7 +426,7 @@ touch fdroid-icon.png
 mkdir repo
 cp $WORKSPACE/tests/urzip.apk $REPOROOT/repo/
 set +e
-$fdroid update --create-metadata
+$fdroid update --create-metadata --verbose
 if [ $? -eq 0 ]; then
     echo "This should have failed because this repo has no keystore!"
     exit 1
@@ -443,7 +442,7 @@ echo 'repo_keyalias = "foo"' >> config.py
 echo 'keystorepass = "foo"' >> config.py
 echo 'keypass = "foo"' >> config.py
 set +e
-$fdroid update --create-metadata
+$fdroid update --create-metadata --verbose
 if [ $? -eq 0 ]; then
     echo "This should have failed because this repo has a bad/fake keystore!"
     exit 1
@@ -462,7 +461,7 @@ cd $REPOROOT
 $fdroid init --keystore $KEYSTORE
 test -e $KEYSTORE
 cp $WORKSPACE/tests/urzip.apk $REPOROOT/repo/
-$fdroid update --create-metadata
+$fdroid update --create-metadata --verbose
 $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar