Merge branch 'move-makebuildserver.config.py-to-Vagrantfile.yaml' into 'master'

buildserver: turn ./makebuildserver into a pure helper script, allow standalone vagrant operation

See merge request fdroid/fdroidserver!1222

.gitlab-ci.yml

@@ -223,7 +223,6 @@ black:
     - pip install black
     - black --check --diff --color
         examples/fdroid_extract_repo_pubkey.py
-        examples/makebuildserver.config.py
         fdroid
         fdroidserver/btlog.py
         fdroidserver/exception.py

MANIFEST.in

@@ -14,10 +14,10 @@ include examples/fdroid_export_keystore_to_nitrokey.py
 include examples/fdroid_extract_repo_pubkey.py
 include examples/fdroid_fetchsrclibs.py
 include examples/fdroid_nitrokeyimport.py
-include examples/makebuildserver.config.py
 include examples/opensc-fdroid.cfg
 include examples/public-read-only-s3-bucket-policy.json
 include examples/template.yml
+include examples/Vagrantfile.yaml
 include gradlew-fdroid
 include LICENSE
 include locale/bo/LC_MESSAGES/fdroidserver.po
@@ -42,7 +42,6 @@ include locale/zh_Hans/LC_MESSAGES/fdroidserver.po
 include locale/zh_Hant/LC_MESSAGES/fdroidserver.po
 include makebuildserver
 include README.md
-include tests/androguard_test.py
 include tests/apk.embedded_1.apk
 include tests/bad-unicode-*.apk
 include tests/build.TestCase

buildserver/Vagrantfile

@@ -1,29 +1,41 @@
 require 'yaml'
 require 'pathname'
+require 'fileutils'
+
+configfile = {
+  'boot_timeout' => 600,
+  'cachedir' => File.join(ENV['HOME'], '.cache', 'fdroidserver'),
+  'cpus' => 1,
+  'debian_mirror' => 'https://deb.debian.org/debian/',
+  'hwvirtex' => 'on',
+  'memory' => 2048,
+  'vm_provider' => 'virtualbox',
+}
 
 srvpath = Pathname.new(File.dirname(__FILE__)).realpath
 configpath = File.join(srvpath, "/Vagrantfile.yaml")
 if File.exists? configpath
-  configfile = YAML.load_file(configpath)
+  c = YAML.load_file(configpath)
+  if c and not c.empty?
+    c.each do |k,v|
+      configfile[k] = v
+    end
+  end
 else
-  puts "#{configpath} does not exist, run ./makebuildserver?"
+  puts "Copying example file to #{configpath}"
-  configfile = Hash.new
+  FileUtils.cp('../examples/Vagrantfile.yaml', configpath)
 end
 
 Vagrant.configure("2") do |config|
 
-  # these two caching methods conflict, so only use one at a time
+  if Vagrant.has_plugin?("vagrant-cachier")
-  if Vagrant.has_plugin?("vagrant-cachier") and not configfile.has_key? "aptcachedir"
     config.cache.scope = :box
     config.cache.auto_detect = false
     config.cache.enable :apt
     config.cache.enable :chef
   end
 
-  config.vm.box = configfile['basebox']
+  config.vm.box = "fdroid/bullseye64"
-  if configfile.has_key? "basebox_version"
-    config.vm.box_version = configfile['basebox_version']
-  end
 
   if not configfile.has_key? "vm_provider" or configfile["vm_provider"] == "virtualbox"
     # default to VirtualBox if not set
@@ -66,23 +78,14 @@ Vagrant.configure("2") do |config|
       args: [configfile["aptproxy"]]
   end
 
-  # buildserver/ is shared to the VM's /vagrant by default so the old
-  # default does not need a custom mount
-  if configfile["cachedir"] != "buildserver/cache"
   config.vm.synced_folder configfile["cachedir"], '/vagrant/cache',
                           create: true, type: synced_folder_type
-  end
+
   # Make sure dir exists to mount to, since buildserver/ is
   # automatically mounted as /vagrant in the guest VM. This is more
   # necessary with 9p synced folders
   Dir.mkdir('cache') unless File.exists?('cache')
 
-  # cache .deb packages on the host via a mount trick
-  if configfile.has_key? "aptcachedir"
-    config.vm.synced_folder configfile["aptcachedir"], "/var/cache/apt/archives",
-      owner: 'root', group: 'root', create: true
-  end
-
   config.vm.provision "shell", name: "setup-env-vars", path: "setup-env-vars",
     args: ["/opt/android-sdk"]
   config.vm.provision "shell", name: "apt-get-install", path: "provision-apt-get-install",

buildserver/provision-apt-get-install

@@ -90,6 +90,8 @@ purge="
  xxd
  xz-utils
 "
+# clean up files packages to be purged, then purge the packages
+rm -rf /var/run/dbus /var/log/unattended-upgrades
 apt-get purge $purge
 
 apt-get upgrade --download-only

examples/Vagrantfile.yaml

@@ -0,0 +1,53 @@
+
+# You may want to alter these before running ./makebuildserver
+
+# In the process of setting up the build server, many gigs of files
+# are downloaded (Android SDK components, gradle, etc).  These are
+# cached so that they are not redownloaded each time. By default,
+# these are stored in ~/.cache/fdroidserver
+#
+# cachedir: buildserver/cache
+
+# To specify which Debian mirror the build server VM should use, by
+# default it uses http.debian.net, which auto-detects which is the
+# best mirror to use.
+#
+# debian_mirror: https://debian.osuosl.org/debian/
+
+# The amount of RAM the build server will have (default: 2048)
+# memory: 3584
+
+# The number of CPUs the build server will have
+# cpus: 1
+
+# Debian package proxy server - if you have one
+# aptproxy: http://192.168.0.19:8000
+
+# If this is running on an older machine or on a virtualized system,
+# it can run a lot slower. If the provisioning fails with a warning
+# about the timeout, extend the timeout here. (default: 600 seconds)
+#
+# boot_timeout: 1200
+
+# By default, this whole process uses VirtualBox as the provider, but
+# QEMU+KVM is also supported via the libvirt plugin to vagrant. If
+# this is run within a KVM guest, then libvirt's QEMU+KVM will be used
+# automatically.  It can also be manually enabled by uncommenting
+# below:
+#
+# vm_provider: libvirt
+
+# By default libvirt uses 'virtio' for both network and disk drivers.
+# Some systems (eg. nesting VMware ESXi) do not support virtio. As a
+# workaround for such rare cases, this setting allows to configure
+# KVM/libvirt to emulate hardware rather than using virtio.
+#
+# libvirt_disk_bus: sata
+# libvirt_nic_model_type: rtl8139
+
+# Sometimes, it is not possible to use the 9p synced folder type with
+# libvirt, like if running a KVM buildserver instance inside of a
+# VMware ESXi guest.  In that case, using NFS or another method is
+# required.
+#
+# synced_folder_type: nfs

examples/makebuildserver.config.py

@@ -1,94 +0,0 @@
-#!/usr/bin/env python3
-#
-# You may want to alter these before running ./makebuildserver
-
-# Name of the Vagrant basebox to use, by default it will be downloaded
-# from Vagrant Cloud.  For release builds setup, generate the basebox
-# locally using https://gitlab.com/fdroid/basebox, add it to Vagrant,
-# then set this to the local basebox name.
-# This defaults to "fdroid/basebox-stretch64" which will download a
-# prebuilt basebox from https://app.vagrantup.com/fdroid.
-#
-# (If you change this value you have to supply the `--clean` option on
-#  your next `makebuildserver` run.)
-#
-# basebox = "basebox-stretch64"
-
-# This allows you to pin your basebox to a specific versions. It defaults
-# the most recent basebox version which can be aumotaically verifyed by
-# `makebuildserver`.
-# Please note that vagrant does not support versioning of locally added
-# boxes, so we can't support that either.
-#
-# (If you change this value you have to supply the `--clean` option on
-#  your next `makebuildserver` run.)
-#
-# basebox_version = "0.1"
-
-# In the process of setting up the build server, many gigs of files
-# are downloaded (Android SDK components, gradle, etc).  These are
-# cached so that they are not redownloaded each time. By default,
-# these are stored in ~/.cache/fdroidserver
-#
-# cachedir = 'buildserver/cache'
-
-# A big part of creating a new instance is downloading packages from Debian.
-# This setups up a folder in ~/.cache/fdroidserver to cache the downloaded
-# packages when rebuilding the build server from scratch.  This requires
-# that virtualbox-guest-utils is installed.
-#
-# apt_package_cache = True
-
-# The buildserver can use some local caches to speed up builds,
-# especially when the internet connection is slow and/or expensive.
-# If enabled, the buildserver setup will look for standard caches in
-# your HOME dir and copy them to the buildserver VM. Be aware: this
-# will reduce the isolation of the buildserver from your host machine,
-# so the buildserver will provide an environment only as trustworthy
-# as the host machine's environment.
-#
-# copy_caches_from_host = True
-
-# To specify which Debian mirror the build server VM should use, by
-# default it uses http.debian.net, which auto-detects which is the
-# best mirror to use.
-#
-# debian_mirror = 'http://ftp.uk.debian.org/debian/'
-
-# The amount of RAM the build server will have (default: 2048)
-# memory = 3584
-
-# The number of CPUs the build server will have
-# cpus = 1
-
-# Debian package proxy server - if you have one
-# aptproxy = "http://192.168.0.19:8000"
-
-# If this is running on an older machine or on a virtualized system,
-# it can run a lot slower. If the provisioning fails with a warning
-# about the timeout, extend the timeout here. (default: 600 seconds)
-#
-# boot_timeout = 1200
-
-# By default, this whole process uses VirtualBox as the provider, but
-# QEMU+KVM is also supported via the libvirt plugin to vagrant. If
-# this is run within a KVM guest, then libvirt's QEMU+KVM will be used
-# automatically.  It can also be manually enabled by uncommenting
-# below:
-#
-# vm_provider = 'libvirt'
-
-# By default libvirt uses 'virtio' for both network and disk drivers.
-# Some systems (eg. nesting VMware ESXi) do not support virtio. As a
-# workaround for such rare cases, this setting allows to configure
-# KVM/libvirt to emulate hardware rather than using virtio.
-#
-# libvirt_disk_bus = 'sata'
-# libvirt_nic_model_type = 'rtl8139'
-
-# Sometimes, it is not possible to use the 9p synced folder type with
-# libvirt, like if running a KVM buildserver instance inside of a
-# VMware ESXi guest.  In that case, using NFS or another method is
-# required.
-#
-# synced_folder_type = 'nfs'

jenkins-setup-build-environment

@@ -66,13 +66,11 @@ if [ `nproc` -le 6 ]; then
 else
     cpus=6
 fi
-cat <<EOF > $WORKSPACE/makebuildserver.config.py
+cat <<EOF > $WORKSPACE/buildserver/Vagrantfile.yaml
-debian_mirror = 'http://deb.debian.org/debian/'
+debian_mirror: https://deb.debian.org/debian/
-boot_timeout = 1200
+boot_timeout: 1200
-apt_package_cache = True
+memory: $memory
-copy_caches_from_host = True
+cpus: $cpus
-memory = $memory
-cpus = $cpus
 EOF
 
 cd $WORKSPACE

makebuildserver

@@ -24,9 +24,13 @@ parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1
 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
 parser.add_option("-c", "--clean", action="store_true", default=False,
                   help="Build from scratch, rather than attempting to update the existing server")
+parser.add_option('--skip-box-verification', action="store_true", default=False,
+                  help="""Skip verifying the downloaded base box.""")
 parser.add_option('--skip-cache-update', action="store_true", default=False,
                   help="""Skip downloading and checking cache."""
                        """This assumes that the cache is already downloaded completely.""")
+parser.add_option('--copy-caches-from-host', action="store_true", default=False,
+                  help="""Copy gradle and pip caches into the buildserver VM""")
 parser.add_option('--keep-box-file', action="store_true", default=False,
                   help="""Box file will not be deleted after adding it to box storage"""
                        """ (KVM-only).""")
@@ -74,36 +78,35 @@ BASEBOX_CHECKSUMS = {
     },
 }
 
-config = {
+configfile = 'buildserver/Vagrantfile.yaml'
-    'basebox': BASEBOX_DEFAULT,
+if not os.path.exists(configfile):
-    'debian_mirror': 'https://deb.debian.org/debian/',
+    logging.warning('%s does not exist, copying template file.' % configfile)
-    'apt_package_cache': False,
+    shutil.copy('examples/Vagrantfile.yaml', configfile)
-    'copy_caches_from_host': False,
+with open(configfile) as fp:
-    'boot_timeout': 600,
+    config = yaml.safe_load(fp)
-    'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'),
+with open('buildserver/Vagrantfile') as fp:
-    'cpus': 1,
+    m = re.search(r"""\.vm\.box\s*=\s*["'](.*)["']""", fp.read())
-    'memory': 2048,
+    if not m:
-    'hwvirtex': 'off',
+        logging.error('Cannot find box name in buildserver/Vagrantfile!')
-    'vm_provider': 'virtualbox',
+        exit(1)
-}
+    config['basebox'] = m.group(1)
-
-# load config file, if present
-if os.path.exists('makebuildserver.config.py'):
-    exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
-elif os.path.exists('makebs.config.py'):
-    # this is the old name for the config file
-    exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
-if '__builtins__' in config:
-    del config['__builtins__']  # added by compile/exec
-logging.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
-if config['basebox'] == BASEBOX_DEFAULT and 'basebox_version' not in config:
 config['basebox_version'] = BASEBOX_VERSION_DEFAULT
-# note: vagrant allows putting '/' into the name of a local box,
+config['cachedir'] = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
-# so this check is not completely reliable, but better than nothing
+
-if 'basebox_version' in config and 'basebox' in config and '/' not in config['basebox']:
+show_config_deprecation = False
-    logging.critical("Can not get version '{version}' for basebox '{box}', "
+if os.path.exists('makebuildserver.config.py'):
-                     "vagrant does not support versioning for locally added boxes."
+    show_config_deprecation = True
-                     .format(box=config['basebox'], version=config['basebox_version']))
+    logging.error('makebuildserver.config.py exists!')
+elif os.path.exists('makebs.config.py'):
+    show_config_deprecation = True
+    # this is the old name for the config file
+    logging.error('makebs.config.py exists!')
+if show_config_deprecation:
+    logging.error('Config is via buildserver/Vagrantfile.yaml and command line flags.')
+    parser.print_help()
+    exit(1)
+
+logging.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
 
 # Update cached files.
 if not os.path.exists(config['cachedir']):
@@ -121,18 +124,6 @@ if config['vm_provider'] == 'libvirt':
         tmp = os.path.dirname(tmp)
     logging.debug('cache dir %s is accessible for libvirt vm.', config['cachedir'])
 
-if config['apt_package_cache']:
-    config['aptcachedir'] = config['cachedir'] + '/apt/archives'
-    logging.debug('aptcachedir is set to %s', config['aptcachedir'])
-    aptcachelock = os.path.join(config['aptcachedir'], 'lock')
-    if os.path.isfile(aptcachelock):
-        logging.info('apt cache dir is locked, removing lock')
-        os.remove(aptcachelock)
-    aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
-    if os.path.isdir(aptcachepartial):
-        logging.info('removing partial downloads from apt cache dir')
-        shutil.rmtree(aptcachepartial)
-
 CACHE_FILES = [
     ('https://services.gradle.org/distributions/gradle-6.8.3-bin.zip',
      '7faa7198769f872826c8ef4f1450f839ec27f0b4d5d1e51bade63667cbccd205'),
@@ -265,17 +256,14 @@ def main():
     # use VirtualBox software virtualization if hardware is not available,
     # like if this is being run in kvm or some other VM platform, like
     # http://jenkins.debian.net, the values are 'on' or 'off'
-    if sys.platform.startswith('darwin'):
+    if config.get('hwvirtex') != 'off' and os.path.exists('/proc/cpuinfo'):
-        # all < 10 year old Macs work, and OSX servers as VM host are very
-        # rare, but this could also be auto-detected if someone codes it
-        config['hwvirtex'] = 'on'
-        logging.info('platform is darwnin -> hwvirtex = \'on\'')
-    elif os.path.exists('/proc/cpuinfo'):
         with open('/proc/cpuinfo') as f:
             contents = f.read()
         if 'vmx' in contents or 'svm' in contents:
-            config['hwvirtex'] = 'on'
+            logging.debug('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
-            logging.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
+        else:
+            logging.error('hwvirtex = \'on\' and no \'vmx\' or \'svm\' found in /proc/cpuinfo!')
+            exit(1)
 
     serverdir = os.path.join(os.getcwd(), 'buildserver')
     logfilename = os.path.join(serverdir, 'up.log')
@@ -324,27 +312,15 @@ def main():
                          "virtualbox, libvirt)"
                          .format(vm_provider=config['cm_provider']))
         sys.exit(1)
-    # Check if selected Vagrant box is available
-    available_boxes_by_provider = [x.name for x in v.box_list() if x.provider == config['vm_provider']]
-    if '/' not in config['basebox'] and config['basebox'] not in available_boxes_by_provider:
-        logging.critical("Vagrant box '{basebox}' not available "
-                         "for '{vm_provider}' VM provider. "
-                         "Please make sure it's added to vagrant. "
-                         "(If you need a basebox to begin with, "
-                         "here is how we're bootstrapping it: "
-                         "https://gitlab.com/fdroid/basebox)"
-                         .format(vm_provider=config['vm_provider'],
-                                 basebox=config['basebox']))
-        sys.exit(1)
 
     # Download and verify pre-built Vagrant boxes
-    if config['basebox'] == BASEBOX_DEFAULT:
+    if not options.skip_box_verification:
         buildserver_not_created = any([True for x in v.status() if x.state == 'not_created' and x.name == 'default'])
         if buildserver_not_created or options.clean:
             # make vagrant download and add basebox
-            target_basebox_installed = any([x for x in v.box_list() if x.name == BASEBOX_DEFAULT and x.provider == config['vm_provider'] and x.version == config['basebox_version']])
+            target_basebox_installed = any([x for x in v.box_list() if x.name == config['basebox'] and x.provider == config['vm_provider'] and x.version == config['basebox_version']])
             if not target_basebox_installed:
-                cmd = [shutil.which('vagrant'), 'box', 'add', BASEBOX_DEFAULT,
+                cmd = [shutil.which('vagrant'), 'box', 'add', config['basebox'],
                        '--box-version=' + config['basebox_version'],
                        '--provider=' + config['vm_provider']]
                 ret_val = subprocess.call(cmd)
@@ -365,7 +341,7 @@ def main():
             for filename, sha256 in BASEBOX_CHECKSUMS[config['basebox_version']][config['vm_provider']].items():
                 verify_file_sha256(os.path.join(get_vagrant_home(),
                                                 'boxes',
-                                                BASEBOX_DEFAULT.replace('/', '-VAGRANTSLASH-'),
+                                                config['basebox'].replace('/', '-VAGRANTSLASH-'),
                                                 config['basebox_version'],
                                                 config['vm_provider'],
                                                 filename),
@@ -375,8 +351,6 @@ def main():
                          .format(box=config['basebox'],
                                  provider=config['vm_provider'],
                                  version=config['basebox_version']))
-        else:
-            logging.debug('not updating basebox ...')
     else:
         logging.debug('using unverified basebox ...')
 
@@ -389,7 +363,7 @@ def main():
         logging.error("'vagrant up' failed.")
         sys.exit(1)
 
-    if config['copy_caches_from_host']:
+    if options.copy_caches_from_host:
         ssh_config = v.ssh_config()
         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)