buildserver: buildserver/Vagrantfile is configed by .yaml file

Vagrantfile is now committed and not changed between configurations. It is
configed by translating the python config file's dict to a YAML file, which
Vagrantfile now loads and uses.  This makes it a lot easier for vagrant
users and python programmers to understand, and hopefully makes it easier
to maintain and test with.

.gitignore

@@ -19,7 +19,6 @@ tmp/
 tests/repo/icons*
 
 # files used in manual testing
-/buildserver/Vagrantfile.yaml
 /config.py
 /tmp/
 /logs/

buildserver/.gitignore

@@ -1,4 +1,4 @@
 .vagrant
 up.log
 cache/
-Vagrantfile
+Vagrantfile.yaml

buildserver/Vagrantfile

@@ -0,0 +1,70 @@
+
+require 'yaml'
+configfile = YAML.load_file('Vagrantfile.yaml')
+
+Vagrant.configure("2") do |config|
+
+  if Vagrant.has_plugin?("vagrant-cachier")
+    config.cache.scope = :box
+    config.cache.auto_detect = false
+    config.cache.enable :apt
+    config.cache.enable :chef
+  end
+
+  config.vm.box = configfile['basebox']
+  config.vm.box_url = configfile['baseboxurl']
+
+  config.vm.provider "virtualbox" do |v|
+    v.customize ["modifyvm", :id, "--memory", configfile['memory']]
+    v.customize ["modifyvm", :id, "--cpus", configfile['cpus']]
+    v.customize ["modifyvm", :id, "--hwvirtex", configfile['hwvirtex']]
+  end
+
+  config.vm.boot_timeout = configfile['boot_timeout']
+
+  config.vm.provision :shell, :path => "fixpaths.sh"
+
+  if configfile.has_key? "aptproxy"
+    config.vm.provision :shell, path: "provision-apt-proxy",
+      args: [configfile["aptproxy"]]
+  end
+
+  # buildserver/ is shared to the VM's /vagrant by default so the old
+  # default does not need a custom mount
+  if configfile["cachedir"] != "buildserver/cache"
+    config.vm.synced_folder configfile["cachedir"], '/vagrant/cache',
+      owner: 'root', group: 'root', create: true
+  end
+
+  # cache .deb packages on the host via a mount trick
+  if configfile.has_key? "aptcachedir"
+    config.vm.synced_folder configfile["aptcachedir"], "/var/cache/apt/archives",
+      owner: 'root', group: 'root', create: true
+  end
+
+  config.vm.provision "shell", path: "setup-env-vars",
+    args: ["/home/vagrant/android-sdk"]
+  config.vm.provision "shell", path: "provision-apt-get-install",
+    args: [configfile['debian_mirror']]
+
+  config.vm.provision :chef_solo do |chef|
+    chef.cookbooks_path = "cookbooks"
+    chef.log_level = :debug
+    chef.add_recipe "kivy"
+  end
+
+  config.vm.provision "shell", path: "provision-android-sdk"
+  config.vm.provision "shell", path: "provision-android-ndk",
+    args: ["/home/vagrant/android-ndk"]
+  config.vm.provision "shell", path: "provision-pip",
+    args: ["compare-locales"]
+  config.vm.provision "shell", path: "provision-gradle"
+  config.vm.provision "file", source: "gradle",
+    destination: "/opt/gradle/bin/gradle"
+
+  # let Ubuntu/trusty's paramiko work with the VM instance
+  if `uname -v`.include? "14.04"
+    config.vm.provision "shell", path: "provision-ubuntu-trusty-paramiko"
+  end
+
+end

hooks/pre-commit

@@ -12,7 +12,7 @@ if [ -z "$files" ]; then
     PY_TEST_FILES="tests/*.TestCase"
     SH_FILES="hooks/pre-commit"
     BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion buildserver/provision-*"
-    RB_FILES="buildserver/cookbooks/*/recipes/*.rb"
+    RB_FILES="buildserver/cookbooks/*/recipes/*.rb buildserver/Vagrantfile"
 else
     # if actually committing right now, then only run on the files
     # that are going to be committed at this moment

makebuildserver

@@ -71,6 +71,7 @@ if os.path.exists('makebuildserver.config.py'):
 elif os.path.exists('makebs.config.py'):
     # this is the old name for the config file
     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
+del(config['__builtins__'])  # added by compile/exec
 
 if not os.path.exists('makebuildserver') or not os.path.exists(serverdir):
     print('This must be run from the correct directory!')
@@ -320,94 +321,15 @@ elif os.path.exists('/proc/cpuinfo'):
     if 'vmx' in contents or 'svm' in contents:
         config['hwvirtex'] = 'on'
 
-del(config['__builtins__'])  # added by compile/exec
+# Check against the existing Vagrantfile.yaml, and if they differ, we
-with open(os.path.join(serverdir, 'Vagrantfile.yaml'), 'w') as f:
+# need to create a new box:
-    yaml.dump(config, f)
+vf = os.path.join(serverdir, 'Vagrantfile.yaml')
-
-# Generate an appropriate Vagrantfile for the buildserver, based on our
-# settings...
-vagrantfile = """
-require 'yaml'
-configfile = YAML.load_file('Vagrantfile.yaml')
-
-Vagrant.configure("2") do |config|
-
-  if Vagrant.has_plugin?("vagrant-cachier")
-    config.cache.scope = :box
-    config.cache.auto_detect = false
-    config.cache.enable :apt
-    config.cache.enable :chef
-  end
-
-  config.vm.box = configfile['basebox']
-  config.vm.box_url = configfile['baseboxurl']
-
-  config.vm.provider "virtualbox" do |v|
-    v.customize ["modifyvm", :id, "--memory", configfile['memory']]
-    v.customize ["modifyvm", :id, "--cpus", configfile['cpus']]
-    v.customize ["modifyvm", :id, "--hwvirtex", configfile['hwvirtex']]
-  end
-
-  config.vm.boot_timeout = configfile['boot_timeout']
-
-  config.vm.provision :shell, :path => "fixpaths.sh"
-
-  if configfile.has_key? "aptproxy"
-    config.vm.provision :shell, path: "provision-apt-proxy",
-      args: [configfile["aptproxy"]]
-  end
-
-  # buildserver/ is shared to the VM's /vagrant by default so the old
-  # default does not need a custom mount
-  if configfile["cachedir"] != "buildserver/cache"
-    config.vm.synced_folder configfile["cachedir"], '/vagrant/cache',
-      owner: 'root', group: 'root', create: true
-  end
-
-  # cache .deb packages on the host via a mount trick
-  if configfile.has_key? "aptcachedir"
-    config.vm.synced_folder configfile["aptcachedir"], "/var/cache/apt/archives",
-      owner: 'root', group: 'root', create: true
-  end
-
-  config.vm.provision "shell", path: "setup-env-vars",
-    args: ["/home/vagrant/android-sdk"]
-  config.vm.provision "shell", path: "provision-apt-get-install",
-    args: [configfile['debian_mirror']]
-
-  config.vm.provision :chef_solo do |chef|
-    chef.cookbooks_path = "cookbooks"
-    chef.log_level = :debug
-    chef.add_recipe "kivy"
-  end
-
-  config.vm.provision "shell", path: "provision-android-sdk"
-  config.vm.provision "shell", path: "provision-android-ndk",
-    args: ["/home/vagrant/android-ndk"]
-  config.vm.provision "shell", path: "provision-pip",
-    args: ["compare-locales"]
-  config.vm.provision "shell", path: "provision-gradle"
-  config.vm.provision "file", source: "gradle",
-    destination: "/opt/gradle/bin/gradle"
-
-  # let Ubuntu/trusty's paramiko work with the VM instance
-  if `uname -v`.include? "14.04"
-    config.vm.provision "shell", path: "provision-ubuntu-trusty-paramiko"
-  end
-
-end
-"""
-
-
-# Check against the existing Vagrantfile, and if they differ, we need to
-# create a new box:
-vf = os.path.join(serverdir, 'Vagrantfile')
 writevf = True
 if os.path.exists(vf):
     vagrant(['halt'], serverdir)
-    with open(vf, 'r') as f:
+    with open(vf, 'r', encoding='utf-8') as f:
-        oldvf = f.read()
+        oldconfig = yaml.load(f)
-    if oldvf != vagrantfile:
+    if config != oldconfig:
         print("Server configuration has changed, rebuild from scratch is required")
         vagrant(['destroy', '-f'], serverdir)
     else:
@@ -416,9 +338,8 @@ if os.path.exists(vf):
 else:
     print("No existing server - building from scratch")
 if writevf:
-    with open(vf, 'w') as f:
+    with open(vf, 'w', encoding='utf-8') as f:
-        f.write(vagrantfile)
+        yaml.dump(config, f)
-
 
 print("Configuring build server VM")
 returncode, out = vagrant(['up', '--provision'], serverdir, printout=True)