overhauled and moved destroying builder vm to vmtools.py

fdroidserver/build.py

@@ -1110,7 +1110,7 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir,
        this is the 'unsigned' directory.
     :param repo_dir: The repo directory - used for checking if the build is
        necessary.
-    :paaram also_check_dir: An additional location for checking if the build
+    :param also_check_dir: An additional location for checking if the build
        is necessary (usually the archive repo)
     :param test: True if building in test mode, in which case the build will
        always happen, even if the output already exists. In test mode, the

fdroidserver/vmtools.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+#
+# vmtools.py - part of the FDroid server tools
+# Copyright (C) 2017 Michael Poehn <michael.poehn@fsfe.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from os.path import isdir, isfile, join as joinpath, basename, abspath
+import time
+import shutil
+import vagrant
+import subprocess
+from .common import FDroidException
+from logging import getLogger
+
+logger = getLogger('fdroidserver-vmtools')
+
+
+def get_build_vm(srvdir, provider=None):
+    """Factory function for getting FDroidBuildVm instances.
+
+    This function tries to figure out what hypervisor should be used
+    and creates an object for controlling a build VM.
+
+    :param srvdir: path to a directory which contains a Vagrantfile
+    :param provider: optionally this parameter allows specifiying an
+        spesific vagrant provider.
+    :returns: FDroidBuildVm instance.
+    """
+    abssrvdir = abspath(srvdir)
+    if provider:
+        if provider == 'libvirt':
+            logger.debug('build vm provider \'libvirt\' selected')
+            return LibvirtBuildVm(abssrvdir)
+        elif provider == 'virtualbox':
+            logger.debug('build vm provider \'virtualbox\' selected')
+            return VirtualboxBuildVm(abssrvdir)
+        else:
+            logger.warn('unsupported provider \'%s\' requested', provider)
+    has_libvirt_machine = isdir(joinpath(abssrvdir, '.vagrant',
+                                         'machines', 'default', 'libvirt'))
+    has_vbox_machine = isdir(joinpath(abssrvdir, '.vagrant',
+                                      'machines', 'default', 'libvirt'))
+    if has_libvirt_machine and has_vbox_machine:
+        logger.info('build vm provider lookup found virtualbox and libvirt, defaulting to \'virtualbox\'')
+        return VirtualboxBuildVm(abssrvdir)
+    elif has_libvirt_machine:
+        logger.debug('build vm provider lookup found \'libvirt\'')
+        return LibvirtBuildVm(abssrvdir)
+    elif has_vbox_machine:
+        logger.debug('build vm provider lookup found \'virtualbox\'')
+        return VirtualboxBuildVm(abssrvdir)
+
+    logger.info('build vm provider lookup could not determine provider, defaulting to \'virtualbox\'')
+    return VirtualboxBuildVm(abssrvdir)
+
+
+class FDroidBuildVmException(FDroidException):
+    pass
+
+
+class FDroidBuildVm():
+    """Abstract base class for working with FDroids build-servers.
+
+    Use the factory method `fdroidserver.vmtools.get_build_vm()` for
+    getting correct instances of this class.
+
+    This is intended to be a hypervisor independant, fault tolerant
+    wrapper around the vagrant functions we use.
+    """
+
+    def __init__(self, srvdir):
+        """Create new server class.
+        """
+        self.srvdir = srvdir
+        self.srvname = basename(srvdir) + '_default'
+        self.vgrntfile = joinpath(srvdir, 'Vagrantfile')
+        if not isdir(srvdir):
+            raise FDroidBuildVmException("Can not init vagrant, directory %s not present" % (srvdir))
+        if not isfile(self.vgrntfile):
+            raise FDroidBuildVmException("Can not init vagrant, '%s' not present" % (self.vgrntfile))
+        self.vgrnt = vagrant.Vagrant(root=srvdir, out_cm=vagrant.stdout_cm, err_cm=vagrant.stdout_cm)
+
+    def isUpAndRunning(self):
+        raise NotImplementedError('TODO implement this')
+
+    def up(self, provision=True):
+        try:
+            self.vgrnt.up(provision=provision)
+        except subprocess.CalledProcessError as e:
+            logger.info('could not bring vm up: %s', e)
+
+    def destroy(self):
+        """Remove every trace of this VM from the system.
+
+        This includes deleting:
+        * hypervisor specific definitions
+        * vagrant state informations (eg. `.vagrant` folder)
+        * images related to this vm
+        """
+        try:
+            self.vgrnt.destroy()
+            logger.debug('vagrant destroy completed')
+        except subprocess.CalledProcessError as e:
+            logger.debug('vagrant destroy failed: %s', e)
+        vgrntdir = joinpath(self.srvdir, '.vagrant')
+        try:
+            shutil.rmtree(vgrntdir)
+            logger.debug('deleted vagrant dir: %s', vgrntdir)
+        except Exception as e:
+            logger.debug("could not delete vagrant dir: %s, %s", vgrntdir, e)
+        try:
+            subprocess.check_call(['vagrant', 'global-status', '--prune'])
+        except subprocess.CalledProcessError as e:
+            logger.debug('pruning global vagrant status failed: %s', e)
+
+
+class LibvirtBuildVm(FDroidBuildVm):
+    def __init__(self, srvdir):
+        super().__init__(srvdir)
+        import libvirt
+
+        try:
+            self.conn = libvirt.open('qemu:///system')
+        except libvirt.libvirtError as e:
+            logger.critical('could not connect to libvirtd: %s', e)
+
+    def destroy(self):
+
+        super().destroy()
+
+        # resorting to virsh instead of libvirt python bindings, because
+        # this is way more easy and therefore fault tolerant.
+        # (eg. lookupByName only works on running VMs)
+        try:
+            logger.debug('virsh -c qemu:///system destroy', self.srvname)
+            subprocess.check_call(('virsh', '-c', 'qemu:///system', 'destroy'))
+            logger.info("...waiting a sec...")
+            time.sleep(10)
+        except subprocess.CalledProcessError as e:
+            logger.info("could not force libvirt domain '%s' off: %s", self.srvname, e)
+        try:
+            # libvirt python bindings do not support all flags required
+            # for undefining domains correctly.
+            logger.debug('virsh -c qemu:///system undefine %s --nvram --managed-save --remove-all-storage --snapshots-metadata', self.srvname)
+            subprocess.check_call(('virsh', '-c', 'qemu:///system', 'undefine', self.srvname, '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata'))
+            logger.info("...waiting a sec...")
+            time.sleep(10)
+        except subprocess.CalledProcessError as e:
+            logger.info("could not undefine libvirt domain '%s': %s", self.srvname, e)
+
+
+class VirtualboxBuildVm(FDroidBuildVm):
+    pass

makebuildserver

@@ -19,6 +19,7 @@ import logging
 from clint.textui import progress
 from optparse import OptionParser
 import fdroidserver.tail
+import fdroidserver.vmtools
 
 
 parser = OptionParser()
@@ -32,7 +33,7 @@ parser.add_option('--skip-cache-update', action="store_true", default=False,
                        """This assumes that the cache is already downloaded completely.""")
 options, args = parser.parse_args()
 
-logger = logging.getLogger('fdroid-makebuildserver')
+logger = logging.getLogger('fdroidserver-makebuildserver')
 if options.verbosity >= 2:
     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
     logger.setLevel(logging.DEBUG)
@@ -320,55 +321,6 @@ def sha256_for_file(path):
         return s.hexdigest()
 
 
-def destroy_current_image(v, serverdir):
-    global config
-
-    logger.info('destroying buildserver vm, removing images and vagrant-configs...')
-
-    try:
-        v.destroy()
-        logger.debug('vagrant destroy completed')
-    except subprocess.CalledProcessError as e:
-        logger.debug('vagrant destroy failed: %s', e)
-    try:
-        subprocess.check_call(['vagrant', 'global-status', '--prune'])
-    except subprocess.CalledProcessError as e:
-        logger.debug('pruning global vagrant status failed: %s', e)
-
-    try:
-        shutil.rmtree(os.path.join(serverdir, '.vagrant'))
-    except Exception as e:
-        logger.debug("could not delete vagrant dir: %s, %s", os.path.join(serverdir, '.vagrant'), e)
-
-    if config['vm_provider'] == 'libvirt':
-        import libvirt
-        try:
-            conn = libvirt.open('qemu:///system')
-            try:
-                dom = conn.lookupByName(config['domain'])
-                try:
-                    logger.debug('virsh -c qemu:///system destroy %s', config['domain'])
-                    subprocess.check_call(['virsh', '-c', 'qemu:///system', 'destroy', config['domain']])
-                    logger.info("...waiting a sec...")
-                    time.sleep(10)
-                except subprocess.CalledProcessError as e:
-                    logger.info("could not force libvirt domain '%s' off: %s", config['domain'], e)
-                try:
-                    # libvirt python bindings do not support all flags required
-                    # for undefining domains correctly.
-                    logger.debug('virsh -c qemu:///system undefine %s --nvram --managed-save --remove-all-storage --snapshots-metadata', config['domain'])
-                    subprocess.check_call(('virsh', '-c', 'qemu:///system', 'undefine', config['domain'], '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata'))
-                    logger.info("...waiting a sec...")
-                    time.sleep(10)
-                except subprocess.CalledProcessError as e:
-                    logger.info("could not undefine libvirt domain '%s': %s", dom.name(), e)
-            except libvirt.libvirtError as e:
-                logger.info("finding libvirt domain '%s' failed. (%s)", config['domain'], e)
-        except libvirt.libvirtError as e:
-            logger.critical('could not connect to libvirtd: %s', e)
-            sys.exit(1)
-
-
 def kvm_package(boxfile):
     '''
     Hack to replace missing `vagrant package` for kvm, based  on the script
@@ -532,8 +484,9 @@ def main():
         tail = fdroidserver.tail.Tail(logfilename)
         tail.start()
 
+    vm = fdroidserver.vmtools.get_build_vm(serverdir)
     if options.clean:
-        destroy_current_image(v, serverdir)
+        vm.destroy()
 
     # Check against the existing Vagrantfile.yaml, and if they differ, we
     # need to create a new box:
@@ -546,7 +499,7 @@ def main():
             oldconfig = yaml.load(f)
         if config != oldconfig:
             logger.info("Server configuration has changed, rebuild from scratch is required")
-            destroy_current_image(v, serverdir)
+            vm.destroy()
         else:
             logger.info("Re-provisioning existing server")
             writevf = False