teuthology 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

teuthology/openstack/__init__.py

@@ -123,7 +123,7 @@ class OpenStackInstance(object):
         with safe_while(sleep=2, tries=30,
                         action="get ip " + self['id']) as proceed:
             while proceed():
-                found = re.match('.*\d+', self['addresses'])
+                found = re.match(r'.*\d+', self['addresses'])
                 if found:
                     return self['addresses']
                 self.set_info()
@@ -165,7 +165,7 @@ class OpenStackInstance(object):
                 self.private_ip = self.get_ip_neutron()
             except Exception as e:
                 log.debug("ignoring get_ip_neutron exception " + str(e))
-                self.private_ip = re.findall(network + '=([\d.]+)',
+                self.private_ip = re.findall(network + r'=([\d.]+)',
                                              self.get_addresses())[0]
         return self.private_ip
 
@@ -1026,7 +1026,8 @@ ssh access           : ssh {identity}{username}@{ip} # logs in /usr/share/nginx/
         cluster, based on a template where the OpenStack credentials
         and a few other values are substituted.
         """
-        path = tempfile.mktemp()
+        fd, path = tempfile.mkstemp()
+        os.close(fd)
 
         with open(os.path.dirname(__file__) + '/bootstrap-teuthology.sh', 'rb') as f:
             b64_bootstrap = base64.b64encode(f.read())

teuthology/openstack/openstack-centos-7.0-user-data.txt

@@ -6,7 +6,7 @@ bootcmd:
  - ( curl --silent http://169.254.169.254/2009-04-04/meta-data/hostname | sed -e 's/[\.-].*//' ; eval printf "%03d%03d%03d%03d.{lab_domain}" $(curl --silent http://169.254.169.254/2009-04-04/meta-data/local-ipv4 | tr . ' ' ) ) | tee /etc/hostname
  - hostname $(cat /etc/hostname)
  - ( echo ; echo "MaxSessions 1000" ) >> /etc/ssh/sshd_config
-# See https://github.com/ceph/ceph-cm-ansible/blob/master/roles/cobbler/templates/snippets/cephlab_user
+# See https://github.com/ceph/ceph-cm-ansible/blob/main/roles/cobbler/templates/snippets/cephlab_user
  - ( echo 'Defaults !requiretty' ; echo 'Defaults visiblepw' ) | tee /etc/sudoers.d/cephlab_sudo ; chmod 0440 /etc/sudoers.d/cephlab_sudo
 preserve_hostname: true
 system_info:

teuthology/openstack/openstack-centos-7.1-user-data.txt

@@ -6,7 +6,7 @@ bootcmd:
  - ( curl --silent http://169.254.169.254/2009-04-04/meta-data/hostname | sed -e 's/[\.-].*//' ; eval printf "%03d%03d%03d%03d.{lab_domain}" $(curl --silent http://169.254.169.254/2009-04-04/meta-data/local-ipv4 | tr . ' ' ) ) | tee /etc/hostname
  - hostname $(cat /etc/hostname)
  - ( echo ; echo "MaxSessions 1000" ) >> /etc/ssh/sshd_config
-# See https://github.com/ceph/ceph-cm-ansible/blob/master/roles/cobbler/templates/snippets/cephlab_user
+# See https://github.com/ceph/ceph-cm-ansible/blob/main/roles/cobbler/templates/snippets/cephlab_user
  - ( echo 'Defaults !requiretty' ; echo 'Defaults visiblepw' ) | tee /etc/sudoers.d/cephlab_sudo ; chmod 0440 /etc/sudoers.d/cephlab_sudo
 preserve_hostname: true
 system_info:

teuthology/openstack/openstack-centos-7.2-user-data.txt

@@ -6,7 +6,7 @@ bootcmd:
  - ( curl --silent http://169.254.169.254/2009-04-04/meta-data/hostname | sed -e 's/[\.-].*//' ; eval printf "%03d%03d%03d%03d.{lab_domain}" $(curl --silent http://169.254.169.254/2009-04-04/meta-data/local-ipv4 | tr . ' ' ) ) | tee /etc/hostname
  - hostname $(cat /etc/hostname)
  - ( echo ; echo "MaxSessions 1000" ) >> /etc/ssh/sshd_config
-# See https://github.com/ceph/ceph-cm-ansible/blob/master/roles/cobbler/templates/snippets/cephlab_user
+# See https://github.com/ceph/ceph-cm-ansible/blob/main/roles/cobbler/templates/snippets/cephlab_user
  - ( echo 'Defaults !requiretty' ; echo 'Defaults visiblepw' ) | tee /etc/sudoers.d/cephlab_sudo ; chmod 0440 /etc/sudoers.d/cephlab_sudo
 preserve_hostname: true
 system_info:

teuthology/openstack/openstack-debian-8.0-user-data.txt

@@ -18,7 +18,7 @@ packages:
  - git
  - ntp
 runcmd:
-# See https://github.com/ceph/ceph-cm-ansible/blob/master/roles/cobbler/templates/snippets/cephlab_user
+# See https://github.com/ceph/ceph-cm-ansible/blob/main/roles/cobbler/templates/snippets/cephlab_user
  - ( echo 'Defaults !requiretty' ; echo 'Defaults visiblepw' ) | tee /etc/sudoers.d/cephlab_sudo ; chmod 0440 /etc/sudoers.d/cephlab_sudo
  - echo '{username} ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 final_message: "{up}, after $UPTIME seconds"

teuthology/openstack/openstack-opensuse-42.1-user-data.txt

@@ -6,7 +6,7 @@ bootcmd:
  - ( curl --silent http://169.254.169.254/2009-04-04/meta-data/hostname | sed -e 's/[\.-].*//' ; eval printf "%03d%03d%03d%03d.{lab_domain}" $(curl --silent http://169.254.169.254/2009-04-04/meta-data/local-ipv4 | tr . ' ' ) ) | tee /etc/hostname
  - hostname $(cat /etc/hostname)
  - ( echo ; echo "MaxSessions 1000" ) >> /etc/ssh/sshd_config
-# See https://github.com/ceph/ceph-cm-ansible/blob/master/roles/cobbler/templates/snippets/cephlab_user
+# See https://github.com/ceph/ceph-cm-ansible/blob/main/roles/cobbler/templates/snippets/cephlab_user
  - ( echo 'Defaults !requiretty' ; echo 'Defaults visiblepw' ) | tee /etc/sudoers.d/cephlab_sudo ; chmod 0440 /etc/sudoers.d/cephlab_sudo
 preserve_hostname: true
 users:

teuthology/openstack/openstack-teuthology.cron

@@ -1,2 +1 @@
 SHELL=/bin/bash
-*/30 * * * * ( date ; source $HOME/openrc.sh ; time timeout 900 $HOME/teuthology/virtualenv/bin/teuthology-nuke --stale-openstack ) >> $HOME/cron.log 2>&1

teuthology/orchestra/cluster.py

@@ -2,8 +2,7 @@
 Cluster definition
 part of context, Cluster is used to save connection information.
 """
-import teuthology.misc
-
+from teuthology.orchestra import run
 
 class Cluster(object):
     """
@@ -50,18 +49,47 @@ class Cluster(object):
                 )
         self.remotes[remote] = list(roles)
 
-    def run(self, **kwargs):
+    def run(self, wait=True, parallel=False, **kwargs):
         """
         Run a command on all the nodes in this cluster.
 
         Goes through nodes in alphabetical order.
 
-        If you don't specify wait=False, this will be sequentially.
+        The default usage is when parallel=False and wait=True,
+        which is a sequential run for each node one by one.
+
+        If you specify parallel=True, it will be in parallel.
+
+        If you specify wait=False, it returns immediately.
+        Since it is not possible to run sequentially and
+        do not wait each command run finished, the parallel value
+        is ignored and treated as True.
 
         Returns a list of `RemoteProcess`.
         """
+        # -+-------+----------+----------+------------+---------------
+        #  | wait  | parallel | run.wait | remote.run | comments
+        # -+-------+----------+----------+------------+---------------
+        # 1|*True  |*False    | no       | wait=True  | sequentially
+        # 2| True  | True     | yes      | wait=False | parallel
+        # 3| False | True     | no       | wait=False | parallel
+        # 4| False | False    | no       | wait=False | same as above
+
+        # We always run in parallel if wait=False,
+        # that is why (4) is equivalent to (3).
+
+        # We wait from remote.run only if run sequentially.
+        _wait = (parallel == False and wait == True)
+
         remotes = sorted(self.remotes.keys(), key=lambda rem: rem.name)
-        return [remote.run(**kwargs) for remote in remotes]
+        procs = [remote.run(**kwargs, wait=_wait) for remote in remotes]
+
+        # We do run.wait only if parallel=True, because if parallel=False,
+        # we have run sequentially and all processes are complete.
+
+        if parallel and wait:
+            run.wait(procs)
+        return procs
 
     def sh(self, script, **kwargs):
         """
@@ -86,11 +114,12 @@ class Cluster(object):
         remotes = sorted(self.remotes.keys(), key=lambda rem: rem.name)
         for remote in remotes:
             if sudo:
-                teuthology.misc.sudo_write_file(remote, file_name, content, perms=perms, owner=owner)
+                remote.write_file(file_name, content,
+                                  sudo=True, mode=perms, owner=owner)
             else:
                 if perms is not None or owner is not None:
                     raise ValueError("To specify perms or owner, sudo must be True")
-                teuthology.misc.write_file(remote, file_name, content)
+                remote.write_file(file_name, content)
 
     def only(self, *roles):
         """
@@ -144,3 +173,16 @@ class Cluster(object):
             if remote not in matches.remotes:
                 c.add(remote, has_roles)
         return c
+
+    def filter(self, func):
+        """
+        Return a cluster whose remotes are filtered by `func`.
+
+        Example::
+            cluster = ctx.cluster.filter(lambda r: r.is_online)
+        """
+        result = self.__class__()
+        for rem, roles in self.remotes.items():
+            if func(rem):
+                result.add(rem, roles)
+        return result

teuthology/orchestra/connection.py

@@ -79,13 +79,17 @@ def connect(user_at_host, host_key=None, keep_alive=False, timeout=60,
         timeout=timeout
     )
 
-    ssh_config_path = os.path.expanduser("~/.ssh/config")
-    if os.path.exists(ssh_config_path):
+    key_filename = key_filename or config.ssh_key
+    ssh_config_path = config.ssh_config_path or "~/.ssh/config"
+    ssh_config_path = os.path.expanduser(ssh_config_path)
+    if not key_filename and os.path.exists(ssh_config_path):
         ssh_config = paramiko.SSHConfig()
         ssh_config.parse(open(ssh_config_path))
         opts = ssh_config.lookup(host)
         if not key_filename and 'identityfile' in opts:
             key_filename = opts['identityfile']
+        if 'hostname' in opts:
+            connect_args['hostname'] = opts['hostname']
 
     if key_filename:
         if not isinstance(key_filename, list):
@@ -98,13 +102,20 @@ def connect(user_at_host, host_key=None, keep_alive=False, timeout=60,
     if not retry:
         ssh.connect(**connect_args)
     else:
-        # Retries are implemented using safe_while
-        with safe_while(sleep=1, action='connect to ' + host) as proceed:
+        with safe_while(sleep=1, increment=3, action='connect to ' + host) as proceed:
             while proceed():
+                auth_err_msg = f"Error authenticating with {host}"
                 try:
                     ssh.connect(**connect_args)
                     break
+                except EOFError:
+                    log.error(f"{auth_err_msg}: EOFError")
                 except paramiko.AuthenticationException as e:
-                    log.error(f"Error authenticating with {host}: {str(e)}")
+                    log.error(f"{auth_err_msg}: {repr(e)}")
+                except paramiko.SSHException as e:
+                    auth_err_msg = f"{auth_err_msg}: {repr(e)}"
+                    if not key_filename:
+                        auth_err_msg = f"{auth_err_msg} (No SSH private key found!)"
+                    log.exception(auth_err_msg)
     ssh.get_transport().set_keepalive(keep_alive)
     return ssh

teuthology/orchestra/console.py

@@ -1,3 +1,4 @@
+import io
 import logging
 import os
 import pexpect
@@ -6,6 +7,8 @@ import subprocess
 import sys
 import time
 
+from typing import Union, Literal, Optional
+
 import teuthology.lock.query
 import teuthology.lock.util
 from teuthology.config import config
@@ -19,6 +22,7 @@ except ImportError:
     libvirt = None
 
 log = logging.getLogger(__name__)
+PowerOnOffState = Union[Literal["on"], Literal["off"]]
 
 
 class RemoteConsole():
@@ -35,11 +39,11 @@ class PhysicalConsole(RemoteConsole):
     Physical Console (set from getRemoteConsole)
     """
     def __init__(self, name, ipmiuser=None, ipmipass=None, ipmidomain=None,
-                 logfile=None, timeout=40):
+                 timeout=120):
         self.name = name
         self.shortname = self.getShortName(name)
+        self.log = log.getChild(self.shortname)
         self.timeout = timeout
-        self.logfile = None
         self.ipmiuser = ipmiuser or config.ipmi_user
         self.ipmipass = ipmipass or config.ipmi_password
         self.ipmidomain = ipmidomain or config.ipmi_domain
@@ -71,11 +75,14 @@ class PhysicalConsole(RemoteConsole):
         """
         Run a command using pexpect.spawn(). Return the child object.
         """
-        log.debug('pexpect command: %s', cmd)
-        return pexpect.spawn(
+        self.log.debug('pexpect command: %s', cmd)
+        p = pexpect.spawn(
             cmd,
-            logfile=self.logfile,
+            encoding='utf-8',
+            codec_errors="backslashreplace",
         )
+        p.logfile_read = io.StringIO()
+        return p
 
     def _get_console(self, readonly=True):
         def start():
@@ -84,7 +91,7 @@ class PhysicalConsole(RemoteConsole):
 
         child = start()
         if self.has_conserver and not child.isalive():
-            log.error("conserver failed to get the console; will try ipmitool")
+            self.log.error("conserver failed to get the console; will try ipmitool")
             self.has_conserver = False
             child = start()
         return child
@@ -114,7 +121,7 @@ class PhysicalConsole(RemoteConsole):
 
     def _check_ipmi_credentials(self):
         if not self.has_ipmi_credentials:
-            log.error(
+            self.log.error(
                 "Must set ipmi_user, ipmi_password, and ipmi_domain in "
                 ".teuthology.yaml"
             )
@@ -129,41 +136,108 @@ class PhysicalConsole(RemoteConsole):
                 timeout=t)
             if r != 0:
                 child.kill(15)
+            self.log.debug('console disconnect output: %s', child.logfile_read.getvalue().strip())
         else:
             child.send('~.')
             r = child.expect(
                 ['terminated ipmitool', pexpect.TIMEOUT, pexpect.EOF],
                 timeout=t)
+            self.log.debug('ipmitool disconnect output: %s', child.logfile_read.getvalue().strip())
             if r != 0:
                 self._pexpect_spawn_ipmi('sol deactivate')
+                self.log.debug('sol deactivate output: %s', child.logfile_read.getvalue().strip())
 
     def _wait_for_login(self, timeout=None, attempts=2):
         """
         Wait for login.  Retry if timeouts occur on commands.
         """
         t = timeout or self.timeout
-        log.debug('Waiting for login prompt on {s}'.format(s=self.shortname))
+        self.log.debug('Waiting for login prompt')
         # wait for login prompt to indicate boot completed
         for i in range(0, attempts):
             start = time.time()
             while time.time() - start < t:
                 child = self._get_console(readonly=False)
                 child.send('\n')
-                log.debug('expect: {s} login'.format(s=self.shortname))
                 r = child.expect(
                     ['{s} login: '.format(s=self.shortname),
                      pexpect.TIMEOUT,
                      pexpect.EOF],
                     timeout=(t - (time.time() - start)))
-                log.debug('expect before: {b}'.format(b=child.before))
-                log.debug('expect after: {a}'.format(a=child.after))
+                self.log.debug('expect before: {b}'.format(b=child.before))
+                self.log.debug('expect after: {a}'.format(a=child.after))
 
                 self._exit_session(child)
                 if r == 0:
                     return
         raise ConsoleError("Did not get a login prompt from %s!" % self.name)
 
-    def check_power(self, state, timeout=None):
+    def check_power(self, state: Literal["on","off"]):
+        c = self._pexpect_spawn_ipmi('power status')
+        r = c.expect(['Chassis Power is {s}'.format(
+            s=state), pexpect.EOF, pexpect.TIMEOUT], timeout=1)
+        self.log.debug('check power output: %s', c.logfile_read.getvalue().strip())
+        return r == 0
+
+    def set_power(self, state: PowerOnOffState, timeout: Optional[int]):
+        self.log.info(f"Power {state}")
+        timeout = timeout or self.timeout
+        sleep_time = 4
+        reissue_after_failures = 5
+        failures = 0
+        issued = False
+        succeeded = False
+        with safe_while(
+                sleep=sleep_time,
+                tries=int(timeout / sleep_time),
+                _raise=False,
+                action='wait for power on') as proceed:
+            while proceed():
+                if not issued:
+                    child = self._pexpect_spawn_ipmi(f"power {state}")
+                    rc = child.expect(
+                        [
+                            "Up/On" if state.lower() == "on" else "Down/Off",
+                            pexpect.EOF
+                        ],
+                        timeout=self.timeout
+                    )
+                    self.log.debug(
+                        f"power {state} output: {child.logfile_read.getvalue().strip()}"
+                    )
+                    if rc == 0:
+                        issued = True
+                    continue
+
+                if not succeeded:
+                    child = self._pexpect_spawn_ipmi('power status')
+                    rc = child.expect(
+                        [
+                            f"Chassis Power is {state}",
+                            pexpect.EOF,
+                            pexpect.TIMEOUT
+                        ],
+                        timeout=1
+                    )
+                    self.log.debug(
+                        f"check power output: {child.logfile_read.getvalue().strip()}"
+                    )
+                    if rc == 0:
+                        succeeded = True
+                        break
+                    failures += 1
+                    if failures == reissue_after_failures:
+                        issued = False
+
+        if issued and succeeded:
+            self.log.info(f"Power {state} completed")
+            return True
+        raise RuntimeError(
+            f"Failed to power {state} {self.shortname} in {self.timeout}s"
+        )
+        return False
+
+    def check_power_retries(self, state, timeout=None):
         """
         Check power.  Retry if EOF encountered on power check read.
         """
@@ -178,6 +252,7 @@ class PhysicalConsole(RemoteConsole):
                 c = self._pexpect_spawn_ipmi('power status')
                 r = c.expect(['Chassis Power is {s}'.format(
                     s=state), pexpect.EOF, pexpect.TIMEOUT], timeout=1)
+                self.log.debug('check power output: %s', c.logfile_read.getvalue().strip())
                 if r == 0:
                     return True
         return False
@@ -190,9 +265,8 @@ class PhysicalConsole(RemoteConsole):
             # check for login prompt at console
             self._wait_for_login(timeout)
             return True
-        except Exception as e:
-            log.info('Failed to get ipmi console status for {s}: {e}'.format(
-                s=self.shortname, e=e))
+        except Exception:
+            self.log.exception('Failed to get ipmi console status')
             return False
 
     def power_cycle(self, timeout=300):
@@ -201,62 +275,45 @@ class PhysicalConsole(RemoteConsole):
 
         :param timeout: How long to wait for login
         """
-        log.info('Power cycling {s}'.format(s=self.shortname))
+        self.log.info('Power cycling')
         child = self._pexpect_spawn_ipmi('power cycle')
         child.expect('Chassis Power Control: Cycle', timeout=self.timeout)
+        self.log.debug('power cycle output: %s', child.logfile_read.getvalue().strip())
         self._wait_for_login(timeout=timeout)
-        log.info('Power cycle for {s} completed'.format(s=self.shortname))
+        self.log.info('Power cycle completed')
 
     def hard_reset(self, wait_for_login=True):
         """
         Perform physical hard reset.  Retry if EOF returned from read
         and wait for login when complete.
         """
-        log.info('Performing hard reset of {s}'.format(s=self.shortname))
+        self.log.info('Performing hard reset')
         start = time.time()
         while time.time() - start < self.timeout:
             child = self._pexpect_spawn_ipmi('power reset')
             r = child.expect(['Chassis Power Control: Reset', pexpect.EOF],
                              timeout=self.timeout)
+            self.log.debug('power reset output: %s', child.logfile_read.getvalue().strip())
             if r == 0:
                 break
         if wait_for_login:
             self._wait_for_login()
-        log.info('Hard reset for {s} completed'.format(s=self.shortname))
+        self.log.info('Hard reset completed')
 
     def power_on(self):
         """
         Physical power on.  Loop checking cmd return.
         """
-        log.info('Power on {s}'.format(s=self.shortname))
-        start = time.time()
-        while time.time() - start < self.timeout:
-            child = self._pexpect_spawn_ipmi('power on')
-            r = child.expect(['Chassis Power Control: Up/On', pexpect.EOF],
-                             timeout=self.timeout)
-            if r == 0:
-                break
-        if self.check_power('on'):
-            log.info('Power on for {s} completed'.format(s=self.shortname))
-        else:
-            log.error('Failed to power on {s}'.format(s=self.shortname))
+        return self.set_power("on", timeout=None)
 
     def power_off(self):
         """
         Physical power off.  Loop checking cmd return.
         """
-        log.info('Power off {s}'.format(s=self.shortname))
-        start = time.time()
-        while time.time() - start < self.timeout:
-            child = self._pexpect_spawn_ipmi('power off')
-            r = child.expect(['Chassis Power Control: Down/Off', pexpect.EOF],
-                             timeout=self.timeout)
-            if r == 0:
-                break
-        if self.check_power('off', 60):
-            log.info('Power off for {s} completed'.format(s=self.shortname))
-        else:
-            log.error('Failed to power off {s}'.format(s=self.shortname))
+        try:
+            return self.set_power("off", timeout=None)
+        except Exception:
+            pass
 
     def power_off_for_interval(self, interval=30):
         """
@@ -264,17 +321,21 @@ class PhysicalConsole(RemoteConsole):
 
         :param interval: Length of power-off period.
         """
-        log.info('Power off {s} for {i} seconds'.format(
-            s=self.shortname, i=interval))
+        self.log.info('Power off for {i} seconds'.format(i=interval))
         child = self._pexpect_spawn_ipmi('power off')
         child.expect('Chassis Power Control: Down/Off', timeout=self.timeout)
 
+        self.log.debug('power off output: %s', child.logfile_read.getvalue().strip())
+        child.logfile_read.seek(0)
+        child.logfile_read.truncate()
+
         time.sleep(interval)
 
         child = self._pexpect_spawn_ipmi('power on')
         child.expect('Chassis Power Control: Up/On', timeout=self.timeout)
+        self.log.debug('power on output: %s', child.logfile_read.getvalue().strip())
         self._wait_for_login()
-        log.info('Power off for {i} seconds completed'.format(i=interval))
+        self.log.info('Power off for {i} seconds completed'.format(i=interval))
 
     def spawn_sol_log(self, dest_path):
         """
@@ -307,7 +368,7 @@ class PhysicalConsole(RemoteConsole):
 
         proc = start()
         if self.has_conserver and proc.poll() is not None:
-            log.error("conserver failed to get the console; will try ipmitool")
+            self.log.error("conserver failed to get the console; will try ipmitool")
             self.has_conserver = False
             proc = start()
         return proc
@@ -322,6 +383,7 @@ class VirtualConsole(RemoteConsole):
             raise RuntimeError("libvirt not found")
 
         self.shortname = self.getShortName(name)
+        self.log = log.getChild(self.shortname)
         status_info = teuthology.lock.query.get_status(self.shortname)
         try:
             if teuthology.lock.query.is_vm(status=status_info):
@@ -379,9 +441,8 @@ class VirtualConsole(RemoteConsole):
         """
         Simiulate power off for an interval.
         """
-        log.info('Power off {s} for {i} seconds'.format(
-            s=self.shortname, i=interval))
+        self.log.info('Power off for {i} seconds'.format(i=interval))
         self.vm_domain.info().destroy()
         time.sleep(interval)
         self.vm_domain.info().create()
-        log.info('Power off for {i} seconds completed'.format(i=interval))
+        self.log.info('Power off for {i} seconds completed'.format(i=interval))

teuthology/orchestra/daemon/cephadmunit.py

@@ -35,9 +35,9 @@ class CephadmUnit(DaemonState):
 
     def kill_cmd(self, sig):
         return ' '.join([
-            'sudo', 'docker', 'kill',
+            'sudo', 'systemctl', 'kill',
             '-s', str(int(sig)),
-            'ceph-%s-%s.%s' % (self.fsid, self.type_, self.id_),
+            'ceph-%s@%s.%s' % (self.fsid, self.type_, self.id_),
         ])
 
     def _start_logger(self):
@@ -112,6 +112,18 @@ class CephadmUnit(DaemonState):
         """
         return self.is_started
 
+    def finished(self):
+        """
+        Is the daemon finished? 
+        Return False if active.
+        """
+        proc = self.remote.run(
+            args=self.status_cmd,
+            check_status=False,
+            quiet=True,
+        )
+        return proc.returncode != 0
+
     def signal(self, sig, silent=False):
         """
         Send a signal to associated remote command
@@ -132,11 +144,12 @@ class CephadmUnit(DaemonState):
         Start this daemon instance.
         """
         if self.running():
-            self.log.warn('Restarting a running daemon')
+            self.log.warning('Restarting a running daemon')
             self.restart()
             return
         self._start_logger()
-        self.remote.run(self.start_cmd)
+        self.remote.run(args=self.start_cmd)
+        self.is_started = True
 
     def stop(self, timeout=300):
         """

teuthology/orchestra/daemon/state.py

@@ -97,6 +97,13 @@ class DaemonState(object):
         """
         return self.proc is not None
 
+    def finished(self):
+        """
+        Is the daemon finished? 
+        Return False if active.
+        """
+        return self.proc.finished if self.proc is not None else False
+
     def signal(self, sig, silent=False):
         """
         Send a signal to associated remote command.
@@ -118,7 +125,7 @@ class DaemonState(object):
         Start this daemon instance.
         """
         if self.running():
-            self.log.warn('Restarting a running daemon')
+            self.log.warning('Restarting a running daemon')
         self.restart()
 
     def stop(self, timeout=300):

teuthology/orchestra/daemon/systemd.py

@@ -86,7 +86,7 @@ class SystemDState(DaemonState):
             self.status_cmd + " | grep 'Main.*code=exited'",
         )
         line = out.strip().split('\n')[-1]
-        exit_code = int(re.match('.*status=(\d+).*', line).groups()[0])
+        exit_code = int(re.match(r'.*status=(\d+).*', line).groups()[0])
         if exit_code:
             self.remote.run(
                 args=self.output_cmd
@@ -156,7 +156,7 @@ class SystemDState(DaemonState):
 
         :param extra_args: Extra keyword arguments to be added.
         """
-        self.log.warn(
+        self.log.warning(
                 "restart_with_args() is not supported with systemd; performing"
                 "normal restart")
         self.restart()
@@ -180,7 +180,7 @@ class SystemDState(DaemonState):
 
         :param sig: signal to send
         """
-        self.log.warn("systemd may restart daemons automatically")
+        self.log.warning("systemd may restart daemons automatically")
         pid = self.pid
         self.log.info("Sending signal %s to process %s", sig, pid)
         sig = '-' + str(sig)
@@ -191,7 +191,7 @@ class SystemDState(DaemonState):
         Start this daemon instance.
         """
         if self.running():
-            self.log.warn('Restarting a running daemon')
+            self.log.warning('Restarting a running daemon')
             self.restart()
             return
         self.remote.run(args=[run.Raw(self.start_cmd)])