teuthology 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

teuthology/orchestra/remote.py

@@ -4,17 +4,21 @@ Support for paramiko remote objects.
 
 import teuthology.lock.query
 import teuthology.lock.util
+from teuthology.contextutil import safe_while
 from teuthology.orchestra import run
 from teuthology.orchestra import connection
 from teuthology.orchestra import console
 from teuthology.orchestra.opsys import OS
+import teuthology.provision
 from teuthology import misc
-from teuthology.exceptions import CommandFailedError
+from teuthology.exceptions import CommandFailedError, UnitTestError
+from teuthology.util.scanner import UnitTestScanner
 from teuthology.misc import host_shortname
-import time
+import errno
 import re
 import logging
 from io import BytesIO
+from io import StringIO
 import os
 import pwd
 import tempfile
@@ -23,8 +27,317 @@ import netaddr
 log = logging.getLogger(__name__)
 
 
-class Remote(object):
+class RemoteShell(object):
+    """
+    Contains methods to run miscellaneous shell commands on remote machines.
+
+    These methods were originally part of orchestra.remote.Remote. The reason
+    for moving these methods from Remote is that applications that use
+    teuthology for testing usually have programs that can run tests locally on
+    a single node machine for development work (for example, vstart_runner.py
+    in case of Ceph). These programs can import and reuse these methods
+    without having to deal SSH stuff. In short, this class serves a shared
+    interface.
+
+    To use these methods, inherit the class here and implement "run()" method in
+    the subclass.
+    """
+
+    def remove(self, path):
+        self.run(args=['rm', '-fr', path])
+
+    def mkdtemp(self, suffix=None, parentdir=None):
+        """
+        Create a temporary directory on remote machine and return it's path.
+        """
+        args = ['mktemp', '-d']
+
+        if suffix:
+            args.append('--suffix=%s' % suffix)
+        if parentdir:
+            args.append('--tmpdir=%s' % parentdir)
+
+        return self.sh(args).strip()
+
+    def mktemp(self, suffix=None, parentdir=None, data=None):
+        """
+        Make a remote temporary file.
+
+        :param suffix:      suffix for the temporary file
+        :param parentdir:   parent dir where temp file should be created
+        :param data:        write data to the file if provided
+
+        Returns: the path of the temp file created.
+        """
+        args = ['mktemp']
+        if suffix:
+            args.append('--suffix=%s' % suffix)
+        if parentdir:
+            args.append('--tmpdir=%s' % parentdir)
+
+        path = self.sh(args).strip()
+
+        if data:
+            self.write_file(path=path, data=data)
+
+        return path
+
+    def sh(self, script, **kwargs):
+        """
+        Shortcut for run method.
+
+        Usage:
+            my_name = remote.sh('whoami')
+            remote_date = remote.sh('date')
+        """
+        if 'stdout' not in kwargs:
+            kwargs['stdout'] = BytesIO()
+        if 'args' not in kwargs:
+            kwargs['args'] = script
+        proc = self.run(**kwargs)
+        out = proc.stdout.getvalue()
+        if isinstance(out, bytes):
+            return out.decode()
+        else:
+            return out
+
+    def sh_file(self, script, label="script", sudo=False, **kwargs):
+        """
+        Run shell script after copying its contents to a remote file
+
+        :param script:  string with script text, or file object
+        :param sudo:    run command with sudo if True,
+                        run as user name if string value (defaults to False)
+        :param label:   string value which will be part of file name
+        Returns: stdout
+        """
+        ftempl = '/tmp/teuthology-remote-$(date +%Y%m%d%H%M%S)-{}-XXXX'\
+                 .format(label)
+        script_file = self.sh("mktemp %s" % ftempl).strip()
+        self.sh("cat - | tee {script} ; chmod a+rx {script}"\
+            .format(script=script_file), stdin=script)
+        if sudo:
+            if isinstance(sudo, str):
+                command="sudo -u %s %s" % (sudo, script_file)
+            else:
+                command="sudo %s" % script_file
+        else:
+            command="%s" % script_file
+
+        return self.sh(command, **kwargs)
+
+    def chmod(self, file_path, permissions):
+        """
+        As super-user, set permissions on the remote file specified.
+        """
+        args = [
+            'sudo',
+            'chmod',
+            permissions,
+            file_path,
+            ]
+        self.run(
+            args=args,
+            )
+
+    def chcon(self, file_path, context):
+        """
+        Set the SELinux context of a given file.
+
+        VMs and non-RPM-based hosts will skip this operation because ours
+        currently have SELinux disabled.
+
+        :param file_path: The path to the file
+        :param context:   The SELinux context to be used
+        """
+        if self.os.package_type != 'rpm' or \
+                self.os.name in ['opensuse', 'sle']:
+            return
+        if teuthology.lock.query.is_vm(self.shortname):
+            return
+        self.run(args="sudo chcon {con} {path}".format(
+            con=context, path=file_path))
+
+    def copy_file(self, src, dst, sudo=False, mode=None, owner=None,
+                                              mkdir=False, append=False):
+        """
+        Copy data to remote file
+
+        :param src:     source file path on remote host
+        :param dst:     destination file path on remote host
+        :param sudo:    use sudo to write file, defaults False
+        :param mode:    set file mode bits if provided
+        :param owner:   set file owner if provided
+        :param mkdir:   ensure the destination directory exists, defaults
+                        False
+        :param append:  append data to the file, defaults False
+        """
+        dd = 'sudo dd' if sudo else 'dd'
+        args = dd + ' if=' + src + ' of=' + dst
+        if append:
+            args += ' conv=notrunc oflag=append'
+        if mkdir:
+            mkdirp = 'sudo mkdir -p' if sudo else 'mkdir -p'
+            dirpath = os.path.dirname(dst)
+            if dirpath:
+                args = mkdirp + ' ' + dirpath + '\n' + args
+        if mode:
+            chmod = 'sudo chmod' if sudo else 'chmod'
+            args += '\n' + chmod + ' ' + mode + ' ' + dst
+        if owner:
+            chown = 'sudo chown' if sudo else 'chown'
+            args += '\n' + chown + ' ' + owner + ' ' + dst
+        args = 'set -ex' + '\n' + args
+        self.run(args=args)
+
+    def move_file(self, src, dst, sudo=False, mode=None, owner=None,
+                                              mkdir=False):
+        """
+        Move data to remote file
+
+        :param src:     source file path on remote host
+        :param dst:     destination file path on remote host
+        :param sudo:    use sudo to write file, defaults False
+        :param mode:    set file mode bits if provided
+        :param owner:   set file owner if provided
+        :param mkdir:   ensure the destination directory exists, defaults
+                        False
+        """
+        mv = 'sudo mv' if sudo else 'mv'
+        args = mv + ' ' + src + ' ' + dst
+        if mkdir:
+            mkdirp = 'sudo mkdir -p' if sudo else 'mkdir -p'
+            dirpath = os.path.dirname(dst)
+            if dirpath:
+                args = mkdirp + ' ' + dirpath + '\n' + args
+        if mode:
+            chmod = 'sudo chmod' if sudo else 'chmod'
+            args += ' && ' + chmod + ' ' + mode + ' ' + dst
+        if owner:
+            chown = 'sudo chown' if sudo else 'chown'
+            args += ' && ' + chown + ' ' + owner + ' ' + dst
+        self.run(args=args)
+
+    def read_file(self, path, sudo=False, stdout=None,
+                              offset=0, length=0):
+        """
+        Read data from remote file
+
+        :param path:    file path on remote host
+        :param sudo:    use sudo to read the file, defaults False
+        :param stdout:  output object, defaults to io.BytesIO()
+        :param offset:  number of bytes to skip from the file
+        :param length:  number of bytes to read from the file
+
+        :raises: :class:`FileNotFoundError`: there is no such file by the path
+        :raises: :class:`RuntimeError`: unexpected error occurred
+
+        :returns: the file contents in bytes, if stdout is `io.BytesIO`, by
+                  default
+        :returns: the file contents in str, if stdout is `io.StringIO`
+        """
+        dd = 'sudo dd' if sudo else 'dd'
+        args = dd + ' if=' + path + ' of=/dev/stdout'
+        iflags=[]
+        # we have to set defaults here instead of the method's signature,
+        # because python is reusing the object from call to call
+        stdout = stdout or BytesIO()
+        if offset:
+            args += ' skip=' + str(offset)
+            iflags += 'skip_bytes'
+        if length:
+            args += ' count=' + str(length)
+            iflags += 'count_bytes'
+        if iflags:
+            args += ' iflag=' + ','.join(iflags)
+        args = 'set -ex' + '\n' + args
+        proc = self.run(args=args, stdout=stdout, stderr=StringIO(),
+                        check_status=False, quiet=True)
+        if proc.returncode:
+            if 'No such file or directory' in proc.stderr.getvalue():
+                raise FileNotFoundError(errno.ENOENT,
+                        f"Cannot find file on the remote '{self.name}'", path)
+            else:
+                raise RuntimeError("Unexpected error occurred while trying to "
+                        f"read '{path}' file on the remote '{self.name}'")
+
+        return proc.stdout.getvalue()
+
+
+    def write_file(self, path, data, sudo=False, mode=None, owner=None,
+                                     mkdir=False, append=False):
+        """
+        Write data to remote file
+
+        :param path:    file path on remote host
+        :param data:    str, binary or fileobj to be written
+        :param sudo:    use sudo to write file, defaults False
+        :param mode:    set file mode bits if provided
+        :param owner:   set file owner if provided
+        :param mkdir:   preliminary create the file directory, defaults False
+        :param append:  append data to the file, defaults False
+        """
+        dd = 'sudo dd' if sudo else 'dd'
+        args = dd + ' of=' + path
+        if append:
+            args += ' conv=notrunc oflag=append'
+        if mkdir:
+            mkdirp = 'sudo mkdir -p' if sudo else 'mkdir -p'
+            dirpath = os.path.dirname(path)
+            if dirpath:
+                args = mkdirp + ' ' + dirpath + '\n' + args
+        if mode:
+            chmod = 'sudo chmod' if sudo else 'chmod'
+            args += '\n' + chmod + ' ' + mode + ' ' + path
+        if owner:
+            chown = 'sudo chown' if sudo else 'chown'
+            args += '\n' + chown + ' ' + owner + ' ' + path
+        args = 'set -ex' + '\n' + args
+        self.run(args=args, stdin=data, quiet=True)
+
+    def sudo_write_file(self, path, data, **kwargs):
+        """
+        Write data to remote file with sudo, for more info see `write_file()`.
+        """
+        self.write_file(path, data, sudo=True, **kwargs)
+
+    def is_mounted(self, path):
+        """
+        Check if the given path is mounted on the remote machine.
+
+        This method checks the contents of "/proc/self/mounts" instead of
+        using "mount" or "findmnt" command since these commands hang when a
+        CephFS client is blocked and its mount point on the remote machine
+        is left unhandled/unmounted.
+
+        :param path: path on remote host
+        """
+        # XXX: matching newline too is crucial so that "/mnt" does not match
+        # "/mnt/cephfs" if it's present in the output.
+        return f'{path}\n' in self.sh("cat /proc/self/mounts | awk '{print $2}'")
+
+    @property
+    def os(self):
+        if not hasattr(self, '_os'):
+            try:
+                os_release = self.sh('cat /etc/os-release').strip()
+                self._os = OS.from_os_release(os_release)
+                return self._os
+            except CommandFailedError:
+                pass
+
+            lsb_release = self.sh('lsb_release -a').strip()
+            self._os = OS.from_lsb_release(lsb_release)
+        return self._os
 
+    @property
+    def arch(self):
+        if not hasattr(self, '_arch'):
+            self._arch = self.sh('uname -m').strip()
+        return self._arch
+
+
+class Remote(RemoteShell):
     """
     A connection to a remote host.
 
@@ -33,6 +346,7 @@ class Remote(object):
 
     # for unit tests to hook into
     _runner = staticmethod(run.run)
+    _reimage_types = None
 
     def __init__(self, name, ssh=None, shortname=None, console=None,
                  host_key=None, keep_alive=True):
@@ -53,6 +367,9 @@ class Remote(object):
         self._console = console
         self.ssh = ssh
 
+        if self._reimage_types is None:
+            Remote._reimage_types = teuthology.provision.get_reimage_types()
+
     def connect(self, timeout=None, create_key=None, context='connect'):
         args = dict(user_at_host=self.name, host_key=self._host_key,
                     keep_alive=self.keep_alive, _create_key=create_key)
@@ -70,7 +387,7 @@ class Remote(object):
         self.ssh = connection.connect(**args)
         return self.ssh
 
-    def reconnect(self, timeout=None, socket_timeout=None, sleep_time=30):
+    def reconnect(self, timeout=30, socket_timeout=None):
         """
         Attempts to re-establish connection. Returns True for success; False
         for failure.
@@ -79,21 +396,18 @@ class Remote(object):
             self.ssh.close()
         if not timeout:
             return self._reconnect(timeout=socket_timeout)
-        start_time = time.time()
-        elapsed_time = lambda: time.time() - start_time
-        while elapsed_time() < timeout:
-            success = self._reconnect(timeout=socket_timeout)
-            if success:
-                log.info('Successfully reconnected to host')
-                break
-            # Don't let time_remaining be < 0
-            time_remaining = max(0, timeout - elapsed_time())
-            sleep_val = min(time_remaining, sleep_time)
-            time.sleep(sleep_val)
-        return success
+        action = "reconnect to {self.shortname}"
+        with safe_while(action=action, timeout=timeout, increment=3, _raise=False) as proceed:
+            success = False
+            while proceed():
+                success = self._reconnect(timeout=socket_timeout)
+                if success:
+                    log.info(f"Successfully reconnected to host '{self.name}'")
+                    return success
+            return success
 
     def _reconnect(self, timeout=None):
-        log.info("Trying to reconnect to host")
+        log.info(f"Trying to reconnect to host '{self.name}'")
         try:
             self.connect(timeout=timeout, context='reconnect')
             return self.is_online
@@ -150,6 +464,10 @@ class Remote(object):
             self._machine_type = remote_info.get("machine_type", None)
         return self._machine_type
 
+    @property
+    def is_reimageable(self):
+        return self.machine_type in self._reimage_types
+
     @property
     def shortname(self):
         if self._shortname is None:
@@ -173,7 +491,7 @@ class Remote(object):
             return
         self.connect()
         if not self.is_online:
-            raise Exception('unable to connect')
+            raise ConnectionError(f'Failed to connect to {self.shortname}')
 
     @property
     def system_type(self):
@@ -200,114 +518,25 @@ class Remote(object):
         if not self.ssh or \
            not self.ssh.get_transport() or \
            not self.ssh.get_transport().is_active():
-            self.reconnect()
+            if not self.reconnect():
+                raise ConnectionError(f'Failed to reconnect to {self.shortname}')
         r = self._runner(client=self.ssh, name=self.shortname, **kwargs)
         r.remote = self
         return r
 
-    def mkdtemp(self, suffix=None, parentdir=None):
-        """
-        Create a temporary directory on remote machine and return it's path.
-        """
-        args = ['mktemp', '-d']
-
-        if suffix:
-            args.append('--suffix=%s' % suffix)
-        if parentdir:
-            args.append('--tmpdir=%s' % parentdir)
-
-        return self.sh(args).strip()
-
-    def mktemp(self, suffix=None, parentdir=None):
-        """
-        Make a remote temporary file
-
-        Returns: the path of the temp file created.
-        """
-        args = ['mktemp']
-
-        if suffix:
-            args.append('--suffix=%s' % suffix)
-        if parentdir:
-            args.append('--tmpdir=%s' % parentdir)
-
-        return self.sh(args).strip()
-
-    def sh(self, script, **kwargs):
-        """
-        Shortcut for run method.
-
-        Usage:
-            my_name = remote.sh('whoami')
-            remote_date = remote.sh('date')
-        """
-        if 'stdout' not in kwargs:
-            kwargs['stdout'] = BytesIO()
-        if 'args' not in kwargs:
-            kwargs['args'] = script
-        proc = self.run(**kwargs)
-        out = proc.stdout.getvalue()
-        if isinstance(out, bytes):
-            return out.decode()
-        else:
-            return out
-
-    def sh_file(self, script, label="script", sudo=False, **kwargs):
-        """
-        Run shell script after copying its contents to a remote file
-
-        :param script:  string with script text, or file object
-        :param sudo:    run command with sudo if True,
-                        run as user name if string value (defaults to False)
-        :param label:   string value which will be part of file name
-        Returns: stdout
-        """
-        ftempl = '/tmp/teuthology-remote-$(date +%Y%m%d%H%M%S)-{}-XXXX'\
-                 .format(label)
-        script_file = self.sh("mktemp %s" % ftempl).strip()
-        self.sh("cat - | tee {script} ; chmod a+rx {script}"\
-            .format(script=script_file), stdin=script)
-        if sudo:
-            if isinstance(sudo, str):
-                command="sudo -u %s %s" % (sudo, script_file)
-            else:
-                command="sudo %s" % script_file
-        else:
-            command="%s" % script_file
-
-        return self.sh(command, **kwargs)
-
-    def chmod(self, file_path, permissions):
-        """
-        As super-user, set permissions on the remote file specified.
-        """
-        args = [
-            'sudo',
-            'chmod',
-            permissions,
-            file_path,
-            ]
-        self.run(
-            args=args,
-            )
-
-    def chcon(self, file_path, context):
-        """
-        Set the SELinux context of a given file.
-
-        VMs and non-RPM-based hosts will skip this operation because ours
-        currently have SELinux disabled.
-
-        :param file_path: The path to the file
-        :param context:   The SELinux context to be used
-        """
-        if self.os.package_type != 'rpm' or \
-                self.os.name in ['opensuse', 'sle']:
-            return
-        if teuthology.lock.query.is_vm(self.shortname):
-            return
-        self.run(args="sudo chcon {con} {path}".format(
-            con=context, path=file_path))
+    def run_unit_test(self, xml_path_regex, output_yaml, **kwargs):
+        try:
+            r = self.run(**kwargs)
+        except CommandFailedError as exc:
+            if xml_path_regex:
+                error_msg = UnitTestScanner(remote=self).scan_and_write(xml_path_regex, output_yaml)
+                if error_msg:
+                    raise UnitTestError(
+                        exitstatus=exc.exitstatus, node=exc.node, 
+                        label=exc.label, message=error_msg
+                    )
+            raise exc
+        return r
 
     def _sftp_put_file(self, local_path, remote_path):
         """
@@ -329,12 +558,14 @@ class Remote(object):
         sftp.get(remote_path, local_path)
         return local_path
 
-    def _sftp_open_file(self, remote_path):
+    def _sftp_open_file(self, remote_path, mode=None):
         """
         Use the paramiko.SFTPClient to open a file. Returns a
         paramiko.SFTPFile object.
         """
         sftp = self.ssh.open_sftp()
+        if mode:
+            return sftp.open(remote_path, mode)
         return sftp.open(remote_path)
 
     def _sftp_get_size(self, remote_path):
@@ -355,9 +586,6 @@ class Remote(object):
             file_size = file_size / 1024.0
         return "{:3.0f}{}".format(file_size, unit)
 
-    def remove(self, path):
-        self.run(args=['rm', '-fr', path])
-
     def put_file(self, path, dest_path, sudo=False):
         """
         Copy a local filename to a remote file
@@ -408,7 +636,7 @@ class Remote(object):
             self.remove(path)
         return local_path
 
-    def get_tar(self, path, to_path, sudo=False):
+    def get_tar(self, path, to_path, sudo=False, compress=True):
         """
         Tar a remote directory and copy it locally
         """
@@ -418,7 +646,7 @@ class Remote(object):
             args.append('sudo')
         args.extend([
             'tar',
-            'cz',
+            'cz' if compress else 'c',
             '-f', '-',
             '-C', path,
             '--',
@@ -431,7 +659,7 @@ class Remote(object):
         self._sftp_get_file(remote_temp_path, to_path)
         self.remove(remote_temp_path)
 
-    def get_tar_stream(self, path, sudo=False):
+    def get_tar_stream(self, path, sudo=False, compress=True):
         """
         Tar-compress a remote directory and return the RemoteProcess
         for streaming
@@ -441,7 +669,7 @@ class Remote(object):
             args.append('sudo')
         args.extend([
             'tar',
-            'cz',
+            'cz' if compress else 'c',
             '-f', '-',
             '-C', path,
             '--',
@@ -449,26 +677,6 @@ class Remote(object):
             ])
         return self.run(args=args, wait=False, stdout=run.PIPE)
 
-    @property
-    def os(self):
-        if not hasattr(self, '_os'):
-            try:
-                os_release = self.sh('cat /etc/os-release').strip()
-                self._os = OS.from_os_release(os_release)
-                return self._os
-            except CommandFailedError:
-                pass
-
-            lsb_release = self.sh('lsb_release -a').strip()
-            self._os = OS.from_lsb_release(lsb_release)
-        return self._os
-
-    @property
-    def arch(self):
-        if not hasattr(self, '_arch'):
-            self._arch = self.sh('uname -m').strip()
-        return self._arch
-
     @property
     def host_key(self):
         if not self._host_key:
@@ -501,6 +709,15 @@ class Remote(object):
             self._is_vm = teuthology.lock.query.is_vm(self.name)
         return self._is_vm
 
+    @property
+    def is_container(self):
+        if not hasattr(self, '_is_container'):
+            self._is_container = not bool(self.run(
+                args="test -f /run/.containerenv -o -f /.dockerenv",
+                check_status=False,
+            ).returncode)
+        return self._is_container
+
     @property
     def init_system(self):
         """
@@ -524,7 +741,7 @@ class Remote(object):
 
 
 def getRemoteConsole(name, ipmiuser=None, ipmipass=None, ipmidomain=None,
-                     logfile=None, timeout=20):
+                     timeout=60):
     """
     Return either VirtualConsole or PhysicalConsole depending on name.
     """
@@ -534,4 +751,4 @@ def getRemoteConsole(name, ipmiuser=None, ipmipass=None, ipmidomain=None,
         except Exception:
             return None
     return console.PhysicalConsole(
-        name, ipmiuser, ipmipass, ipmidomain, logfile, timeout)
+        name, ipmiuser, ipmipass, ipmidomain, timeout)