teuthology 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

teuthology/misc.py

@@ -7,6 +7,7 @@ import os
 import logging
 import configobj
 import getpass
+import shutil
 import socket
 import subprocess
 import tarfile
@@ -14,7 +15,13 @@ import time
 import yaml
 import json
 import re
+from sys import stdin
 import pprint
+import datetime
+
+from tarfile import ReadError
+
+from typing import Optional, TypeVar
 
 from teuthology.util.compat import urljoin, urlopen, HTTPError
 
@@ -31,13 +38,12 @@ from teuthology.orchestra.opsys import DEFAULT_OS_VERSION
 
 log = logging.getLogger(__name__)
 
-import datetime
 stamp = datetime.datetime.now().strftime("%y%m%d%H%M")
 
 is_arm = lambda x: x.startswith('tala') or x.startswith(
     'ubuntu@tala') or x.startswith('saya') or x.startswith('ubuntu@saya')
 
-hostname_expr_templ = '(?P<user>.*@)?(?P<shortname>.*)\.{lab_domain}'
+hostname_expr_templ = '(?P<user>.*@)?(?P<shortname>.*){lab_domain}'
 
 def host_shortname(hostname):
     if _is_ipv4(hostname) or _is_ipv6(hostname):
@@ -45,9 +51,9 @@ def host_shortname(hostname):
     else:
         return hostname.split('.', 1)[0]
 
-def canonicalize_hostname(hostname, user='ubuntu'):
+def canonicalize_hostname(hostname, user: Optional[str] ='ubuntu'):
     hostname_expr = hostname_expr_templ.format(
-        lab_domain=config.lab_domain.replace('.', '\.'))
+        lab_domain=config.lab_domain.replace('.', r'\.'))
     match = re.match(hostname_expr, hostname)
     if _is_ipv4(hostname) or _is_ipv6(hostname):
         return "%s@%s" % (user, hostname)
@@ -63,18 +69,22 @@ def canonicalize_hostname(hostname, user='ubuntu'):
         user_ = user
 
     user_at = user_.strip('@') + '@' if user_ else ''
-
-    ret = '{user_at}{short}.{lab_domain}'.format(
+    domain = config.lab_domain
+    if domain and not shortname.endswith('.'):
+        domain = '.' + domain
+    ret = '{user_at}{short}{domain}'.format(
         user_at=user_at,
         short=shortname,
-        lab_domain=config.lab_domain,
+        domain=domain,
     )
     return ret
 
 
 def decanonicalize_hostname(hostname):
-    hostname_expr = hostname_expr_templ.format(
-        lab_domain=config.lab_domain.replace('.', '\.'))
+    lab_domain = ''
+    if config.lab_domain:
+        lab_domain=r'\.' + config.lab_domain.replace('.', r'\.')
+    hostname_expr = hostname_expr_templ.format(lab_domain=lab_domain)
     match = re.match(hostname_expr, hostname)
     if match:
         hostname = match.groupdict()['shortname']
@@ -99,31 +109,20 @@ def config_file(string):
     return config_dict
 
 
-class MergeConfig(argparse.Action):
-    """
-    Used by scripts to mergeg configurations.   (nuke, run, and
-    schedule, for example)
-    """
-    def __call__(self, parser, namespace, values, option_string=None):
-        """
-        Perform merges of all the day in the config dictionaries.
-        """
-        config_dict = getattr(namespace, self.dest)
-        for new in values:
-            deep_merge(config_dict, new)
-
-
-def merge_configs(config_paths):
+def merge_configs(config_paths) -> dict:
     """ Takes one or many paths to yaml config files and merges them
         together, returning the result.
     """
     conf_dict = dict()
     for conf_path in config_paths:
-        if not os.path.exists(conf_path):
+        if conf_path == "-":
+            partial_dict = yaml.safe_load(stdin)
+        elif not os.path.exists(conf_path):
             log.debug("The config path {0} does not exist, skipping.".format(conf_path))
             continue
-        with open(conf_path) as partial_file:
-            partial_dict = yaml.safe_load(partial_file)
+        else:
+            with open(conf_path) as partial_file:
+                partial_dict: dict = yaml.safe_load(partial_file)
         try:
             conf_dict = deep_merge(conf_dict, partial_dict)
         except Exception:
@@ -220,13 +219,13 @@ def get_ceph_binary_url(package=None,
         assert tag is None, "cannot set both sha1 and tag"
     else:
         # gitbuilder uses remote-style ref names for branches, mangled to
-        # have underscores instead of slashes; e.g. origin_master
+        # have underscores instead of slashes; e.g. origin_main
         if tag is not None:
             ref = tag
             assert branch is None, "cannot set both branch and tag"
         else:
             if branch is None:
-                branch = 'master'
+                branch = 'main'
             ref = branch
 
         sha1_url = urljoin(BASE, 'ref/{ref}/sha1'.format(ref=ref))
@@ -521,14 +520,7 @@ def write_file(remote, path, data):
     :param path: Path on the remote being written to.
     :param data: Data to be written.
     """
-    remote.run(
-        args=[
-            'cat',
-            run.Raw('>'),
-            path,
-        ],
-        stdin=data,
-    )
+    remote.write_file(path, data)
 
 
 def sudo_write_file(remote, path, data, perms=None, owner=None):
@@ -543,21 +535,7 @@ def sudo_write_file(remote, path, data, perms=None, owner=None):
 
     Both perms and owner are passed directly to chmod.
     """
-    permargs = []
-    if perms:
-        permargs = [run.Raw('&&'), 'sudo', 'chmod', perms, path]
-    owner_args = []
-    if owner:
-        owner_args = [run.Raw('&&'), 'sudo', 'chown', owner, path]
-    remote.run(
-        args=[
-            'sudo',
-            'sh',
-            '-c',
-            'cat > ' + path,
-        ] + owner_args + permargs,
-        stdin=data,
-    )
+    remote.sudo_write_file(path, data, mode=perms, owner=owner)
 
 
 def copy_file(from_remote, from_path, to_remote, to_path=None):
@@ -645,7 +623,7 @@ def remove_lines_from_file(remote, path, line_is_valid_test,
     on when the main site goes up and down.
     """
     # read in the specified file
-    in_data = get_file(remote, path, False).decode()
+    in_data = remote.read_file(path, False).decode()
     out_data = ""
 
     first_line = True
@@ -677,22 +655,8 @@ def remove_lines_from_file(remote, path, line_is_valid_test,
 def append_lines_to_file(remote, path, lines, sudo=False):
     """
     Append lines to a file.
-    An intermediate file is used in the same manner as in
-    Remove_lines_from_list.
     """
-
-    temp_file_path = remote.mktemp()
-
-    data = get_file(remote, path, sudo).decode()
-
-    # add the additional data and write it back out, using a temp file
-    # in case of connectivity of loss, and then mv it to the
-    # actual desired location
-    data += lines
-    write_file(remote, temp_file_path, data)
-
-    # then do a 'mv' to the actual file location
-    move_file(remote, temp_file_path, path, sudo)
+    remote.write_file(path, lines, append=True, sudo=sudo)
 
 def prepend_lines_to_file(remote, path, lines, sudo=False):
     """
@@ -702,17 +666,9 @@ def prepend_lines_to_file(remote, path, lines, sudo=False):
     """
 
     temp_file_path = remote.mktemp()
-
-    data = get_file(remote, path, sudo).decode()
-
-    # add the additional data and write it back out, using a temp file
-    # in case of connectivity of loss, and then mv it to the
-    # actual desired location
-    data = lines + data
-    write_file(remote, temp_file_path, data)
-
-    # then do a 'mv' to the actual file location
-    move_file(remote, temp_file_path, path, sudo)
+    remote.write_file(temp_file_path, lines)
+    remote.copy_file(path, temp_file_path, append=True, sudo=sudo)
+    remote.move_file(temp_file_path, path, sudo=sudo)
 
 
 def create_file(remote, path, data="", permissions=str(644), sudo=False):
@@ -753,16 +709,31 @@ def get_file(remote, path, sudo=False, dest_dir='/tmp'):
     return file_data
 
 
-def pull_directory(remote, remotedir, localdir):
+def copy_fileobj(src, tarinfo, local_path):
+    with open(local_path, 'wb') as dest:
+        shutil.copyfileobj(src, dest)
+
+
+def pull_directory(remote, remotedir, localdir, write_to=copy_fileobj):
     """
     Copy a remote directory to a local directory.
+
+    :param remote: the remote object representing the remote host from where
+                   the specified directory is pulled
+    :param remotedir: the source directory on remote host
+    :param localdir: the destination directory on localhost
+    :param write_to: optional function to write the file to localdir.
+                     its signature should be:
+                     func(src: fileobj,
+                          tarinfo: tarfile.TarInfo,
+                          local_path: str)
     """
     log.debug('Transferring archived files from %s:%s to %s',
               remote.shortname, remotedir, localdir)
     if not os.path.exists(localdir):
         os.mkdir(localdir)
-    r = remote.get_tar_stream(remotedir, sudo=True)
-    tar = tarfile.open(mode='r|gz', fileobj=r.stdout)
+    r = remote.get_tar_stream(remotedir, sudo=True, compress=False)
+    tar = tarfile.open(mode='r|', fileobj=r.stdout)
     while True:
         ti = tar.next()
         if ti is None:
@@ -775,7 +746,8 @@ def pull_directory(remote, remotedir, localdir):
         elif ti.isfile():
             sub = safepath.munge(ti.name)
             safepath.makedirs(root=localdir, path=os.path.dirname(sub))
-            tar.makefile(ti, targetpath=os.path.join(localdir, sub))
+            with tar.extractfile(ti) as src:
+                write_to(src, ti, os.path.join(localdir, sub))
         else:
             if ti.isdev():
                 type_ = 'device'
@@ -798,7 +770,7 @@ def pull_directory_tarball(remote, remotedir, localfile):
 
 
 def get_wwn_id_map(remote, devs):
-    log.warn("Entering get_wwn_id_map, a deprecated function that will be removed")
+    log.warning("Entering get_wwn_id_map, a deprecated function that will be removed")
     return dict((d, d) for d in devs)
 
 
@@ -808,7 +780,7 @@ def get_scratch_devices(remote):
     """
     devs = []
     try:
-        file_data = get_file(remote, "/scratch_devs").decode()
+        file_data = remote.read_file("/scratch_devs").decode()
         devs = file_data.split()
     except Exception:
         devs = remote.sh('ls /dev/[sv]d?').strip().split('\n')
@@ -817,33 +789,31 @@ def get_scratch_devices(remote):
     for dev in devs:
         if 'vda' in dev:
             devs.remove(dev)
-            log.warn("Removing root device: %s from device list" % dev)
+            log.warning("Removing root device: %s from device list" % dev)
 
     log.debug('devs={d}'.format(d=devs))
 
     retval = []
     for dev in devs:
-        try:
-            # FIXME: Split this into multiple calls.
-            remote.run(
-                args=[
-                    # node exists
-                    'stat',
-                    dev,
-                    run.Raw('&&'),
-                    # readable
-                    'sudo', 'dd', 'if=%s' % dev, 'of=/dev/null', 'count=1',
-                    run.Raw('&&'),
-                    # not mounted
-                    run.Raw('!'),
-                    'mount',
-                    run.Raw('|'),
-                    'grep', '-q', dev,
-                ]
-            )
+        dev_checks = [
+            [['stat', dev], "does not exist"],
+            [['sudo', 'dd', 'if=%s' % dev, 'of=/dev/null', 'count=1'], "is not readable"],
+            [
+                [run.Raw('!'), 'mount', run.Raw('|'), 'grep', '-v', 'devtmpfs', run.Raw('|'),
+                'grep', '-q', dev],
+                "is in use"
+            ],
+        ]
+        for args, msg in dev_checks:
+            try:
+                remote.run(args=args)
+            except CommandFailedError:
+                log.debug(f"get_scratch_devices: {dev} {msg}")
+                break
+        else:
             retval.append(dev)
-        except CommandFailedError:
-            log.debug("get_scratch_devices: %s is in use" % dev)
+            continue
+        break
     return retval
 
 
@@ -1015,7 +985,8 @@ def replace_all_with_clients(cluster, config):
     return norm_config
 
 
-def deep_merge(a, b):
+DeepMerge = TypeVar('DeepMerge')
+def deep_merge(a: DeepMerge, b: DeepMerge) -> DeepMerge:
     """
     Deep Merge.  If a and b are both lists, all elements in b are
     added into a.  If a and b are both dictionaries, elements in b are
@@ -1023,10 +994,10 @@ def deep_merge(a, b):
     :param a: object items will be merged into
     :param b: object items will be merged from
     """
-    if a is None:
-        return b
     if b is None:
         return a
+    if a is None:
+        return deep_merge(b.__class__(), b)
     if isinstance(a, list):
         assert isinstance(b, list)
         a.extend(b)
@@ -1034,15 +1005,12 @@ def deep_merge(a, b):
     if isinstance(a, dict):
         assert isinstance(b, dict)
         for (k, v) in b.items():
-            if k in a:
-                a[k] = deep_merge(a[k], v)
-            else:
-                a[k] = v
+            a[k] = deep_merge(a.get(k), v)
         return a
     return b
 
 
-def get_valgrind_args(testdir, name, preamble, v):
+def get_valgrind_args(testdir, name, preamble, v, exit_on_first_error=True):
     """
     Build a command line for running valgrind.
 
@@ -1067,28 +1035,31 @@ def get_valgrind_args(testdir, name, preamble, v):
             'valgrind',
             '--trace-children=no',
             '--child-silent-after-fork=yes',
+            '--soname-synonyms=somalloc=*tcmalloc*',
             '--num-callers=50',
             '--suppressions={tdir}/valgrind.supp'.format(tdir=testdir),
             '--xml=yes',
             '--xml-file={vdir}/{n}.log'.format(vdir=val_path, n=name),
             '--time-stamp=yes',
             '--vgdb=yes',
-            # at least Valgrind 3.14 is required
-            '--exit-on-first-error=yes',
-            '--error-exitcode=42',
         ]
     else:
         extra_args = [
             'valgrind',
             '--trace-children=no',
             '--child-silent-after-fork=yes',
+            '--soname-synonyms=somalloc=*tcmalloc*',
             '--suppressions={tdir}/valgrind.supp'.format(tdir=testdir),
             '--log-file={vdir}/{n}.log'.format(vdir=val_path, n=name),
             '--time-stamp=yes',
             '--vgdb=yes',
+        ]
+    if exit_on_first_error:
+        extra_args.extend([
+            # at least Valgrind 3.14 is required
             '--exit-on-first-error=yes',
             '--error-exitcode=42',
-        ]
+        ])
     args = [
         'cd', testdir,
         run.Raw('&&'),
@@ -1113,7 +1084,8 @@ def ssh_keyscan(hostnames, _raise=True):
     for hostname in hostnames:
         with safe_while(
             sleep=1,
-            tries=5 if _raise else 1,
+            tries=15 if _raise else 1,
+            increment=1,
             _raise=_raise,
             action="ssh_keyscan " + hostname,
         ) as proceed:
@@ -1126,7 +1098,7 @@ def ssh_keyscan(hostnames, _raise=True):
         missing = set(hostnames) - set(keys_dict.keys())
         msg = "Unable to scan these host keys: %s" % ' '.join(missing)
         if not _raise:
-            log.warn(msg)
+            log.warning(msg)
         else:
             raise RuntimeError(msg)
     return keys_dict
@@ -1139,7 +1111,7 @@ def _ssh_keyscan(hostname):
     :param hostname: The hostname
     :returns: The host key
     """
-    args = ['ssh-keyscan', '-T', '1', '-t', 'rsa', hostname]
+    args = ['ssh-keyscan', '-T', '1', hostname]
     p = subprocess.Popen(
         args=args,
         stdout=subprocess.PIPE,
@@ -1151,9 +1123,12 @@ def _ssh_keyscan(hostname):
         line = line.strip()
         if line and not line.startswith('#'):
             log.error(line)
+    keys = list()
     for line in p.stdout:
         host, key = line.strip().decode().split(' ', 1)
-        return key
+        keys.append(key)
+    if len(keys) > 0:
+        return sorted(keys)[0]
 
 
 def ssh_keyscan_wait(hostname):
@@ -1194,28 +1169,19 @@ def stop_daemons_of_type(ctx, type_, cluster='ceph'):
 def get_system_type(remote, distro=False, version=False):
     """
     If distro, return distro.
-    If version, return version (lsb_release -rs)
+    If version, return version
     If both, return both.
     If neither, return 'deb' or 'rpm' if distro is known to be one of those
-    Finally, if unknown, return the unfiltered distro (from lsb_release -is)
     """
-    system_value = remote.sh('sudo lsb_release -is').strip()
-    log.debug("System to be installed: %s" % system_value)
     if version:
-        version = remote.sh('sudo lsb_release -rs').strip()
+        version = remote.os.version
     if distro and version:
-        return system_value.lower(), version
+        return remote.os.name, version
     if distro:
-        return system_value.lower()
+        return remote.os.name
     if version:
         return version
-    if system_value in ['Ubuntu', 'Debian']:
-        return "deb"
-    if system_value in ['CentOS', 'Fedora', 'RedHatEnterpriseServer',
-                        'RedHatEnterprise',
-                        'openSUSE', 'openSUSE project', 'SUSE', 'SUSE LINUX']:
-        return "rpm"
-    return system_value
+    return remote.os.package_type
 
 def get_pkg_type(os_type):
     if os_type in ('centos', 'fedora', 'opensuse', 'rhel', 'sle'):
@@ -1324,7 +1290,7 @@ def sh(command, log_limit=1024, cwd=None, env=None):
         for line in proc.stdout:
             line = line.decode()
             lines.append(line)
-            line = line.strip()
+            line = line.rstrip()
             if len(line) > log_limit:
                 truncated = True
                 log.debug(line[:log_limit] +
@@ -1345,3 +1311,52 @@ def sh(command, log_limit=1024, cwd=None, env=None):
             output=output
         )
     return output
+
+
+def add_remote_path(ctx, local_dir, remote_dir):
+    """
+    Add key/value pair (local_dir: remote_dir) to job's info.yaml.
+    These key/value pairs are read to archive them in case of job timeout.
+    """
+    if ctx.archive is None:
+        return
+    with open(os.path.join(ctx.archive, 'info.yaml'), 'r+') as info_file:
+        info_yaml = yaml.safe_load(info_file)
+        info_file.seek(0)
+        if 'archive' in info_yaml:
+            info_yaml['archive'][local_dir] = remote_dir
+        else:
+            info_yaml['archive'] = {local_dir: remote_dir}
+        yaml.safe_dump(info_yaml, info_file, default_flow_style=False)
+
+
+def archive_logs(ctx, remote_path, log_path):
+    """
+    Archive directories from all nodes in a cliuster. It pulls all files in
+    remote_path dir to job's archive dir under log_path dir.
+    """
+    if ctx.archive is None:
+        return
+    path = os.path.join(ctx.archive, 'remote')
+    os.makedirs(path, exist_ok=True)
+    for remote in ctx.cluster.remotes.keys():
+        sub = os.path.join(path, remote.shortname)
+        os.makedirs(sub, exist_ok=True)
+        try:
+            pull_directory(remote, remote_path, os.path.join(sub, log_path))
+        except ReadError:
+            pass
+
+
+def compress_logs(ctx, remote_dir):
+    """
+    Compress all files in remote_dir from all nodes in a cluster.
+    """
+    log.info('Compressing logs...')
+    run.wait(
+        ctx.cluster.run(
+            args=(f"sudo find {remote_dir} -name *.log -print0 | "
+                  f"sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose --"),
+            wait=False,
+        ),
+    )