teuthology 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

teuthology/task/ceph_ansible.py

@@ -22,7 +22,7 @@ class CephAnsible(Task):
 
     - ceph-ansible:
         repo: {git_base}ceph-ansible.git
-        branch: mybranch # defaults to master
+        branch: mybranch # defaults to main
         ansible-version: 2.4 # defaults to 2.5
         vars:
           ceph_dev: True ( default)
@@ -73,7 +73,7 @@ class CephAnsible(Task):
         if 'ceph_dev_key' not in vars:
             vars['ceph_dev_key'] = 'https://download.ceph.com/keys/autobuild.asc'
         if 'ceph_dev_branch' not in vars:
-            vars['ceph_dev_branch'] = ctx.config.get('branch', 'master')
+            vars['ceph_dev_branch'] = ctx.config.get('branch', 'main')
         self.cluster_name = vars.get('cluster', 'ceph')
 
     def setup(self):
@@ -296,8 +296,11 @@ class CephAnsible(Task):
             roles = self.ctx.cluster.remotes[remote]
             dev_needed = len([role for role in roles
                               if role.startswith('osd')])
-            if teuth_config.get('ceph_ansible') and \
-                    self.ctx.machine_type in teuth_config['ceph_ansible']['has_lvm_scratch_disks']:
+            if (
+                    teuth_config.get('ceph_ansible') and 
+                    hasattr(self.ctx, "machine_type") and
+                    self.ctx.machine_type in teuth_config['ceph_ansible']['has_lvm_scratch_disks']
+                ):
                 devices = get_file(remote, "/scratch_devs").decode().split()
                 vols = []
 
@@ -375,7 +378,7 @@ class CephAnsible(Task):
                 'python-dev'
             ])
         ansible_repo = self.config['repo']
-        branch = 'master'
+        branch = 'main'
         if self.config.get('branch'):
             branch = self.config.get('branch')
         ansible_ver = 'ansible==2.5'
@@ -404,7 +407,6 @@ class CephAnsible(Task):
             run.Raw('cd ~/ceph-ansible'),
             run.Raw(';'),
             'virtualenv',
-            run.Raw('--system-site-packages'),
             run.Raw('--python=python3'),
             'venv',
             run.Raw(';'),
@@ -425,6 +427,11 @@ class CephAnsible(Task):
             run.Raw('setuptools>=11.3'),
             run.Raw('notario>=0.0.13'), # FIXME: use requirements.txt
             run.Raw('netaddr'),
+            run.Raw('six'),
+            run.Raw(';'),
+            'LANG=en_US.utf8',
+            'pip',
+            'install',
             run.Raw(ansible_ver),
             run.Raw(';'),
             run.Raw(str_args)

teuthology/task/cephmetrics.py

@@ -5,8 +5,9 @@ import time
 
 from teuthology.config import config as teuth_config
 from teuthology.exceptions import CommandFailedError
+from teuthology.task.ansible import Ansible
+from teuthology.util.loggerfile import LoggerFile
 
-from teuthology.ansible import Ansible, LoggerFile
 
 log = logging.getLogger(__name__)
 

teuthology/task/clock.py

@@ -8,6 +8,13 @@ from teuthology.orchestra import run
 
 log = logging.getLogger(__name__)
 
+def filter_out_containers(cluster):
+    """
+    Returns a cluster that excludes remotes which should skip this task.
+    Currently, only skips containerized remotes.
+    """
+    return cluster.filter(lambda r: not r.is_container)
+
 @contextlib.contextmanager
 def task(ctx, config):
     """
@@ -30,8 +37,9 @@ def task(ctx, config):
     """
 
     log.info('Syncing clocks and checking initial clock skew...')
-    for rem in ctx.cluster.remotes.keys():
-        rem.run(
+    cluster = filter_out_containers(ctx.cluster)
+    run.wait(
+        cluster.run(
             args = [
                 'sudo', 'systemctl', 'stop', 'ntp.service', run.Raw('||'),
                 'sudo', 'systemctl', 'stop', 'ntpd.service', run.Raw('||'),
@@ -50,22 +58,27 @@ def task(ctx, config):
                 'true'
             ],
             timeout = 360,
+            wait=False,
         )
+    )
 
     try:
         yield
 
     finally:
         log.info('Checking final clock skew...')
-        for rem in ctx.cluster.remotes.keys():
-            rem.run(
+        cluster = filter_out_containers(ctx.cluster)
+        run.wait(
+            cluster.run(
                 args=[
                     'PATH=/usr/bin:/usr/sbin', 'ntpq', '-p', run.Raw('||'),
                     'PATH=/usr/bin:/usr/sbin', 'chronyc', 'sources',
                     run.Raw('||'),
                     'true'
-                    ],
-                )
+                ],
+                wait=False,
+            )
+        )
 
 
 @contextlib.contextmanager
@@ -77,27 +90,33 @@ def check(ctx, config):
     :param config: Configuration
     """
     log.info('Checking initial clock skew...')
-    for rem in ctx.cluster.remotes.keys():
-        rem.run(
+    cluster = filter_out_containers(ctx.cluster)
+    run.wait(
+        cluster.run(
             args=[
                 'PATH=/usr/bin:/usr/sbin', 'ntpq', '-p', run.Raw('||'),
                 'PATH=/usr/bin:/usr/sbin', 'chronyc', 'sources',
                 run.Raw('||'),
                 'true'
-                ],
-            )
+            ],
+            wait=False,
+        )
+    )
 
     try:
         yield
 
     finally:
         log.info('Checking final clock skew...')
-        for rem in ctx.cluster.remotes.keys():
-            rem.run(
+        cluster = filter_out_containers(ctx.cluster)
+        run.wait(
+            cluster.run(
                 args=[
                     'PATH=/usr/bin:/usr/sbin', 'ntpq', '-p', run.Raw('||'),
                     'PATH=/usr/bin:/usr/sbin', 'chronyc', 'sources',
                     run.Raw('||'),
                     'true'
-                    ],
-                )
+                ],
+                wait=False,
+            )
+        )

teuthology/task/exec.py

@@ -23,6 +23,16 @@ def task(ctx, config):
     It stops and fails with the first command that does not return on success. It means
     that if the first command fails, the second won't run at all.
 
+    You can run a command on all hosts `all-hosts`, or all roles with `all-roles`:
+
+        tasks:
+        - exec:
+            all-hosts:
+              - touch /etc/passwd
+        - exec:
+            all-roles:
+              - pwd
+
     To avoid confusion it is recommended to explicitly enclose the commands in 
     double quotes. For instance if the command is false (without double quotes) it will
     be interpreted as a boolean by the YAML parser.
@@ -39,6 +49,14 @@ def task(ctx, config):
         a = config['all']
         roles = teuthology.all_roles(ctx.cluster)
         config = dict((id_, a) for id_ in roles)
+    elif 'all-roles' in config and len(config) == 1:
+        a = config['all-roles']
+        roles = teuthology.all_roles(ctx.cluster)
+        config = dict((id_, a) for id_ in roles)
+    elif 'all-hosts' in config and len(config) == 1:
+        a = config['all-hosts']
+        roles = [roles[0] for roles in ctx.cluster.remotes.values()]
+        config = dict((id_, a) for id_ in roles)
 
     for role, ls in config.items():
         (remote,) = ctx.cluster.only(role).remotes.keys()

teuthology/task/hadoop.py

@@ -259,8 +259,8 @@ def install_hadoop(ctx, config):
                 format = "jar",
                 dist = "precise",
                 arch = "x86_64",
-                flavor = "basic",
-                branch = "master")
+                flavor = "default",
+                branch = "main")
 
         run.wait(
             hadoops.run(

teuthology/task/install/__init__.py

@@ -9,6 +9,7 @@ from teuthology import misc as teuthology
 from teuthology import contextutil, packaging
 from teuthology.parallel import parallel
 from teuthology.task import ansible
+from teuthology.exceptions import ConfigError
 
 from distutils.version import LooseVersion
 from teuthology.task.install.util import (
@@ -282,7 +283,7 @@ def upgrade_remote_to_config(ctx, config):
             # take any remote in the dict
             remote = next(iter(remotes_dict))
             if remote in remotes:
-                log.warn('remote %s came up twice (role %s)', remote, role)
+                log.warning('remote %s came up twice (role %s)', remote, role)
                 continue
             remotes[remote] = config.get(role)
 
@@ -458,6 +459,12 @@ def task(ctx, config):
     are welcome to add support for other distros.
 
 
+    Enable Fedora copr repositories using enable_coprs:
+
+    - install:
+        enable_coprs: [ceph/el9]
+
+
     Overrides are project specific:
 
     overrides:
@@ -542,8 +549,8 @@ def task(ctx, config):
           sha1: 1234
 
     where sha1 matches the --ceph argument. For instance if
-    teuthology-suite is called with --ceph master, the sha1 will be
-    the tip of master. If called with --ceph v0.94.1, the sha1 will be
+    teuthology-suite is called with --ceph main, the sha1 will be
+    the tip of main. If called with --ceph v0.94.1, the sha1 will be
     the v0.94.1 (as returned by git rev-parse v0.94.1 which is not to
     be confused with git rev-parse v0.94.1^{commit})
 
@@ -559,11 +566,25 @@ def task(ctx, config):
     log.debug('project %s' % project)
     overrides = ctx.config.get('overrides')
     repos = None
+
     if overrides:
-        install_overrides = overrides.get('install', {})
-        teuthology.deep_merge(config, install_overrides.get(project, {}))
-        repos = install_overrides.get('repos', None)
-        log.debug('INSTALL overrides: %s' % install_overrides)
+        try:
+            install_overrides = overrides.get('install', {})
+            log.debug('INSTALL overrides: %s' % install_overrides)
+            teuthology.deep_merge(config, install_overrides.get(project, {}))
+            overrides_extra_system_packages = install_overrides.get('extra_system_packages')
+            if overrides_extra_system_packages:
+                extra_system_packages = config.get('extra_system_packages')
+                config['extra_system_packages'] = teuthology.deep_merge(extra_system_packages, overrides_extra_system_packages)
+            repos = install_overrides.get('repos', None)
+        except AssertionError:
+            raise ConfigError(
+                "'install' task config and its overrides contain" \
+                "conflicting types for the same config key. Ensure that " \
+                "the configuration is of the same type (dict, list, etc.) " \
+                "in both the task definition and its overrides."
+            )
+
     log.debug('config %s' % config)
 
     rhbuild = None
@@ -599,6 +620,7 @@ def task(ctx, config):
                 extra_packages=config.get('extra_packages', []),
                 extra_system_packages=config.get('extra_system_packages', []),
                 extras=config.get('extras', None),
+                enable_coprs=config.get('enable_coprs', []),
                 flavor=flavor,
                 install_ceph_packages=config.get('install_ceph_packages', True),
                 packages=config.get('packages', dict()),

teuthology/task/install/bin/adjust-ulimits

@@ -0,0 +1,16 @@
+#!/bin/sh
+# If we're running as root, allow large amounts of open files.
+USER=$(whoami)
+
+# If a ulimit call fails, exit immediately.
+set -e
+
+if [ "$USER" = "root" ]
+then
+    # Enable large number of open files
+    ulimit -n 65536
+fi
+
+# Enable core dumps for everything
+ulimit -c unlimited
+exec "$@"

teuthology/task/install/bin/daemon-helper

@@ -0,0 +1,114 @@
+#!/usr/bin/python3
+
+"""
+Helper script for running long-living processes.
+
+(Name says daemon, but that is intended to mean "long-living", we
+assume child process does not double-fork.)
+
+We start the command passed as arguments, with /dev/null as stdin, and
+then wait for EOF on stdin.
+
+When EOF is seen on stdin, the child process is killed.
+
+When the child process exits, this helper exits too.
+
+Usage:
+    daemon-helper <signal> [--kill-group] [nostdin] COMMAND ...
+"""
+
+from __future__ import print_function
+
+import fcntl
+import os
+import select
+import signal
+import struct
+import subprocess
+import sys
+from argparse import ArgumentParser
+
+parser = ArgumentParser(epilog=
+    'The remaining parameters are the command to be run.  If these\n' +
+    'parameters start wih nostdin, then no stdin input is expected.')
+parser.add_argument('signal')
+parser.add_argument('--kill-group', action='store_true', 
+                    help='kill all processes in the group')
+parser.add_argument('--nostdin', action='store_true', 
+                    help='no stdin input expected')
+parsed, args = parser.parse_known_args()
+end_signal = signal.SIGKILL
+if parsed.signal == 'term':
+    end_signal = signal.SIGTERM
+group = parsed.kill_group
+nostdin = parsed.nostdin
+skip_nostdin = 0
+try:
+    if args[0] == 'nostdin':
+        nostdin = True
+        skip_nostdin = 1
+except IndexError:
+    print('No command specified')
+    sys.exit(1)
+
+
+proc = None
+if nostdin:
+    if len(args) - skip_nostdin == 0:
+        print('No command specified')
+        sys.exit(1)
+    proc = subprocess.Popen(
+        args=args[skip_nostdin:],
+        )
+else:
+    with open('/dev/null', 'rb') as devnull:
+        proc = subprocess.Popen(
+            args=args,
+            stdin=devnull,
+            preexec_fn=os.setsid,
+            )
+
+flags = fcntl.fcntl(0, fcntl.F_GETFL)
+fcntl.fcntl(0, fcntl.F_SETFL, flags | os.O_NDELAY)
+
+saw_eof = False
+while True:
+    r,w,x = select.select([0], [], [0], 0.2)
+    if r:
+        data = os.read(0, 1)
+        if not data:
+            saw_eof = True
+            if not group:
+                proc.send_signal(end_signal)
+            else:
+                os.killpg(proc.pid, end_signal)
+            break
+        else:
+            sig, = struct.unpack('!b', data)
+            if not group:
+                proc.send_signal(sig)
+            else:
+                os.killpg(proc.pid, end_signal)
+
+
+    if proc.poll() is not None:
+        # child exited
+        break
+
+exitstatus = proc.wait()
+if exitstatus > 0:
+    print('{me}: command failed with exit status {exitstatus:d}'.format(
+        me=os.path.basename(sys.argv[0]),
+        exitstatus=exitstatus,
+        ), file=sys.stderr)
+    sys.exit(exitstatus)
+elif exitstatus < 0:
+    if saw_eof and exitstatus == -end_signal:
+        # suppress error from the exit we intentionally caused
+        pass
+    else:
+        print('{me}: command crashed with signal {signal:d}'.format(
+            me=os.path.basename(sys.argv[0]),
+            signal=-exitstatus,
+            ), file=sys.stderr)
+        sys.exit(1)

teuthology/task/install/bin/stdin-killer

@@ -0,0 +1,263 @@
+#!/bin/python3
+
+# Forward stdin to a subcommand. If EOF is read from stdin or
+# stdin/stdout/stderr are closed or hungup, then give the command "timeout"
+# seconds to complete before it is killed.
+#
+# The command is run in a separate process group. This is mostly to simplify
+# killing the set of processes (if well-behaving). You can configure that with
+# --setpgrp switch.
+
+# usage: stdin-killer [-h] [--timeout TIMEOUT] [--debug DEBUG] [--signal SIGNAL] [--verbose] [--setpgrp {no,self,child}] command [arguments ...]
+#
+# wait for stdin EOF then kill forked subcommand
+#
+# positional arguments:
+#   command            command to execute
+#   arguments          arguments to command
+#
+# options:
+#   -h, --help         show this help message and exit
+#   --timeout TIMEOUT  time to wait for forked subcommand to willing terminate
+#   --debug DEBUG      debug file
+#   --signal SIGNAL    signal to send
+#   --verbose          increase debugging
+#  --setpgrp {no,self,child}
+#                        create process group
+
+
+import argparse
+import fcntl
+import logging
+import os
+import select
+import signal
+import struct
+import subprocess
+import sys
+import time
+
+NAME = "stdin-killer"
+
+log = logging.getLogger(NAME)
+PAGE_SIZE = 4096
+
+POLL_HANGUP = select.POLLHUP | (select.POLLRDHUP if hasattr(select, 'POLLRDHUP') else 0) | select.POLLERR
+
+
+def handle_event(poll, buffer, fd, event, p):
+    if sigfdr == fd:
+        b = os.read(sigfdr, 1)
+        (signum,) = struct.unpack("B", b)
+        log.debug("got signal %d", signum)
+        try:
+            p.wait(timeout=0)
+            return True
+        except subprocess.TimeoutExpired:
+            pass
+    elif 0 == fd:
+        if event & POLL_HANGUP:
+            log.debug("peer closed connection, waiting for process exit")
+            poll.unregister(0)
+            sys.stdin.close()
+            if len(buffer) == 0 and p.stdin is not None:
+                p.stdin.close()
+                p.stdin = None
+            return True
+        elif event & select.POLLIN:
+            b = os.read(0, PAGE_SIZE)
+            if b == b"":
+                log.debug("read EOF")
+                poll.unregister(0)
+                sys.stdin.close()
+                if len(buffer) == 0:
+                    p.stdin.close()
+                return True
+            if p.stdin is not None:
+                buffer += b
+                # ignore further POLLIN until buffer is written to p.stdin
+                poll.register(0, POLL_HANGUP)
+                poll.register(p.stdin.fileno(), select.POLLOUT)
+    elif p.stdin is not None and p.stdin.fileno() == fd:
+        assert event & select.POLLOUT
+        b = buffer[:PAGE_SIZE]
+        log.debug("sending %d bytes to process", len(b))
+        try:
+            n = p.stdin.write(b)
+            p.stdin.flush()
+            log.debug("wrote %d bytes", n)
+            buffer = buffer[n:]
+            poll.register(0, select.POLLIN | POLL_HANGUP)
+            poll.unregister(p.stdin.fileno())
+        except BrokenPipeError:
+            log.debug("got SIGPIPE")
+            poll.unregister(p.stdin.fileno())
+            p.stdin.close()
+            p.stdin = None
+            return True
+        except BlockingIOError:
+            poll.register(p.stdin.fileno(), select.POLLOUT | POLL_HANGUP)
+    elif 1 == fd:
+        assert event & POLL_HANGUP
+        log.debug("stdout pipe has closed")
+        poll.unregister(1)
+        return True
+    elif 2 == fd:
+        assert event & POLL_HANGUP
+        log.debug("stderr pipe has closed")
+        poll.unregister(2)
+        return True
+    else:
+        assert False
+    return False
+
+
+def listen_for_events(sigfdr, p, timeout):
+    poll = select.poll()
+    # listen for data on stdin
+    poll.register(0, select.POLLIN | POLL_HANGUP)
+    # listen for stdout/stderr to be closed, if they are closed then my parent
+    # is gone and I should expire the command and myself.
+    poll.register(1, POLL_HANGUP)
+    poll.register(2, POLL_HANGUP)
+    # for SIGCHLD
+    poll.register(sigfdr, select.POLLIN)
+    buffer = bytearray()
+    expired = 0.0
+    while True:
+        if expired > 0.0:
+            since = time.monotonic() - expired
+            wait = int((timeout - since) * 1000.0)
+            if wait <= 0:
+                return
+        else:
+            wait = 5000
+        log.debug("polling for %d milliseconds", wait)
+        events = poll.poll(wait)
+        for fd, event in events:
+            log.debug("event: (%d, %d)", fd, event)
+            if handle_event(poll, buffer, fd, event, p):
+                if p.returncode is not None:
+                    return
+                if expired == 0.0:
+                    expired = time.monotonic()
+                    log.info(
+                        "expiration expected; waiting %d seconds for command to complete",
+                        NS.timeout,
+                    )
+
+
+if __name__ == "__main__":
+    signal.signal(signal.SIGPIPE, signal.SIG_IGN)
+    try:
+        (sigfdr, sigfdw) = os.pipe2(os.O_NONBLOCK | os.O_CLOEXEC)
+    except AttributeError:
+        # pipe2 is only available on "some flavors of Unix"
+        # https://docs.python.org/3.10/library/os.html?highlight=pipe2#os.pipe2
+        pipe_ends = os.pipe()
+        for fd in pipe_ends:
+            flags = fcntl.fcntl(fd, fcntl.F_GETFL)
+            fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK | os.O_CLOEXEC)
+        (sigfdr, sigfdw) = pipe_ends
+
+    signal.set_wakeup_fd(sigfdw)
+
+    def do_nothing(signum, frame):
+        pass
+
+    signal.signal(signal.SIGCHLD, do_nothing)
+
+    P = argparse.ArgumentParser(
+        description="wait for stdin EOF then kill forked subcommand"
+    )
+    P.add_argument(
+        "--timeout",
+        action="store",
+        default=5,
+        help="time to wait for forked subcommand to willing terminate",
+        type=int,
+    )
+    P.add_argument("--debug", action="store", help="debug file", type=str)
+    P.add_argument(
+        "--signal",
+        action="store",
+        help="signal to send",
+        type=int,
+        default=signal.SIGKILL,
+    )
+    P.add_argument("--verbose", action="store_true", help="increase debugging")
+    P.add_argument(
+        "--setpgrp",
+        action="store",
+        choices=["no", "self", "child"],
+        default="self",
+        help="create process group",
+    )
+    P.add_argument(
+        "cmd", metavar="command", type=str, nargs=1, help="command to execute"
+    )
+    P.add_argument(
+        "args", metavar="arguments", type=str, nargs="*", help="arguments to command"
+    )
+    NS = P.parse_args()
+
+    logargs = {}
+    if NS.debug is not None:
+        logargs["filename"] = NS.debug
+    else:
+        logargs["stream"] = sys.stderr
+    if NS.verbose:
+        logargs["level"] = logging.DEBUG
+    else:
+        logargs["level"] = logging.INFO
+    logargs["format"] = f"%(asctime)s {NAME} %(levelname)s: %(message)s"
+    logargs["datefmt"] = "%Y-%m-%dT%H:%M:%S"
+    logging.basicConfig(**logargs)
+
+    cargs = NS.cmd + NS.args
+    popen_kwargs = {
+        "stdin": subprocess.PIPE,
+    }
+
+    if NS.setpgrp == "self":
+        pgrp = os.getpgrp()
+        if pgrp != os.getpid():
+            os.setpgrp()
+            pgrp = os.getpgrp()
+    elif NS.setpgrp == "child":
+        popen_kwargs["preexec_fn"] = os.setpgrp
+        pgrp = None
+    elif NS.setpgrp == "no":
+        pgrp = 0
+    else:
+        assert False
+
+    log.debug("executing %s", cargs)
+    p = subprocess.Popen(cargs, **popen_kwargs)
+    if pgrp is None:
+        pgrp = p.pid
+    flags = fcntl.fcntl(p.stdin.fileno(), fcntl.F_GETFL)
+    fcntl.fcntl(p.stdin.fileno(), fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
+    listen_for_events(sigfdr, p, NS.timeout)
+
+    if p.returncode is None:
+        log.error("timeout expired: sending signal %d to command and myself", NS.signal)
+        if pgrp == 0:
+            os.kill(p.pid, NS.signal)
+        else:
+            os.killpg(pgrp, NS.signal)  # should kill me too
+        os.kill(os.getpid(), NS.signal)  # to exit abnormally with same signal
+        log.error("signal did not cause termination, sending myself SIGKILL")
+        os.kill(os.getpid(), signal.SIGKILL)  # failsafe
+    rc = p.returncode
+    log.debug("rc = %d", rc)
+    assert rc is not None
+    if rc < 0:
+        log.error("command terminated with signal %d: sending same signal to myself!", -rc)
+        os.kill(os.getpid(), -rc)  # kill myself with the same signal
+        log.error("signal did not cause termination, sending myself SIGKILL")
+        os.kill(os.getpid(), signal.SIGKILL)  # failsafe
+    else:
+        log.info("command exited with status %d: exiting normally with same code!", rc)
+        sys.exit(rc)