teuthology 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

teuthology/task/ceph_ansible.py

@@ -22,7 +22,7 @@ class CephAnsible(Task):
 
     - ceph-ansible:
         repo: {git_base}ceph-ansible.git
-        branch: mybranch # defaults to master
+        branch: mybranch # defaults to main
         ansible-version: 2.4 # defaults to 2.5
         vars:
           ceph_dev: True ( default)
@@ -73,7 +73,7 @@ class CephAnsible(Task):
         if 'ceph_dev_key' not in vars:
             vars['ceph_dev_key'] = 'https://download.ceph.com/keys/autobuild.asc'
         if 'ceph_dev_branch' not in vars:
-            vars['ceph_dev_branch'] = ctx.config.get('branch', 'master')
+            vars['ceph_dev_branch'] = ctx.config.get('branch', 'main')
         self.cluster_name = vars.get('cluster', 'ceph')
 
     def setup(self):
@@ -296,8 +296,11 @@ class CephAnsible(Task):
             roles = self.ctx.cluster.remotes[remote]
             dev_needed = len([role for role in roles
                               if role.startswith('osd')])
-            if teuth_config.get('ceph_ansible') and \
-                    self.ctx.machine_type in teuth_config['ceph_ansible']['has_lvm_scratch_disks']:
+            if (
+                    teuth_config.get('ceph_ansible') and 
+                    hasattr(self.ctx, "machine_type") and
+                    self.ctx.machine_type in teuth_config['ceph_ansible']['has_lvm_scratch_disks']
+                ):
                 devices = get_file(remote, "/scratch_devs").decode().split()
                 vols = []
 
@@ -375,7 +378,7 @@ class CephAnsible(Task):
                 'python-dev'
             ])
         ansible_repo = self.config['repo']
-        branch = 'master'
+        branch = 'main'
         if self.config.get('branch'):
             branch = self.config.get('branch')
         ansible_ver = 'ansible==2.5'
@@ -404,7 +407,6 @@ class CephAnsible(Task):
             run.Raw('cd ~/ceph-ansible'),
             run.Raw(';'),
             'virtualenv',
-            run.Raw('--system-site-packages'),
             run.Raw('--python=python3'),
             'venv',
             run.Raw(';'),
@@ -425,6 +427,11 @@ class CephAnsible(Task):
             run.Raw('setuptools>=11.3'),
             run.Raw('notario>=0.0.13'), # FIXME: use requirements.txt
             run.Raw('netaddr'),
+            run.Raw('six'),
+            run.Raw(';'),
+            'LANG=en_US.utf8',
+            'pip',
+            'install',
             run.Raw(ansible_ver),
             run.Raw(';'),
             run.Raw(str_args)

teuthology/task/cephmetrics.py

@@ -5,8 +5,9 @@ import time
 
 from teuthology.config import config as teuth_config
 from teuthology.exceptions import CommandFailedError
+from teuthology.task.ansible import Ansible
+from teuthology.util.loggerfile import LoggerFile
 
-from teuthology.ansible import Ansible, LoggerFile
 
 log = logging.getLogger(__name__)
 

teuthology/task/clock.py

@@ -8,6 +8,13 @@ from teuthology.orchestra import run
 
 log = logging.getLogger(__name__)
 
+def filter_out_containers(cluster):
+    """
+    Returns a cluster that excludes remotes which should skip this task.
+    Currently, only skips containerized remotes.
+    """
+    return cluster.filter(lambda r: not r.is_container)
+
 @contextlib.contextmanager
 def task(ctx, config):
     """
@@ -30,8 +37,9 @@ def task(ctx, config):
     """
 
     log.info('Syncing clocks and checking initial clock skew...')
-    for rem in ctx.cluster.remotes.keys():
-        rem.run(
+    cluster = filter_out_containers(ctx.cluster)
+    run.wait(
+        cluster.run(
             args = [
                 'sudo', 'systemctl', 'stop', 'ntp.service', run.Raw('||'),
                 'sudo', 'systemctl', 'stop', 'ntpd.service', run.Raw('||'),
@@ -50,22 +58,27 @@ def task(ctx, config):
                 'true'
             ],
             timeout = 360,
+            wait=False,
         )
+    )
 
     try:
         yield
 
     finally:
         log.info('Checking final clock skew...')
-        for rem in ctx.cluster.remotes.keys():
-            rem.run(
+        cluster = filter_out_containers(ctx.cluster)
+        run.wait(
+            cluster.run(
                 args=[
                     'PATH=/usr/bin:/usr/sbin', 'ntpq', '-p', run.Raw('||'),
                     'PATH=/usr/bin:/usr/sbin', 'chronyc', 'sources',
                     run.Raw('||'),
                     'true'
-                    ],
-                )
+                ],
+                wait=False,
+            )
+        )
 
 
 @contextlib.contextmanager
@@ -77,27 +90,33 @@ def check(ctx, config):
     :param config: Configuration
     """
     log.info('Checking initial clock skew...')
-    for rem in ctx.cluster.remotes.keys():
-        rem.run(
+    cluster = filter_out_containers(ctx.cluster)
+    run.wait(
+        cluster.run(
             args=[
                 'PATH=/usr/bin:/usr/sbin', 'ntpq', '-p', run.Raw('||'),
                 'PATH=/usr/bin:/usr/sbin', 'chronyc', 'sources',
                 run.Raw('||'),
                 'true'
-                ],
-            )
+            ],
+            wait=False,
+        )
+    )
 
     try:
         yield
 
     finally:
         log.info('Checking final clock skew...')
-        for rem in ctx.cluster.remotes.keys():
-            rem.run(
+        cluster = filter_out_containers(ctx.cluster)
+        run.wait(
+            cluster.run(
                 args=[
                     'PATH=/usr/bin:/usr/sbin', 'ntpq', '-p', run.Raw('||'),
                     'PATH=/usr/bin:/usr/sbin', 'chronyc', 'sources',
                     run.Raw('||'),
                     'true'
-                    ],
-                )
+                ],
+                wait=False,
+            )
+        )

teuthology/task/exec.py

@@ -23,6 +23,16 @@ def task(ctx, config):
     It stops and fails with the first command that does not return on success. It means
     that if the first command fails, the second won't run at all.
 
+    You can run a command on all hosts `all-hosts`, or all roles with `all-roles`:
+
+        tasks:
+        - exec:
+            all-hosts:
+              - touch /etc/passwd
+        - exec:
+            all-roles:
+              - pwd
+
     To avoid confusion it is recommended to explicitly enclose the commands in 
     double quotes. For instance if the command is false (without double quotes) it will
     be interpreted as a boolean by the YAML parser.
@@ -39,6 +49,14 @@ def task(ctx, config):
         a = config['all']
         roles = teuthology.all_roles(ctx.cluster)
         config = dict((id_, a) for id_ in roles)
+    elif 'all-roles' in config and len(config) == 1:
+        a = config['all-roles']
+        roles = teuthology.all_roles(ctx.cluster)
+        config = dict((id_, a) for id_ in roles)
+    elif 'all-hosts' in config and len(config) == 1:
+        a = config['all-hosts']
+        roles = [roles[0] for roles in ctx.cluster.remotes.values()]
+        config = dict((id_, a) for id_ in roles)
 
     for role, ls in config.items():
         (remote,) = ctx.cluster.only(role).remotes.keys()

teuthology/task/hadoop.py

@@ -259,8 +259,8 @@ def install_hadoop(ctx, config):
                 format = "jar",
                 dist = "precise",
                 arch = "x86_64",
-                flavor = "basic",
-                branch = "master")
+                flavor = "default",
+                branch = "main")
 
         run.wait(
             hadoops.run(

teuthology/task/install/__init__.py

@@ -9,6 +9,7 @@ from teuthology import misc as teuthology
 from teuthology import contextutil, packaging
 from teuthology.parallel import parallel
 from teuthology.task import ansible
+from teuthology.exceptions import ConfigError
 
 from distutils.version import LooseVersion
 from teuthology.task.install.util import (
@@ -99,11 +100,13 @@ def remove_packages(ctx, config, pkgs):
         "deb": deb._remove,
         "rpm": rpm._remove,
     }
+    cleanup = config.get('cleanup', False)
     with parallel() as p:
         for remote in ctx.cluster.remotes.keys():
-            system_type = teuthology.get_system_type(remote)
-            p.spawn(remove_pkgs[
-                    system_type], ctx, config, remote, pkgs[system_type])
+            if not remote.is_reimageable or cleanup:
+                system_type = teuthology.get_system_type(remote)
+                p.spawn(remove_pkgs[
+                        system_type], ctx, config, remote, pkgs[system_type])
 
 
 def remove_sources(ctx, config):
@@ -117,13 +120,15 @@ def remove_sources(ctx, config):
         'deb': deb._remove_sources_list,
         'rpm': rpm._remove_sources_list,
     }
+    cleanup = config.get('cleanup', False)
+    project = config.get('project', 'ceph')
     with parallel() as p:
-        project = config.get('project', 'ceph')
-        log.info("Removing {proj} sources lists".format(
-            proj=project))
         for remote in ctx.cluster.remotes.keys():
-            remove_fn = remove_sources_pkgs[remote.os.package_type]
-            p.spawn(remove_fn, ctx, config, remote)
+            if not remote.is_reimageable or cleanup:
+                log.info("Removing {p} sources lists on {r}"
+                         .format(p=project,r=remote))
+                remove_fn = remove_sources_pkgs[remote.os.package_type]
+                p.spawn(remove_fn, ctx, config, remote)
 
 
 def get_package_list(ctx, config):
@@ -278,7 +283,7 @@ def upgrade_remote_to_config(ctx, config):
             # take any remote in the dict
             remote = next(iter(remotes_dict))
             if remote in remotes:
-                log.warn('remote %s came up twice (role %s)', remote, role)
+                log.warning('remote %s came up twice (role %s)', remote, role)
                 continue
             remotes[remote] = config.get(role)
 
@@ -454,6 +459,12 @@ def task(ctx, config):
     are welcome to add support for other distros.
 
 
+    Enable Fedora copr repositories using enable_coprs:
+
+    - install:
+        enable_coprs: [ceph/el9]
+
+
     Overrides are project specific:
 
     overrides:
@@ -538,8 +549,8 @@ def task(ctx, config):
           sha1: 1234
 
     where sha1 matches the --ceph argument. For instance if
-    teuthology-suite is called with --ceph master, the sha1 will be
-    the tip of master. If called with --ceph v0.94.1, the sha1 will be
+    teuthology-suite is called with --ceph main, the sha1 will be
+    the tip of main. If called with --ceph v0.94.1, the sha1 will be
     the v0.94.1 (as returned by git rev-parse v0.94.1 which is not to
     be confused with git rev-parse v0.94.1^{commit})
 
@@ -555,11 +566,25 @@ def task(ctx, config):
     log.debug('project %s' % project)
     overrides = ctx.config.get('overrides')
     repos = None
+
     if overrides:
-        install_overrides = overrides.get('install', {})
-        teuthology.deep_merge(config, install_overrides.get(project, {}))
-        repos = install_overrides.get('repos', None)
-        log.debug('INSTALL overrides: %s' % install_overrides)
+        try:
+            install_overrides = overrides.get('install', {})
+            log.debug('INSTALL overrides: %s' % install_overrides)
+            teuthology.deep_merge(config, install_overrides.get(project, {}))
+            overrides_extra_system_packages = install_overrides.get('extra_system_packages')
+            if overrides_extra_system_packages:
+                extra_system_packages = config.get('extra_system_packages')
+                config['extra_system_packages'] = teuthology.deep_merge(extra_system_packages, overrides_extra_system_packages)
+            repos = install_overrides.get('repos', None)
+        except AssertionError:
+            raise ConfigError(
+                "'install' task config and its overrides contain" \
+                "conflicting types for the same config key. Ensure that " \
+                "the configuration is of the same type (dict, list, etc.) " \
+                "in both the task definition and its overrides."
+            )
+
     log.debug('config %s' % config)
 
     rhbuild = None
@@ -588,23 +613,27 @@ def task(ctx, config):
     else:
         nested_config = dict(
                 branch=config.get('branch'),
-                tag=config.get('tag'),
-                sha1=config.get('sha1'),
+                cleanup=config.get('cleanup'),
                 debuginfo=config.get('debuginfo'),
-                flavor=flavor,
                 downgrade_packages=config.get('downgrade_packages', []),
+                exclude_packages=config.get('exclude_packages', []),
                 extra_packages=config.get('extra_packages', []),
                 extra_system_packages=config.get('extra_system_packages', []),
-                exclude_packages=config.get('exclude_packages', []),
                 extras=config.get('extras', None),
-                wait_for_package=config.get('wait_for_package', False),
-                project=project,
-                packages=config.get('packages', dict()),
+                enable_coprs=config.get('enable_coprs', []),
+                flavor=flavor,
                 install_ceph_packages=config.get('install_ceph_packages', True),
+                packages=config.get('packages', dict()),
+                project=project,
                 repos_only=config.get('repos_only', False),
+                sha1=config.get('sha1'),
+                tag=config.get('tag'),
+                wait_for_package=config.get('wait_for_package', False),
         )
         if repos:
             nested_config['repos'] = repos
+        if 'shaman' in config:
+            nested_config['shaman'] = config['shaman']
         with contextutil.nested(
             lambda: install(ctx=ctx, config=nested_config),
             lambda: ship_utilities(ctx=ctx, config=None),

teuthology/task/install/bin/adjust-ulimits

@@ -0,0 +1,16 @@
+#!/bin/sh
+# If we're running as root, allow large amounts of open files.
+USER=$(whoami)
+
+# If a ulimit call fails, exit immediately.
+set -e
+
+if [ "$USER" = "root" ]
+then
+    # Enable large number of open files
+    ulimit -n 65536
+fi
+
+# Enable core dumps for everything
+ulimit -c unlimited
+exec "$@"

teuthology/task/install/bin/daemon-helper

@@ -0,0 +1,114 @@
+#!/usr/bin/python3
+
+"""
+Helper script for running long-living processes.
+
+(Name says daemon, but that is intended to mean "long-living", we
+assume child process does not double-fork.)
+
+We start the command passed as arguments, with /dev/null as stdin, and
+then wait for EOF on stdin.
+
+When EOF is seen on stdin, the child process is killed.
+
+When the child process exits, this helper exits too.
+
+Usage:
+    daemon-helper <signal> [--kill-group] [nostdin] COMMAND ...
+"""
+
+from __future__ import print_function
+
+import fcntl
+import os
+import select
+import signal
+import struct
+import subprocess
+import sys
+from argparse import ArgumentParser
+
+parser = ArgumentParser(epilog=
+    'The remaining parameters are the command to be run.  If these\n' +
+    'parameters start wih nostdin, then no stdin input is expected.')
+parser.add_argument('signal')
+parser.add_argument('--kill-group', action='store_true', 
+                    help='kill all processes in the group')
+parser.add_argument('--nostdin', action='store_true', 
+                    help='no stdin input expected')
+parsed, args = parser.parse_known_args()
+end_signal = signal.SIGKILL
+if parsed.signal == 'term':
+    end_signal = signal.SIGTERM
+group = parsed.kill_group
+nostdin = parsed.nostdin
+skip_nostdin = 0
+try:
+    if args[0] == 'nostdin':
+        nostdin = True
+        skip_nostdin = 1
+except IndexError:
+    print('No command specified')
+    sys.exit(1)
+
+
+proc = None
+if nostdin:
+    if len(args) - skip_nostdin == 0:
+        print('No command specified')
+        sys.exit(1)
+    proc = subprocess.Popen(
+        args=args[skip_nostdin:],
+        )
+else:
+    with open('/dev/null', 'rb') as devnull:
+        proc = subprocess.Popen(
+            args=args,
+            stdin=devnull,
+            preexec_fn=os.setsid,
+            )
+
+flags = fcntl.fcntl(0, fcntl.F_GETFL)
+fcntl.fcntl(0, fcntl.F_SETFL, flags | os.O_NDELAY)
+
+saw_eof = False
+while True:
+    r,w,x = select.select([0], [], [0], 0.2)
+    if r:
+        data = os.read(0, 1)
+        if not data:
+            saw_eof = True
+            if not group:
+                proc.send_signal(end_signal)
+            else:
+                os.killpg(proc.pid, end_signal)
+            break
+        else:
+            sig, = struct.unpack('!b', data)
+            if not group:
+                proc.send_signal(sig)
+            else:
+                os.killpg(proc.pid, end_signal)
+
+
+    if proc.poll() is not None:
+        # child exited
+        break
+
+exitstatus = proc.wait()
+if exitstatus > 0:
+    print('{me}: command failed with exit status {exitstatus:d}'.format(
+        me=os.path.basename(sys.argv[0]),
+        exitstatus=exitstatus,
+        ), file=sys.stderr)
+    sys.exit(exitstatus)
+elif exitstatus < 0:
+    if saw_eof and exitstatus == -end_signal:
+        # suppress error from the exit we intentionally caused
+        pass
+    else:
+        print('{me}: command crashed with signal {signal:d}'.format(
+            me=os.path.basename(sys.argv[0]),
+            signal=-exitstatus,
+            ), file=sys.stderr)
+        sys.exit(1)