teuthology 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

teuthology/task/install/bin/stdin-killer

@@ -0,0 +1,263 @@
+#!/bin/python3
+
+# Forward stdin to a subcommand. If EOF is read from stdin or
+# stdin/stdout/stderr are closed or hungup, then give the command "timeout"
+# seconds to complete before it is killed.
+#
+# The command is run in a separate process group. This is mostly to simplify
+# killing the set of processes (if well-behaving). You can configure that with
+# --setpgrp switch.
+
+# usage: stdin-killer [-h] [--timeout TIMEOUT] [--debug DEBUG] [--signal SIGNAL] [--verbose] [--setpgrp {no,self,child}] command [arguments ...]
+#
+# wait for stdin EOF then kill forked subcommand
+#
+# positional arguments:
+#   command            command to execute
+#   arguments          arguments to command
+#
+# options:
+#   -h, --help         show this help message and exit
+#   --timeout TIMEOUT  time to wait for forked subcommand to willing terminate
+#   --debug DEBUG      debug file
+#   --signal SIGNAL    signal to send
+#   --verbose          increase debugging
+#  --setpgrp {no,self,child}
+#                        create process group
+
+
+import argparse
+import fcntl
+import logging
+import os
+import select
+import signal
+import struct
+import subprocess
+import sys
+import time
+
+NAME = "stdin-killer"
+
+log = logging.getLogger(NAME)
+PAGE_SIZE = 4096
+
+POLL_HANGUP = select.POLLHUP | (select.POLLRDHUP if hasattr(select, 'POLLRDHUP') else 0) | select.POLLERR
+
+
+def handle_event(poll, buffer, fd, event, p):
+    if sigfdr == fd:
+        b = os.read(sigfdr, 1)
+        (signum,) = struct.unpack("B", b)
+        log.debug("got signal %d", signum)
+        try:
+            p.wait(timeout=0)
+            return True
+        except subprocess.TimeoutExpired:
+            pass
+    elif 0 == fd:
+        if event & POLL_HANGUP:
+            log.debug("peer closed connection, waiting for process exit")
+            poll.unregister(0)
+            sys.stdin.close()
+            if len(buffer) == 0 and p.stdin is not None:
+                p.stdin.close()
+                p.stdin = None
+            return True
+        elif event & select.POLLIN:
+            b = os.read(0, PAGE_SIZE)
+            if b == b"":
+                log.debug("read EOF")
+                poll.unregister(0)
+                sys.stdin.close()
+                if len(buffer) == 0:
+                    p.stdin.close()
+                return True
+            if p.stdin is not None:
+                buffer += b
+                # ignore further POLLIN until buffer is written to p.stdin
+                poll.register(0, POLL_HANGUP)
+                poll.register(p.stdin.fileno(), select.POLLOUT)
+    elif p.stdin is not None and p.stdin.fileno() == fd:
+        assert event & select.POLLOUT
+        b = buffer[:PAGE_SIZE]
+        log.debug("sending %d bytes to process", len(b))
+        try:
+            n = p.stdin.write(b)
+            p.stdin.flush()
+            log.debug("wrote %d bytes", n)
+            buffer = buffer[n:]
+            poll.register(0, select.POLLIN | POLL_HANGUP)
+            poll.unregister(p.stdin.fileno())
+        except BrokenPipeError:
+            log.debug("got SIGPIPE")
+            poll.unregister(p.stdin.fileno())
+            p.stdin.close()
+            p.stdin = None
+            return True
+        except BlockingIOError:
+            poll.register(p.stdin.fileno(), select.POLLOUT | POLL_HANGUP)
+    elif 1 == fd:
+        assert event & POLL_HANGUP
+        log.debug("stdout pipe has closed")
+        poll.unregister(1)
+        return True
+    elif 2 == fd:
+        assert event & POLL_HANGUP
+        log.debug("stderr pipe has closed")
+        poll.unregister(2)
+        return True
+    else:
+        assert False
+    return False
+
+
+def listen_for_events(sigfdr, p, timeout):
+    poll = select.poll()
+    # listen for data on stdin
+    poll.register(0, select.POLLIN | POLL_HANGUP)
+    # listen for stdout/stderr to be closed, if they are closed then my parent
+    # is gone and I should expire the command and myself.
+    poll.register(1, POLL_HANGUP)
+    poll.register(2, POLL_HANGUP)
+    # for SIGCHLD
+    poll.register(sigfdr, select.POLLIN)
+    buffer = bytearray()
+    expired = 0.0
+    while True:
+        if expired > 0.0:
+            since = time.monotonic() - expired
+            wait = int((timeout - since) * 1000.0)
+            if wait <= 0:
+                return
+        else:
+            wait = 5000
+        log.debug("polling for %d milliseconds", wait)
+        events = poll.poll(wait)
+        for fd, event in events:
+            log.debug("event: (%d, %d)", fd, event)
+            if handle_event(poll, buffer, fd, event, p):
+                if p.returncode is not None:
+                    return
+                if expired == 0.0:
+                    expired = time.monotonic()
+                    log.info(
+                        "expiration expected; waiting %d seconds for command to complete",
+                        NS.timeout,
+                    )
+
+
+if __name__ == "__main__":
+    signal.signal(signal.SIGPIPE, signal.SIG_IGN)
+    try:
+        (sigfdr, sigfdw) = os.pipe2(os.O_NONBLOCK | os.O_CLOEXEC)
+    except AttributeError:
+        # pipe2 is only available on "some flavors of Unix"
+        # https://docs.python.org/3.10/library/os.html?highlight=pipe2#os.pipe2
+        pipe_ends = os.pipe()
+        for fd in pipe_ends:
+            flags = fcntl.fcntl(fd, fcntl.F_GETFL)
+            fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK | os.O_CLOEXEC)
+        (sigfdr, sigfdw) = pipe_ends
+
+    signal.set_wakeup_fd(sigfdw)
+
+    def do_nothing(signum, frame):
+        pass
+
+    signal.signal(signal.SIGCHLD, do_nothing)
+
+    P = argparse.ArgumentParser(
+        description="wait for stdin EOF then kill forked subcommand"
+    )
+    P.add_argument(
+        "--timeout",
+        action="store",
+        default=5,
+        help="time to wait for forked subcommand to willing terminate",
+        type=int,
+    )
+    P.add_argument("--debug", action="store", help="debug file", type=str)
+    P.add_argument(
+        "--signal",
+        action="store",
+        help="signal to send",
+        type=int,
+        default=signal.SIGKILL,
+    )
+    P.add_argument("--verbose", action="store_true", help="increase debugging")
+    P.add_argument(
+        "--setpgrp",
+        action="store",
+        choices=["no", "self", "child"],
+        default="self",
+        help="create process group",
+    )
+    P.add_argument(
+        "cmd", metavar="command", type=str, nargs=1, help="command to execute"
+    )
+    P.add_argument(
+        "args", metavar="arguments", type=str, nargs="*", help="arguments to command"
+    )
+    NS = P.parse_args()
+
+    logargs = {}
+    if NS.debug is not None:
+        logargs["filename"] = NS.debug
+    else:
+        logargs["stream"] = sys.stderr
+    if NS.verbose:
+        logargs["level"] = logging.DEBUG
+    else:
+        logargs["level"] = logging.INFO
+    logargs["format"] = f"%(asctime)s {NAME} %(levelname)s: %(message)s"
+    logargs["datefmt"] = "%Y-%m-%dT%H:%M:%S"
+    logging.basicConfig(**logargs)
+
+    cargs = NS.cmd + NS.args
+    popen_kwargs = {
+        "stdin": subprocess.PIPE,
+    }
+
+    if NS.setpgrp == "self":
+        pgrp = os.getpgrp()
+        if pgrp != os.getpid():
+            os.setpgrp()
+            pgrp = os.getpgrp()
+    elif NS.setpgrp == "child":
+        popen_kwargs["preexec_fn"] = os.setpgrp
+        pgrp = None
+    elif NS.setpgrp == "no":
+        pgrp = 0
+    else:
+        assert False
+
+    log.debug("executing %s", cargs)
+    p = subprocess.Popen(cargs, **popen_kwargs)
+    if pgrp is None:
+        pgrp = p.pid
+    flags = fcntl.fcntl(p.stdin.fileno(), fcntl.F_GETFL)
+    fcntl.fcntl(p.stdin.fileno(), fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
+    listen_for_events(sigfdr, p, NS.timeout)
+
+    if p.returncode is None:
+        log.error("timeout expired: sending signal %d to command and myself", NS.signal)
+        if pgrp == 0:
+            os.kill(p.pid, NS.signal)
+        else:
+            os.killpg(pgrp, NS.signal)  # should kill me too
+        os.kill(os.getpid(), NS.signal)  # to exit abnormally with same signal
+        log.error("signal did not cause termination, sending myself SIGKILL")
+        os.kill(os.getpid(), signal.SIGKILL)  # failsafe
+    rc = p.returncode
+    log.debug("rc = %d", rc)
+    assert rc is not None
+    if rc < 0:
+        log.error("command terminated with signal %d: sending same signal to myself!", -rc)
+        os.kill(os.getpid(), -rc)  # kill myself with the same signal
+        log.error("signal did not cause termination, sending myself SIGKILL")
+        os.kill(os.getpid(), signal.SIGKILL)  # failsafe
+    else:
+        log.info("command exited with status %d: exiting normally with same code!", rc)
+        sys.exit(rc)

teuthology/task/install/deb.py

@@ -4,12 +4,32 @@ import os
 from io import StringIO
 
 from teuthology.orchestra import run
+from teuthology.contextutil import safe_while
 
 from teuthology.task.install.util import _get_builder_project, _get_local_dir
 
 
 log = logging.getLogger(__name__)
 
+def _retry_if_eagain_in_output(remote, args):
+    # wait at most 5 minutes
+    with safe_while(sleep=10, tries=30) as proceed:
+        while proceed():
+            stderr = StringIO()
+            try:
+                return remote.run(args=args, stderr=stderr)
+            except run.CommandFailedError:
+                if "could not get lock" in stderr.getvalue().lower():
+                    stdout = StringIO()
+                    args = ['sudo', 'fuser', '-v', '/var/lib/dpkg/lock-frontend']
+                    remote.run(args=args, stdout=stdout)
+                    log.info("The processes holding 'lock-frontend':\n{}".format(stdout.getvalue()))
+                    continue
+                else:
+                    raise
+
+def install_dep_packages(remote, args):
+    _retry_if_eagain_in_output(remote, args)
 
 def _update_package_list_and_install(ctx, remote, debs, config):
     """
@@ -71,11 +91,11 @@ def _update_package_list_and_install(ctx, remote, debs, config):
                 'Dpkg::Options::="--force-confold"'),
             'install',
         ]
-    remote.run(
+    install_dep_packages(remote,
         args=install_cmd + ['%s=%s' % (d, version) for d in debs],
     )
     if system_pkglist:
-        remote.run(
+        install_dep_packages(remote,
             args=install_cmd + system_pkglist,
         )
     ldir = _get_local_dir(config, remote)
@@ -122,7 +142,7 @@ def _remove(ctx, config, remote, debs):
             run.Raw('|'),
             # Any package that is unpacked or half-installed and also requires
             # reinstallation
-            'grep', '^.\(U\|H\)R',
+            'grep', r'^.\(U\|H\)R',
             run.Raw('|'),
             'awk', '{print $2}',
             run.Raw('|'),
@@ -195,7 +215,7 @@ def _upgrade_packages(ctx, config, remote, debs):
     builder.install_repo()
 
     remote.run(args=['sudo', 'apt-get', 'update'], check_status=False)
-    remote.run(
+    install_dep_packages(remote,
         args=[
             'sudo',
             'DEBIAN_FRONTEND=noninteractive', 'apt-get', '-y', '--force-yes',

teuthology/task/install/redhat.py

@@ -6,6 +6,7 @@ import os
 from teuthology import packaging
 from teuthology.orchestra import run
 from teuthology.parallel import parallel
+from teuthology.config import config as teuth_config
 
 log = logging.getLogger(__name__)
 
@@ -40,28 +41,21 @@ def install(ctx, config):
              - ceph-osd
              - ceph-mds
     """
-    yaml_path = None
-    # Look for rh specific packages in <suite_path>/rh/downstream.yaml
-    if 'suite_path' in ctx.config:
-        ds_yaml = os.path.join(
-            ctx.config['suite_path'],
-            'rh',
-            'downstream.yaml',
-        )
-        if os.path.exists(ds_yaml):
-            yaml_path = ds_yaml
-    # default to user home dir if one exists
-    default_yaml = os.path.expanduser('~/downstream.yaml')
-    if os.path.exists(default_yaml):
-        yaml_path = default_yaml
-    log.info("using yaml path %s", yaml_path)
-    downstream_config = yaml.safe_load(open(yaml_path))
+    # Look for rh specific packages
+    ds_yaml = os.path.join(
+        teuth_config.get('ds_yaml_dir'),
+        config.get('rhbuild') + ".yaml",
+    )
+    if not os.path.exists(ds_yaml):
+        raise FileNotFoundError(f'Downstream rh version yaml file missing: {ds_yaml}')
+    log.info("using yaml path %s", ds_yaml)
+    downstream_config = yaml.safe_load(open(ds_yaml))
     rh_versions = downstream_config.get('versions', dict()).get('supported', [])
-    external_config = dict(extra_system_packages=config.get('extra_system_packages', {}),
-                           extra_packages=config.get('extra_packages', {}),
+    external_config = dict(extra_system_packages=config.get('extra_system_packages'),
+                           extra_packages=config.get('extra_packages'),
                            )
     downstream_config.update(external_config)
-    version = config['rhbuild']
+    version = config.get('rhbuild')
     if version in rh_versions:
         log.info("%s is a supported version", version)
     else:
@@ -95,10 +89,16 @@ def install_pkgs(ctx, remote, version, downstream_config):
     :param downstream_config the dict object that has downstream pkg info
     """
     rh_version_check = downstream_config.get('versions').get('rpm').get('mapped')
-    rh_rpm_pkgs = downstream_config.get('pkgs').get('rpm') + \
-                  downstream_config.get('extra_system_packages').get('rpm', []) + \
-                  downstream_config.get('extra_packages').get('rpm', [])
+    rh_rpm_pkgs = downstream_config.get('pkgs').get('rpm')
+    extras = [downstream_config.get('extra_system_packages'),
+              downstream_config.get('extra_packages')]
+    for extra in extras:
+        if isinstance(extra, dict):
+            rh_rpm_pkgs += extra.get('rpm', [])
+        elif isinstance(extra, list):
+            rh_rpm_pkgs += extra
     pkgs = str.join(' ', rh_rpm_pkgs)
+
     log.info("Remove any epel packages installed on node %s", remote.shortname)
     # below packages can come from epel and still work, ensure we use cdn pkgs
     remote.run(
@@ -172,9 +172,14 @@ def install_deb_pkgs(
     : param downstream_config the dict object that has downstream pkg info
     """
     rh_version_check = downstream_config.get('versions').get('deb').get('mapped')
-    rh_deb_pkgs = downstream_config.get('pkgs').get('deb') + \
-                  downstream_config.get('extra_system_packages').get('deb', []) + \
-                  downstream_config.get('extra_packages').get('deb', [])
+    rh_deb_pkgs = downstream_config.get('pkgs').get('deb')
+    extras = [downstream_config.get('extra_system_packages'),
+              downstream_config.get('extra_packages')]
+    for extra in extras:
+        if isinstance(extra, dict):
+            rh_deb_pkgs += extra.get('deb', [])
+        elif isinstance(extra, list):
+            rh_deb_pkgs += extra
     pkgs = str.join(' ', rh_deb_pkgs)
     log.info("Installing redhat ceph packages")
     remote.run(args=['sudo', 'apt-get', '-y', 'install',
@@ -200,14 +205,13 @@ def uninstall_pkgs(ctx, remote, downstream_config):
     :param remote: the teuthology.orchestra.remote.Remote object
     :param downstream_config the dict object that has downstream pkg info
     """
-    rh_rpm_pkgs = downstream_config.get('pkgs').get('rpm')
-    rpm_pkgs = str.join(' ', rh_rpm_pkgs)
-
-    rh_deb_pkgs = downstream_config.get('pkgs').get('deb')
-    deb_pkgs = str.join(' ', rh_deb_pkgs)
 
     if remote.os.name == 'rhel':
-        remote.run(args=['sudo', 'yum', 'remove', run.Raw(rpm_pkgs), '-y'])
+        pkgs = downstream_config.get('pkgs').get('rpm')
+        if pkgs:
+            remote.sh(['sudo', 'yum', 'remove'] + pkgs + ['-y'])
     else:
-        remote.run(args=['sudo', 'apt-get', 'remove', run.Raw(deb_pkgs), '-y'])
+        pkgs = downstream_config.get('pkgs').get('deb') 
+        if pkgs:
+            remote.sh(['sudo', 'apt-get', 'remove'] + pkgs + ['-y'])
     remote.run(args=['sudo', 'rm', '-rf', '/var/lib/ceph'])

teuthology/task/install/rpm.py

@@ -1,9 +1,11 @@
 import logging
 import os.path
+from io import StringIO
 
 from distutils.version import LooseVersion
 
 from teuthology.config import config as teuth_config
+from teuthology.contextutil import safe_while
 from teuthology.orchestra import run
 from teuthology import packaging
 
@@ -143,6 +145,21 @@ def _downgrade_packages(ctx, remote, pkgs, pkg_version, config):
     remote.run(args='sudo yum -y downgrade {}'.format(' '.join(pkgs_opt)))
     return [pkg for pkg in pkgs if pkg not in downgrade_pkgs]
 
+def _retry_if_failures_are_recoverable(remote, args):
+    # wait at most 5 minutes
+    with safe_while(sleep=10, tries=30) as proceed:
+        while proceed():
+            stdout = StringIO()
+            stderr = StringIO()
+            try:
+                return remote.run(args=args, stderr=stderr, stdout=stdout)
+            except run.CommandFailedError:
+                if "status code: 503" in stdout.getvalue().lower():
+                    continue
+                if "failed to download metadata for repo" in stderr.getvalue().lower():
+                    continue
+                else:
+                    raise
 
 def _update_package_list_and_install(ctx, remote, rpm, config):
     """
@@ -156,6 +173,13 @@ def _update_package_list_and_install(ctx, remote, rpm, config):
     :param rpm: list of packages names to install
     :param config: the config dict
     """
+
+    enable_coprs = config.get('enable_coprs', [])
+    if len(enable_coprs):
+        remote.run(args=['sudo', 'dnf', '-y', 'install', 'dnf-command(copr)'])
+        for copr in enable_coprs:
+            remote.run(args=['sudo', 'dnf', '-y', 'copr', 'enable', copr])
+
     # rpm does not force installation of a particular version of the project
     # packages, so we can put extra_system_packages together with the rest
     system_pkglist = config.get('extra_system_packages')
@@ -236,26 +260,27 @@ def _update_package_list_and_install(ctx, remote, rpm, config):
         rpm = _downgrade_packages(ctx, remote, rpm, pkg_version, config)
 
     if system_pkglist:
-        remote.run(
+        _retry_if_failures_are_recoverable(remote,
             args='{install_cmd} {rpms}'
                  .format(install_cmd=install_cmd, rpms=' '.join(rpm))
             )
     else:
         for cpack in rpm:
             if ldir:
-                remote.run(args='''
+                _retry_if_failures_are_recoverable(remote,
+                  args='''
                   if test -e {pkg} ; then
                     {remove_cmd} {pkg} ;
                     {install_cmd} {pkg} ;
                   else
                     {install_cmd} {cpack} ;
                   fi
-                '''.format(remove_cmd=remove_cmd,
-                           install_cmd=install_cmd,
-                           pkg=os.path.join(ldir, cpack),
-                           cpack=cpack))
+                  '''.format(remove_cmd=remove_cmd,
+                             install_cmd=install_cmd,
+                             pkg=os.path.join(ldir, cpack),
+                             cpack=cpack))
             else:
-                remote.run(
+                _retry_if_failures_are_recoverable(remote,
                     args='{install_cmd} {cpack}'
                          .format(install_cmd=install_cmd, cpack=cpack)
                     )
@@ -350,6 +375,8 @@ def _remove_sources_list(ctx, config, remote):
     if remote.os.name not in ['opensuse', 'sle']:
         _yum_unset_check_obsoletes(remote)
 
+    for copr in config.get('enable_coprs', []):
+        remote.run(args=['sudo', 'dnf', '-y', 'copr', 'disable', copr])
 
 def _upgrade_packages(ctx, config, remote, pkgs):
     """
@@ -401,14 +428,14 @@ def _upgrade_packages(ctx, config, remote, pkgs):
     # Actually upgrade the project packages
     if builder.dist_release in ['opensuse', 'sle']:
         pkg_mng_opts = '-n'
-        pkg_mng_subcommand_opts = '--capability'
-        pkg_mng_install_opts = '--no-recommends'
+        pkg_mng_subcommand = 'install'
+        pkg_mng_subcommand_opts = ['--capability', '--no-recommends']
     else:
         pkg_mng_opts = '-y'
-        pkg_mng_subcommand_opts = ''
-        pkg_mng_install_opts = ''
-    args = ['sudo', pkg_mng_cmd, pkg_mng_opts,
-            'install', pkg_mng_subcommand_opts,
-            pkg_mng_install_opts]
+        pkg_mng_subcommand = 'upgrade'
+        pkg_mng_subcommand_opts = []
+    args = ['sudo', pkg_mng_cmd, pkg_mng_opts, pkg_mng_subcommand]
+    if pkg_mng_subcommand_opts:
+        args += pkg_mng_subcommand_opts
     args += pkgs
     remote.run(args=args)

teuthology/task/install/util.py

@@ -38,7 +38,7 @@ def get_flavor(config):
     Determine the flavor to use.
     """
     config = config or dict()
-    flavor = config.get('flavor', 'basic')
+    flavor = config.get('flavor', 'default')
 
     if config.get('path'):
         # local dir precludes any other flavors
@@ -51,18 +51,14 @@ def get_flavor(config):
                 flavor = 'gcov'
     return flavor
 
-
-@contextlib.contextmanager
-def ship_utilities(ctx, config):
+def _ship_utilities(ctx):
     """
     Write a copy of valgrind.supp to each of the remote sites.  Set executables
     used by Ceph in /usr/local/bin.  When finished (upon exit of the teuthology
     run), remove these files.
 
     :param ctx: Context
-    :param config: Configuration
     """
-    assert config is None
     testdir = teuthology.get_testdir(ctx)
     filenames = []
 
@@ -85,11 +81,11 @@ def ship_utilities(ctx, config):
     except IOError as e:
         log.info('Cannot ship supression file for valgrind: %s...', e.strerror)
 
-    FILES = ['daemon-helper', 'adjust-ulimits']
+    FILES = ['daemon-helper', 'adjust-ulimits', 'stdin-killer']
     destdir = '/usr/bin'
     for filename in FILES:
         log.info('Shipping %r...', filename)
-        src = os.path.join(os.path.dirname(__file__), filename)
+        src = os.path.join(os.path.dirname(__file__), 'bin', filename)
         dst = os.path.join(destdir, filename)
         filenames.append(dst)
         with open(src, 'rb') as f:
@@ -109,19 +105,49 @@ def ship_utilities(ctx, config):
                         dst,
                     ],
                 )
+    return filenames
 
-    try:
+def _remove_utilities(ctx, filenames):
+    """
+    Remove the shipped utilities.
+
+    :param ctx: Context
+    :param filenames: The utilities install paths
+    """
+    log.info('Removing shipped files: %s...', ' '.join(filenames))
+    if filenames == []:
+        return
+    run.wait(
+        ctx.cluster.run(
+            args=[
+                'sudo',
+                'rm',
+                '-f',
+                '--',
+            ] + list(filenames),
+            wait=False,
+        ),
+    )
+
+@contextlib.contextmanager
+def ship_utilities(ctx, config):
+    """
+    Ship utilities during the first call, and skip it in the following ones.
+    See also `_ship_utilities`.
+
+    :param ctx: Context
+    :param config: Configuration
+    """
+    assert config is None
+
+    do_ship_utilities = ctx.get('do_ship_utilities', True)
+    if do_ship_utilities:
+        ctx['do_ship_utilities'] = False
+        filenames = _ship_utilities(ctx)
+        try:
+            yield
+        finally:
+            _remove_utilities(ctx, filenames)
+    else:
+        log.info('Utilities already shipped, skip it...')
         yield
-    finally:
-        log.info('Removing shipped files: %s...', ' '.join(filenames))
-        run.wait(
-            ctx.cluster.run(
-                args=[
-                    'sudo',
-                    'rm',
-                    '-f',
-                    '--',
-                ] + list(filenames),
-                wait=False,
-            ),
-        )