teuthology 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

teuthology/util/time.py

@@ -0,0 +1,52 @@
+import re
+
+from datetime import datetime, timedelta, timezone
+
+# When we're not using ISO format, we're using this
+TIMESTAMP_FMT = "%Y-%m-%d_%H:%M:%S"
+
+def parse_timestamp(timestamp: str) -> datetime:
+    """
+    timestamp: A string either in ISO 8601 format or TIMESTAMP_FMT.
+               If no timezone is specified, UTC is assumed.
+
+    :returns: a datetime object
+    """
+    try:
+        dt = datetime.fromisoformat(timestamp)
+    except ValueError:
+        dt = datetime.strptime(timestamp, TIMESTAMP_FMT)
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    return dt
+
+def parse_offset(offset: str) -> timedelta:
+    """
+    offset: A string consisting of digits followed by one of the following
+            characters:
+                s: seconds
+                m: minutes
+                h: hours
+                d: days
+                w: weeks
+    """
+    err_msg = "Offsets must either be an ISO 8601-formatted timestamp or " \
+        f"a relative value like '2w', '1d', '7h', '45m', '90s'. Got: {offset}"
+    match = re.match(r'(\d+)(s|m|h|d|w)$', offset)
+    if match is None:
+        raise ValueError(err_msg)
+    num = int(match.groups()[0])
+    unit = match.groups()[1]
+    match unit:
+        case 's':
+            return timedelta(seconds=num)
+        case 'm':
+            return timedelta(minutes=num)
+        case 'h':
+            return timedelta(hours=num)
+        case 'd':
+            return timedelta(days=num)
+        case 'w':
+            return timedelta(weeks=num)
+        case _:
+            raise ValueError(err_msg)

teuthology-1.2.0.data/scripts/adjust-ulimits

@@ -0,0 +1,16 @@
+#!/bin/sh
+# If we're running as root, allow large amounts of open files.
+USER=$(whoami)
+
+# If a ulimit call fails, exit immediately.
+set -e
+
+if [ "$USER" = "root" ]
+then
+    # Enable large number of open files
+    ulimit -n 65536
+fi
+
+# Enable core dumps for everything
+ulimit -c unlimited
+exec "$@"

teuthology-1.2.0.data/scripts/daemon-helper

@@ -0,0 +1,114 @@
+#!python
+
+"""
+Helper script for running long-living processes.
+
+(Name says daemon, but that is intended to mean "long-living", we
+assume child process does not double-fork.)
+
+We start the command passed as arguments, with /dev/null as stdin, and
+then wait for EOF on stdin.
+
+When EOF is seen on stdin, the child process is killed.
+
+When the child process exits, this helper exits too.
+
+Usage:
+    daemon-helper <signal> [--kill-group] [nostdin] COMMAND ...
+"""
+
+from __future__ import print_function
+
+import fcntl
+import os
+import select
+import signal
+import struct
+import subprocess
+import sys
+from argparse import ArgumentParser
+
+parser = ArgumentParser(epilog=
+    'The remaining parameters are the command to be run.  If these\n' +
+    'parameters start wih nostdin, then no stdin input is expected.')
+parser.add_argument('signal')
+parser.add_argument('--kill-group', action='store_true', 
+                    help='kill all processes in the group')
+parser.add_argument('--nostdin', action='store_true', 
+                    help='no stdin input expected')
+parsed, args = parser.parse_known_args()
+end_signal = signal.SIGKILL
+if parsed.signal == 'term':
+    end_signal = signal.SIGTERM
+group = parsed.kill_group
+nostdin = parsed.nostdin
+skip_nostdin = 0
+try:
+    if args[0] == 'nostdin':
+        nostdin = True
+        skip_nostdin = 1
+except IndexError:
+    print('No command specified')
+    sys.exit(1)
+
+
+proc = None
+if nostdin:
+    if len(args) - skip_nostdin == 0:
+        print('No command specified')
+        sys.exit(1)
+    proc = subprocess.Popen(
+        args=args[skip_nostdin:],
+        )
+else:
+    with open('/dev/null', 'rb') as devnull:
+        proc = subprocess.Popen(
+            args=args,
+            stdin=devnull,
+            preexec_fn=os.setsid,
+            )
+
+flags = fcntl.fcntl(0, fcntl.F_GETFL)
+fcntl.fcntl(0, fcntl.F_SETFL, flags | os.O_NDELAY)
+
+saw_eof = False
+while True:
+    r,w,x = select.select([0], [], [0], 0.2)
+    if r:
+        data = os.read(0, 1)
+        if not data:
+            saw_eof = True
+            if not group:
+                proc.send_signal(end_signal)
+            else:
+                os.killpg(proc.pid, end_signal)
+            break
+        else:
+            sig, = struct.unpack('!b', data)
+            if not group:
+                proc.send_signal(sig)
+            else:
+                os.killpg(proc.pid, end_signal)
+
+
+    if proc.poll() is not None:
+        # child exited
+        break
+
+exitstatus = proc.wait()
+if exitstatus > 0:
+    print('{me}: command failed with exit status {exitstatus:d}'.format(
+        me=os.path.basename(sys.argv[0]),
+        exitstatus=exitstatus,
+        ), file=sys.stderr)
+    sys.exit(exitstatus)
+elif exitstatus < 0:
+    if saw_eof and exitstatus == -end_signal:
+        # suppress error from the exit we intentionally caused
+        pass
+    else:
+        print('{me}: command crashed with signal {signal:d}'.format(
+            me=os.path.basename(sys.argv[0]),
+            signal=-exitstatus,
+            ), file=sys.stderr)
+        sys.exit(1)

teuthology-1.2.0.data/scripts/stdin-killer

@@ -0,0 +1,263 @@
+#!python
+
+# Forward stdin to a subcommand. If EOF is read from stdin or
+# stdin/stdout/stderr are closed or hungup, then give the command "timeout"
+# seconds to complete before it is killed.
+#
+# The command is run in a separate process group. This is mostly to simplify
+# killing the set of processes (if well-behaving). You can configure that with
+# --setpgrp switch.
+
+# usage: stdin-killer [-h] [--timeout TIMEOUT] [--debug DEBUG] [--signal SIGNAL] [--verbose] [--setpgrp {no,self,child}] command [arguments ...]
+#
+# wait for stdin EOF then kill forked subcommand
+#
+# positional arguments:
+#   command            command to execute
+#   arguments          arguments to command
+#
+# options:
+#   -h, --help         show this help message and exit
+#   --timeout TIMEOUT  time to wait for forked subcommand to willing terminate
+#   --debug DEBUG      debug file
+#   --signal SIGNAL    signal to send
+#   --verbose          increase debugging
+#  --setpgrp {no,self,child}
+#                        create process group
+
+
+import argparse
+import fcntl
+import logging
+import os
+import select
+import signal
+import struct
+import subprocess
+import sys
+import time
+
+NAME = "stdin-killer"
+
+log = logging.getLogger(NAME)
+PAGE_SIZE = 4096
+
+POLL_HANGUP = select.POLLHUP | (select.POLLRDHUP if hasattr(select, 'POLLRDHUP') else 0) | select.POLLERR
+
+
+def handle_event(poll, buffer, fd, event, p):
+    if sigfdr == fd:
+        b = os.read(sigfdr, 1)
+        (signum,) = struct.unpack("B", b)
+        log.debug("got signal %d", signum)
+        try:
+            p.wait(timeout=0)
+            return True
+        except subprocess.TimeoutExpired:
+            pass
+    elif 0 == fd:
+        if event & POLL_HANGUP:
+            log.debug("peer closed connection, waiting for process exit")
+            poll.unregister(0)
+            sys.stdin.close()
+            if len(buffer) == 0 and p.stdin is not None:
+                p.stdin.close()
+                p.stdin = None
+            return True
+        elif event & select.POLLIN:
+            b = os.read(0, PAGE_SIZE)
+            if b == b"":
+                log.debug("read EOF")
+                poll.unregister(0)
+                sys.stdin.close()
+                if len(buffer) == 0:
+                    p.stdin.close()
+                return True
+            if p.stdin is not None:
+                buffer += b
+                # ignore further POLLIN until buffer is written to p.stdin
+                poll.register(0, POLL_HANGUP)
+                poll.register(p.stdin.fileno(), select.POLLOUT)
+    elif p.stdin is not None and p.stdin.fileno() == fd:
+        assert event & select.POLLOUT
+        b = buffer[:PAGE_SIZE]
+        log.debug("sending %d bytes to process", len(b))
+        try:
+            n = p.stdin.write(b)
+            p.stdin.flush()
+            log.debug("wrote %d bytes", n)
+            buffer = buffer[n:]
+            poll.register(0, select.POLLIN | POLL_HANGUP)
+            poll.unregister(p.stdin.fileno())
+        except BrokenPipeError:
+            log.debug("got SIGPIPE")
+            poll.unregister(p.stdin.fileno())
+            p.stdin.close()
+            p.stdin = None
+            return True
+        except BlockingIOError:
+            poll.register(p.stdin.fileno(), select.POLLOUT | POLL_HANGUP)
+    elif 1 == fd:
+        assert event & POLL_HANGUP
+        log.debug("stdout pipe has closed")
+        poll.unregister(1)
+        return True
+    elif 2 == fd:
+        assert event & POLL_HANGUP
+        log.debug("stderr pipe has closed")
+        poll.unregister(2)
+        return True
+    else:
+        assert False
+    return False
+
+
+def listen_for_events(sigfdr, p, timeout):
+    poll = select.poll()
+    # listen for data on stdin
+    poll.register(0, select.POLLIN | POLL_HANGUP)
+    # listen for stdout/stderr to be closed, if they are closed then my parent
+    # is gone and I should expire the command and myself.
+    poll.register(1, POLL_HANGUP)
+    poll.register(2, POLL_HANGUP)
+    # for SIGCHLD
+    poll.register(sigfdr, select.POLLIN)
+    buffer = bytearray()
+    expired = 0.0
+    while True:
+        if expired > 0.0:
+            since = time.monotonic() - expired
+            wait = int((timeout - since) * 1000.0)
+            if wait <= 0:
+                return
+        else:
+            wait = 5000
+        log.debug("polling for %d milliseconds", wait)
+        events = poll.poll(wait)
+        for fd, event in events:
+            log.debug("event: (%d, %d)", fd, event)
+            if handle_event(poll, buffer, fd, event, p):
+                if p.returncode is not None:
+                    return
+                if expired == 0.0:
+                    expired = time.monotonic()
+                    log.info(
+                        "expiration expected; waiting %d seconds for command to complete",
+                        NS.timeout,
+                    )
+
+
+if __name__ == "__main__":
+    signal.signal(signal.SIGPIPE, signal.SIG_IGN)
+    try:
+        (sigfdr, sigfdw) = os.pipe2(os.O_NONBLOCK | os.O_CLOEXEC)
+    except AttributeError:
+        # pipe2 is only available on "some flavors of Unix"
+        # https://docs.python.org/3.10/library/os.html?highlight=pipe2#os.pipe2
+        pipe_ends = os.pipe()
+        for fd in pipe_ends:
+            flags = fcntl.fcntl(fd, fcntl.F_GETFL)
+            fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK | os.O_CLOEXEC)
+        (sigfdr, sigfdw) = pipe_ends
+
+    signal.set_wakeup_fd(sigfdw)
+
+    def do_nothing(signum, frame):
+        pass
+
+    signal.signal(signal.SIGCHLD, do_nothing)
+
+    P = argparse.ArgumentParser(
+        description="wait for stdin EOF then kill forked subcommand"
+    )
+    P.add_argument(
+        "--timeout",
+        action="store",
+        default=5,
+        help="time to wait for forked subcommand to willing terminate",
+        type=int,
+    )
+    P.add_argument("--debug", action="store", help="debug file", type=str)
+    P.add_argument(
+        "--signal",
+        action="store",
+        help="signal to send",
+        type=int,
+        default=signal.SIGKILL,
+    )
+    P.add_argument("--verbose", action="store_true", help="increase debugging")
+    P.add_argument(
+        "--setpgrp",
+        action="store",
+        choices=["no", "self", "child"],
+        default="self",
+        help="create process group",
+    )
+    P.add_argument(
+        "cmd", metavar="command", type=str, nargs=1, help="command to execute"
+    )
+    P.add_argument(
+        "args", metavar="arguments", type=str, nargs="*", help="arguments to command"
+    )
+    NS = P.parse_args()
+
+    logargs = {}
+    if NS.debug is not None:
+        logargs["filename"] = NS.debug
+    else:
+        logargs["stream"] = sys.stderr
+    if NS.verbose:
+        logargs["level"] = logging.DEBUG
+    else:
+        logargs["level"] = logging.INFO
+    logargs["format"] = f"%(asctime)s {NAME} %(levelname)s: %(message)s"
+    logargs["datefmt"] = "%Y-%m-%dT%H:%M:%S"
+    logging.basicConfig(**logargs)
+
+    cargs = NS.cmd + NS.args
+    popen_kwargs = {
+        "stdin": subprocess.PIPE,
+    }
+
+    if NS.setpgrp == "self":
+        pgrp = os.getpgrp()
+        if pgrp != os.getpid():
+            os.setpgrp()
+            pgrp = os.getpgrp()
+    elif NS.setpgrp == "child":
+        popen_kwargs["preexec_fn"] = os.setpgrp
+        pgrp = None
+    elif NS.setpgrp == "no":
+        pgrp = 0
+    else:
+        assert False
+
+    log.debug("executing %s", cargs)
+    p = subprocess.Popen(cargs, **popen_kwargs)
+    if pgrp is None:
+        pgrp = p.pid
+    flags = fcntl.fcntl(p.stdin.fileno(), fcntl.F_GETFL)
+    fcntl.fcntl(p.stdin.fileno(), fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
+    listen_for_events(sigfdr, p, NS.timeout)
+
+    if p.returncode is None:
+        log.error("timeout expired: sending signal %d to command and myself", NS.signal)
+        if pgrp == 0:
+            os.kill(p.pid, NS.signal)
+        else:
+            os.killpg(pgrp, NS.signal)  # should kill me too
+        os.kill(os.getpid(), NS.signal)  # to exit abnormally with same signal
+        log.error("signal did not cause termination, sending myself SIGKILL")
+        os.kill(os.getpid(), signal.SIGKILL)  # failsafe
+    rc = p.returncode
+    log.debug("rc = %d", rc)
+    assert rc is not None
+    if rc < 0:
+        log.error("command terminated with signal %d: sending same signal to myself!", -rc)
+        os.kill(os.getpid(), -rc)  # kill myself with the same signal
+        log.error("signal did not cause termination, sending myself SIGKILL")
+        os.kill(os.getpid(), signal.SIGKILL)  # failsafe
+    else:
+        log.info("command exited with status %d: exiting normally with same code!", rc)
+        sys.exit(rc)

teuthology-1.2.0.dist-info/METADATA

@@ -0,0 +1,89 @@
+Metadata-Version: 2.1
+Name: teuthology
+Version: 1.2.0
+Summary: Ceph test framework
+Home-page: https://github.com/ceph/teuthology
+Author: Ceph Community
+License: MIT
+Keywords: teuthology,test,ceph,cluster
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: English
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Classifier: Topic :: System :: Distributed Computing
+Classifier: Topic :: System :: Filesystems
+Requires-Python: >=3.10
+Description-Content-Type: text/x-rst
+License-File: LICENSE
+Requires-Dist: PyYAML
+Requires-Dist: ansible-core==2.16.6
+Requires-Dist: apache-libcloud
+Requires-Dist: backports.ssl-match-hostname
+Requires-Dist: beanstalkc3>=0.4.0
+Requires-Dist: configobj
+Requires-Dist: configparser
+Requires-Dist: cryptography==41.0.7
+Requires-Dist: docopt
+Requires-Dist: gevent
+Requires-Dist: httplib2
+Requires-Dist: humanfriendly
+Requires-Dist: lupa
+Requires-Dist: lxml
+Requires-Dist: ndg-httpsclient
+Requires-Dist: netaddr
+Requires-Dist: paramiko
+Requires-Dist: pexpect
+Requires-Dist: pip-tools
+Requires-Dist: prettytable
+Requires-Dist: prometheus-client>=0.16.0
+Requires-Dist: psutil>=2.1.0
+Requires-Dist: pyasn1
+Requires-Dist: pynacl>=1.5.0
+Requires-Dist: pyopenssl>=0.13
+Requires-Dist: python-dateutil
+Requires-Dist: requests>2.13.0
+Requires-Dist: sentry-sdk
+Requires-Dist: types-psutil
+Requires-Dist: urllib3<1.27,>=1.25.4
+Provides-Extra: manhole
+Requires-Dist: manhole; extra == "manhole"
+Provides-Extra: openstack
+Requires-Dist: python-openstackclient; extra == "openstack"
+Requires-Dist: python-novaclient; extra == "openstack"
+Provides-Extra: rocketchat
+Requires-Dist: rocket-python>=1.2.15; extra == "rocketchat"
+Provides-Extra: sentry
+Requires-Dist: sentry-sdk; extra == "sentry"
+Provides-Extra: test
+Requires-Dist: PyJWT; extra == "test"
+Requires-Dist: boto>=2.0b4; extra == "test"
+Requires-Dist: boto3; extra == "test"
+Requires-Dist: coverage; extra == "test"
+Requires-Dist: ipy; extra == "test"
+Requires-Dist: mock; extra == "test"
+Requires-Dist: pynose; extra == "test"
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
+Requires-Dist: toml; extra == "test"
+Requires-Dist: tox; extra == "test"
+Requires-Dist: xmltodict; extra == "test"
+
+===================================================
+`Teuthology` -- The Ceph integration test framework
+===================================================
+
+
+Welcome! Teuthology's documentation is primarily hosted at `docs.ceph.com
+<https://docs.ceph.com/projects/teuthology/>`__.
+
+You can also look at docs `inside this repository <docs/>`__, but note that
+GitHub's `RST <http://docutils.sourceforge.net/rst.html>`__ rendering is quite
+limited. Mainly that means that links between documents will be broken.