teuthology 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

teuthology/config.py

@@ -1,7 +1,10 @@
 import os
 import yaml
 import logging
-import collections
+try:
+    from collections.abc import MutableMapping
+except ImportError:
+    from collections import MutableMapping
 
 
 def init_logging():
@@ -11,7 +14,7 @@ def init_logging():
 log = init_logging()
 
 
-class YamlConfig(collections.MutableMapping):
+class YamlConfig(MutableMapping):
     """
     A configuration object populated by parsing a yaml file, with optional
     default values.
@@ -30,12 +33,13 @@ class YamlConfig(collections.MutableMapping):
             self._conf = dict()
 
     def load(self, conf=None):
-        if conf:
+        if conf is not None:
             if isinstance(conf, dict):
                 self._conf = conf
-            else:
+                return
+            elif conf:
                 self._conf = yaml.safe_load(conf)
-            return
+                return
         if os.path.exists(self.yaml_path):
             with open(self.yaml_path) as f:
                 self._conf = yaml.safe_load(f)
@@ -149,10 +153,12 @@ class TeuthologyConfig(YamlConfig):
         'conserver_master': 'conserver.front.sepia.ceph.com',
         'conserver_port': 3109,
         'gitbuilder_host': 'gitbuilder.ceph.com',
-        'githelper_base_url': 'http://git.ceph.com:8080',
+        'githelper_base_url': 'http://githelper.ceph.com',
         'check_package_signatures': True,
+        'job_threshold': 500,
         'lab_domain': 'front.sepia.ceph.com',
         'lock_server': 'http://paddles.front.sepia.ceph.com/',
+        'max_job_age': 1209600,  # 2 weeks
         'max_job_time': 259200,  # 3 days
         'nsupdate_url': 'http://nsupdate.front.sepia.ceph.com/update',
         'results_server': 'http://paddles.front.sepia.ceph.com/',
@@ -162,6 +168,8 @@ class TeuthologyConfig(YamlConfig):
         'src_base_path': os.path.expanduser('~/src'),
         'verify_host_keys': True,
         'watchdog_interval': 120,
+        'fog_reimage_timeout': 1800,
+        'fog_wait_for_ssh_timeout': 600,
         'kojihub_url': 'http://koji.fedoraproject.org/kojihub',
         'kojiroot_url': 'http://kojipkgs.fedoraproject.org/packages',
         'koji_task_url': 'https://kojipkgs.fedoraproject.org/work/',
@@ -187,6 +195,8 @@ class TeuthologyConfig(YamlConfig):
         },
         'rocketchat': None,
         'sleep_before_teardown': 0,
+        'ssh_key': None,
+        'active_machine_types': [],
     }
 
     def __init__(self, yaml_path=None):

teuthology/contextutil.py

@@ -2,7 +2,6 @@ import contextlib
 import sys
 import logging
 import time
-import itertools
 
 from teuthology.config import config
 from teuthology.exceptions import MaxWhileTries
@@ -58,8 +57,8 @@ def nested(*managers):
 class safe_while(object):
     """
     A context manager to remove boiler plate code that deals with `while` loops
-    that need a given number of tries and some seconds to sleep between each
-    one of those tries.
+    that need a given number of tries or total timeout and some seconds to sleep
+    between each one of those tries.
 
     The most simple example possible will try 10 times sleeping for 6 seconds:
 
@@ -82,6 +81,8 @@ class safe_while(object):
     :param increment: The amount to add to the sleep value on each try.
                       Default 0.
     :param tries:     The amount of tries before giving up. Default 10.
+    :param timeout:   Total seconds to try for, overrides the tries parameter
+                      if specified. Default 0.
     :param action:    The name of the action being attempted. Default none.
     :param _raise:    Whether to raise an exception (or log a warning).
                       Default True.
@@ -89,28 +90,24 @@ class safe_while(object):
                       Default time.sleep
     """
 
-    def __init__(self, sleep=6, increment=0, tries=10, action=None,
+    def __init__(self, sleep=6, increment=0, tries=10, timeout=0, action=None,
                  _raise=True, _sleeper=None):
         self.sleep = sleep
         self.increment = increment
         self.tries = tries
+        self.timeout = timeout
         self.counter = 0
         self.sleep_current = sleep
         self.action = action
         self._raise = _raise
         self.sleeper = _sleeper or time.sleep
+        self.total_seconds = sleep
 
     def _make_error_msg(self):
         """
         Sum the total number of seconds we waited while providing the number
         of tries we attempted
         """
-        total_seconds_waiting = sum(
-            itertools.islice(
-                itertools.count(self.sleep, self.increment),
-                self.tries
-            )
-        )
         msg = 'reached maximum tries ({tries})' + \
             ' after waiting for {total} seconds'
         if self.action:
@@ -118,8 +115,8 @@ class safe_while(object):
 
         msg = msg.format(
             action=self.action,
-            tries=self.tries,
-            total=total_seconds_waiting,
+            tries=self.counter - 1,
+            total=self.total_seconds,
         )
         return msg
 
@@ -127,15 +124,22 @@ class safe_while(object):
         self.counter += 1
         if self.counter == 1:
             return True
-        if self.counter > self.tries:
+        def must_stop():
+            return self.tries > 0 and self.counter > self.tries
+        if ((self.timeout > 0 and
+             self.total_seconds >= self.timeout) or
+            (self.timeout == 0 and must_stop())):
             error_msg = self._make_error_msg()
             if self._raise:
                 raise MaxWhileTries(error_msg)
             else:
                 log.warning(error_msg)
                 return False
-        self.sleeper(self.sleep_current)
         self.sleep_current += self.increment
+        if self.timeout > 0:
+            self.sleep_current = min(self.timeout - self.total_seconds, self.sleep_current)
+        self.total_seconds += self.sleep_current
+        self.sleeper(self.sleep_current)
         return True
 
     def __enter__(self):

teuthology/describe_tests.py

@@ -13,7 +13,7 @@ from distutils.util import strtobool
 from teuthology.exceptions import ParseError
 from teuthology.suite.build_matrix import \
         build_matrix, generate_combinations, _get_matrix
-from teuthology.suite import util
+from teuthology.suite import util, merge
 
 def main(args):
     try:
@@ -57,6 +57,7 @@ def describe_tests(args):
                                          limit=conf['limit'],
                                          seed=conf['seed'],
                                          subset=conf['subset'],
+                                         no_nested_subset=conf['no_nested_subset'],
                                          fields=conf['fields'],
                                          filter_in=conf['filter_in'],
                                          filter_out=conf['filter_out'],
@@ -69,6 +70,7 @@ def describe_tests(args):
                        limit=conf['limit'],
                        seed=conf['seed'],
                        subset=conf['subset'],
+                       no_nested_subset=conf['no_nested_subset'],
                        show_desc=conf['print_description'],
                        show_frag=conf['print_fragments'],
                        filter_in=conf['filter_in'],
@@ -109,6 +111,7 @@ def output_results(headers, rows, output_format, hrule):
 def output_summary(path, limit=0,
                          seed=None,
                          subset=None,
+                         no_nested_subset=None,
                          show_desc=True,
                          show_frag=False,
                          show_matrix=False,
@@ -124,17 +127,19 @@ def output_summary(path, limit=0,
     """
 
     random.seed(seed)
-    mat, first, matlimit = _get_matrix(path, subset)
+    mat, first, matlimit = _get_matrix(path, subset=subset, no_nested_subset=no_nested_subset)
     configs = generate_combinations(path, mat, first, matlimit)
     count = 0
+    total = len(configs)
     suite = os.path.basename(path)
-    config_list = util.filter_configs(configs,
-                                      suite_name=suite,
-                                      filter_in=filter_in,
-                                      filter_out=filter_out,
-                                      filter_all=filter_all,
-                                      filter_fragments=filter_fragments)
-    for c in config_list:
+    configs = merge.config_merge(configs,
+                                 suite_name=suite,
+                                 filter_in=filter_in,
+                                 filter_out=filter_out,
+                                 filter_all=filter_all,
+                                 filter_fragments=filter_fragments,
+                                 seed=seed)
+    for c in configs:
         if limit and count >= limit:
             break
         count += 1
@@ -145,12 +150,13 @@ def output_summary(path, limit=0,
                     print("    {}".format(util.strip_fragment_path(path)))
     if show_matrix:
        print(mat.tostr(1))
-    print("# {}/{} {}".format(count, len(configs), path))
+    print("# {}/{} {}".format(count, total, path))
 
 def get_combinations(suite_dir,
                      limit=0,
                      seed=None,
                      subset=None,
+                     no_nested_subset=False,
                      fields=[],
                      filter_in=None,
                      filter_out=None,
@@ -166,7 +172,7 @@ def get_combinations(suite_dir,
     of strings.
     """
     suite = os.path.basename(suite_dir)
-    configs = build_matrix(suite_dir, subset, seed)
+    configs = build_matrix(suite_dir, subset=subset, no_nested_subset=no_nested_subset, seed=seed)
 
     num_listed = 0
     rows = []
@@ -175,13 +181,14 @@ def get_combinations(suite_dir,
     dirs = {}
     max_dir_depth = 0
 
-    configs = util.filter_configs(configs,
-                                  suite_name=suite,
-                                  filter_in=filter_in,
-                                  filter_out=filter_out,
-                                  filter_all=filter_all,
-                                  filter_fragments=filter_fragments)
-    for _, fragment_paths in configs:
+    configs = merge.config_merge(configs,
+                                 suite_name=suite,
+                                 filter_in=filter_in,
+                                 filter_out=filter_out,
+                                 filter_all=filter_all,
+                                 filter_fragments=filter_fragments,
+                                 seed=seed)
+    for _, fragment_paths, __ in configs:
         if limit > 0 and num_listed >= limit:
             break
 

teuthology/dispatcher/__init__.py

@@ -1,25 +1,32 @@
+import datetime
 import logging
 import os
+import psutil
 import subprocess
 import sys
 import yaml
 
-from datetime import datetime
-
-from teuthology import setup_log_file, install_except_hook
-from teuthology import beanstalk
-from teuthology import report
+from typing import Dict, List
+
+from teuthology import (
+    # non-modules
+    setup_log_file,
+    install_except_hook,
+    # modules
+    beanstalk,
+    exporter,
+    report,
+    repo_utils,
+)
 from teuthology.config import config as teuth_config
-from teuthology.exceptions import SkipJob
-from teuthology.repo_utils import fetch_qa_suite, fetch_teuthology
-from teuthology.lock.ops import block_and_lock_machines
 from teuthology.dispatcher import supervisor
-from teuthology.worker import prep_job
+from teuthology.exceptions import BranchNotFoundError, CommitNotFoundError, SkipJob, MaxWhileTries
+from teuthology.lock import ops as lock_ops
+from teuthology.util.time import parse_timestamp
 from teuthology import safepath
-from teuthology.nuke import nuke
 
 log = logging.getLogger(__name__)
-start_time = datetime.utcnow()
+start_time = datetime.datetime.now(datetime.timezone.utc)
 restart_file_path = '/tmp/teuthology-restart-dispatcher'
 stop_file_path = '/tmp/teuthology-stop-dispatcher'
 
@@ -27,11 +34,14 @@ stop_file_path = '/tmp/teuthology-stop-dispatcher'
 def sentinel(path):
     if not os.path.exists(path):
         return False
-    file_mtime = datetime.utcfromtimestamp(os.path.getmtime(path))
+    file_mtime = datetime.datetime.fromtimestamp(
+        os.path.getmtime(path),
+        datetime.timezone.utc,
+    )
     return file_mtime > start_time
 
 
-def restart():
+def restart(log=log):
     log.info('Restarting...')
     args = sys.argv[:]
     args.insert(0, sys.executable)
@@ -56,38 +66,39 @@ def load_config(archive_dir=None):
 
 
 def main(args):
-    # run dispatcher in job supervisor mode if --supervisor passed
-    if args["--supervisor"]:
-        return supervisor.main(args)
-
-    verbose = args["--verbose"]
-    tube = args["--tube"]
-    log_dir = args["--log-dir"]
-    archive_dir = args["--archive-dir"]
+    archive_dir = args.archive_dir or teuth_config.archive_base
 
-    if archive_dir is None:
-        archive_dir = teuth_config.archive_base
+    # Refuse to start more than one dispatcher per machine type
+    procs = find_dispatcher_processes().get(args.tube)
+    if procs:
+        raise RuntimeError(
+            "There is already a teuthology-dispatcher process running:"
+            f" {procs}"
+        )
 
     # setup logging for disoatcher in {log_dir}
     loglevel = logging.INFO
-    if verbose:
+    if args.verbose:
         loglevel = logging.DEBUG
+    logging.getLogger().setLevel(loglevel)
     log.setLevel(loglevel)
-    log_file_path = os.path.join(log_dir, f"dispatcher.{tube}.{os.getpid()}")
+    log_file_path = os.path.join(args.log_dir, f"dispatcher.{args.tube}.{os.getpid()}")
     setup_log_file(log_file_path)
     install_except_hook()
 
     load_config(archive_dir=archive_dir)
 
     connection = beanstalk.connect()
-    beanstalk.watch_tube(connection, tube)
+    beanstalk.watch_tube(connection, args.tube)
     result_proc = None
 
     if teuth_config.teuthology_path is None:
-        fetch_teuthology('master')
-    fetch_qa_suite('master')
+        repo_utils.fetch_teuthology('main')
+    repo_utils.fetch_qa_suite('main')
 
     keep_running = True
+    job_procs = set()
+    worst_returncode = 0
     while keep_running:
         # Check to see if we have a teuthology-results process hanging around
         # and if so, read its return code so that it can exit.
@@ -102,9 +113,16 @@ def main(args):
             stop()
 
         load_config()
-
+        for proc in list(job_procs):
+            rc = proc.poll()
+            if rc is not None:
+                worst_returncode = max([worst_returncode, rc])
+                job_procs.remove(proc)
         job = connection.reserve(timeout=60)
         if job is None:
+            if args.exit_on_empty_queue and not job_procs:
+                log.info("Queue is empty and no supervisor processes running; exiting!")
+                break
             continue
 
         # bury the job so it won't be re-run if it fails
@@ -132,8 +150,7 @@ def main(args):
             job_config = lock_machines(job_config)
 
         run_args = [
-            os.path.join(teuth_bin_path, 'teuthology-dispatcher'),
-            '--supervisor',
+            os.path.join(teuth_bin_path, 'teuthology-supervisor'),
             '-v',
             '--bin-path', teuth_bin_path,
             '--archive-dir', archive_dir,
@@ -153,13 +170,24 @@ def main(args):
         run_args.extend(["--job-config", job_config_path])
 
         try:
-            job_proc = subprocess.Popen(run_args)
+            job_proc = subprocess.Popen(
+                run_args,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+            job_procs.add(job_proc)
             log.info('Job supervisor PID: %s', job_proc.pid)
         except Exception:
             error_message = "Saw error while trying to spawn supervisor."
             log.exception(error_message)
             if 'targets' in job_config:
-                nuke(supervisor.create_fake_context(job_config), True)
+                node_names = job_config["targets"].keys()
+                lock_ops.unlock_safe(
+                    node_names,
+                    job_config["owner"],
+                    job_config["name"],
+                    job_config["job_id"]
+                )
             report.try_push_job_info(job_config, dict(
                 status='fail',
                 failure_reason=error_message))
@@ -171,12 +199,159 @@ def main(args):
         except Exception:
             log.exception("Saw exception while trying to delete job")
 
+    return worst_returncode
+
+
+def find_dispatcher_processes() -> Dict[str, List[psutil.Process]]:
+    def match(proc):
+        try:
+            cmdline = proc.cmdline()
+        except psutil.AccessDenied:
+            return False
+        except psutil.ZombieProcess:
+            return False
+        if len(cmdline) < 3:
+            return False
+        if not cmdline[1].endswith("/teuthology-dispatcher"):
+            return False
+        if cmdline[2] == "--supervisor":
+            return False
+        if "--tube" not in cmdline:
+            return False
+        if proc.pid == os.getpid():
+            return False
+        return True
+
+    procs = {}
+    attrs = ["pid", "cmdline"]
+    for proc in psutil.process_iter(attrs=attrs):
+        if not match(proc):
+            continue
+        cmdline = proc.cmdline()
+        machine_type = cmdline[cmdline.index("--tube") + 1]
+        procs.setdefault(machine_type, []).append(proc)
+    return procs
+
+
+def prep_job(job_config, log_file_path, archive_dir):
+    job_id = job_config['job_id']
+    check_job_expiration(job_config)
+
+    safe_archive = safepath.munge(job_config['name'])
+    job_config['worker_log'] = log_file_path
+    archive_path_full = os.path.join(
+        archive_dir, safe_archive, str(job_id))
+    job_config['archive_path'] = archive_path_full
+
+    # If the teuthology branch was not specified, default to main and
+    # store that value.
+    teuthology_branch = job_config.get('teuthology_branch', 'main')
+    job_config['teuthology_branch'] = teuthology_branch
+    teuthology_sha1 = job_config.get('teuthology_sha1')
+    if not teuthology_sha1:
+        repo_url = repo_utils.build_git_url('teuthology', 'ceph')
+        try:
+            teuthology_sha1 = repo_utils.ls_remote(repo_url, teuthology_branch)
+        except Exception as exc:
+            log.exception(f"Could not get teuthology sha1 for branch {teuthology_branch}")
+            report.try_push_job_info(
+                job_config,
+                dict(status='dead', failure_reason=str(exc))
+            )
+            raise SkipJob()
+        if not teuthology_sha1:
+            reason = "Teuthology branch {} not found; marking job as dead".format(teuthology_branch)
+            log.error(reason)
+            report.try_push_job_info(
+                job_config,
+                dict(status='dead', failure_reason=reason)
+            )
+            raise SkipJob()
+        if teuth_config.teuthology_path is None:
+            log.info('Using teuthology sha1 %s', teuthology_sha1)
+
+    try:
+        if teuth_config.teuthology_path is not None:
+            teuth_path = teuth_config.teuthology_path
+        else:
+            teuth_path = repo_utils.fetch_teuthology(branch=teuthology_branch,
+                                          commit=teuthology_sha1)
+        # For the teuthology tasks, we look for suite_branch, and if we
+        # don't get that, we look for branch, and fall back to 'main'.
+        # last-in-suite jobs don't have suite_branch or branch set.
+        ceph_branch = job_config.get('branch', 'main')
+        suite_branch = job_config.get('suite_branch', ceph_branch)
+        suite_sha1 = job_config.get('suite_sha1')
+        suite_repo = job_config.get('suite_repo')
+        if suite_repo:
+            teuth_config.ceph_qa_suite_git_url = suite_repo
+        job_config['suite_path'] = os.path.normpath(os.path.join(
+            repo_utils.fetch_qa_suite(suite_branch, suite_sha1),
+            job_config.get('suite_relpath', ''),
+        ))
+    except (BranchNotFoundError, CommitNotFoundError) as exc:
+        log.exception("Requested version not found; marking job as dead")
+        report.try_push_job_info(
+            job_config,
+            dict(status='dead', failure_reason=str(exc))
+        )
+        raise SkipJob()
+    except MaxWhileTries as exc:
+        log.exception("Failed to fetch or bootstrap; marking job as dead")
+        report.try_push_job_info(
+            job_config,
+            dict(status='dead', failure_reason=str(exc))
+        )
+        raise SkipJob()
+
+    teuth_bin_path = os.path.join(teuth_path, 'virtualenv', 'bin')
+    if not os.path.isdir(teuth_bin_path):
+        raise RuntimeError("teuthology branch %s at %s not bootstrapped!" %
+                           (teuthology_branch, teuth_bin_path))
+    return job_config, teuth_bin_path
+
+
+def check_job_expiration(job_config):
+    job_id = job_config['job_id']
+    expired = False
+    now = datetime.datetime.now(datetime.timezone.utc)
+    if expire_str := job_config.get('timestamp'):
+        expire = parse_timestamp(expire_str) + \
+            datetime.timedelta(seconds=teuth_config.max_job_age)
+        expired = expire < now
+    if not expired and (expire_str := job_config.get('expire')):
+        try:
+            expire = parse_timestamp(expire_str)
+            expired = expired or expire < now
+        except ValueError:
+            log.warning(f"Failed to parse job expiration: {expire_str=}")
+            pass
+    if expired:
+        log.info(f"Skipping job {job_id} because it is expired: {expire_str} is in the past")
+        report.try_push_job_info(
+            job_config,
+            # TODO: Add a 'canceled' status to paddles, and use that.
+            dict(status='dead'),
+        )
+        raise SkipJob()
+
 
 def lock_machines(job_config):
     report.try_push_job_info(job_config, dict(status='running'))
     fake_ctx = supervisor.create_fake_context(job_config, block=True)
-    block_and_lock_machines(fake_ctx, len(job_config['roles']),
-                            job_config['machine_type'], reimage=False)
+    machine_type = job_config["machine_type"]
+    count = len(job_config['roles'])
+    with exporter.NodeLockingTime().time(
+        machine_type=machine_type,
+        count=count,
+    ):
+        lock_ops.block_and_lock_machines(
+            fake_ctx,
+            count,
+            machine_type,
+            tries=-1,
+            reimage=False,
+        )
     job_config = fake_ctx.config
     return job_config