teuthology 1.1.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

teuthology/dispatcher/supervisor.py

@@ -1,65 +1,79 @@
+import datetime
 import logging
 import os
 import subprocess
 import time
 import yaml
+import requests
 
-from datetime import datetime
+from urllib.parse import urljoin
 
-import teuthology
-from teuthology import report
-from teuthology import safepath
+from teuthology import exporter, dispatcher, kill, report, safepath
 from teuthology.config import config as teuth_config
-from teuthology.exceptions import SkipJob
+from teuthology.exceptions import SkipJob, MaxWhileTries
 from teuthology import setup_log_file, install_except_hook
-from teuthology.lock.ops import reimage_machines
 from teuthology.misc import get_user, archive_logs, compress_logs
 from teuthology.config import FakeNamespace
-from teuthology.job_status import get_status
-from teuthology.nuke import nuke
-from teuthology.kill import kill_job
-from teuthology.task.internal import add_remotes
+from teuthology.lock import ops as lock_ops
+from teuthology.task import internal
 from teuthology.misc import decanonicalize_hostname as shortname
 from teuthology.lock import query
+from teuthology.util import sentry
 
 log = logging.getLogger(__name__)
 
 
 def main(args):
-
-    verbose = args["--verbose"]
-    archive_dir = args["--archive-dir"]
-    teuth_bin_path = args["--bin-path"]
-    config_file_path = args["--job-config"]
-
-    with open(config_file_path, 'r') as config_file:
+    with open(args.job_config, 'r') as config_file:
         job_config = yaml.safe_load(config_file)
 
     loglevel = logging.INFO
-    if verbose:
+    if args.verbose:
         loglevel = logging.DEBUG
+    logging.getLogger().setLevel(loglevel)
     log.setLevel(loglevel)
 
     log_file_path = os.path.join(job_config['archive_path'],
                                  f"supervisor.{job_config['job_id']}.log")
     setup_log_file(log_file_path)
     install_except_hook()
+    try:
+        dispatcher.check_job_expiration(job_config)
+    except SkipJob:
+        return 0
 
     # reimage target machines before running the job
     if 'targets' in job_config:
-        reimage(job_config)
-        with open(config_file_path, 'w') as f:
+        node_count = len(job_config["targets"])
+        # If a job (e.g. from the nop suite) doesn't need nodes, avoid
+        # submitting a zero here.
+        if node_count:
+            with exporter.NodeReimagingTime().time(
+                machine_type=job_config["machine_type"],
+                node_count=node_count,
+            ):
+                reimage(job_config)
+        else:
+            reimage(job_config)
+        with open(args.job_config, 'w') as f:
             yaml.safe_dump(job_config, f, default_flow_style=False)
 
-    try:
-        run_job(
+    suite = job_config.get("suite")
+    if suite:
+        with exporter.JobTime().time(suite=suite):
+            return run_job(
+                job_config,
+                args.bin_path,
+                args.archive_dir,
+                args.verbose
+            )
+    else:
+        return run_job(
             job_config,
-            teuth_bin_path,
-            archive_dir,
-            verbose
+            args.bin_path,
+            args.archive_dir,
+            args.verbose
         )
-    except SkipJob:
-        return
 
 
 def run_job(job_config, teuth_bin_path, archive_dir, verbose):
@@ -77,6 +91,8 @@ def run_job(job_config, teuth_bin_path, archive_dir, verbose):
                 args.extend(['--seed', job_config['seed']])
             if job_config.get('subset'):
                 args.extend(['--subset', job_config['subset']])
+            if job_config.get('no_nested_subset'):
+                args.extend(['--no-nested-subset'])
         else:
             log.info('Generating results for %s', job_config['name'])
             timeout = job_config.get('results_timeout',
@@ -107,7 +123,7 @@ def run_job(job_config, teuth_bin_path, archive_dir, verbose):
     if 'config' in job_config:
         inner_config = job_config.pop('config')
         if not isinstance(inner_config, dict):
-            log.warn("run_job: job_config['config'] isn't a dict, it's a %s",
+            log.warning("run_job: job_config['config'] isn't a dict, it's a %s",
                      str(type(inner_config)))
         else:
             job_config.update(inner_config)
@@ -126,7 +142,11 @@ def run_job(job_config, teuth_bin_path, archive_dir, verbose):
     arg.extend(['--', job_archive])
 
     log.debug("Running: %s" % ' '.join(arg))
-    p = subprocess.Popen(args=arg)
+    p = subprocess.Popen(
+        args=arg,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
     log.info("Job archive: %s", job_config['archive_path'])
     log.info("Job PID: %s", str(p.pid))
 
@@ -150,6 +170,49 @@ def run_job(job_config, teuth_bin_path, archive_dir, verbose):
         log.info('Success!')
     if 'targets' in job_config:
         unlock_targets(job_config)
+    return p.returncode
+
+def failure_is_reimage(failure_reason):
+    if not failure_reason:
+        return False
+    reimage_failure = "Error reimaging machines:"
+    if reimage_failure in failure_reason:
+        return True
+    else:
+        return False
+
+
+def check_for_reimage_failures_and_mark_down(targets, count=10):
+    # Grab paddles history of jobs in the machine
+    # and count the number of reimaging errors
+    # if it fails N times then mark the machine down
+    base_url = teuth_config.results_server
+    for k, _ in targets.items():
+        machine = k.split('@')[-1]
+        url = urljoin(
+            base_url,
+            '/nodes/{0}/jobs/?count={1}'.format(machine, count)
+        )
+        resp = requests.get(url)
+        jobs = resp.json()
+        if len(jobs) < count:
+            continue
+        reimage_failures = list(filter(
+            lambda j: failure_is_reimage(j['failure_reason']),
+            jobs
+        ))
+        if len(reimage_failures) < count:
+            continue
+        # Mark machine down
+        machine_name = shortname(k)
+        lock_ops.update_lock(
+            machine_name,
+            description='reimage failed {0} times'.format(count),
+            status='down',
+        )
+        log.error(
+            'Reimage failed {0} times ... marking machine down'.format(count)
+        )
 
 
 def reimage(job_config):
@@ -160,12 +223,21 @@ def reimage(job_config):
     report.try_push_job_info(ctx.config, dict(status='waiting'))
     targets = job_config['targets']
     try:
-        reimaged = reimage_machines(ctx, targets, job_config['machine_type'])
+        reimaged = lock_ops.reimage_machines(ctx, targets, job_config['machine_type'])
     except Exception as e:
         log.exception('Reimaging error. Nuking machines...')
         # Reimage failures should map to the 'dead' status instead of 'fail'
-        report.try_push_job_info(ctx.config, dict(status='dead', failure_reason='Error reimaging machines: ' + str(e)))
-        nuke(ctx, True)
+        report.try_push_job_info(
+            ctx.config,
+            dict(status='dead', failure_reason='Error reimaging machines: ' + str(e))
+        )
+        # There isn't an actual task called "reimage", but it doesn't seem
+        # necessary to create a whole new Sentry tag for this.
+        ctx.summary = {
+            'sentry_event': sentry.report_error(job_config, e, task_name="reimage")
+        }
+        # Machine that fails to reimage after 10 times will be marked down
+        check_for_reimage_failures_and_mark_down(targets)
         raise
     ctx.config['targets'] = reimaged
     # change the status to running after the reimaging process
@@ -175,29 +247,30 @@ def reimage(job_config):
 def unlock_targets(job_config):
     serializer = report.ResultsSerializer(teuth_config.archive_base)
     job_info = serializer.job_info(job_config['name'], job_config['job_id'])
-    machine_status = query.get_statuses(job_info['targets'].keys())
-    # only unlock/nuke targets if locked in the first place
-    locked = [shortname(_['name'])
-              for _ in machine_status if _['locked']]
+    machine_statuses = query.get_statuses(job_info['targets'].keys())
+    # only unlock targets if locked and description matches
+    locked = []
+    for status in machine_statuses:
+        name = shortname(status['name'])
+        description = status['description']
+        if not status['locked']:
+            continue
+        if description != job_info['archive_path']:
+            log.warning(
+                "Was going to unlock %s but it was locked by another job: %s",
+                name, description
+            )
+            continue
+        locked.append(name)
     if not locked:
         return
-    job_status = get_status(job_info)
-    if job_status == 'pass' or \
-            (job_config.get('unlock_on_failure', False) and not job_config.get('nuke-on-error', False)):
+    if job_config.get("unlock_on_failure", True):
         log.info('Unlocking machines...')
-        fake_ctx = create_fake_context(job_config)
-        for machine in locked:
-            teuthology.lock.ops.unlock_one(fake_ctx,
-                                           machine, job_info['owner'],
-                                           job_info['archive_path'])
-    if job_status != 'pass' and job_config.get('nuke-on-error', False):
-        log.info('Nuking machines...')
-        fake_ctx = create_fake_context(job_config)
-        nuke(fake_ctx, True)
+        lock_ops.unlock_safe(locked, job_info["owner"], job_info["name"], job_info["job_id"])
 
 
 def run_with_watchdog(process, job_config):
-    job_start_time = datetime.utcnow()
+    job_start_time = datetime.datetime.now(datetime.timezone.utc)
 
     # Only push the information that's relevant to the watchdog, to save db
     # load
@@ -211,7 +284,7 @@ def run_with_watchdog(process, job_config):
     hit_max_timeout = False
     while process.poll() is None:
         # Kill jobs that have been running longer than the global max
-        run_time = datetime.utcnow() - job_start_time
+        run_time = datetime.datetime.now(datetime.timezone.utc) - job_start_time
         total_seconds = run_time.days * 60 * 60 * 24 + run_time.seconds
         if total_seconds > teuth_config.max_job_time:
             hit_max_timeout = True
@@ -220,9 +293,11 @@ def run_with_watchdog(process, job_config):
             try:
                 # kill processes but do not unlock yet so we can save
                 # the logs, coredumps, etc.
-                kill_job(job_info['name'], job_info['job_id'],
-                         teuth_config.archive_base, job_config['owner'],
-                         save_logs=True)
+                kill.kill_job(
+                    job_info['name'], job_info['job_id'],
+                    teuth_config.archive_base, job_config['owner'],
+                    skip_unlock=True
+                )
             except Exception:
                 log.exception('Failed to kill job')
 
@@ -234,13 +309,18 @@ def run_with_watchdog(process, job_config):
 
             try:
                 # this time remove everything and unlock the machines
-                kill_job(job_info['name'], job_info['job_id'],
-                         teuth_config.archive_base, job_config['owner'])
+                kill.kill_job(
+                    job_info['name'], job_info['job_id'],
+                    teuth_config.archive_base, job_config['owner']
+                )
             except Exception:
                 log.exception('Failed to kill job and unlock machines')
 
         # calling this without a status just updates the jobs updated time
-        report.try_push_job_info(job_info)
+        try:
+            report.try_push_job_info(job_info)
+        except MaxWhileTries:
+            log.exception("Failed to report job status; ignoring")
         time.sleep(teuth_config.watchdog_interval)
 
     # we no longer support testing theses old branches
@@ -254,7 +334,8 @@ def run_with_watchdog(process, job_config):
     extra_info = dict(status='dead')
     if hit_max_timeout:
         extra_info['failure_reason'] = 'hit max job timeout'
-    report.try_push_job_info(job_info, extra_info)
+    if not (job_config.get('first_in_suite') or job_config.get('last_in_suite')):
+        report.try_push_job_info(job_info, extra_info)
 
 
 def create_fake_context(job_config, block=False):
@@ -270,6 +351,7 @@ def create_fake_context(job_config, block=False):
         'os_type': job_config.get('os_type', 'ubuntu'),
         'os_version': os_version,
         'name': job_config['name'],
+        'job_id': job_config['job_id'],
     }
 
     return FakeNamespace(ctx_args)
@@ -281,7 +363,7 @@ def transfer_archives(run_name, job_id, archive_base, job_config):
 
     if 'archive' in job_info:
         ctx = create_fake_context(job_config)
-        add_remotes(ctx, job_config)
+        internal.add_remotes(ctx, job_config)
 
         for log_type, log_path in job_info['archive'].items():
             if log_type == 'init':

teuthology/exceptions.py

@@ -12,6 +12,18 @@ class BranchNotFoundError(ValueError):
             branch=self.branch, repo_str=repo_str)
 
 
+class BranchMismatchError(ValueError):
+    def __init__(self, branch, repo, reason=None):
+        self.branch = branch
+        self.repo = repo
+        self.reason = reason
+
+    def __str__(self):
+        msg = f"Cannot use branch {self.branch} with repo {self.repo}"
+        if self.reason:
+            msg = f"{msg} because {self.reason}"
+        return msg
+
 class CommitNotFoundError(ValueError):
     def __init__(self, commit, repo=None):
         self.commit = commit
@@ -93,6 +105,13 @@ class AnsibleFailedError(Exception):
             failures=self.failures,
         )
 
+    def fingerprint(self):
+        """
+        Sentry will use this to group events by their failure reasons, rather
+        than lumping all AnsibleFailedErrors together
+        """
+        return self.failures
+
 
 class CommandCrashedError(Exception):
 
@@ -193,3 +212,27 @@ class NoRemoteError(Exception):
 
     def __str__(self):
         return self.message
+
+
+class UnitTestError(Exception):
+    """
+    Exception thrown on unit test failure
+    """
+    def __init__(self, exitstatus=None, node=None, label=None, message=None):
+        self.exitstatus = exitstatus
+        self.node = node
+        self.label = label
+        self.message = message
+
+    def __str__(self):
+        prefix = "Unit test failed"
+        if self.label:
+            prefix += " ({label})".format(label=self.label)
+        if self.node:
+            prefix += " on {node}".format(node=self.node)
+        if self.exitstatus:
+            prefix += " with status {status}".format(status=self.exitstatus)
+        return "{prefix}: '{message}'".format(
+            prefix=prefix,
+            message=self.message,
+        )