teuthology 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

teuthology/dispatcher/supervisor.py

@@ -0,0 +1,374 @@
+import datetime
+import logging
+import os
+import subprocess
+import time
+import yaml
+import requests
+
+from urllib.parse import urljoin
+
+from teuthology import exporter, dispatcher, kill, report, safepath
+from teuthology.config import config as teuth_config
+from teuthology.exceptions import SkipJob, MaxWhileTries
+from teuthology import setup_log_file, install_except_hook
+from teuthology.misc import get_user, archive_logs, compress_logs
+from teuthology.config import FakeNamespace
+from teuthology.lock import ops as lock_ops
+from teuthology.task import internal
+from teuthology.misc import decanonicalize_hostname as shortname
+from teuthology.lock import query
+from teuthology.util import sentry
+
+log = logging.getLogger(__name__)
+
+
+def main(args):
+    with open(args.job_config, 'r') as config_file:
+        job_config = yaml.safe_load(config_file)
+
+    loglevel = logging.INFO
+    if args.verbose:
+        loglevel = logging.DEBUG
+    logging.getLogger().setLevel(loglevel)
+    log.setLevel(loglevel)
+
+    log_file_path = os.path.join(job_config['archive_path'],
+                                 f"supervisor.{job_config['job_id']}.log")
+    setup_log_file(log_file_path)
+    install_except_hook()
+    try:
+        dispatcher.check_job_expiration(job_config)
+    except SkipJob:
+        return 0
+
+    # reimage target machines before running the job
+    if 'targets' in job_config:
+        node_count = len(job_config["targets"])
+        # If a job (e.g. from the nop suite) doesn't need nodes, avoid
+        # submitting a zero here.
+        if node_count:
+            with exporter.NodeReimagingTime().time(
+                machine_type=job_config["machine_type"],
+                node_count=node_count,
+            ):
+                reimage(job_config)
+        else:
+            reimage(job_config)
+        with open(args.job_config, 'w') as f:
+            yaml.safe_dump(job_config, f, default_flow_style=False)
+
+    suite = job_config.get("suite")
+    if suite:
+        with exporter.JobTime().time(suite=suite):
+            return run_job(
+                job_config,
+                args.bin_path,
+                args.archive_dir,
+                args.verbose
+            )
+    else:
+        return run_job(
+            job_config,
+            args.bin_path,
+            args.archive_dir,
+            args.verbose
+        )
+
+
+def run_job(job_config, teuth_bin_path, archive_dir, verbose):
+    safe_archive = safepath.munge(job_config['name'])
+    if job_config.get('first_in_suite') or job_config.get('last_in_suite'):
+        job_archive = os.path.join(archive_dir, safe_archive)
+        args = [
+            os.path.join(teuth_bin_path, 'teuthology-results'),
+            '--archive-dir', job_archive,
+            '--name', job_config['name'],
+        ]
+        if job_config.get('first_in_suite'):
+            log.info('Generating memo for %s', job_config['name'])
+            if job_config.get('seed'):
+                args.extend(['--seed', job_config['seed']])
+            if job_config.get('subset'):
+                args.extend(['--subset', job_config['subset']])
+            if job_config.get('no_nested_subset'):
+                args.extend(['--no-nested-subset'])
+        else:
+            log.info('Generating results for %s', job_config['name'])
+            timeout = job_config.get('results_timeout',
+                                     teuth_config.results_timeout)
+            args.extend(['--timeout', str(timeout)])
+            if job_config.get('email'):
+                args.extend(['--email', job_config['email']])
+        # Execute teuthology-results, passing 'preexec_fn=os.setpgrp' to
+        # make sure that it will continue to run if this worker process
+        # dies (e.g. because of a restart)
+        result_proc = subprocess.Popen(args=args, preexec_fn=os.setpgrp)
+        log.info("teuthology-results PID: %s", result_proc.pid)
+        # Remove unnecessary logs for first and last jobs in run
+        log.info('Deleting job\'s archive dir %s', job_config['archive_path'])
+        for f in os.listdir(job_config['archive_path']):
+            os.remove(os.path.join(job_config['archive_path'], f))
+        os.rmdir(job_config['archive_path'])
+        return
+
+    log.info('Running job %s', job_config['job_id'])
+
+    arg = [
+        os.path.join(teuth_bin_path, 'teuthology'),
+    ]
+    # The following is for compatibility with older schedulers, from before we
+    # started merging the contents of job_config['config'] into job_config
+    # itself.
+    if 'config' in job_config:
+        inner_config = job_config.pop('config')
+        if not isinstance(inner_config, dict):
+            log.warning("run_job: job_config['config'] isn't a dict, it's a %s",
+                     str(type(inner_config)))
+        else:
+            job_config.update(inner_config)
+
+    if verbose or job_config['verbose']:
+        arg.append('-v')
+
+    arg.extend([
+        '--owner', job_config['owner'],
+        '--archive', job_config['archive_path'],
+        '--name', job_config['name'],
+    ])
+    if job_config['description'] is not None:
+        arg.extend(['--description', job_config['description']])
+    job_archive = os.path.join(job_config['archive_path'], 'orig.config.yaml')
+    arg.extend(['--', job_archive])
+
+    log.debug("Running: %s" % ' '.join(arg))
+    p = subprocess.Popen(
+        args=arg,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+    log.info("Job archive: %s", job_config['archive_path'])
+    log.info("Job PID: %s", str(p.pid))
+
+    if teuth_config.results_server:
+        log.info("Running with watchdog")
+        try:
+            run_with_watchdog(p, job_config)
+        except Exception:
+            log.exception("run_with_watchdog had an unhandled exception")
+            raise
+    else:
+        log.info("Running without watchdog")
+        # This sleep() is to give the child time to start up and create the
+        # archive dir.
+        time.sleep(5)
+        p.wait()
+
+    if p.returncode != 0:
+        log.error('Child exited with code %d', p.returncode)
+    else:
+        log.info('Success!')
+    if 'targets' in job_config:
+        unlock_targets(job_config)
+    return p.returncode
+
+def failure_is_reimage(failure_reason):
+    if not failure_reason:
+        return False
+    reimage_failure = "Error reimaging machines:"
+    if reimage_failure in failure_reason:
+        return True
+    else:
+        return False
+
+
+def check_for_reimage_failures_and_mark_down(targets, count=10):
+    # Grab paddles history of jobs in the machine
+    # and count the number of reimaging errors
+    # if it fails N times then mark the machine down
+    base_url = teuth_config.results_server
+    for k, _ in targets.items():
+        machine = k.split('@')[-1]
+        url = urljoin(
+            base_url,
+            '/nodes/{0}/jobs/?count={1}'.format(machine, count)
+        )
+        resp = requests.get(url)
+        jobs = resp.json()
+        if len(jobs) < count:
+            continue
+        reimage_failures = list(filter(
+            lambda j: failure_is_reimage(j['failure_reason']),
+            jobs
+        ))
+        if len(reimage_failures) < count:
+            continue
+        # Mark machine down
+        machine_name = shortname(k)
+        lock_ops.update_lock(
+            machine_name,
+            description='reimage failed {0} times'.format(count),
+            status='down',
+        )
+        log.error(
+            'Reimage failed {0} times ... marking machine down'.format(count)
+        )
+
+
+def reimage(job_config):
+    # Reimage the targets specified in job config
+    # and update their keys in config after reimaging
+    ctx = create_fake_context(job_config)
+    # change the status during the reimaging process
+    report.try_push_job_info(ctx.config, dict(status='waiting'))
+    targets = job_config['targets']
+    try:
+        reimaged = lock_ops.reimage_machines(ctx, targets, job_config['machine_type'])
+    except Exception as e:
+        log.exception('Reimaging error. Nuking machines...')
+        # Reimage failures should map to the 'dead' status instead of 'fail'
+        report.try_push_job_info(
+            ctx.config,
+            dict(status='dead', failure_reason='Error reimaging machines: ' + str(e))
+        )
+        # There isn't an actual task called "reimage", but it doesn't seem
+        # necessary to create a whole new Sentry tag for this.
+        ctx.summary = {
+            'sentry_event': sentry.report_error(job_config, e, task_name="reimage")
+        }
+        # Machine that fails to reimage after 10 times will be marked down
+        check_for_reimage_failures_and_mark_down(targets)
+        raise
+    ctx.config['targets'] = reimaged
+    # change the status to running after the reimaging process
+    report.try_push_job_info(ctx.config, dict(status='running'))
+
+
+def unlock_targets(job_config):
+    serializer = report.ResultsSerializer(teuth_config.archive_base)
+    job_info = serializer.job_info(job_config['name'], job_config['job_id'])
+    machine_statuses = query.get_statuses(job_info['targets'].keys())
+    # only unlock targets if locked and description matches
+    locked = []
+    for status in machine_statuses:
+        name = shortname(status['name'])
+        description = status['description']
+        if not status['locked']:
+            continue
+        if description != job_info['archive_path']:
+            log.warning(
+                "Was going to unlock %s but it was locked by another job: %s",
+                name, description
+            )
+            continue
+        locked.append(name)
+    if not locked:
+        return
+    if job_config.get("unlock_on_failure", True):
+        log.info('Unlocking machines...')
+        lock_ops.unlock_safe(locked, job_info["owner"], job_info["name"], job_info["job_id"])
+
+
+def run_with_watchdog(process, job_config):
+    job_start_time = datetime.datetime.now(datetime.timezone.utc)
+
+    # Only push the information that's relevant to the watchdog, to save db
+    # load
+    job_info = dict(
+        name=job_config['name'],
+        job_id=job_config['job_id'],
+    )
+
+    # Sleep once outside of the loop to avoid double-posting jobs
+    time.sleep(teuth_config.watchdog_interval)
+    hit_max_timeout = False
+    while process.poll() is None:
+        # Kill jobs that have been running longer than the global max
+        run_time = datetime.datetime.now(datetime.timezone.utc) - job_start_time
+        total_seconds = run_time.days * 60 * 60 * 24 + run_time.seconds
+        if total_seconds > teuth_config.max_job_time:
+            hit_max_timeout = True
+            log.warning("Job ran longer than {max}s. Killing...".format(
+                max=teuth_config.max_job_time))
+            try:
+                # kill processes but do not unlock yet so we can save
+                # the logs, coredumps, etc.
+                kill.kill_job(
+                    job_info['name'], job_info['job_id'],
+                    teuth_config.archive_base, job_config['owner'],
+                    skip_unlock=True
+                )
+            except Exception:
+                log.exception('Failed to kill job')
+
+            try:
+                transfer_archives(job_info['name'], job_info['job_id'],
+                                  teuth_config.archive_base, job_config)
+            except Exception:
+                log.exception('Could not save logs')
+
+            try:
+                # this time remove everything and unlock the machines
+                kill.kill_job(
+                    job_info['name'], job_info['job_id'],
+                    teuth_config.archive_base, job_config['owner']
+                )
+            except Exception:
+                log.exception('Failed to kill job and unlock machines')
+
+        # calling this without a status just updates the jobs updated time
+        try:
+            report.try_push_job_info(job_info)
+        except MaxWhileTries:
+            log.exception("Failed to report job status; ignoring")
+        time.sleep(teuth_config.watchdog_interval)
+
+    # we no longer support testing theses old branches
+    assert(job_config.get('teuthology_branch') not in ('argonaut', 'bobtail',
+                                                       'cuttlefish', 'dumpling'))
+
+    # Let's make sure that paddles knows the job is finished. We don't know
+    # the status, but if it was a pass or fail it will have already been
+    # reported to paddles. In that case paddles ignores the 'dead' status.
+    # If the job was killed, paddles will use the 'dead' status.
+    extra_info = dict(status='dead')
+    if hit_max_timeout:
+        extra_info['failure_reason'] = 'hit max job timeout'
+    if not (job_config.get('first_in_suite') or job_config.get('last_in_suite')):
+        report.try_push_job_info(job_info, extra_info)
+
+
+def create_fake_context(job_config, block=False):
+    owner = job_config.get('owner', get_user())
+    os_version = job_config.get('os_version', None)
+
+    ctx_args = {
+        'config': job_config,
+        'block': block,
+        'owner': owner,
+        'archive': job_config['archive_path'],
+        'machine_type': job_config['machine_type'],
+        'os_type': job_config.get('os_type', 'ubuntu'),
+        'os_version': os_version,
+        'name': job_config['name'],
+        'job_id': job_config['job_id'],
+    }
+
+    return FakeNamespace(ctx_args)
+
+
+def transfer_archives(run_name, job_id, archive_base, job_config):
+    serializer = report.ResultsSerializer(archive_base)
+    job_info = serializer.job_info(run_name, job_id, simple=True)
+
+    if 'archive' in job_info:
+        ctx = create_fake_context(job_config)
+        internal.add_remotes(ctx, job_config)
+
+        for log_type, log_path in job_info['archive'].items():
+            if log_type == 'init':
+                log_type = ''
+            compress_logs(ctx, log_path)
+            archive_logs(ctx, log_path, log_type)
+    else:
+        log.info('No archives to transfer.')

teuthology/exceptions.py

@@ -12,6 +12,18 @@ class BranchNotFoundError(ValueError):
             branch=self.branch, repo_str=repo_str)
 
 
+class BranchMismatchError(ValueError):
+    def __init__(self, branch, repo, reason=None):
+        self.branch = branch
+        self.repo = repo
+        self.reason = reason
+
+    def __str__(self):
+        msg = f"Cannot use branch {self.branch} with repo {self.repo}"
+        if self.reason:
+            msg = f"{msg} because {self.reason}"
+        return msg
+
 class CommitNotFoundError(ValueError):
     def __init__(self, commit, repo=None):
         self.commit = commit
@@ -68,6 +80,17 @@ class CommandFailedError(Exception):
             prefix=prefix,
             )
 
+    def fingerprint(self):
+        """
+        Returns a list of strings to group failures with.
+        Used by sentry instead of grouping by backtrace.
+        """
+        return [
+            self.label or self.command,
+            'exit status {}'.format(self.exitstatus),
+            '{{ type }}',
+        ]
+
 
 class AnsibleFailedError(Exception):
 
@@ -82,6 +105,13 @@ class AnsibleFailedError(Exception):
             failures=self.failures,
         )
 
+    def fingerprint(self):
+        """
+        Sentry will use this to group events by their failure reasons, rather
+        than lumping all AnsibleFailedErrors together
+        """
+        return self.failures
+
 
 class CommandCrashedError(Exception):
 
@@ -182,3 +212,27 @@ class NoRemoteError(Exception):
 
     def __str__(self):
         return self.message
+
+
+class UnitTestError(Exception):
+    """
+    Exception thrown on unit test failure
+    """
+    def __init__(self, exitstatus=None, node=None, label=None, message=None):
+        self.exitstatus = exitstatus
+        self.node = node
+        self.label = label
+        self.message = message
+
+    def __str__(self):
+        prefix = "Unit test failed"
+        if self.label:
+            prefix += " ({label})".format(label=self.label)
+        if self.node:
+            prefix += " on {node}".format(node=self.node)
+        if self.exitstatus:
+            prefix += " with status {status}".format(status=self.exitstatus)
+        return "{prefix}: '{message}'".format(
+            prefix=prefix,
+            message=self.message,
+        )