teuthology 1.1.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

teuthology/report.py

@@ -4,11 +4,14 @@ import json
 import re
 import requests
 import logging
+import random
 import socket
 from datetime import datetime
 
 import teuthology
+import teuthology.exporter
 from teuthology.config import config
+from teuthology.contextutil import safe_while
 from teuthology.job_status import get_status, set_status
 
 report_exceptions = (requests.exceptions.RequestException, socket.error)
@@ -141,7 +144,7 @@ class ResultsSerializer(object):
             return {}
         jobs = {}
         for item in os.listdir(archive_dir):
-            if not re.match('\d+$', item):
+            if not re.match(r'\d+$', item):
                 continue
             job_id = item
             job_dir = os.path.join(archive_dir, job_id)
@@ -197,7 +200,7 @@ class ResultsReporter(object):
 
         if not self.base_uri:
             msg = "No results_server set in {yaml}; cannot report results"
-            self.log.warn(msg.format(yaml=config.yaml_path))
+            self.log.warning(msg.format(yaml=config.yaml_path))
 
     def _make_session(self, max_retries=10):
         session = requests.Session()
@@ -289,37 +292,42 @@ class ResultsReporter(object):
             set_status(job_info, 'dead')
         job_json = json.dumps(job_info)
         headers = {'content-type': 'application/json'}
-        response = self.session.post(run_uri, data=job_json, headers=headers)
 
-        if response.status_code == 200:
-            return job_id
+        inc = random.uniform(0, 1)
+        with safe_while(
+                sleep=1, increment=inc, action=f'report job {job_id}') as proceed:
+            while proceed():
+                response = self.session.post(run_uri, data=job_json, headers=headers)
 
-        # This call is wrapped in a try/except because of:
-        #  http://tracker.ceph.com/issues/8166
-        try:
-            resp_json = response.json()
-        except ValueError:
-            resp_json = dict()
+                if response.status_code == 200:
+                    return
 
-        if resp_json:
-            msg = resp_json.get('message', '')
-        else:
-            msg = response.text
-
-        if msg and msg.endswith('already exists'):
-            job_uri = os.path.join(run_uri, job_id, '')
-            response = self.session.put(job_uri, data=job_json,
-                                        headers=headers)
-        elif msg:
-            self.log.error(
-                "POST to {uri} failed with status {status}: {msg}".format(
-                    uri=run_uri,
-                    status=response.status_code,
-                    msg=msg,
-                ))
-        response.raise_for_status()
+                # This call is wrapped in a try/except because of:
+                #  http://tracker.ceph.com/issues/8166
+                try:
+                    resp_json = response.json()
+                except ValueError:
+                    resp_json = dict()
 
-        return job_id
+                if resp_json:
+                    msg = resp_json.get('message', '')
+                else:
+                    msg = response.text
+
+                if msg and msg.endswith('already exists'):
+                    job_uri = os.path.join(run_uri, job_id, '')
+                    response = self.session.put(job_uri, data=job_json,
+                                                headers=headers)
+                    if response.status_code == 200:
+                        return
+                elif msg:
+                    self.log.error(
+                        "POST to {uri} failed with status {status}: {msg}".format(
+                            uri=run_uri,
+                            status=response.status_code,
+                            msg=msg,
+                        ))
+        response.raise_for_status()
 
     @property
     def last_run(self):
@@ -394,6 +402,7 @@ class ResultsReporter(object):
         log_path = os.path.join(self.archive_base, run_name, 'results.log')
         # parse the log file generated by teuthology.results.results()
         subset = None
+        no_nested_subset = None
         seed = None
         with open(log_path) as results_log:
             for line in results_log:
@@ -403,15 +412,17 @@ class ResultsReporter(object):
                 line = line.strip()
                 if subset is None:
                     subset = self._parse_log_line(line, 'subset:')
-                elif seed is None:
+                if no_nested_subset is None:
+                    no_nested_subset = self._parse_log_line(line, 'no_nested_subset:')
+                if seed is None:
                     seed = self._parse_log_line(line, 'seed:')
-                else:
-                    break
         if subset is not None:
             subset = tuple(int(i) for i in subset.split('/'))
+        if no_nested_subset is not None:
+            no_nested_subset = bool(no_nested_subset)
         if seed is not None:
             seed = int(seed)
-        return subset, seed
+        return subset, no_nested_subset, seed
 
     def delete_job(self, run_name, job_id):
         """
@@ -461,6 +472,12 @@ def push_job_info(run_name, job_id, job_info, base_uri=None):
     if not reporter.base_uri:
         return
     reporter.report_job(run_name, job_id, job_info)
+    status = get_status(job_info)
+    if status in ["pass", "fail", "dead"] and "machine_type" in job_info:
+        teuthology.exporter.JobResults().record(
+            machine_type=job_info["machine_type"],
+            status=status,
+        )
 
 
 def try_push_job_info(job_config, extra_info=None):
@@ -488,8 +505,8 @@ def try_push_job_info(job_config, extra_info=None):
     job_id = job_config['job_id']
 
     if extra_info is not None:
-        job_info = extra_info.copy()
-        job_info.update(job_config)
+        job_info = job_config.copy()
+        job_info.update(extra_info)
     else:
         job_info = job_config
 
@@ -569,6 +586,11 @@ def try_mark_run_dead(run_name):
             try:
                 log.info("Marking job {job_id} as dead".format(job_id=job_id))
                 reporter.report_job(run_name, job['job_id'], dead=True)
+                if "machine_type" in job:
+                    teuthology.exporter.JobResults().record(
+                        machine_type=job["machine_type"],
+                        status=job["status"],
+                    )
             except report_exceptions:
                 log.exception("Could not mark job as dead: {job_id}".format(
                     job_id=job_id))

teuthology/results.py

@@ -28,7 +28,7 @@ def main(args):
 
     try:
         if args['--seed']:
-            note_rerun_params(args['--subset'], args['--seed'])
+            note_rerun_params(args['--subset'], args['--no-nested-subset'], args['--seed'])
         else:
             results(args['--archive-dir'], args['--name'], args['--email'],
                     int(args['--timeout']), args['--dry-run'])
@@ -37,9 +37,11 @@ def main(args):
         raise
 
 
-def note_rerun_params(subset, seed):
+def note_rerun_params(subset, no_nested_subset, seed):
     if subset:
         log.info('subset: %r', subset)
+    if no_nested_subset:
+        log.info('no_nested_subset: %r', no_nested_subset)
     if seed:
         log.info('seed: %r', seed)
 
@@ -53,7 +55,7 @@ def results(archive_dir, name, email, timeout, dry_run):
     reporter = ResultsReporter()
     while timeout > 0:
         if time.time() - starttime > timeout:
-            log.warn('test(s) did not finish before timeout of %d seconds',
+            log.warning('test(s) did not finish before timeout of %d seconds',
                      timeout)
             break
         jobs = reporter.get_jobs(name, fields=['job_id', 'status'])

teuthology/run.py

@@ -8,7 +8,6 @@ from teuthology import install_except_hook
 from teuthology import report
 from teuthology.job_status import get_status
 from teuthology.misc import get_user, merge_configs
-from teuthology.nuke import nuke
 from teuthology.run_tasks import run_tasks
 from teuthology.repo_utils import fetch_qa_suite
 from teuthology.results import email_results
@@ -80,7 +79,7 @@ def fetch_tasks_if_needed(job_config):
     except ImportError:
         log.info("Tasks not found; will attempt to fetch")
 
-    ceph_branch = job_config.get('branch', 'master')
+    ceph_branch = job_config.get('branch', 'main')
     suite_repo = job_config.get('suite_repo')
     if suite_repo:
         teuth_config.ceph_qa_suite_git_url = suite_repo
@@ -88,13 +87,13 @@ def fetch_tasks_if_needed(job_config):
     suite_sha1 = job_config.get('suite_sha1')
     suite_path = os.path.normpath(os.path.join(
         fetch_qa_suite(suite_branch, commit=suite_sha1),
-        job_config.get('suite_relpath', ''),
+        job_config.get('suite_relpath', 'qa'),
     ))
     sys.path.insert(1, suite_path)
     return suite_path
 
 
-def setup_config(config_paths):
+def setup_config(config_paths) -> dict:
     """
     Takes a list of config yaml files and combines them
     into a single dictionary. Processes / validates the dictionary and then
@@ -250,7 +249,9 @@ def get_initial_tasks(lock, config, machine_type):
             {'internal.git_ignore_ssl': None},
             {'internal.setup_cdn_repo': None},
             {'internal.setup_base_repo': None},
-            {'internal.setup_additional_repo': None}
+            {'internal.setup_additional_repo': None},
+            {'internal.setup_container_registry': None},
+            {'install': None},
         ])
         # Install latest kernel task for redhat downstream runs
         if config.get('redhat').get('install_latest_rh_kernel', False):
@@ -259,15 +260,11 @@ def get_initial_tasks(lock, config, machine_type):
     return init_tasks
 
 
-def report_outcome(config, archive, summary, fake_ctx):
+def report_outcome(config, archive, summary):
     """ Reports on the final outcome of the command. """
     status = get_status(summary)
     passed = status == 'pass'
 
-    if not passed and bool(config.get('nuke-on-error')):
-        # only unlock if we locked them in the first place
-        nuke(fake_ctx, fake_ctx.lock)
-
     if archive is not None:
         with open(os.path.join(archive, 'summary.yaml'), 'w') as f:
             yaml.safe_dump(summary, f, default_flow_style=False)
@@ -329,8 +326,6 @@ def main(args):
     os_version = args["--os-version"]
     interactive_on_error = args["--interactive-on-error"]
 
-    set_up_logging(verbose, archive)
-
     # print the command being ran
     log.debug("Teuthology command: {0}".format(get_teuthology_command(args)))
 
@@ -341,6 +336,10 @@ def main(args):
 
     if archive is not None and 'archive_path' not in config:
         config['archive_path'] = archive
+    elif archive is None and 'archive_path' in config:
+        archive = args['--archive'] = config['archive_path']
+
+    set_up_logging(verbose, archive)
 
     write_initial_metadata(archive, config, name, description, owner)
     report.try_push_job_info(config, dict(status='running'))
@@ -403,10 +402,10 @@ def main(args):
     # FIXME this should become more generic, and the keys should use
     # '_' uniformly
     if fake_ctx.config.get('interactive-on-error'):
-        teuthology.config.config.ctx = fake_ctx
+        teuth_config.config.ctx = fake_ctx
 
     try:
         run_tasks(tasks=config['tasks'], ctx=fake_ctx)
     finally:
         # print to stdout the results and possibly send an email on any errors
-        report_outcome(config, archive, fake_ctx.summary, fake_ctx)
+        report_outcome(config, archive, fake_ctx.summary)

teuthology/run_tasks.py

@@ -1,3 +1,4 @@
+import importlib
 import jinja2
 import logging
 import os
@@ -6,15 +7,16 @@ import time
 import types
 import yaml
 
-from copy import deepcopy
 from humanfriendly import format_timespan
-import sentry_sdk
+
+import teuthology.exporter as exporter
 
 from teuthology.config import config as teuth_config
 from teuthology.exceptions import ConnectionLostError
 from teuthology.job_status import set_status, get_status
 from teuthology.misc import get_http_log_path, get_results_url
 from teuthology.timer import Timer
+from teuthology.util import sentry
 
 log = logging.getLogger(__name__)
 
@@ -60,6 +62,17 @@ def _import(from_package, module_name, task_name, fail_on_import_error=False):
         if fail_on_import_error:
             raise
         else:
+            if (
+                importlib.util.find_spec(from_package) is not None and
+                importlib.util.find_spec(full_module_name) is not None
+            ):
+                # If we get here, it means we could _find_ both the module and
+                # the package that contains it, but still got an ImportError.
+                # Typically that means the module failed to import because it
+                # could not find one of its dependencies; if we don't raise
+                # here it will look like we just could't find the module,
+                # making the dependency issue difficult to discover.
+                raise
             return None
     return module
 
@@ -80,6 +93,7 @@ def run_tasks(tasks, ctx):
     else:
         timer = Timer()
     stack = []
+    taskname = ""
     try:
         for taskdict in tasks:
             try:
@@ -91,7 +105,11 @@ def run_tasks(tasks, ctx):
             manager = run_one_task(taskname, ctx=ctx, config=config)
             if hasattr(manager, '__enter__'):
                 stack.append((taskname, manager))
-                manager.__enter__()
+                with exporter.TaskTime().time(
+                    name=taskname,
+                    phase="enter"
+                ):
+                    manager.__enter__()
     except BaseException as e:
         if isinstance(e, ConnectionLostError):
             # Prevent connection issues being flagged as failures
@@ -104,45 +122,7 @@ def run_tasks(tasks, ctx):
             ctx.summary['failure_reason'] = str(e)
         log.exception('Saw exception from tasks.')
 
-        if teuth_config.sentry_dsn:
-            sentry_sdk.init(teuth_config.sentry_dsn)
-            config = deepcopy(ctx.config)
-
-            tags = {
-                'task': taskname,
-                'owner': ctx.owner,
-            }
-            optional_tags = ('teuthology_branch', 'branch', 'suite',
-                             'machine_type', 'os_type', 'os_version')
-            for tag in optional_tags:
-                if tag in config:
-                    tags[tag] = config[tag]
-
-            # Remove ssh keys from reported config
-            if 'targets' in config:
-                targets = config['targets']
-                for host in targets.keys():
-                    targets[host] = '<redacted>'
-
-            job_id = ctx.config.get('job_id')
-            archive_path = ctx.config.get('archive_path')
-            extras = dict(config=config,
-                         )
-            if job_id:
-                extras['logs'] = get_http_log_path(archive_path, job_id)
-
-            fingerprint = e.fingerprint() if hasattr(e, 'fingerprint') else None
-            exc_id = sentry_sdk.capture_exception(
-                error=e,
-                tags=tags,
-                extras=extras,
-                fingerprint=fingerprint,
-            )
-            event_url = "{server}/?query={id}".format(
-                server=teuth_config.sentry_server.strip('/'), id=exc_id)
-            log.exception(" Sentry event: %s" % event_url)
-            ctx.summary['sentry_event'] = event_url
-
+        ctx.summary['sentry_event'] = sentry.report_error(ctx.config, e, taskname)
         if ctx.config.get('interactive-on-error'):
             ctx.config['interactive-on-error'] = False
             from teuthology.task import interactive
@@ -173,7 +153,11 @@ def run_tasks(tasks, ctx):
                 log.debug('Unwinding manager %s', taskname)
                 timer.mark('%s exit' % taskname)
                 try:
-                    suppress = manager.__exit__(*exc_info)
+                    with exporter.TaskTime().time(
+                        name=taskname,
+                        phase="exit"
+                    ):
+                        suppress = manager.__exit__(*exc_info)
                 except Exception as e:
                     if isinstance(e, ConnectionLostError):
                         # Prevent connection issues being flagged as failures

teuthology/schedule.py

@@ -8,11 +8,11 @@ from teuthology import report
 
 def main(args):
     if not args['--first-in-suite']:
-        first_job_args = ['subset', 'seed']
+        first_job_args = ['subset', 'no-nested-subset', 'seed']
         for arg in first_job_args:
             opt = '--{arg}'.format(arg=arg)
             msg_fmt = '{opt} is only applicable to the first job in a suite'
-            if args[opt]:
+            if args.get(opt):
                 raise ValueError(msg_fmt.format(opt=opt))
 
     if not args['--last-in-suite']:
@@ -78,7 +78,8 @@ def build_config(args):
     job_config.update(conf_dict)
     for arg,conf in {'--timeout':'results_timeout',
                      '--seed': 'seed',
-                     '--subset': 'subset'}.items():
+                     '--subset': 'subset',
+                     '--no-nested-subset': 'no_nested_subset'}.items():
         val = args.get(arg, None)
         if val is not None:
             job_config[conf] = val

teuthology/scrape.py

@@ -12,7 +12,6 @@ import re
 import logging
 import subprocess
 
-import six
 
 log = logging.getLogger('scrape')
 log.addHandler(logging.StreamHandler())
@@ -31,11 +30,12 @@ def grep(path, expr):
     """
     Call out to native grep rather than feeding massive log files through python line by line
     """
-    p = subprocess.Popen(["grep", expr, path], stdout=subprocess.PIPE)
+    p = subprocess.Popen(["grep", expr, path], stdout=subprocess.PIPE,
+                         universal_newlines=True)
     p.wait()
     out, err = p.communicate()
     if p.returncode == 0:
-        return six.ensure_str(out).split("\n")
+        return out.split("\n")
     else:
         return []
 
@@ -83,9 +83,9 @@ class GenericReason(Reason):
         else:
             if "Test failure:" in self.failure_reason:
                 return self.failure_reason == job.get_failure_reason()
-            elif re.search("workunit test (.*)\) on ", self.failure_reason):
-                workunit_name = re.search("workunit test (.*)\) on ", self.failure_reason).group(1)
-                other_match = re.search("workunit test (.*)\) on ", job.get_failure_reason())
+            elif re.search(r"workunit test (.*)\) on ", self.failure_reason):
+                workunit_name = re.search(r"workunit test (.*)\) on ", self.failure_reason).group(1)
+                other_match = re.search(r"workunit test (.*)\) on ", job.get_failure_reason())
                 return other_match is not None and workunit_name == other_match.group(1)
             else:
                 reason_ratio = difflib.SequenceMatcher(None, self.failure_reason, job.get_failure_reason()).ratio()
@@ -198,12 +198,13 @@ class DeadReason(Reason):
             else:
                 # I have BT but he doesn't, so we're different
                 return False
-
-        if self.last_tlog_line or job.get_last_tlog_line():
+        last_tlog_line = job.get_last_tlog_line()
+        if self.last_tlog_line is not None and last_tlog_line is not None:
             ratio = difflib.SequenceMatcher(None, self.last_tlog_line,
-                                            job.get_last_tlog_line()).ratio()
+                                            last_tlog_line).ratio()
             return ratio > 0.5
-        return True
+        else:
+            return self.last_tlog_line == last_tlog_line
 
 
 class TimeoutReason(Reason):
@@ -302,29 +303,34 @@ class Job(object):
         for line in file_obj:
             # Log prefix from teuthology.log
             if ".stderr:" in line:
+                # Only captures the error and disregard what ever comes before
                 line = line.split(".stderr:")[1]
 
-            if "FAILED assert" in line:
+            if "FAILED assert" in line or "__ceph_assert_fail" in line:
                 assertion = line.strip()
 
             if line.startswith(" ceph version"):
                 # The start of a backtrace!
                 bt_lines = [line]
-            elif line.startswith(" NOTE: a copy of the executable"):
-                # The backtrace terminated, if we have a buffer return it
-                if len(bt_lines):
-                    return ("".join(bt_lines)).strip(), assertion
-                else:
-                    log.warning("Saw end of BT but not start")
+            elif line.startswith(" NOTE: a copy of the executable") or \
+                line.strip().endswith("clone()") or \
+                "clone()+0x" in line:
+                    # The backtrace terminated, if we have a buffer return it
+                    if len(bt_lines):
+                        return ("".join(bt_lines)).strip(), assertion
+                    else:
+                        log.warning("Saw end of BT but not start")
             elif bt_lines:
                 # We're in a backtrace, push the line onto the list
                 if len(bt_lines) > MAX_BT_LINES:
-                    # Something wrong with our parsing, drop it
-                    log.warning("Ignoring malparsed backtrace: {0}".format(
+                    # We exceeded MAX_BT_LINES, drop it
+                    log.warning("Ignoring backtrace that exceeds MAX_BACKTRACE_LINES: {0}".format(
                         ", ".join(bt_lines[0:3])
                     ))
+                    log.warning("Either the backtrace is too long or we did a bad job of checking for end of backtrace!")
                     bt_lines = []
-                bt_lines.append(line)
+                else:
+                    bt_lines.append(line)
 
         return None, assertion
 
@@ -357,7 +363,7 @@ class Job(object):
         for line in grep(tlog_path, "command crashed with signal"):
             log.debug("Found a crash indication: {0}".format(line))
             # tasks.ceph.osd.1.plana82.stderr
-            match = re.search("tasks.ceph.([^\.]+).([^\.]+).([^\.]+).stderr", line)
+            match = re.search(r"tasks.ceph.([^\.]+).([^\.]+).([^\.]+).stderr", line)
             if not match:
                 log.warning("Not-understood crash indication {0}".format(line))
                 continue
@@ -410,7 +416,7 @@ class ValgrindReason(Reason):
                 log.warning("Misunderstood line: {0}".format(line))
                 continue
             err_typ, log_basename = match.groups()
-            svc_typ = six.ensure_str(log_basename).split(".")[0]
+            svc_typ = log_basename.split(".")[0]
             if err_typ not in result[svc_typ]:
                 result[svc_typ].append(err_typ)
                 result[svc_typ] = sorted(result[svc_typ])