teuthology 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

teuthology/task/tests/__init__.py

@@ -1,31 +1,28 @@
 """
-This task is used to integration test teuthology. Including this
-task in your yaml config will execute pytest which finds any tests in
-the current directory.  Each test that is discovered will be passed the
-teuthology ctx and config args that each teuthology task usually gets.
-This allows the tests to operate against the cluster.
+This task runs teuthology's unit tests and integration tests.
+It can run in one of two modes: "py" or "cli". The latter executes py.test in a
+separate process, whereas the former invokes it in the teuthology job's python
+process.
+If the running job has remotes available to it, it will attempt to run integration tests.
+Note that this requires running in "py" mode - the default.
 
 An example::
 
     tasks
       - tests:
-
 """
 import logging
+import os
+import pathlib
+import pexpect
 import pytest
 
-
-log = logging.getLogger(__name__)
-
-
-@pytest.fixture
-def ctx():
-    return {}
+from teuthology.job_status import set_status
+from teuthology.task import Task
+from teuthology.util.loggerfile import LoggerFile
 
 
-@pytest.fixture
-def config():
-    return []
+log = logging.getLogger(__name__)
 
 
 class TeuthologyContextPlugin(object):
@@ -33,78 +30,141 @@ class TeuthologyContextPlugin(object):
         self.ctx = ctx
         self.config = config
         self.failures = list()
+        self.stats = dict()
 
     # this is pytest hook for generating tests with custom parameters
     def pytest_generate_tests(self, metafunc):
         # pass the teuthology ctx and config to each test method
-        metafunc.parametrize(["ctx", "config"], [(self.ctx, self.config),])
-
-    @pytest.mark.trylast
-    def pytest_configure(self, config):
-        # removes the default pytest TerminalReporter
-        # this fixes failures with scheduled jobs; when run by a worker
-        # there is no terminal to report to and pytest dies
-        standard_reporter = config.pluginmanager.getplugin('terminalreporter')
-        config.pluginmanager.unregister(standard_reporter)
-        log.info("removing pytest terminal reporter")
+        if "ctx" in metafunc.fixturenames and \
+                "config" in metafunc.fixturenames:
+            metafunc.parametrize(["ctx", "config"], [(self.ctx, self.config),])
 
     # log the outcome of each test
-    def pytest_runtest_makereport(self, __multicall__, item, call):
-        report = __multicall__.execute()
-
-        # after the test has been called, get it's report and log it
-        if call.when == 'call':
-            # item.location[0] is a slash delimeted path to the test file
-            # being ran. We only want the portion after teuthology.task.tests
-            test_path = item.location[0].replace("/", ".").split(".")
-            test_path = ".".join(test_path[4:-1])
-            # removes the string '[ctx0, config0]' after the test name
-            test_name = item.location[2].split("[")[0]
-            name = "{path}:{name}".format(path=test_path, name=test_name)
-            if report.passed:
-                log.info("{name} Passed".format(name=name))
-            elif report.skipped:
-                log.info("{name} {info}".format(
-                    name=name,
-                    info=call.excinfo.exconly()
-                ))
+    @pytest.hookimpl(hookwrapper=True)
+    def pytest_runtest_makereport(self, item: pytest.Item, call: pytest.CallInfo):
+        outcome = yield
+        report = outcome.get_result()
+        test_path = item.location[0]
+        line_no = item.location[1]
+        test_name = item.location[2]
+        name = f"{test_path}:{line_no}:{test_name}"
+        log_msg = f"{report.outcome.upper()} {name}"
+        outcome_str = report.outcome.lower()
+        self.stats.setdefault(outcome_str, 0)
+        self.stats[outcome_str] += 1
+        if outcome_str in ['passed', 'skipped']:
+            if call.when == 'call':
+                log.info(log_msg)
+            else:
+                log.info(f"----- {name} {call.when} -----")
+        else:
+            log_msg = f"{log_msg}:{call.when}"
+            if call.excinfo:
+                self.failures.append(name)
+                log_msg = f"{log_msg}: {call.excinfo.getrepr()}"
             else:
-                # TODO: figure out a way to log the traceback
-                log.error("{name} Failed:\n {info}".format(
-                    name=name,
-                    info=call.excinfo.exconly()
-                ))
-                failure = "{name}: {err}".format(
-                    name=name,
-                    err=call.excinfo.exconly().replace("\n", "")
-                )
-                self.failures.append(failure)
-                self.ctx.summary['failure_reason'] = self.failures
-
-        return report
-
-
-def task(ctx, config):
+                self.failures.append(log_msg)
+            log.error(log_msg)
+
+        return
+
+
+# https://docs.pytest.org/en/stable/reference/exit-codes.html
+exit_codes = {
+    0: "All tests were collected and passed successfully",
+    1: "Tests were collected and run but some of the tests failed",
+    2: "Test execution was interrupted by the user",
+    3: "Internal error happened while executing tests",
+    4: "pytest command line usage error",
+    5: "No tests were collected",
+}
+
+
+class Tests(Task):
     """
     Use pytest to recurse through this directory, finding any tests
     and then executing them with the teuthology ctx and config args.
     Your tests must follow standard pytest conventions to be discovered.
+
+    If config["mode"] == "py", (the default), it will be run in the job's process.
+    If config["mode"] == "cli" py.test will be invoked as a subprocess.
     """
-    try:
-        status = pytest.main(
-            args=[
-                '-q',
-                '--pyargs', __name__
-            ],
-            plugins=[TeuthologyContextPlugin(ctx, config)]
-        )
-    except Exception:
-        log.exception("Saw failure running pytest")
-        ctx.summary["status"] = "dead"
-    else:
-        if status == 0:
+    base_args = ['-v', '--color=no']
+
+    def setup(self):
+        super().setup()
+        mode = self.config.get("mode", "py")
+        assert mode in ["py", "cli"], "mode must either be 'py' or 'cli'"
+        if mode == "cli":
+            # integration tests need ctx from this process, so we need to invoke
+            # pytest via python to be able to pass them
+            assert len(self.cluster.remotes) == 0, \
+                "Tests requiring remote nodes conflicts with CLI mode"
+        self.mode = mode
+        self.stats = dict()
+        self.orig_curdir = os.curdir
+
+    def begin(self):
+        super().begin()
+        try:
+            if self.mode == "py":
+                self.status, self.failures = self.run_py()
+            else:
+                self.status, self.failures = self.run_cli()
+        except Exception as e:
+            log.exception("Saw non-test failure!")
+            self.ctx.summary['failure_reason'] = str(e)
+            set_status(self.ctx.summary, "dead")
+
+    def end(self):
+        if os.curdir != self.orig_curdir:
+            os.chdir(self.orig_curdir)
+        if self.stats:
+            log.info(f"Stats: {self.stats}")
+        if self.status == 0:
             log.info("OK. All tests passed!")
-            ctx.summary["status"] = "pass"
+            set_status(self.ctx.summary, "pass")
         else:
-            log.error("FAIL. Saw test failures...")
-            ctx.summary["status"] = "fail"
+            status_msg = str(self.status)
+            if self.status in exit_codes:
+                status_msg = f"{status_msg}: {exit_codes[self.status]}"
+            log.error(f"FAIL (exit code {status_msg})")
+            if self.failures:
+                msg = f"{len(self.failures)} Failures: {self.failures}"
+                self.ctx.summary['failure_reason'] = msg
+                log.error(msg)
+            set_status(self.ctx.summary, "fail")
+        super().end()
+
+    def run_cli(self):
+        pytest_args = self.base_args + ['./teuthology/test', './scripts']
+        if len(self.cluster.remotes):
+            pytest_args.append('./teuthology/task/tests')
+        self.log.info(f"pytest args: {pytest_args}")
+        cwd = str(pathlib.Path(__file__).parents[3])
+        log.info(f"pytest cwd: {cwd}")
+        _, status = pexpect.run(
+            "py.test " + " ".join(pytest_args),
+            cwd=cwd,
+            withexitstatus=True,
+            timeout=None,
+            logfile=LoggerFile(self.log, logging.INFO),
+        )
+        return status, []
+
+    def run_py(self):
+        pytest_args = self.base_args + ['--pyargs', 'teuthology', 'scripts']
+        if len(self.cluster.remotes):
+            pytest_args.append(__name__)
+        self.log.info(f"pytest args: {pytest_args}")
+        context_plugin = TeuthologyContextPlugin(self.ctx, self.config)
+        # the cwd needs to change so that FakeArchive can find files in this repo
+        os.chdir(str(pathlib.Path(__file__).parents[3]))
+        status = pytest.main(
+            args=pytest_args,
+            plugins=[context_plugin],
+        )
+        self.stats = context_plugin.stats
+        return status, context_plugin.failures
+
+task = Tests

teuthology/task/tests/test_fetch_coredumps.py

@@ -0,0 +1,116 @@
+from teuthology.task.internal import fetch_binaries_for_coredumps
+from unittest.mock import patch, Mock
+import gzip
+import os
+
+class TestFetchCoreDumps(object):
+    class MockDecode(object):
+        def __init__(self, ret):
+            self.ret = ret
+            pass
+
+        def decode(self):
+            return self.ret
+
+    class MockPopen(object):
+        def __init__(self, ret):
+            self.ret = ret
+
+        def communicate(self, input=None):
+            return [TestFetchCoreDumps.MockDecode(self.ret)]
+
+    def setup_method(self):
+        self.the_function = fetch_binaries_for_coredumps
+        with gzip.open('file.gz', 'wb') as f:
+            f.write(b'Hello world!')
+        self.core_dump_path = "file.gz"
+        self.m_remote = Mock()
+        self.uncompressed_correct = self.MockPopen(
+            "ELF 64-bit LSB core file,"\
+            " x86-64, version 1 (SYSV), SVR4-style, from 'ceph_test_rados_api_io',"\
+            " real uid: 1194, effective uid: 1194, real gid: 1194,"\
+            " effective gid: 1194, execfn: '/usr/bin/ceph_test_rados_api_io', platform: 'x86_64'"
+        )
+        self.uncompressed_incorrect = self.MockPopen("ASCII text")
+        self.compressed_correct = self.MockPopen(
+            "gzip compressed data, was "\
+            "'correct.format.core', last modified: Wed Jun 29"\
+            " 19:55:29 2022, from Unix, original size modulo 2^32 3167080"
+        )
+
+        self.compressed_incorrect = self.MockPopen(
+            "gzip compressed data, was "\
+            "'incorrect.format.core', last modified: Wed Jun 29"\
+            " 19:56:56 2022, from Unix, original size modulo 2^32 11"
+        )
+
+    # Core is not compressed and file is in the correct format
+    @patch('teuthology.task.internal.subprocess.Popen')
+    @patch('teuthology.task.internal.os')
+    def test_uncompressed_correct_format(self, m_os, m_subproc_popen):
+        m_subproc_popen.side_effect = [
+                self.uncompressed_correct,
+                Exception("We shouldn't be hitting this!")
+            ]
+        m_os.path.join.return_value = self.core_dump_path
+        m_os.path.sep = self.core_dump_path
+        m_os.path.isdir.return_value = True
+        m_os.path.dirname.return_value = self.core_dump_path
+        m_os.path.exists.return_value = True
+        m_os.listdir.return_value = [self.core_dump_path]
+        self.the_function(None, self.m_remote)
+        assert self.m_remote.get_file.called
+
+    # Core is not compressed and file is in the wrong format
+    @patch('teuthology.task.internal.subprocess.Popen')
+    @patch('teuthology.task.internal.os')
+    def test_uncompressed_incorrect_format(self, m_os, m_subproc_popen):
+        m_subproc_popen.side_effect = [
+                self.uncompressed_incorrect,
+                Exception("We shouldn't be hitting this!")
+            ]
+        m_os.path.join.return_value = self.core_dump_path
+        m_os.path.sep = self.core_dump_path
+        m_os.path.isdir.return_value = True
+        m_os.path.dirname.return_value = self.core_dump_path
+        m_os.path.exists.return_value = True
+        m_os.listdir.return_value = [self.core_dump_path]
+        self.the_function(None, self.m_remote)
+        assert self.m_remote.get_file.called == False
+
+    # Core is compressed and file is in the correct format
+    @patch('teuthology.task.internal.subprocess.Popen')
+    @patch('teuthology.task.internal.os')
+    def test_compressed_correct_format(self, m_os, m_subproc_popen):
+        m_subproc_popen.side_effect = [
+                self.compressed_correct,
+                self.uncompressed_correct
+            ]
+        m_os.path.join.return_value = self.core_dump_path
+        m_os.path.sep = self.core_dump_path
+        m_os.path.isdir.return_value = True
+        m_os.path.dirname.return_value = self.core_dump_path
+        m_os.path.exists.return_value = True
+        m_os.listdir.return_value = [self.core_dump_path]
+        self.the_function(None, self.m_remote)
+        assert self.m_remote.get_file.called
+
+    # Core is compressed and file is in the wrong format
+    @patch('teuthology.task.internal.subprocess.Popen')
+    @patch('teuthology.task.internal.os')
+    def test_compressed_incorrect_format(self, m_os, m_subproc_popen):
+        m_subproc_popen.side_effect = [
+                self.compressed_incorrect,
+                self.uncompressed_incorrect
+            ]
+        m_os.path.join.return_value = self.core_dump_path
+        m_os.path.sep = self.core_dump_path
+        m_os.path.isdir.return_value = True
+        m_os.path.dirname.return_value = self.core_dump_path
+        m_os.path.exists.return_value = True
+        m_os.listdir.return_value = [self.core_dump_path]
+        self.the_function(None, self.m_remote)
+        assert self.m_remote.get_file.called == False
+
+    def teardown(self):
+        os.remove(self.core_dump_path)

teuthology/task/tests/test_run.py

@@ -1,7 +1,7 @@
 import logging
 import pytest
 
-from StringIO import StringIO
+from io import StringIO
 
 from teuthology.exceptions import CommandFailedError
 
@@ -17,7 +17,7 @@ class TestRun(object):
         result = ""
         try:
             ctx.cluster.run(
-                args=["python", "-c", "assert False"],
+                args=["python3", "-c", "assert False"],
                 label="working as expected, nothing to see here"
             )
         except CommandFailedError as e:
@@ -28,13 +28,13 @@ class TestRun(object):
     def test_command_failed_no_label(self, ctx, config):
         with pytest.raises(CommandFailedError):
             ctx.cluster.run(
-                args=["python", "-c", "assert False"],
+                args=["python3", "-c", "assert False"],
             )
 
     def test_command_success(self, ctx, config):
         result = StringIO()
         ctx.cluster.run(
-            args=["python", "-c", "print('hi')"],
+            args=["python3", "-c", "print('hi')"],
             stdout=result
         )
         assert result.getvalue().strip() == "hi"

teuthology/timer.py

@@ -2,7 +2,7 @@ import logging
 import time
 import yaml
 
-from datetime import datetime
+import datetime
 
 log = logging.getLogger(__name__)
 
@@ -68,8 +68,8 @@ class Timer(object):
 
         :param time: Time in seconds; like from time.time()
         """
-        _datetime = datetime.utcfromtimestamp(time)
-        return datetime.strftime(
+        _datetime = datetime.datetime.fromtimestamp(time, datetime.timezone.utc)
+        return datetime.datetime.strftime(
             _datetime,
             self.datetime_format,
         )

teuthology/util/loggerfile.py

@@ -0,0 +1,19 @@
+import logging
+
+class LoggerFile(object):
+    """
+    A thin wrapper around a logging.Logger instance that provides a file-like
+    interface.
+
+    Used by Ansible.execute_playbook() when it calls pexpect.run()
+    """
+    def __init__(self, logger: logging.Logger, level: int):
+        self.logger = logger
+        self.level = level
+
+    def write(self, string):
+        self.logger.log(self.level, string.decode('utf-8', 'ignore'))
+
+    def flush(self):
+        pass
+

teuthology/util/scanner.py

@@ -0,0 +1,159 @@
+import logging
+import yaml
+from typing import Optional, Tuple
+from collections import defaultdict
+from lxml import etree
+
+log = logging.getLogger(__name__)
+
+
+class Scanner():
+    def __init__(self, remote=None) -> None:
+        self.summary_data = []
+        self.remote = remote
+
+    def _parse(self, file_content) -> Tuple[str, dict]:
+        """
+        This parses file_content and returns:
+        :returns: a message string 
+        :returns: data dictionary with additional info
+
+        Just an abstract method in Scanner class, 
+        to be defined in inherited classes. 
+        """
+        raise NotImplementedError
+    
+    def scan_file(self, path: str) -> Optional[str]:
+        if not path:
+            return None
+        try:
+            file = self.remote._sftp_open_file(path, 'r')
+            file_content = file.read()
+            txt, data = self._parse(file_content)
+            if data:
+                data["file_path"] = path
+                self.summary_data += [data]
+            file.close()
+            return txt
+        except Exception as exc:
+            log.error(str(exc))
+
+    def scan_all_files(self, path_regex: str) -> [str]:
+        """
+        Scans all files matching path_regex
+        and collect additional data in self.summary_data 
+
+        :param path_regex: Regex string to find all the files which have to be scanned. 
+                           Example: /path/to/dir/*.xml
+        """
+        (_, stdout, _) = self.remote.ssh.exec_command(f'ls -d {path_regex}', timeout=200)
+        
+        files = stdout.read().decode().split('\n')
+        
+        extracted_txts = []
+        for fpath in files:
+            txt = self.scan_file(fpath)
+            if txt:
+                extracted_txts += [txt]
+        return extracted_txts
+    
+    def write_summary(self, yaml_path: str) -> None:
+        """
+        Create yaml file locally 
+        with self.summary_data.
+        """
+        if self.summary_data and yaml_path:
+            with open(yaml_path, 'a') as f:
+                yaml.safe_dump(self.summary_data, f, default_flow_style=False)
+        else:
+            log.info("summary_data or yaml_file is empty!")
+
+
+class UnitTestScanner(Scanner):
+    def __init__(self, remote=None) -> None:
+        super().__init__(remote)
+
+    def _parse(self, file_content: str) -> Tuple[Optional[str], Optional[dict]]:
+        xml_tree = etree.fromstring(file_content)
+
+        failed_testcases = xml_tree.xpath('.//failure/.. | .//error/..')
+        if len(failed_testcases) == 0:
+            return None, None
+
+        exception_txt = ""
+        error_data = defaultdict(list)
+        for testcase in failed_testcases:
+            testcase_name = testcase.get("name", "test-name")
+            testcase_suitename = testcase.get("classname", "suite-name")
+            for child in testcase:
+                if child.tag in ['failure', 'error']:
+                    fault_kind = child.tag
+                    reason = child.get('message', 'No message found in xml output, check logs.')
+                    short_reason = (reason[:200].strip() + '...') if len(reason) > 200 else reason.strip()
+                    error_data[testcase_suitename] += [{
+                            "kind": fault_kind, 
+                            "testcase": testcase_name,
+                            "message": reason,
+                        }]
+                    if not exception_txt:
+                        exception_txt = f'{fault_kind.upper()}: Test `{testcase_name}` of `{testcase_suitename}`. Reason: {short_reason}.'
+        
+        return exception_txt, { "failed_testsuites": dict(error_data), "num_of_failures": len(failed_testcases) }
+
+    @property
+    def num_of_total_failures(self):
+        total_failed_testcases = 0
+        if self.summary_data:
+            for file_data in self.summary_data:
+                failed_tests = file_data.get("num_of_failures", 0)
+                total_failed_testcases += failed_tests
+        return total_failed_testcases
+
+    def scan_and_write(self, path_regex: str, summary_path: str) -> Optional[str]:
+        """
+        Scan all files matching 'path_regex'
+        and write summary in 'summary_path'.
+        """
+        try:
+            errors = self.scan_all_files(path_regex)
+            self.write_summary(summary_path)
+            if errors:
+                count = self.num_of_total_failures
+                return f"(total {count} failed) " + errors[0]
+        except Exception as scanner_exc:
+            log.error(str(scanner_exc))
+
+
+class ValgrindScanner(Scanner):
+    def __init__(self, remote=None) -> None:
+        super().__init__(remote)
+
+    def _parse(self, file_content: str) -> Tuple[Optional[str], Optional[dict]]:
+        xml_tree = etree.fromstring(file_content)
+        if xml_tree is None:
+            return None, None
+        
+        error_tree = xml_tree.find('error')
+        if error_tree is None:
+            return None, None
+        
+        error_data = {
+            "kind": error_tree.findtext("kind"),
+            "traceback": [],
+        }
+        for frame in error_tree.xpath("stack/frame"):
+            if len(error_data["traceback"]) >= 5:
+                break
+            curr_frame = {
+                "file": f"{frame.findtext('dir', '')}/{frame.findtext('file', '')}",
+                "line": frame.findtext("line", ''),
+                "function": frame.findtext("fn", ''),
+            }
+            error_data["traceback"].append(curr_frame)
+
+        traceback_functions = "\n".join(
+                frame.get("function", "N/A") 
+                for frame in error_data["traceback"][:3]
+            )
+        exception_text = f"valgrind error: {error_data['kind']}\n{traceback_functions}"
+        return exception_text, error_data

teuthology/util/sentry.py

@@ -0,0 +1,52 @@
+import logging
+import sentry_sdk
+
+from copy import deepcopy
+
+from teuthology.config import config as teuth_config
+from teuthology.misc import get_http_log_path
+
+log = logging.getLogger(__name__)
+
+
+def report_error(job_config, exception, task_name=None):
+    if not teuth_config.sentry_dsn:
+        return None
+    sentry_sdk.init(teuth_config.sentry_dsn)
+    job_config = deepcopy(job_config)
+
+    tags = {
+        'task': task_name,
+        'owner': job_config.get("owner"),
+    }
+    optional_tags = ('teuthology_branch', 'branch', 'suite',
+                     'machine_type', 'os_type', 'os_version')
+    for tag in optional_tags:
+        if tag in job_config:
+            tags[tag] = job_config[tag]
+
+    # Remove ssh keys from reported config
+    if 'targets' in job_config:
+        targets = job_config['targets']
+        for host in targets.keys():
+            targets[host] = '<redacted>'
+
+    job_id = job_config.get('job_id')
+    archive_path = job_config.get('archive_path')
+    extras = dict(config=job_config)
+    if job_id:
+        extras['logs'] = get_http_log_path(archive_path, job_id)
+
+    fingerprint = exception.fingerprint() if hasattr(exception, 'fingerprint') else None
+    exc_id = sentry_sdk.capture_exception(
+        error=exception,
+        tags=tags,
+        extras=extras,
+        fingerprint=fingerprint,
+    )
+    event_url = "{server}/?query={id}".format(
+        server=teuth_config.sentry_server.strip('/'), id=exc_id)
+    log.exception(" Sentry event: %s" % event_url)
+    return event_url
+
+