smart-tests-cli 2.0.0__py3-none-any.whl

smart_tests/test_runners/maven.py

@@ -0,0 +1,149 @@
+import glob
+import os
+from typing import Annotated, Dict, List
+
+import click
+
+import smart_tests.args4p.typer as typer
+from smart_tests.utils import glob as uglob
+from smart_tests.utils.java import junit5_nested_class_path_builder
+
+from ..commands.record.tests import RecordTests
+from ..commands.subset import Subset
+from . import smart_tests
+
+# Surefire has the default inclusion pattern
+# https://maven.apache.org/surefire/maven-surefire-plugin/test-mojo.html#includes
+# and the default exclusion pattern
+# https://maven.apache.org/surefire/maven-surefire-plugin/test-mojo.html#excludes
+# these variables emulates those effects.
+# TODO: inclusion/exclusion are user configurable patterns, so it should be user configurable
+# beyond that and to fully generalize this, there's internal discussion of
+# this at https://launchableinc.atlassian.net/l/c/TXDJnn09
+includes = [uglob.compile(x) for x in [
+    # HACK: we check extensions outside the glob. We seem to allow both source
+    # file enumeration and class file enumeration
+    '**/Test*.*',
+    '**/*Test.*',
+    '**/*Spec.*',
+    '**/*Tests.*',
+    '**/*TestCase.*'
+]]
+excludes = [uglob.compile(x) for x in [
+    '**/*$*'
+]]
+
+# Test if a given path name is a test that Surefire recognizes
+
+
+def is_file(f: str) -> bool:
+    if not (f.endswith('.java') or f.endswith(".scala") or f.endswith(".kt") or f.endswith(".class") or f.endswith(".groovy")):
+        return False
+    for p in excludes:
+        if p.fullmatch(f):
+            return False
+    for p in includes:
+        if p.fullmatch(f):
+            return True
+    return False
+
+
+@smart_tests.subset
+def subset(
+    client: Subset,
+    source_roots: Annotated[List[str] | None, typer.Argument(
+        multiple=True,
+        required=False,
+        help="Source root directories to scan for tests"
+    )] = None,
+    test_compile_created_file: Annotated[List[str] | None, typer.Option(
+        "--test-compile-created-file",
+        multiple=True,
+        help="Please run `mvn test-compile` command to create input file for this option"
+    )] = None,
+    is_scan_test_compile_lst: Annotated[bool, typer.Option(
+        "--scan-test-compile-lst",
+        help="Scan testCompile/default-testCompile/createdFiles.lst for *.lst files generated by `mvn compile` and "
+             "use them as test inputs."
+    )] = False,
+):
+
+    def file2class_test_path(f: str) -> List[Dict[str, str]]:
+        # remove extension
+        f, _ = os.path.splitext(f)
+
+        # directory -> package name conversion
+        cls_name = f.replace(os.path.sep, '.')
+        return [{"type": "class", "name": cls_name}]
+
+    def file2test(f: str) -> List | None:
+        if is_file(f):
+            return file2class_test_path(f)
+        else:
+            return None
+
+    # Handle None values
+    if test_compile_created_file is None:
+        test_compile_created_file = []
+    if source_roots is None:
+        source_roots = []
+
+    files_to_read = list(test_compile_created_file)
+    if is_scan_test_compile_lst:
+        if len(test_compile_created_file) > 0:
+            click.secho(
+                "Warning: --test-compile-created-file is overridden by --scan-test-compile-lst",
+                fg='yellow', err=True)
+
+        pattern = os.path.join('**', 'createdFiles.lst')
+        files_to_read = glob.glob(pattern, recursive=True)
+
+        if not files_to_read:
+            click.secho(
+                "Warning: No .lst files. Please run after executing `mvn test-compile`",
+                fg='yellow',
+                err=True)
+            return
+
+    if files_to_read:
+        for file in files_to_read:
+            with open(file, 'r') as f:
+                lines = f.readlines()
+                if len(lines) == 0:
+                    click.secho(
+                        f"Warning: --test-compile-created-file {file} is empty",
+                        fg='yellow',
+                        err=True)
+
+                for line in lines:
+                    # trim trailing newline
+                    line = line.strip()
+
+                    path = file2test(line)
+                    if path:
+                        client.test_paths.append(path)
+    else:
+        for root in source_roots:
+            client.scan(root, '**/*', file2test)
+
+    client.run()
+
+
+# TestNG produces surefire-reports/testng-results.xml in TestNG's native format.
+# Surefire produces TEST-*.xml in JUnit format (see Surefire's StatelessXmlReporter.getReportFile)
+# In addition, TestNG also produces surefire-reports/junitreports/TEST-*.xml
+# (see TestNG's JUnitReportReporter.getFileName)
+# And there are more test reports in play.
+#
+# So to collectly find tests without duplications, we need to find surefire-reports/TEST-*.xml
+# not surefire-reports/**/TEST-*.xml nor surefire-reports/*.xml
+@smart_tests.record.tests
+def record_tests(
+    client: RecordTests,
+    reports: Annotated[List[str], typer.Argument(
+        multiple=True,
+        help="Test report files to process"
+    )],
+):
+    client.path_builder = junit5_nested_class_path_builder(client.path_builder)
+    smart_tests.CommonRecordTestImpls.load_report_files(client=client, source_roots=reports)

smart_tests/test_runners/minitest.py

@@ -0,0 +1,40 @@
+from typing import Annotated, List
+
+from junitparser import TestCase, TestSuite  # type: ignore
+
+import smart_tests.args4p.typer as typer
+
+from ..commands.record.tests import RecordTests
+from ..testpath import TestPath
+from . import smart_tests
+
+subset = smart_tests.CommonSubsetImpls(__name__).scan_files('*_test.rb')
+
+TEST_PATH_ORDER = {"file": 1, "class": 2, "testcase": 3}
+
+
+@smart_tests.record.tests
+def record_tests(
+    client: RecordTests,
+    reports: Annotated[List[str], typer.Argument(
+        multiple=True,
+        help="Test report files to process"
+    )],
+):
+    default_path_builder = client.path_builder
+
+    def path_builder(case: TestCase, suite: TestSuite, report_file: str) -> TestPath:
+        test_path = default_path_builder(case, suite, report_file)
+        if not any(item.get("type") == "class" for item in test_path):
+            # mintiest-ci sets a class name in name attribute in testsuite tag.
+            # https://github.com/CircleCI-Public/minitest-ci/blob/v3.4.0/lib/minitest/ci_plugin.rb#L86-L87
+            # https://github.com/CircleCI-Public/minitest-ci/blob/v3.4.0/lib/minitest/ci_plugin.rb#L132
+            classname = suite._elem.attrib.get("name")
+            if classname:
+                test_path.append({"type": "class", "name": classname})
+                test_path = sorted(test_path, key=lambda x: TEST_PATH_ORDER[x["type"]])
+        return test_path
+
+    client.path_builder = path_builder
+
+    smart_tests.CommonRecordTestImpls.load_report_files(client=client, source_roots=reports)

smart_tests/test_runners/nunit.py

@@ -0,0 +1,190 @@
+from typing import Annotated, Callable, Dict, List
+from xml.etree import ElementTree as ET
+
+import smart_tests.args4p.typer as typer
+from smart_tests.commands.record.case_event import CaseEvent
+from smart_tests.testpath import TestPath, parse_test_path, unparse_test_path
+
+from ..commands.record.tests import RecordTests
+from ..commands.subset import Subset
+from . import smart_tests
+
+# common code between 'subset' & 'record tests' to build up test path from
+# nested <test-suite>s
+
+"""
+Nested class name handling in .NET
+---------------------------------
+
+Nested class 'Zot' in the following example gets the full name "Foo.Bar+Zot":
+
+    namespace Foo {
+        class Bar {
+            class Zot {
+    }}}
+
+This is incontrast to how you refer to this class from the source code. For example,
+"new Foo.Bar.Zot()"
+
+The subset command expects the list of tests to be passed to "nunit --testlist" option,
+and this option expects test names to be in "Foo.Bar+Zot" format.
+
+"""
+
+
+def build_path(e: ET.Element, parent: ET.Element):
+    pp: TestPath = []
+    parent_start_time = parent.attrib.get('start-time')
+    if parent_start_time is not None and e.attrib.get('start-time') is None:
+        # the 'start-time' attribute is normally on <test-case> but apparently not always,
+        # so we try to use the nearest ancestor as an approximate
+        e.set('start-time', parent_start_time)
+    parent_path = parent.attrib.get('path')
+    if parent_path is not None:
+        pp = parse_test_path(parent_path)
+    if e.tag == "test-suite":
+        # <test-suite>s form a nested tree structure so capture those in path
+        pp = pp + [{'type': e.attrib['type'], 'name': e.attrib['name']}]
+    if e.tag == "test-case":
+        # work around a bug in NUnitXML.Logger.
+        # see nunit-reporter-bug-with-nested-type.xml test case
+        methodname = e.attrib['methodname']
+        bra = methodname.find("(")
+        idx = methodname.rfind(".", 0, bra)
+        if idx >= 0:
+            # when things are going well, method name cannot contain '.' since it's not a valid character in a symbol.
+            # but when NUnitXML.Logger messes up, it ends up putting the class name and the method name, like
+            # <test-case name="TheTest" fullname="Launchable.NUnit.Test.Outer+Inner.TheTest"
+            #   methodname="Outer+Inner.TheTest" classname="Test"
+
+            pp = pp[0:-1] + [
+                # NUnitXML.Logger mistreats the last portion of the namespace as a test fixture when
+                # it really should be test suite. So we patch that up too. This is going beyond what's minimally required
+                # to make subset work, because type information won't impact how the test path is printed, but
+                # when NUnitXML.Logger eventually fixes this bug, we don't want that to produce different test paths.
+                {'type': 'TestSuite', 'name': pp[-1]['name']},
+                # Here, we need to insert the missing TestFixture=Outer+Inner.
+                # I chose TestFixture because that's what nunit console runner (which we believe is handling it correctly)
+                # chooses as its type.
+                {'type': 'TestFixture', 'name': methodname[0:idx]}
+            ]
+
+        pp = pp + [{'type': 'TestCase', 'name': e.attrib['name']}]
+
+    if len(pp) > 0:
+        def split_filepath(path: str) -> List[str]:
+            # Supports Linux and Windows
+            if '/' in path:
+                return path.split('/')
+            else:
+                return path.split('\\')
+
+        # "Assembly" type contains full path at a customer's environment
+        # remove file path prefix in Assembly
+        e.set('path', unparse_test_path([
+            {**path, 'name': split_filepath(path['name'])[-1]}
+            if path['type'] == 'Assembly'
+            else path
+            for path in pp
+        ]))
+
+
+def nunit_parse_func(report: str):
+    events = []
+
+    # parse <test-case> element into CaseEvent
+    def on_element(e: ET.Element, parent: ET.Element):
+        build_path(e, parent)
+        if e.tag == "test-case":
+            result = e.attrib.get('result')
+            status = CaseEvent.TEST_FAILED
+            stderr: List[str] = []
+            if result == 'Passed':
+                status = CaseEvent.TEST_PASSED
+            elif result == 'Skipped':
+                status = CaseEvent.TEST_SKIPPED
+            else:
+                failure = e.find('failure')
+                if failure is not None:
+                    message = failure.find('message')
+                    if message is not None and message.text is not None:
+                        stderr.append(message.text)
+                    stack_trace = failure.find('stack-trace')
+                    if stack_trace is not None and stack_trace.text is not None:
+                        stderr.append(stack_trace.text)
+
+            events.append(CaseEvent.create(
+                test_path=_replace_fixture_to_suite(e.attrib['path']),  # type: ignore
+                duration_secs=float(e.attrib['duration']),
+                status=status,
+                timestamp=str(e.attrib.get('start-time')),
+                stderr='\n'.join(stderr)))  # timestamp is already iso-8601 formatted
+
+    _parse_dfs_element(report=report, on_element=on_element)
+    # return the obtained events as a generator
+    return (x for x in events)
+
+
+@smart_tests.subset
+def subset(
+    client: Subset,
+    report_xmls: Annotated[List[str], typer.Argument(
+        multiple=True,
+        required=False,
+        help="Test report XML files to process"
+    )] = [],
+):
+    """
+    Parse an XML file produced from NUnit --explore option to list up all the viable test cases
+    """
+
+    def on_element(e: ET.Element, parent: ET.Element):
+        build_path(e, parent)
+        if e.tag == "test-case":
+            client.test_path(_replace_fixture_to_suite(e.attrib['path']))
+
+    for report_xml in report_xmls:
+        _parse_dfs_element(report=report_xml, on_element=on_element)
+
+    # join all the names except when the type is ParameterizedMethod, because in that case test cases have
+    # the name of the test method in it and ends up creating duplicates
+    client.formatter = lambda x: '.'.join([c['name'] for c in x if c['type'] not in ['ParameterizedMethod', 'Assembly']])
+    client.run()
+
+
+@smart_tests.record.tests
+def record_tests(
+    client: RecordTests,
+    report_xml: Annotated[List[str], typer.Argument(
+        multiple=True,
+        help="Test report XML files to process"
+    )],
+):
+    client.parse_func = nunit_parse_func
+    smart_tests.CommonRecordTestImpls.load_report_files(client=client, source_roots=report_xml)
+
+
+"""
+    Nunit produces different XML structure report between without --explore option and without it.
+    So we replace TestFixture to TestSuite to avid this difference problem.
+"""
+
+
+def _replace_fixture_to_suite(test_path_str: str) -> List[Dict[str, str]]:
+    paths = parse_test_path(test_path_str)
+    for path in paths:
+        if path["type"] == "TestFixture":
+            path["type"] = "TestSuite"
+
+    return paths
+
+
+def _parse_dfs_element(report: str, on_element: Callable[[ET.Element, ET.Element], None]):
+    tree = ET.parse(source=report)
+    root = tree.getroot()
+    stack: List[ET.Element] = [root]
+    while len(stack) > 0:
+        element = stack.pop()
+        for test_suite in element.findall('test-suite') + element.findall('test-case'):
+            on_element(test_suite, element)
+            stack.append(test_suite)

smart_tests/test_runners/playwright.py

@@ -0,0 +1,252 @@
+#
+# The the test runner to support playwright junit and JSON report format.
+# https://playwright.dev/
+#
+import json
+from typing import Annotated, Dict, List
+
+import click
+from junitparser import TestCase, TestSuite  # type: ignore
+
+import smart_tests.args4p.typer as typer
+
+from ..args4p.exceptions import BadCmdLineException
+from ..commands.record.case_event import CaseEvent, CaseEventGenerator
+from ..commands.record.tests import RecordTests
+from ..commands.subset import Subset
+from ..testpath import TestPath
+from . import smart_tests
+
+TEST_CASE_DELIMITER = " › "
+
+
+@smart_tests.record.tests
+def record_tests(
+    client: RecordTests,
+    reports: Annotated[List[str], typer.Argument(
+        multiple=True,
+        help="Test report files to process"
+    )],
+    json_format: Annotated[bool, typer.Option(
+        "--json",
+        help="use JSON report format"
+    )] = False,
+):
+    def path_builder(case: TestCase, suite: TestSuite,
+                     report_file: str) -> TestPath:
+        """
+        The playwright junit report sets a file name to the name attribute in a testsuite element and the classname attribute in a testcase element. # noqa: E501
+        This playwright plugin uses a testsuite attribute value.
+        e.g.)
+        <testsuite name="tests/demo-todo-app.spec.ts" ...>
+            <testcase name="New Todo › should allow me to add todo items" classname="tests/demo-todo-app.spec.ts"></testcase>
+            <testcase name="New Todo › should clear text input field when an item is added" classname="tests/demo-todo-app.spec.ts"></testcase>
+        </testsuite>
+        """
+        filepath = suite.name
+        if not filepath:
+            raise BadCmdLineException("No file name found in %s" % report_file)
+
+        test_path = [client.make_file_path_component(filepath)]
+        if case.name:
+            test_path.append({"type": "testcase", "name": case.name})
+        return test_path
+
+    if json_format:
+        client.parse_func = JSONReportParser(client).parse_func
+    else:
+        client.path_builder = path_builder
+
+    for r in reports:
+        client.report(r)
+
+    client.run()
+
+
+@smart_tests.subset
+def subset(client: Subset):
+    # read lines as test file names
+    for t in client.stdin():
+        client.test_path(t.rstrip("\n"))
+
+    client.run()
+
+
+class JSONReportParser:
+    """
+    example of JSON reporter format:
+    {
+        "suites": [
+            {
+                "title": "tests/demo-todo-app.spec.ts",
+                "file": "tests/demo-todo-app.spec.ts",
+                "column": 0,
+                "line": 0,
+                "specs": [],
+                "suites": [
+                    {
+                        "title": "New Todo",
+                        "file": "tests/demo-todo-app.spec.ts",
+                        "line": 13,
+                        "column": 6,
+                        "specs": [
+                            {
+                                "title": "should allow me to add todo items",
+                                "ok": true,
+                                "tags": [],
+                                "tests": [
+                                    {
+                                        "timeout": 30000,
+                                        "annotations": [],
+                                        "expectedStatus": "passed",
+                                        "projectId": "chromium",
+                                        "projectName": "chromium",
+                                        "results": [
+                                            {
+                                                "workerIndex": 0,
+                                                "status": "passed",
+                                                "duration": 1254,
+                                                "errors": [],
+                                                "stdout": [],
+                                                "stderr": [],
+                                                "retry": 0,
+                                                "startTime": "2024-04-08T07:42:17.455Z",
+                                                "attachments": []
+                                            }
+                                        ],
+                                        "status": "expected"
+                                    },
+                                    {
+                                        "timeout": 30000,
+                                        "annotations": [],
+                                        "expectedStatus": "passed",
+                                        "projectId": "firefox",
+                                        "projectName": "firefox",
+                                        "results": [
+                                            {
+                                                "workerIndex": 5,
+                                                "status": "passed",
+                                                "duration": 1320,
+                                                "errors": [],
+                                                "stdout": [],
+                                                "stderr": [],
+                                                "retry": 0,
+                                                "startTime": "2024-04-08T07:42:21.122Z",
+                                                "attachments": []
+                                            }
+                                        ],
+                                        "status": "expected"
+                                    },
+                                    {
+                                        "timeout": 30000,
+                                        "annotations": [],
+                                        "expectedStatus": "passed",
+                                        "projectId": "webkit",
+                                        "projectName": "webkit",
+                                        "results": [
+                                            {
+                                                "workerIndex": 10,
+                                                "status": "passed",
+                                                "duration": 805,
+                                                "errors": [],
+                                                "stdout": [],
+                                                "stderr": [],
+                                                "retry": 0,
+                                                "startTime": "2024-04-08T07:42:26.319Z",
+                                                "attachments": []
+                                            }
+                                        ],
+                                        "status": "expected"
+                                    }
+                                ],
+                                "id": "f8b37aaddd8ecd14ecd4-3367671d4d7af1046962",
+                                "file": "tests/demo-todo-app.spec.ts",
+                                "line": 14,
+                                "column": 7
+                            }
+                        ]
+                    }
+                ]
+            }
+        ]
+    }
+    """
+
+    def __init__(self, client):
+        self.client = client
+
+    def parse_func(self, report_file: str) -> CaseEventGenerator:
+        data: Dict[str, Dict]
+        with open(report_file, 'r') as json_file:
+            try:
+                data = json.load(json_file)
+            except Exception as e:
+                raise Exception(f"Can't read JSON format report file {report_file}. Make sure to confirm report file.") from e
+
+        suites: List[Dict[str, Dict]] = list(data.get("suites", []))
+        if len(suites) == 0:
+            click.echo(f"Can't find test results from {report_file}. Make sure to confirm report file.", err=True)
+
+        for s in suites:
+            # The title of the root suite object contains the file name.
+            test_file = str(s.get("title", ""))
+
+            for event in self._parse_suites(test_file, s, []):
+                yield event
+
+    def _parse_suites(self, test_file: str, suite: Dict[str, Dict], test_case_names: List[str] = []) -> List:
+        events = []
+
+        # In some cases, suites are nested.
+        for s in suite.get("suites", []):
+            for e in self._parse_suites(test_file, s, test_case_names + [s.get("title")]):
+                events.append(e)
+
+        for spec in suite.get("specs", []):
+            spec_name = spec.get("title", "")
+            line_no = spec.get("line", None)
+            for test in spec.get("tests", []):
+                for result in test.get("results", []):
+                    test_path: TestPath = [
+                        self.client.make_file_path_component(test_file),
+                        {"type": "testcase", "name": TEST_CASE_DELIMITER.join(test_case_names + [spec_name])}
+                    ]
+
+                    duration_msec = result.get("duration", 0)
+                    status = self._case_event_status_from_str(result.get("status", ""))
+                    stdout = self._parse_stdout(result.get("stdout", []))
+                    stderr = self._parse_stderr(result.get("errors", []))
+
+                    events.append(CaseEvent.create(
+                        test_path=test_path,
+                        duration_secs=duration_msec / 1000,  # convert msec to sec,
+                        status=status,
+                        stdout=stdout,
+                        stderr=stderr,
+                        data={"lineNumber": line_no} if line_no else {}
+                    ))
+
+        return events
+
+    def _case_event_status_from_str(self, status_str: str) -> int:
+        # see: https://playwright.dev/docs/api/class-testresult#test-result-status
+        if status_str == "passed":
+            return CaseEvent.TEST_PASSED
+        elif status_str == "failed" or status_str == "timedOut" or status_str == "interrupted":
+            return CaseEvent.TEST_FAILED
+        elif status_str == "skipped":
+            return CaseEvent.TEST_SKIPPED
+        else:
+            return CaseEvent.TEST_PASSED
+
+    def _parse_stdout(self, stdout: List[Dict[str, Dict]]) -> str:
+        if len(stdout) == 0:
+            return ""
+
+        return "\n".join([str(out.get("text", "")) for out in stdout])
+
+    def _parse_stderr(self, stderr: List[Dict[str, Dict]]) -> str:
+        if len(stderr) == 0:
+            return ""
+
+        return "\n".join([str(err.get("message", "")) for err in stderr])