smart-tests-cli 2.0.0__py3-none-any.whl

smart_tests/test_runners/prove.py

@@ -0,0 +1,74 @@
+import re
+from typing import Annotated, List
+
+from junitparser import TestCase, TestSuite  # type: ignore
+
+import smart_tests.args4p.typer as typer
+
+from ..args4p.exceptions import BadCmdLineException
+from ..commands.record.tests import RecordTests
+from ..commands.subset import Subset
+from ..testpath import TestPath
+from . import smart_tests
+
+TEARDOWN = "(teardown)"
+
+
+def remove_leading_number_and_dash(input_string: str) -> str:
+    result = re.sub(r'^\d+ - ', '', input_string)
+    return result
+
+
+@smart_tests.subset
+def subset(client: Subset):
+    # read lines as test file names
+    for t in client.stdin():
+        client.test_path(t.rstrip("\n"))
+
+    client.run()
+
+
+@smart_tests.record.tests
+def record_tests(
+    client: RecordTests,
+    reports: Annotated[List[str], typer.Argument(
+        multiple=True,
+        help="Test report files to process"
+    )],
+):
+    def path_builder(case: TestCase, suite: TestSuite,
+                     report_file: str) -> TestPath:
+        def find_filename():
+            # find file path from test suite attribute first.
+            filepath = suite._elem.attrib.get("name")
+            if filepath:
+                return filepath
+            # find file path from test case attribute.
+            filepath = case._elem.attrib.get("classname")
+            if filepath:
+                return filepath
+            return None  # failing to find a test name
+
+        filepath = find_filename()
+        if not filepath:
+            raise BadCmdLineException(
+                "No file name found in %s."
+                "Perl prove profile is made to take Junit report produced by "
+                "TAP::Formatter::JUnit (https://github.com/bleargh45/TAP-Formatter-JUnit), "
+                "which exports the report in stdout."
+                "Please export the stdout result to XML report file."
+                "If you are not using TAP::Formatter::JUnit, "
+                "please change your reporting to TAP::Formatter::JUnit." % report_file)
+
+        # default test path in `subset` expects to have this file name
+        test_path = [client.make_file_path_component(filepath)]
+        if case.name:
+            case_name = remove_leading_number_and_dash(input_string=case.name)
+            test_path.append({"type": "testcase", "name": case_name})
+        return test_path
+
+    client.path_builder = path_builder
+
+    for r in reports:
+        client.report(r)
+    client.run()

smart_tests/test_runners/pytest.py

@@ -0,0 +1,358 @@
+import glob
+import json
+import os
+import pathlib
+import subprocess
+from typing import Annotated, Generator, Iterable, List
+
+import click
+from junitparser import Properties, TestCase  # type: ignore
+
+import smart_tests.args4p.typer as typer
+from smart_tests.commands.record.case_event import CaseEvent, CaseEventType, MetadataTestCase
+from smart_tests.testpath import TestPath
+
+from ..args4p.exceptions import BadCmdLineException
+from ..commands.record.tests import RecordTests
+from ..commands.subset import Subset
+from . import smart_tests
+
+
+# Please specify junit_family=legacy for pytest report format. if using pytest version 6 or higher.
+# - pytest has changed its default test report format from xunit1 to xunit2 since version 6.
+#     - https://docs.pytest.org/en/latest/deprecations.html#junit-family-default-value-change-to-xunit2
+# - The xunit2 format no longer includes file names.
+#     - It is possible to output in xunit1 format by specifying junit_family=legacy.
+#     - The xunit1 format includes the file name.
+# The format of pytest changes depending on the existence of the class. They are also incompatible with
+# the junit format.
+# Therefore, it converts to junit format at the timing of sending the subset, and converts the returned
+# value to pytest format.
+# for Example
+# $ pytest --collect-only -q
+# > tests/test_mod.py::TestClass::test__can_print_aaa
+# > tests/fooo/func4_test.py::test_func6
+# >
+# > 2 tests collected in 0.02s
+# result.xml(junit)
+# <testcase classname="tests.fooo.func4_test" name="test_func6" file="tests/fooo/func4_test.py"
+# line="0" time="0.000" />
+# <testcase classname="tests.test_mod.TestClass" name="test__can_print_aaa" file="tests/test_mod.py"
+# line="3" time="0.001" />
+#
+@smart_tests.subset
+def subset(
+    client: Subset,
+    source_roots: Annotated[List[str] | None, typer.Argument(
+        help="Source root directories for pytest test collection",
+        multiple=True,
+        required=False,
+    )] = None,
+):
+    def _add_testpaths(lines: Iterable[str]):
+        for line in lines:
+            line = line.rstrip()
+            # When an empty line comes, it's done.
+            if not line:
+                break
+
+            test_path = _parse_pytest_nodeid(line)
+            client.test_path(test_path)
+
+    if not source_roots:
+        _add_testpaths(client.stdin())
+    else:
+        command = ["pytest", "--collect-only", "-q"]
+        command.extend(source_roots)
+        try:
+            result = subprocess.run(command, stdout=subprocess.PIPE, universal_newlines=True)
+            _add_testpaths(result.stdout.split(os.linesep))
+        except FileNotFoundError:
+            raise BadCmdLineException("pytest command not found. Please check the path.")
+
+    client.formatter = _pytest_formatter
+    client.run()
+
+
+def _parse_pytest_nodeid(nodeid: str) -> TestPath:
+    data = nodeid.split("::")
+    file = data[0]
+    class_name = _path_to_class_name(file)
+    normalized_file = os.path.normpath(file)
+
+    # file name only
+    if len(data) == 1:
+        return [
+            {"type": "file", "name": normalized_file},
+            {"type": "class", "name": class_name},
+        ]
+    # file + testcase, or file + class + testcase
+    else:
+        testcase = data[-1]
+        if len(data) == 3:
+            class_name += "." + data[1]
+
+        return [
+            {"type": "file", "name": normalized_file},
+            {"type": "class", "name": class_name},
+            {"type": "testcase", "name": testcase},
+        ]
+
+
+def _path_to_class_name(path):
+    '''
+    tests/fooo/func4_test.py -> tests.fooo.func4_test
+    '''
+    return os.path.splitext(os.path.normpath(path))[0].replace(os.sep, ".")
+
+
+def _pytest_formatter(test_path):
+    for path in test_path:
+        t = path.get('type', '')
+        n = path.get('name', '')
+        if t == 'class':
+            cls_name = n
+        elif t == 'testcase':
+            case = n
+        elif t == 'file':
+            file = n
+    # If there is no class, junitformat use package name, but pytest will be omitted
+    # pytest -> tests/fooo/func4_test.py::test_func6
+    # junitformat -> <testcase classname="tests.fooo.func4_test"
+    # name="test_func6" file="tests/fooo/func4_test.py" line="0" time="0.000"
+    # />
+
+    if cls_name == _path_to_class_name(file):
+        return f"{file}::{case}"
+
+    else:
+        # junitformat's class name includes package, but pytest does not
+        # pytest -> tests/test_mod.py::TestClass::test__can_print_aaa
+        # junitformat -> <testcase classname="tests.test_mod.TestClass"
+        # name="test__can_print_aaa" file="tests/test_mod.py" line="3"
+        # time="0.001" />
+        if cls_name:
+            return f"{file}::{cls_name.split('.')[-1]}::{case}"
+        else:
+            return f"{file}::{case}"
+
+
+@smart_tests.record.tests
+def record_tests(
+    client: RecordTests,
+    source_roots: Annotated[List[str], typer.Argument(
+        multiple=True,
+        help="Source directories containing test report files"
+    )],
+    json_report: Annotated[bool, typer.Option(
+        "--json",
+        help="use JSON report files produced by pytest-dev/pytest-reportlog"
+    )] = False,
+):
+
+    def data_builder(case: TestCase):
+        props = case.child(Properties)
+        result = {}
+        if props is not None:
+            """
+                Here is an example of an XML file with markers.
+                ```
+                    <properties>
+                        <property name="name" value="parametrize" />
+                        <property name="args" value="('y', [2, 3])" />
+                        <property name="kwargs" value="{}" />
+                        <property name="name" value="parametrize" />
+                        <property name="args" value="('x', [0, 1])" />
+                        <property name="kwargs" value="{}" />
+                    </properties>
+                ```
+            """
+            markers = [{"name": prop.name, "value": prop.value} for prop in props]
+            result["markers"] = markers if markers else []
+
+        metadata = MetadataTestCase.fromelem(case)
+        if metadata and metadata.line is not None:
+            # Please note that line numbers start from 0.
+            # https://github.com/pytest-dev/pytest/blob/8.1.1/src/_pytest/_code/source.py#L93
+            result["lineNumber"] = metadata.line + 1
+        return result
+
+    ext = "json" if json_report else "xml"
+    for root in source_roots:
+        match = False
+        for t in glob.iglob(root, recursive=True):
+            match = True
+            if os.path.isdir(t):
+                client.scan(t, f"*.{ext}")
+            else:
+                client.report(t)
+
+        if not match:
+            click.echo(f"No matches found: {root}", err=True)
+            return
+
+    if json_report:
+        client.parse_func = PytestJSONReportParser(client).parse_func
+
+    client.metadata_builder = data_builder
+
+    client.run()
+
+
+"""
+If you want to use --json option, please install pytest-dev/pytest-reportlog.
+(https://github.com/pytest-dev/pytest-reportlog)
+
+Usage
+
+```
+$ pip install -U pytest-reportlog
+$ pytest --report-log report.json
+$ smart-tests record tests --json report.json
+```
+"""
+
+
+class PytestJSONReportParser:
+    def __init__(self, client):
+        self.client = client
+
+    def parse_func(
+            self, report_file: str) -> Generator[CaseEventType, None, None]:
+        with open(report_file, 'r') as json_file:
+            for line in json_file:
+                try:
+                    data = json.loads(line)
+                except Exception as e:
+                    raise Exception(f"Can't read JSON format report file {report_file}. Make sure to confirm report file.") from e
+
+                nodeid = data.get("nodeid", "")
+                if nodeid == "":
+                    continue
+
+                when = data.get("when", "")
+                outcome = data.get("outcome", "")
+
+                if not (when == "call" or (when == "setup" and outcome == "skipped")):
+                    continue
+
+                status = CaseEvent.TEST_FAILED
+                if outcome == "passed":
+                    status = CaseEvent.TEST_PASSED
+                elif outcome == "skipped":
+                    status = CaseEvent.TEST_SKIPPED
+
+                """example json
+                  "longrepr": {
+                    "reprcrash": {
+                        "lineno": 6,
+                        "message": "assert 1 == False",
+                        "path": "/Users/yabuki-ryosuke/src/github.com/launchableinc/cli/tests/data/pytest/tests/test_funcs1.py"
+                        },
+                    "reprtraceback": {
+                    "extraline": null,
+                    "reprentries": [
+                        {
+                        "data": {
+                            "lines": [
+                            "    def test_func2():",
+                            ">       assert 1 == False",
+                            "E       assert 1 == False"
+                            ],
+                            "reprfileloc": {
+                            "lineno": 6,
+                            "message": "AssertionError",
+                            "path": "tests/test_funcs1.py"
+                            },
+                            "reprfuncargs": {
+                            "args": []
+                            },
+                            "reprlocals": null,
+                            "style": "long"
+                        },
+                        "type": "ReprEntry"
+                        }
+                    ],
+                    "style": "long"
+                    },
+                    "sections": []
+                  }
+                """
+                stdout = ""
+                stderr = ""
+                longrepr = data.get("longrepr", None)
+                if longrepr:
+                    # https://github.com/pytest-dev/pytest/blob/1d7d63555e431d4562bcacbdc97038b0613d20ba/src/_pytest/reports.py#L60
+                    if isinstance(longrepr, dict):
+                        # https://github.com/pytest-dev/pytest/blob/1d7d63555e431d4562bcacbdc97038b0613d20ba/src/_pytest/reports.py#L361
+                        message = None
+                        reprcrash = longrepr.get("reprcrash", None)
+                        if reprcrash:
+                            message = reprcrash.get("message", None)
+
+                        text = None
+                        reprtraceback = longrepr.get("reprtraceback", None)
+                        if reprtraceback:
+                            reprentries = reprtraceback.get("reprentries", None)
+                            if reprentries:
+                                for r in reprentries:
+                                    d = r.get("data", None)
+                                    if d:
+                                        text = "\n".join(d.get("lines", []))
+
+                        if message and text:
+                            stderr = message + "\n" + text
+                        elif message:
+                            stderr = stderr + message
+                        elif text:
+                            stderr = stderr + text
+                    elif isinstance(longrepr, list):
+                        # [path, lineno, messge]
+                        # https://github.com/pytest-dev/pytest/blob/1d7d63555e431d4562bcacbdc97038b0613d20ba/src/_pytest/reports.py#L371
+                        if len(longrepr) == 3:
+                            stderr = longrepr[2]
+
+                    elif isinstance(longrepr, str):
+                        # When longrepr is a string, it is the same as the stderr.
+                        # https://github.com/pytest-dev/pytest/blob/1d7d63555e431d4562bcacbdc97038b0613d20ba/src/_pytest/reports.py#L377
+                        # https://github.com/pytest-dev/pytest/blob/1d7d63555e431d4562bcacbdc97038b0613d20ba/src/_pytest/nodes.py#L470
+                        stderr = longrepr
+
+                """example json
+                "user_properties": [
+                    ["name", "dependency"],
+                    ["args", []],
+                    ["kwargs", { "name": "c", "depends": ["a"] }],
+                    ["name", "order"],
+                    ["args", [2]],
+                    ["kwargs", {}]
+                ]
+                """
+                props = data.get('user_properties')
+                if isinstance(props, list):
+                    markers = []
+                    for prop in props:
+                        if isinstance(prop, list) and len(prop) == 2:
+                            # prop is like ["name", "value"]
+                            # prop[0] is name, prop[1] is value
+                            if isinstance(prop[1], str):
+                                markers.append({"name": prop[0], "value": prop[1]})
+                            else:
+                                markers.append({"name": prop[0], "value": json.dumps(prop[1])})
+                    if len(props) > 0:
+                        props = {'markers': markers}
+                    else:
+                        props = None
+
+                test_path = _parse_pytest_nodeid(nodeid)
+                for path in test_path:
+                    if path.get("type") == "file":
+                        path["name"] = pathlib.Path(path["name"]).as_posix()
+
+                yield CaseEvent.create(
+                    test_path=test_path,
+                    duration_secs=data.get("duration", 0),
+                    status=status,
+                    stdout=stdout,
+                    stderr=stderr,
+                    data=props)

smart_tests/test_runners/raw.py

@@ -0,0 +1,238 @@
+import datetime
+import json
+import sys
+from typing import Annotated, Generator, List
+
+import click
+import dateutil.parser
+
+import smart_tests.args4p.typer as typer
+
+from ..args4p.exceptions import BadCmdLineException
+from ..commands.record.case_event import CaseEvent, CaseEventType
+from ..commands.record.tests import RecordTests
+from ..commands.subset import Subset
+from ..testpath import TestPath, parse_test_path, unparse_test_path
+from . import smart_tests
+
+
+@smart_tests.subset
+def subset(
+    client: Subset,
+    test_path_file: Annotated[str | None, typer.Argument(
+        required=False,
+        help="File containing test paths, one per line"
+    )] = None,
+):
+    """Subset tests
+
+    TEST_PATH_FILE is a file that contains test paths (e.g.
+    "file=a.py#class=classA") one per line. Lines start with a hash ('#') are
+    considered as a comment and ignored.
+    """
+
+    if not client.is_get_tests_from_previous_sessions and test_path_file is None:
+        raise BadCmdLineException("Missing argument 'TEST_PATH_FILE'.")
+
+    if client.is_output_exclusion_rules:
+        raise BadCmdLineException(
+            "Don't need to use `--output-exclusion-rules` option. Please use `--rest` option and use it for exclusion"
+        )
+
+    if not client.is_get_tests_from_previous_sessions:
+        assert test_path_file is not None  # Guaranteed by earlier check
+        with open(test_path_file, 'r') as f:
+            tps = [s.strip() for s in f.readlines()]
+        for tp_str in tps:
+            if not tp_str or tp_str.startswith('#'):
+                continue
+            try:
+                tp = parse_test_path(tp_str)
+            except ValueError as e:
+                sys.exit(e.args[0])
+            client.test_path(tp)
+
+    client.formatter = unparse_test_path
+    client.separator = '\n'
+    client.run()
+
+
+@smart_tests.record.tests
+def record_tests(
+    client: RecordTests,
+    test_result_files: Annotated[List[str], typer.Argument(
+        multiple=True,
+        help="Test result files (JSON or JUnit XML)"
+    )],
+):
+    """Record test results
+
+    TEST_RESULT_FILE is a file that contains a JSON document or JUnit XML file
+    that describes the test results.
+
+    ## Example JSON document
+
+    {
+      "testCases": [
+         {
+           "testPath": "file=a.py#class=classA",
+           "duration": 42,
+           "status": "TEST_PASSED",
+           "stdout": "This is stdout",
+           "stderr": "This is stderr",
+           "createdAt": "2021-10-05T12:34:00"
+         }
+      ]
+    }
+
+    ## JSON schema
+
+    {
+      "$id": "https://launchableinc.com/schema/RecordTestInput",
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "title": "RecordTestInput",
+      "description": "The input to record test",
+      "type": "object",
+      "properties": {
+        "testCases": {
+          "description": "Result of test cases",
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "testPath": {
+                "description": "TestPath for the test",
+                "type": "string"
+              },
+              "testPathComponents": {
+                "description": "TestPath for the test",
+                "type": "array",
+                "items": {
+                  "type": "object"
+                }
+              },
+              "duration": {
+                "description": "Time taken to finish the test in seconds. If unspecified, assume 0 sec.",
+                "type": "number",
+                "minimum": 0
+              },
+              "status": {
+                "description": "Test result",
+                "type": "string",
+                "enum": [
+                  "TEST_PASSED",
+                  "TEST_FAILED",
+                  "TEST_SKIPPED"
+                ]
+              },
+              "stdout": {
+                "description": "Standard output of the test. If unspecified, assume empty.",
+                "type": "string"
+              },
+              "stderr": {
+                "description": "Standard error of the test. If unspecified, assume empty",
+                "type": "string"
+              },
+              "createdAt": {
+                "description": "The timestamp that the test started at. If unspecified, assume the current timestamp
+                that the CLI is invoked.",
+                "type": "string",
+                "format": "date-time"
+              },
+              "data": {
+                "description": "Metadata of the test, e.g. line number.",
+                "type": "object",
+                "properties": {
+                  "lineNumber": {
+                    "description": "Line number of the test (1-based).",
+                    "type": "number"
+                  }
+                }
+              }
+            },
+            "required": [
+              "status"
+            ],
+            "oneOf": [
+              {
+                "required": [
+                  "testPath"
+                ]
+              },
+              {
+                "required": [
+                  "testPathComponents"
+                ]
+              }
+            ]
+          }
+        }
+      },
+      "required": [
+        "testCases"
+      ]
+    }
+
+    ## JUnit XML TestPath mapping
+
+    If the file path ends with '.xml', the command parses the file as a JUnit XML file. When this mode is used the subset input
+    TestPath should look like 'class={classname}#testcase={testcase}'.
+    """
+
+    def fail(msg):
+        click.secho(msg, fg='red', err=True)
+        raise typer.Exit(1)
+
+    def parse_json(test_result_file: str) -> Generator[CaseEventType, None, None]:
+        with open(test_result_file, 'r') as f:
+            doc = json.load(f)
+        default_created_at = datetime.datetime.now(datetime.timezone.utc).isoformat()
+        for case in doc['testCases']:
+            test_path_components: TestPath = case.get('testPathComponents', None)
+            test_path: str = case.get('testPath', None)
+            if test_path_components is None and test_path is None:
+                fail("Missing testPath or testPathComponents field in the test case.")
+            if test_path_components and test_path:
+                fail("Specifying both testPath and testPathComponents fields is invalid.")
+            if test_path:
+                test_path_components = parse_test_path(test_path)
+            status = case['status']
+            duration_secs = case.get('duration', 0)
+            if isinstance(duration_secs, str):
+                try:
+                    duration_secs = float(duration_secs)
+                except ValueError:
+                    fail(f"The duration of {test_path_components} in {test_result_file} isn't a valid format (was {duration_secs}). Make sure set a valid duration")  # noqa
+
+            created_at = case.get('createdAt', default_created_at)
+
+            if status not in CaseEvent.STATUS_MAP:
+                fail(
+                    f"The status of {test_path_components} should be one of {list(CaseEvent.STATUS_MAP.keys())} (was {status})")
+
+            if duration_secs < 0:
+                fail(f"The duration of {test_path_components} should be positive (was {duration_secs})")
+            dateutil.parser.parse(created_at)
+            metadata = case.get('data', None)
+
+            yield CaseEvent.create(
+                test_path=test_path_components,
+                duration_secs=duration_secs,
+                status=CaseEvent.STATUS_MAP[status],
+                stdout=case.get('stdout', ''),
+                stderr=case.get('stderr', ''),
+                timestamp=created_at,
+                data=metadata)
+
+    for test_result_file in test_result_files:
+        if not test_result_file.endswith('.xml'):
+            client.parse_func = parse_json
+            break
+
+    for test_result_file in test_result_files:
+        client.report(test_result_file)
+
+    client.run()
+
+
+smart_tests.CommonDetectFlakesImpls(__name__).detect_flakes()