smart-tests-cli 2.0.0__py3-none-any.whl

smart_tests/test_runners/robot.py

@@ -0,0 +1,125 @@
+from datetime import datetime
+from typing import Annotated, List
+from xml.etree import ElementTree as ET
+
+from junitparser import JUnitXml  # type: ignore
+
+import smart_tests.args4p.typer as typer
+
+from ..commands.record.tests import RecordTests
+from ..commands.subset import Subset
+from ..testpath import TestPath
+from . import smart_tests
+
+
+def parse_func(p: str) -> ET.ElementTree:
+
+    def parse_suite(suite: ET.Element):
+        DATETIME_FORMAT = '%Y%m%d %H:%M:%S.%f'
+
+        suite_name = suite.get('name')
+        for test in suite.iter("test"):
+            test_name = test.get('name')
+
+            status_node = test.find('status')
+            status = status_node.get(
+                'status') if status_node is not None else None
+
+            nested_status_node = test.find('./kw/status')
+            nested_status = nested_status_node.get('status') if nested_status_node is not None else None
+
+            if status is not None:
+                start_time_str = status_node.get('starttime') if status_node is not None else ''
+                end_time_str = status_node.get('endtime') if status_node is not None else ''
+
+                if start_time_str != '' and end_time_str != '':
+                    start_time = datetime.strptime(str(start_time_str), DATETIME_FORMAT)
+                    end_time = datetime.strptime(str(end_time_str), DATETIME_FORMAT)
+
+                    duration = end_time - start_time
+
+                testcase = ET.SubElement(testsuite, "testcase", {
+                    "name": str(test_name),
+                    "classname": str(suite_name),
+                    "time": str(duration.total_seconds()) if duration is not None else '0',
+                })
+
+                if status == "FAIL":
+                    failure = ET.SubElement(testcase, 'failure')
+
+                    msg = test.find('kw/msg')
+                    failure.text = msg.text if msg is not None else ''
+                if status == "NOT_RUN" or nested_status == 'NOT_RUN':
+                    skipped = ET.SubElement(testcase, "skipped")  # noqa: F841
+
+    original_tree = ET.parse(p)
+    testsuite = ET.Element("testsuite", {"name": "robot"})
+
+    SUITE_TAG_NAME = "suite"
+    for suites in original_tree.findall(SUITE_TAG_NAME):
+        nested_suites = suites.findall(SUITE_TAG_NAME)
+
+        if nested_suites:
+            # Run tests in a directory
+            for suite in nested_suites:
+                parse_suite(suite)
+        else:
+            # Run tests in a single file
+            parse_suite(suites)
+
+    return ET.ElementTree(testsuite)
+
+
+@smart_tests.record.tests
+def record_tests(
+    client: RecordTests,
+    reports: Annotated[List[str], typer.Argument(
+        multiple=True,
+        help="Test report files to process"
+    )],
+):
+    for r in reports:
+        client.report(r)
+
+    client.junitxml_parse_func = parse_func
+    client.run()
+
+
+@smart_tests.subset
+def subset(
+    client: Subset,
+    reports: Annotated[List[str], typer.Argument(
+        multiple=True,
+        required=False,
+        help="Test report files to process"
+    )] = [],
+):
+    for r in reports:
+        xml = JUnitXml.fromfile(r, parse_func)
+
+        for suite in xml:
+            for case in suite:
+                cls_name = case._elem.attrib.get("classname")
+                name = case._elem.attrib.get('name')
+                if cls_name != '' and name != '':
+                    client.test_path([{'type': 'class', 'name': cls_name}, {'type': 'testcase', 'name': name}])
+
+    client.formatter = robot_formatter
+    client.separator = " "
+    client.run()
+
+
+def robot_formatter(x: TestPath):
+    cls_name = ''
+    case = ''
+    for path in x:
+        t = path['type']
+        if t == 'class':
+            cls_name = path['name']
+        if t == 'testcase':
+            case = path['name']
+
+    if cls_name != '' and case != '':
+        return f"-s '{cls_name}' -t '{case}'"
+
+    return ''

smart_tests/test_runners/rspec.py

@@ -0,0 +1,5 @@
+from . import smart_tests
+
+subset = smart_tests.CommonSubsetImpls(__name__).scan_files('*_spec.rb')
+record_tests = smart_tests.CommonRecordTestImpls(__name__).report_files()
+detect_flakes = smart_tests.CommonDetectFlakesImpls(__name__).detect_flakes()

smart_tests/test_runners/smart_tests.py

@@ -0,0 +1,235 @@
+import glob
+import os
+import sys
+import types
+from typing import Annotated
+
+import click
+from junitparser import TestCase, TestSuite  # type: ignore
+
+import smart_tests.args4p.typer as typer
+from smart_tests import args4p
+from smart_tests.args4p import decorator
+from smart_tests.args4p.command import Group
+from smart_tests.commands.detect_flakes import DetectFlakes
+from smart_tests.commands.detect_flakes import detect_flakes as detect_flakes_cmd
+from smart_tests.commands.record.tests import RecordTests
+from smart_tests.commands.record.tests import tests as record_tests_cmd
+from smart_tests.commands.subset import Subset
+from smart_tests.commands.subset import subset as subset_cmd
+
+from ..testpath import TestPath
+
+
+def cmdname(m):
+    """figure out the sub-command name from a test runner function"""
+
+    # a.b.cde -> cde
+    # xyz -> xyz
+    #
+    # In python module name the conventional separator is '_' but in command name,
+    # it is '-', so we do replace that
+    return m[m.rfind('.') + 1:].replace('_', '-')
+
+
+def wrap(f, group: Group, name=None):
+    """
+    Wraps a 'plugin' function into a click command and registers it to the given group.
+
+    a plugin function receives the scanner object in its first argument
+    """
+    if not name:
+        name = cmdname(f.__module__)
+    d = args4p.command(name=name)
+    cmd = d(f)
+    group.add_command(cmd)
+    return cmd
+
+
+@decorator
+def subset(f):
+    return wrap(f, subset_cmd)
+
+
+record = types.SimpleNamespace()
+# this is also meant to be used as a decorator, e.g., @record.tests
+record.tests = lambda f: wrap(f, record_tests_cmd)
+
+
+@decorator
+def flake_detection(f):
+    return wrap(f, detect_flakes_cmd)
+
+
+# TODO
+# @decorator
+# def split_subset(f):
+#     return wrap(f, split_subset_cmd)
+
+
+class CommonSubsetImpls:
+    """
+    Typical 'subset' implementations that are reusable.
+    """
+
+    def __init__(self, module_name):
+        self.cmdname = cmdname(module_name)
+
+    def scan_files(self, pattern):
+        """
+        Suitable for test runners that use files as unit of tests where file names follow a naming pattern.
+
+        :param pattern: file masks that identify test files, such as '*_spec.rb'
+        """
+        def subset(
+            client: Subset,
+            files: Annotated[list[str], typer.Argument(
+                multiple=True,
+                required=False,
+                help="Test files or directories to include in the subset"
+            )] = []
+        ):
+            # client type: Optimize in def lauchable.commands.subset.subset
+            def parse(fname: str):
+                if os.path.isdir(fname):
+                    client.scan(fname, '**/' + pattern)
+                elif fname == '@-':
+                    # read stdin
+                    for line in sys.stdin:
+                        parse(line.rstrip())
+                elif fname.startswith('@'):
+                    # read response file
+                    with open(fname[1:]) as f:
+                        for line in f:
+                            parse(line.rstrip())
+                else:
+                    # assume it's a file
+                    client.test_path(fname)
+
+            for f in files:
+                parse(f)
+
+            client.run()
+
+        return wrap(subset, subset_cmd, self.cmdname)
+
+
+class CommonRecordTestImpls:
+    """
+    Typical 'record tests' implementations that are reusable.
+    """
+
+    def __init__(self, module_name):
+        self.cmdname = cmdname(module_name)
+
+    def report_files(self, file_mask="*.xml"):
+        """
+        Suitable for test runners that create a directory full of JUnit report files.
+
+        'record tests' expect JUnit report/XML file names.
+        """
+
+        def record_tests(
+            client: RecordTests,
+            source_roots: Annotated[list[str], typer.Argument(
+                multiple=True,
+                help="Source directories containing test report files"
+            )]
+        ):
+            CommonRecordTestImpls.load_report_files(client=client, source_roots=source_roots, file_mask=file_mask)
+
+        return wrap(record_tests, record_tests_cmd, self.cmdname)
+
+    def file_profile_report_files(self):
+        """
+        Suitable for test runners that create a directory full of JUnit report files.
+
+        'record tests' expect JUnit report/XML file names.
+        """
+
+        def record_tests(client: RecordTests,
+                         source_roots: Annotated[list[str], typer.Argument(
+                             multiple=True,
+                             help="Source directories containing test report files"
+                         )]):
+            def path_builder(
+                case: TestCase, suite: TestSuite, report_file: str
+            ) -> TestPath:
+                def find_filename():
+                    """look for what looks like file names from test reports"""
+                    for e in [case, suite]:
+                        for a in ["file", "filepath"]:
+                            filepath = e._elem.attrib.get(a)
+                            if filepath:
+                                return filepath
+                    return None  # failing to find a test name
+
+                filepath = find_filename()
+                if not filepath:
+                    raise click.ClickException("No file name found in %s" % report_file)
+
+                # default test path in `subset` expects to have this file name
+                test_path = [client.make_file_path_component(filepath)]
+                if suite.name:
+                    test_path.append({"type": "testsuite", "name": suite.name})
+                if case.name:
+                    test_path.append({"type": "testcase", "name": case.name})
+                return test_path
+
+            client.path_builder = path_builder
+
+            for r in source_roots:
+                client.report(r)
+            client.run()
+
+        return wrap(record_tests, record_tests_cmd, self.cmdname)
+
+    @classmethod
+    def load_report_files(cls, client: RecordTests, source_roots, file_mask="*.xml"):
+        # client type: RecordTests in def launchable.commands.record.tests.tests
+        # Accept both file names and GLOB patterns
+        # Simple globs like '*.xml' can be dealt with by shell, but
+        # not all shells consistently deal with advanced GLOBS like '**'
+        # so it's worth supporting it here.
+        for root in source_roots:
+            match = False
+            for t in glob.iglob(root, recursive=True):
+                match = True
+                if os.path.isdir(t):
+                    client.scan(t, file_mask)
+                else:
+                    client.report(t)
+
+            if not match:
+                # By following the shell convention, if the file doesn't exist or GLOB doesn't match anything,
+                # raise it as an error. Note this can happen for reasons other than a configuration error.
+                # For example, if a build catastrophically failed and no
+                # tests got run.
+                click.echo(f"No matches found: {root}", err=True)
+                # intentionally exiting with zero
+                return
+
+        client.run()
+
+
+class CommonDetectFlakesImpls:
+    def __init__(
+            self,
+            module_name,
+            formatter=None,
+            separator="\n",
+    ):
+        self.cmdname = cmdname(module_name)
+        self._formatter = formatter
+        self._separator = separator
+
+    def detect_flakes(self):
+        def detect_flakes(client: DetectFlakes):
+            if self._formatter:
+                client.formatter = self._formatter
+            if self._separator:
+                client.separator = self._separator
+
+            client.run()
+
+        return wrap(detect_flakes, detect_flakes_cmd, self.cmdname)

smart_tests/test_runners/vitest.py

@@ -0,0 +1,49 @@
+import xml.etree.ElementTree as ET
+from typing import Annotated, List, cast
+
+import smart_tests.args4p.typer as typer
+
+from ..commands.record.tests import RecordTests
+from ..commands.subset import Subset
+from . import smart_tests
+
+
+@smart_tests.record.tests
+def record_tests(
+    client: RecordTests,
+    reports: Annotated[List[str], typer.Argument(
+        multiple=True,
+        help="Test report files to process"
+    )],
+):
+    def parse_func(report: str) -> ET.ElementTree:
+        """
+        Vitest junit report doesn't set file/filepath attributes on test cases, and it's set as a classname attribute instead.
+        So, set the classname value as the file name in this function.
+        e.g.) <testcase classname="src/components/Hello.test.tsx" name="renders hello message" time="0.008676833">
+          """
+        tree = cast(ET.ElementTree, ET.parse(report))
+        root = tree.getroot()
+
+        if root is None:
+            return tree
+
+        for test_suite in root.findall('testsuite'):
+            for test_case in test_suite.findall('testcase'):
+                classname = test_case.get('classname', '')
+                test_case.set('file', classname)
+                test_case.attrib.pop('classname', None)
+
+        return tree
+
+    client.junitxml_parse_func = parse_func
+    smart_tests.CommonRecordTestImpls.load_report_files(client=client, source_roots=reports)
+
+
+@smart_tests.subset
+def subset(client: Subset):
+    # read lines as test file names
+    for t in client.stdin():
+        client.test_path(t.rstrip("\n"))
+
+    client.run()

smart_tests/test_runners/xctest.py

@@ -0,0 +1,79 @@
+import html
+import xml.etree.ElementTree as ET  # type: ignore
+from typing import Annotated, List, cast
+
+import click
+from junitparser import TestCase, TestSuite
+
+import smart_tests.args4p.typer as typer
+
+from ..commands.record.tests import RecordTests
+from ..commands.subset import Subset
+from ..testpath import TestPath
+from . import smart_tests
+
+
+@smart_tests.record.tests
+def record_tests(
+    client: RecordTests,
+    reports: Annotated[List[str], typer.Argument(
+        multiple=True,
+        help="Test report files to process"
+    )],
+):
+
+    def parse_func(p: str) -> ET.ElementTree:
+        tree = cast(ET.ElementTree, ET.parse(p))
+        root = tree.getroot()
+        if root is None:
+            return tree
+
+        for testsuite in root.findall('testsuite'):
+            for testcase in testsuite.findall('testcase'):
+                failure = testcase.find('failure')
+                if failure is not None:
+                    # Note(Konboi): XCTest escape `"` to `"` in the message, so save as unescaped text
+                    message = html.unescape(failure.get('message', ''))
+                    body = failure.text or ''
+                    failure.text = f"{message}\n{body}"
+
+        return tree
+
+    def path_builder(case: TestCase, suite: TestSuite, report_path: str) -> TestPath:
+        class_attr = case.classname
+        [target, *rest] = class_attr.split('.')
+        class_name = '/'.join(rest)
+        return [
+            {"type": "target", "name": target},
+            {"type": "class", "name": class_name},
+            {"type": "testcase", "name": case.name},
+        ]
+
+    client.junitxml_parse_func = parse_func
+    client.path_builder = path_builder
+    smart_tests.CommonRecordTestImpls.load_report_files(client=client, source_roots=reports)
+
+
+@smart_tests.subset
+def subset(client: Subset):
+    if not client.is_get_tests_from_previous_sessions or not client.is_output_exclusion_rules:
+        click.secho(
+            "XCTest profile only supports the subset with `--get-tests-from-previous-sessions` and `--output-exclusion-rules` options",  # noqa: E501
+            fg='red',
+            err=True,
+        )
+
+    def formatter(test_path: TestPath) -> str:
+        if len(test_path) == 0:
+            return ""
+
+        # only target case
+        if len(test_path) == 1:
+            return f"-skip-testing:{test_path[0]['name']}"
+
+        # default target/class format
+        return f"-skip-testing:{test_path[0]['name']}/{test_path[1]['name']}"
+
+    client.formatter = formatter
+    client.separator = "\n"
+    client.run()

smart_tests/testpath.py

@@ -0,0 +1,154 @@
+import os
+import pathlib
+import subprocess
+import urllib
+
+# No additional typing imports needed
+
+# Path component is a node in a tree.
+# It's the equivalent of a short file/directory name in a file system.
+# In our abstraction, it's represented as arbitrary bag of attributes
+TestPathComponent = dict[str, str]
+
+# TestPath is a full path to a node in a tree from the root
+# It's the equivalent of an absolute file name in a file system
+TestPath = list[TestPathComponent]
+
+
+def parse_test_path(tp_str: str) -> TestPath:
+    """Parse a string representation of TestPath."""
+    if tp_str == '':
+        return []
+    ret: TestPath = []
+    for component_str in tp_str.split('#'):
+        if component_str == '&':
+            # Technically, this should be mapped to {None:None}. But because the
+            # TestPath definition is now dict[str, str], not dict[str | None,
+            # Optinal[str]], we cannot add it. Fixing this definition needs to
+            # fix callers not to assume they are always str. In practice, this
+            # is a rare case. Do not appent {None: None} now...
+            # ret.append({None: None})
+            continue
+        first = True
+        component = {}
+        for kv in component_str.split('&'):
+            if first:
+                first = False
+                if kv:
+                    (component['type'], component['name']) = _parse_kv(kv)
+            else:
+                (k, v) = _parse_kv(kv)
+                component[k] = v
+        ret.append(component)
+    return ret
+
+
+def _parse_kv(kv: str) -> tuple[str, str]:
+    kvs = kv.split('=')
+    if len(kvs) != 2:
+        raise ValueError('Malformed TestPath component: ' + kv)
+    return (_decode_str(kvs[0]), _decode_str(kvs[1]))
+
+
+def unparse_test_path(tp: TestPath) -> str:
+    """Create a string representation of TestPath."""
+    ret = []
+    for component in tp:
+        s = ''
+        pairs = []
+        if component.get('type', None) and component.get('name', None):
+            s += _encode_str(component['type']) + '=' + _encode_str(component['name'])
+            for k, v in component.items():
+                if k not in ('type', 'name'):
+                    pairs.append((k, v))
+        else:
+            for k, v in component.items():
+                if not k or not v:
+                    continue
+                pairs.append((k, v))
+            if len(pairs) == 0:
+                s = '&'
+        pairs = sorted(pairs, key=lambda p: p[0])
+        for (k, v) in pairs:
+            s += '&'
+            s += _encode_str(k) + '=' + _encode_str(v)
+        ret.append(s)
+    return '#'.join(ret)
+
+
+def _decode_str(s: str) -> str:
+    return urllib.parse.unquote(s)
+
+
+def _encode_str(s: str) -> str:
+    return s.replace('%', '%25').replace('=', '%3D').replace('#', '%23').replace('&', '%26')
+
+
+def _relative_to(p: pathlib.Path, base: str) -> pathlib.Path:
+    return p.resolve(strict=False).relative_to(base)
+
+
+class FilePathNormalizer:
+    """Normalize file paths based on the Git repository root
+
+    Some test runners output absolute file paths. This is not preferrable when
+    making statistical data on tests as the absolute paths can vary per machine
+    or per run. FilePathNormalizer guesses the relative paths based on the Git
+    repository root.
+    """
+
+    def __init__(self, base_path: str | None = None, no_base_path_inference: bool = False):
+        self._base_path = base_path
+        self._no_base_path_inference = no_base_path_inference
+        self._inferred_base_path = None  # type: str | None
+
+    def relativize(self, p: str) -> str:
+        return str(self._relativize(pathlib.Path(os.path.normpath(p))))
+
+    def _relativize(self, p: pathlib.Path) -> pathlib.Path:
+        if not p.is_absolute():
+            return p
+
+        if self._base_path:
+            return _relative_to(p, self._base_path)
+
+        if self._no_base_path_inference:
+            return p
+
+        if not self._inferred_base_path:
+            self._inferred_base_path = self._auto_infer_base_path(p)
+
+        if self._inferred_base_path:
+            return _relative_to(p, self._inferred_base_path)
+
+        return p
+
+    def get_effective_base_path(self) -> str | None:
+        """Get the effective base path, either explicitly set or inferred."""
+        if self._base_path:
+            return self._base_path
+        return self._inferred_base_path
+
+    def _auto_infer_base_path(self, p: pathlib.Path) -> str | None:
+        # If p is a file, start from its parent directory
+        if p.is_file():
+            p = p.parent
+
+        while p != p.root and not p.exists():
+            p = p.parent
+        try:
+            toplevel = subprocess.check_output(
+                ['git', 'rev-parse', '--show-superproject-working-tree'],
+                cwd=str(p),
+                stderr=subprocess.DEVNULL,
+                universal_newlines=True).strip()
+            if toplevel:
+                return toplevel
+            return subprocess.check_output(
+                ['git', 'rev-parse', '--show-toplevel'],
+                cwd=str(p),
+                stderr=subprocess.DEVNULL,
+                universal_newlines=True).strip()
+        except subprocess.CalledProcessError:
+            # Cannot infer the Git repo. Continue with the abs path...
+            return None