smart-tests-cli 2.0.0__py3-none-any.whl

smart_tests/test_runners/cts.py

@@ -0,0 +1,189 @@
+from typing import Annotated, List
+from xml.etree import ElementTree as ET
+
+import click
+
+import smart_tests.args4p.typer as typer
+from smart_tests.commands.record.case_event import CaseEvent
+
+from ..commands.record.tests import RecordTests
+from ..commands.subset import Subset
+from . import smart_tests
+
+# https://source.android.com/docs/compatibility/cts/command-console-v2
+include_option = "--include-filter"
+exclude_option = "--exclude-filter"
+
+
+def parse_func(p: str):
+    """  # noqa: E501
+    # sample report format
+    success case:
+    <Module name="CtsAbiOverrideHostTestCases" abi="arm64-v8a" runtime="1171" done="true" pass="1" total_tests="1">
+      <TestCase name="android.abioverride.cts.AbiOverrideTest">
+        <Test result="pass" name="testAbiIs32bit" />
+      </TestCase>
+    </Module>
+
+    failure case:
+    <Module name="CtsAccountManagerTestCases" abi="arm64-v8a" runtime="13414" done="false" pass="2" total_tests="151">
+      <Reason message="Instrumentation run failed due to 'Process crashed.'" error_name="INSTRUMENTATION_CRASH" error_code="520200" />
+      <TestCase name="android.accounts.cts.AbstractAuthenticatorTests">
+        <Test result="fail" name="testFinishSessionAndStartAddAccountSessionDefaultImpl">
+          <Failure message="android.accounts.OperationCanceledException&#13;">
+            <StackTrace>android.accounts.OperationCanceledException
+          at android.accounts.AccountManager$AmsTask.internalGetResult(AccountManager.java:2393)
+          at android.accounts.AccountManager$AmsTask.getResult(AccountManager.java:2422)
+          at android.accounts.AccountManager$AmsTask.getResult(AccountManager.java:2337)
+          at android.accounts.cts.AbstractAuthenticatorTests.testFinishSessionAndStartAddAccountSessionDefaultImpl(AbstractAuthenticatorTests.java:165)
+            </StackTrace>
+          </Failure>
+        </Test>
+      </TestCase>
+    </Module>
+    """
+    class TestResult:
+        def __init__(self, test_case_name: str, test_name: str, result: str, stdout: str, stderr: str):
+            self.test_case_name = test_case_name
+            self.test_name = test_name
+            self.result = result
+            self.stdout = stdout
+            self.stderr = stderr
+
+        def case_event_status(self):
+            if self.result == "fail":
+                return CaseEvent.TEST_FAILED
+            elif self.result == "ASSUMPTION_FAILURE" or self.result == "IGNORED":
+                return CaseEvent.TEST_SKIPPED
+
+            return CaseEvent.TEST_PASSED
+
+    def build_record_test_path(module: str, test_case: str, test: str):
+        return [
+            {"type": "Module", "name": module},
+            {"type": "TestCase", "name": test_case},
+            {"type": "Test", "name": test},
+        ]
+
+    events = []
+    tree = ET.parse(p)
+
+    for module in tree.iter('Module'):
+        test_results = []
+        total_duration = module.get("runtime", "0")
+        module_name = module.get("name", "")
+
+        for test_case in module.iter("TestCase"):
+            test_case_name = test_case.get('name', "")
+
+            for test in test_case.iter("Test"):
+                result = test.get('result', "")
+                test_name = test.get("name", "")
+                stdout = ""
+                stderr = ""
+
+                failure = test.find('Failure')
+                if failure:
+                    stack_trace = ""
+                    stack_trace_element = failure.find("StackTrace")
+                    if stack_trace_element is not None:
+                        stack_trace = str(stack_trace_element.text)
+
+                    stdout = failure.get("message", "")
+                    stderr = stack_trace
+
+                test_results.append(TestResult(
+                    test_case_name=test_case_name,
+                    test_name=test_name,
+                    result=result,
+                    stdout=stdout,
+                    stderr=stderr))
+
+        if len(test_results) == 0:
+            continue
+
+        test_duration_msec_per_test = int(total_duration) / len(test_results)
+
+        for test_result in test_results:  # type: TestResult
+            if module_name == "" or test_result.test_case_name == "" or test_result.test_name == "":
+                continue
+
+            events.append(CaseEvent.create(
+                test_path=build_record_test_path(module_name, test_result.test_case_name, test_result.test_name),
+                duration_secs=float(test_duration_msec_per_test / 1000),
+                status=test_result.case_event_status(),
+                stdout=test_result.stdout,
+                stderr=test_result.stderr,
+            ))
+
+    return (x for x in events)
+
+
+@smart_tests.record.tests
+def record_tests(
+    client: RecordTests,
+    reports: Annotated[List[str], typer.Argument(
+        multiple=True,
+        help="Test report files to process"
+    )],
+):
+    """
+    Beta: Report test result that Compatibility Test Suite (CTS) produced. Supports only CTS v2
+    """
+    for r in reports:
+        client.report(r)
+
+    client.parse_func = parse_func
+    client.run()
+
+
+@smart_tests.subset
+def subset(client: Subset):
+    """
+    Beta: Produces test list from previous test sessions for Compatibility Test Suite (CTS). Supports only CTS v2
+    """
+    start_module = False
+
+    """ # noqa: E501
+    # This is sample output of `cts-tradefed list modules`
+    ==================
+    Notice:
+    We collect anonymous usage statistics in accordance with our Content Licenses (https://source.android.com/setup/start/licenses), Contributor License Agreement (https://opensource.google.com/docs/cla/), Privacy Policy (https://policies.google.com/privacy) and Terms of Service (https://policies.google.com/terms).
+    ==================
+    Android Compatibility Test Suite 12.1_r5 (9566553)
+    Use "help" or "help all" to get more information on running commands.
+    Non-interactive mode: Running initial command then exiting.
+    Using commandline arguments as starting command: [list, modules]
+    arm64-v8a CtsAbiOverrideHostTestCases[instant]
+    arm64-v8a CtsAbiOverrideHostTestCases[secondary_user]
+    arm64-v8a CtsAbiOverrideHostTestCases
+    armeabi-v7a CtsAbiOverrideHostTestCases
+    """
+
+    for t in client.stdin():
+        line = t.rstrip("\n") if isinstance(t, str) else str(t).rstrip("\n")
+
+        if "starting command" in line:
+            start_module = True
+            continue
+
+        if not start_module:
+            continue
+
+        # e.g) armeabi-v7a CtsAbiOverrideHostTestCases
+        device_and_module = line.split()
+        if len(device_and_module) != 2:
+            click.secho(
+                f"Warning: {line} is not expected Module format and skipped",
+                fg='yellow',
+                err=True)
+            continue
+
+        client.test_path([{"type": "Module", "name": device_and_module[1]}])
+
+    option = include_option
+    if client.is_output_exclusion_rules:
+        option = exclude_option
+
+    client.formatter = lambda x: "{option} \"{module}\"".format(option=option, module=x[0]['name'])
+    client.run()

smart_tests/test_runners/cucumber.py

@@ -0,0 +1,451 @@
+import json
+import os
+import pathlib
+import re
+from copy import deepcopy
+from enum import Enum
+from pathlib import Path
+from typing import Annotated, Dict, Generator, List, cast
+from xml.etree import ElementTree as ET
+
+import click
+
+import smart_tests.args4p.typer as typer
+from smart_tests.testpath import FilePathNormalizer, TestPath
+
+from ..commands.record.case_event import CaseEvent, CaseEventType
+from ..commands.record.tests import RecordTests
+from . import smart_tests
+
+subset = smart_tests.CommonSubsetImpls(__name__).scan_files('*_feature')
+
+
+REPORT_FILE_PREFIX = "TEST-"
+
+
+@smart_tests.record.tests
+def record_tests(
+    client: RecordTests,
+    reports: Annotated[List[str], typer.Argument(
+        multiple=True,
+        help="Test report files to process"
+    )],
+    json_format: Annotated[bool, typer.Option(
+        "--json",
+        help="use JSON report format"
+    )] = False,
+):
+    if json_format:
+        for r in reports:
+            client.report(r)
+        client.parse_func = JSONReportParser(client).parse_func
+    else:
+        report_file_and_test_file_map: dict[str, str] = {}
+        _record_tests_from_xml(client, reports, report_file_and_test_file_map)
+
+        def parse_func(report: str) -> ET.ElementTree:
+            tree = cast(ET.ElementTree, ET.parse(report))
+            for case in tree.findall("testcase"):
+                case.attrib["file"] = str(report_file_and_test_file_map[report])
+
+            return tree
+
+        client.junitxml_parse_func = parse_func
+
+    client.run()
+
+
+def _record_tests_from_xml(client, reports, report_file_and_test_file_map: Dict[str, str]):
+    base_path = client.base_path if client.base_path else os.getcwd()
+    for report in reports:
+        if REPORT_FILE_PREFIX not in report:
+            click.echo(f"{report} was load skipped because it doesn't look like a report file.", err=True)
+            continue
+
+        test_file = _find_test_file_from_report_file(base_path, report)
+        if test_file:
+            report_file_and_test_file_map[report] = str(test_file)
+            client.report(report)
+        else:
+            click.echo(f"Cannot find test file of {report}", err=True)
+
+
+class JSONReportParser:
+    """
+        client: smart_tests.RecordTests
+    """
+
+    def __init__(self, client):
+        self.client = client
+        self.file_path_normalizer = FilePathNormalizer(base_path=client.base_path,
+                                                       no_base_path_inference=client.no_base_path_inference)
+
+    def parse_func(self, report_file: str) -> Generator[CaseEventType, None, None]:
+        """
+        example of JSON format report
+        [
+        {
+            "uri": "features/foo/is_it_friday_yet.feature",
+            "id": "is-it-friday-yet?",
+            "keyword": "Feature",
+            "name": "Is it Friday yet?",
+            "description": "  Everybody wants to know when it's Friday",
+            "line": 1,
+            "elements": [
+            {
+                "keyword": "Background",
+                "name": "",
+                "description": "",
+                "line": 4,
+                "type": "background",
+                "steps": [
+                {
+                    "keyword": "Given ",
+                    "name": "this year is 2023",
+                    "line": 5,
+                    "match": {
+                    "location": "features/step_definitions/stepdefs.rb:12"
+                    },
+                    "result": {
+                    "status": "passed",
+                    "duration": 30000
+                    },
+                    "after": [
+                        {
+                            "match": {
+                            "location": "features/support/env.rb:22"
+                            },
+                            "result": {
+                            "status": "passed",
+                            "duration": 10181000
+                            }
+                        }
+                    ]
+                },
+                {
+                    "keyword": "And ",
+                    "name": "this month is January",
+                    "line": 6,
+                    "match": {
+                    "location": "features/step_definitions/stepdefs.rb:17"
+                    },
+                    "result": {
+                    "status": "passed",
+                    "duration": 9000
+                    }
+                }
+                ]
+            },
+            {
+                "id": "is-it-friday-yet?;today-is-or-is-not-friday;;2",
+                "keyword": "Scenario Outline",
+                "name": "Today is or is not Friday",
+                "description": "",
+                "line": 11,
+                "type": "scenario",
+                "steps": [
+                {
+                    "keyword": "Given ",
+                    "name": "today is \"Friday\"",
+                    "line": 5,
+                    "match": {
+                    "location": "features/step_definitions/stepdefs.rb:12"
+                    },
+                    "result": {
+                    "status": "passed",
+                    "duration": 101000
+                    }
+                },
+                {
+                    "keyword": "When ",
+                    "name": "I ask whether it's Friday yet",
+                    "line": 6,
+                    "match": {
+                    "location": "features/step_definitions/stepdefs.rb:24"
+                    },
+                    "result": {
+                    "status": "passed",
+                    "duration": 11000
+                    }
+                },
+                {
+                    "keyword": "Then ",
+                    "name": "I should be told \"TGIF\"",
+                    "line": 7,
+                    "match": {
+                    "location": "features/step_definitions/stepdefs.rb:28"
+                    },
+                    "result": {
+                    "status": "passed",
+                    "duration": 481000
+                    }
+                }
+                ],
+                "before": [
+                    {
+                        "match": {
+                        "location": "features/support/env.rb:10"
+                        },
+                        "result": {
+                        "status": "passed",
+                        "duration": 11092000
+                        }
+                    },
+                    {
+                        "match": {
+                        "location": "features/support/env.rb:14"
+                        },
+                        "result": {
+                        "status": "passed",
+                        "duration": 11081000
+                        }
+                    }
+                    ],
+                    "after": [
+                    {
+                        "match": {
+                        "location": "features/support/env.rb:18"
+                        },
+                        "result": {
+                        "status": "passed",
+                        "duration": 11072000
+                        }
+                    }
+                ]
+            }
+        ]
+        """
+        with open(report_file, 'r') as json_file:
+            try:
+                data = json.load(json_file)
+            except Exception as e:
+                raise Exception(f"Can't read JSON format report file {report_file}. Make sure to confirm report file.") from e
+
+        if len(data) == 0:
+            click.echo(f"Can't find test reports from {report_file}. Make sure to confirm report file.", err=True)
+
+        for d in data:
+            file_name = clean_uri(d.get("uri", ""))
+            class_name = d.get("name", "")
+
+            # Cucumber can define repeating the same `Given` steps as a `Background`
+            # https://cucumber.io/docs/gherkin/reference/#background
+            background: TestCaseInfo | None = None
+
+            for element in d.get("elements", []):
+                test_case = element.get("name", "")
+                # Scenario hooks run for every scenario.
+                # https://cucumber.io/docs/cucumber/api/?lang=java#hooks
+                scenario_hook_information = _parse_hook_from_element(element)
+
+                if element.get("type", "") == CucumberElementType.BACKGROUND.value:
+                    # `Background` can be defined once per scenario so won't available multiple times.
+                    background = _parse_test_case_info_from_element(element=element)
+                    background.append_hook_info(scenario_hook_information)
+                    continue
+
+                test_case_info = _parse_test_case_info_from_element(element=element)
+                if background:
+                    test_case_info.append_background_results(background)
+                    # Initialize background for next scenario
+                    background = None
+
+                test_case_info.append_hook_info(scenario_hook_information)
+
+                if test_case_info.is_failed():
+                    status = CaseEvent.TEST_FAILED
+                elif test_case_info.is_skipped():
+                    status = CaseEvent.TEST_SKIPPED
+                else:
+                    status = CaseEvent.TEST_PASSED
+
+                test_path: TestPath = [
+                    {"type": "file", "name": pathlib.Path(self.file_path_normalizer.relativize(file_name)).as_posix()},
+                    {"type": "class", "name": class_name},
+                    {"type": "testcase", "name": test_case},
+                ]
+                test_path.extend(test_case_info.test_path())
+
+                yield CaseEvent.create(
+                    test_path=test_path,
+                    duration_secs=test_case_info.duration_sec(),
+                    status=status,
+                    stderr="\n".join(test_case_info.stderr()))
+
+
+def _find_test_file_from_report_file(base_path: str, report: str) -> Path | None:
+    """
+    Find test file from cucumber report file path format
+    e.g) Test-features-foo-hoge.xml -> features/foo/hoge.feature or features/foo-hoge.feature
+    """
+
+    report_file = os.path.basename(report)
+    report_file = report_file.lstrip(REPORT_FILE_PREFIX)
+    report_file = os.path.splitext(report_file)[0]
+
+    file_candidates = _create_file_candidate_list(report_file)
+    for candidate in file_candidates:
+        f = Path(base_path, candidate + ".feature")
+        if f.exists():
+            return f
+
+    return None
+
+
+def _create_file_candidate_list(file: str) -> List[str]:
+    candidates = [""]
+    for c in file:
+        if c == "-":
+            list_length = len(candidates)
+            candidates += deepcopy(candidates)
+            for i in range(list_length):
+                candidates[i] += '-'
+            for i in range(list_length, len(candidates)):
+                candidates[i] += '/'
+        else:
+            for i in range(len(candidates)):
+                candidates[i] += c
+
+    return candidates
+
+
+class Result:
+    def __init__(self, statuses: List[str], duration_nano_sec: int, error_message: List[str]) -> None:
+        self._statuses = statuses
+        self._duration_nano_sec = duration_nano_sec
+        self._error_message = error_message
+
+
+class TestCaseHookInfo(Result):
+    def __init__(self, duration_nano_sec: int, statuses: List[str], stderr: List[str]) -> None:
+        super().__init__(statuses=statuses, duration_nano_sec=duration_nano_sec, error_message=stderr)
+
+
+def _parse_hook_from_element(element: Dict[str, List]) -> TestCaseHookInfo:
+    duration_nano_sec: int = 0
+    statuses: List[str] = []
+    stderr: List[str] = []
+
+    def parse_steps(step: Dict[str, Dict]):
+        result = step.get("result", None)
+        if result:
+            nonlocal duration_nano_sec
+            duration_nano_sec += result.get("duration", 0)
+            if result.get("status", None):
+                statuses.append(result["status"])
+            if result.get("error_message", None):
+                stderr.append(result["error_message"])
+
+    for step in element.get("before", []):
+        parse_steps(step)
+
+    for step in element.get("after", []):
+        parse_steps(step)
+
+    return TestCaseHookInfo(
+        duration_nano_sec=duration_nano_sec,
+        statuses=statuses,
+        stderr=stderr
+    )
+
+
+class TestCaseInfo(Result):
+    def __init__(
+            self, steps: List[List[str]],
+            duration_nano_sec: int, statuses: List[str],
+            stderr: List[str],
+            hooks: List[TestCaseHookInfo] = []) -> None:
+        super().__init__(statuses=statuses, duration_nano_sec=duration_nano_sec, error_message=stderr)
+        self._steps = steps
+        self._hooks = hooks
+
+    def steps(self) -> List[List[str]]:
+        return self._steps
+
+    def duration_nano(self) -> int:
+        return self._duration_nano_sec + sum(h._duration_nano_sec for h in self._hooks)
+
+    def duration_sec(self) -> float:
+        return self.duration_nano() / 1000 / 1000 / 1000
+
+    def statuses(self) -> List[str]:
+        return self._statuses + sum([h._statuses for h in self._hooks], [])
+
+    def stderr(self) -> List[str]:
+        return self._error_message + sum([h._error_message for h in self._hooks], [])
+
+    def append_hook_info(self, other: TestCaseHookInfo) -> None:
+        self._hooks.append(other)
+
+    def is_failed(self) -> bool:
+        return "failed" in self.statuses()
+
+    def is_skipped(self) -> bool:
+        return "undefined" in self.statuses()
+
+    def test_path(self) -> TestPath:
+        test_path: TestPath = []
+        for step in self._steps:
+            if len(step) == 2:
+                # While there isn't any cases that the size of step is not 2, we check the size just in case.
+                test_path.append({"type": step[0], "name": step[1]})
+        return test_path
+
+    # This method only for append_background_results method
+    def _to_hook(self) -> TestCaseHookInfo:
+        return TestCaseHookInfo(duration_nano_sec=self.duration_nano(), statuses=self.statuses(), stderr=self.stderr())
+
+    # Type of other TestCaseInfo (Self)..  Python3.6 cannot support `Self`` type even if used typing_extensions module
+    def append_background_results(self, other) -> None:
+        # Need to merge Background steps to main test scenario to calculate correct test duration,
+        # then, we don't need step information of Background. So append it as hooks
+        self.append_hook_info(other._to_hook())
+
+
+def _parse_test_case_info_from_element(element: Dict[str, List]) -> TestCaseInfo:
+    steps: List[List[str]] = []
+    duration = 0  # nano sec
+    statuses = []
+    stderr = []
+    hooks: List[TestCaseHookInfo] = []
+
+    for step in element.get("steps", []):
+        steps.append([step.get("keyword", "").strip(), step.get("name", "").strip()])
+        result = step.get("result", None)
+
+        if result:
+            # duration's unit is nano sec
+            # ref:
+            # https://github.com/cucumber/cucumber-ruby/blob/main/lib/cucumber/formatter/json.rb#L222
+            duration = duration + result.get("duration", 0)
+            statuses.append(result.get("status"))
+            if result.get("error_message", None):
+                stderr.append(result["error_message"])
+
+        # Step hooks are invoked before and after a step.
+        # https://cucumber.io/docs/cucumber/api/?lang=java#hooks
+        # When Step hooks are executed, the information about each step is registered in each element.
+        hooks.append(_parse_hook_from_element(step))
+
+    return TestCaseInfo(
+        steps=steps,
+        duration_nano_sec=duration,
+        statuses=statuses,
+        stderr=stderr,
+        hooks=hooks
+    )
+
+# This type refer to https://github.com/cucumber/json-formatter/blob/v19.0.0/go/json_elements.go#L23.
+
+
+class CucumberElementType(Enum):
+    BACKGROUND = 'background'
+    SCENARIO = 'scenario'
+
+
+def clean_uri(uri: str) -> str:
+    """
+    Trim unused prefix from the URI string.
+    For example, if the URI is "file:features/foo.feature", it will return "features/foo.feature".
+    """
+    return re.sub(r'^[a-zA-Z]+:', '', uri)

smart_tests/test_runners/cypress.py

@@ -0,0 +1,46 @@
+from typing import Annotated, List, cast
+from xml.etree import ElementTree as ET
+
+import smart_tests.args4p.typer as typer
+
+from ..commands.record.tests import RecordTests
+from ..commands.subset import Subset
+from . import smart_tests
+
+
+@smart_tests.record.tests
+def record_tests(
+    client: RecordTests,
+    reports: Annotated[List[str], typer.Argument(
+        multiple=True,
+        help="Test report files to process"
+    )],
+):
+    for r in reports:
+        client.report(r)
+
+    def parse_func(p: str) -> ET.ElementTree:
+        tree = cast(ET.ElementTree, ET.parse(p))
+        for suites in tree.iter("testsuites"):
+            if len(suites) == 0:
+                continue
+            root_suite = suites.find('./testsuite[@name="Root Suite"]')
+            if root_suite is not None:
+                filepath = root_suite.get("file")
+                if filepath is not None:
+                    for suite in suites:
+                        suite.attrib.update({"filepath": filepath})
+        return tree
+
+    client.junitxml_parse_func = parse_func
+    client.run()
+
+
+@smart_tests.subset
+def subset(client: Subset):
+    # read lines as test file names
+    for t in client.stdin():
+        client.test_path(t.rstrip("\n"))
+
+    client.separator = ','
+    client.run()