smart-tests-cli 2.2.0__tar.gz → 2.3.0__tar.gz

{smart_tests_cli-2.2.0 → smart_tests_cli-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: smart-tests-cli
-Version: 2.2.0
+Version: 2.3.0
 Summary: Smart Tests CLI
 Author-email: CloudBees <support@cloudbees.com>
 License: Apache Software License v2

{smart_tests_cli-2.2.0 → smart_tests_cli-2.3.0}/smart_tests/commands/record/build.py

@@ -337,7 +337,7 @@ def build(
     def report(ws: List[Workspace], build_id: str):
         org, workspace = get_org_workspace()
         click.echo(
-            f"Launchable recorded build {build_name} to workspace {org} / {workspace} with commits from "
+            f"Smart Tests recorded build {build_name} to workspace {org} / {workspace} with commits from "
             f"{len(ws)} {'repositories' if len(ws) > 1 else 'repository'}: \n")
 
         header = ["Name", "Path", "HEAD Commit"]

{smart_tests_cli-2.2.0 → smart_tests_cli-2.3.0}/smart_tests/commands/record/session.py

@@ -24,19 +24,18 @@ TEST_SESSION_NAME_RULE = re.compile("^[a-zA-Z0-9][a-zA-Z0-9_.-]*$")
 @args4p.command(help="Record session information")
 def session(
     app: Application,
-    build_name: Annotated[str, typer.Option(
+    build_name: Annotated[str | None, typer.Option(
         "--build",
         help="Build name",
         metavar="NAME",
-        required=True
-    )],
+    )] = None,
     test_suite: Annotated[str, typer.Option(
         "--test-suite",
         help="Set test suite name. A test suite is a collection of test sessions. Setting a test suite allows you to "
              "manage data over test sessions and lineages.",
         metavar="NAME",
         required=True
-    )],
+    )] = "",
     flavors: Annotated[List[KeyValue], typer.Option(
         "--flavor",
         help="Flavors",

{smart_tests_cli-2.2.0 → smart_tests_cli-2.3.0}/smart_tests/commands/subset.py

@@ -28,8 +28,9 @@ from ..testpath import FilePathNormalizer, TestPath
 from ..utils.env_keys import REPORT_ERROR_KEY
 from ..utils.fail_fast_mode import (FailFastModeValidateParams, fail_fast_mode_validate,
                                     set_fail_fast_mode, warn_and_exit_if_fail_fast_mode)
+from ..utils.input_snapshot import InputSnapshotId
 from ..utils.smart_tests_client import SmartTestsClient
-from ..utils.typer_types import Duration, Percentage, parse_duration, parse_percentage
+from ..utils.typer_types import Duration, Fraction, Percentage, parse_duration, parse_fraction, parse_percentage
 from .test_path_writer import TestPathWriter
 
 
@@ -174,6 +175,23 @@ class Subset(TestPathWriter):
                 type=fileText(mode="r"),
                 metavar="FILE"
             )] = None,
+            input_snapshot_id: Annotated[InputSnapshotId | None, InputSnapshotId.as_option()] = None,
+            print_input_snapshot_id: Annotated[bool, typer.Option(
+                "--print-input-snapshot-id",
+                help="Print the input snapshot ID returned from the server instead of the subset results"
+            )] = False,
+            bin_target: Annotated[Fraction | None, typer.Option(
+                "--bin",
+                help="Split subset into bins, e.g. --bin 1/4",
+                metavar="INDEX/COUNT",
+                type=parse_fraction
+            )] = None,
+            same_bin_files: Annotated[List[str], typer.Option(
+                "--same-bin",
+                help="Keep all tests listed in the file together when splitting; one test per line",
+                metavar="FILE",
+                multiple=True
+            )] = [],
             is_get_tests_from_guess: Annotated[bool, typer.Option(
                 "--get-tests-from-guess",
                 help="Get subset list from guessed tests"
@@ -255,9 +273,15 @@ class Subset(TestPathWriter):
         self.ignore_flaky_tests_above = ignore_flaky_tests_above
         self.prioritize_tests_failed_within_hours = prioritize_tests_failed_within_hours
         self.prioritized_tests_mapping_file = prioritized_tests_mapping_file
+        self.input_snapshot_id = input_snapshot_id.value if input_snapshot_id else None
+        self.print_input_snapshot_id = print_input_snapshot_id
+        self.bin_target = bin_target
+        self.same_bin_files = list(same_bin_files)
         self.is_get_tests_from_guess = is_get_tests_from_guess
         self.use_case = use_case
 
+        self._validate_print_input_snapshot_option()
+
         self.file_path_normalizer = FilePathNormalizer(base_path, no_base_path_inference=no_base_path_inference)
 
         self.test_paths: list[list[dict[str, str]]] = []
@@ -305,7 +329,7 @@ class Subset(TestPathWriter):
         """
 
         # To avoid the cli continue to wait from stdin
-        if self.is_get_tests_from_previous_sessions or self.is_get_tests_from_guess:
+        if self._should_skip_stdin():
             return []
 
         if sys.stdin.isatty():
@@ -404,8 +428,103 @@ class Subset(TestPathWriter):
         if self.use_case:
             payload['changesUnderTest'] = self.use_case.value
 
+        if self.input_snapshot_id is not None:
+            payload['subsettingId'] = self.input_snapshot_id
+
+        split_subset = self._build_split_subset_payload()
+        if split_subset:
+            payload['splitSubset'] = split_subset
+
         return payload
 
+    def _build_split_subset_payload(self) -> dict[str, Any] | None:
+        if self.bin_target is None:
+            if self.same_bin_files:
+                print_error_and_die(
+                    "--same-bin option requires --bin option.\nPlease set --bin option to use --same-bin",
+                    self.tracking_client,
+                    Tracking.ErrorEvent.USER_ERROR,
+                )
+            return None
+
+        slice_index = self.bin_target.numerator
+        slice_count = self.bin_target.denominator
+
+        if slice_index <= 0 or slice_count <= 0:
+            print_error_and_die(
+                "Invalid --bin value. Both index and count must be positive integers.",
+                self.tracking_client,
+                Tracking.ErrorEvent.USER_ERROR,
+            )
+
+        if slice_count < slice_index:
+            print_error_and_die(
+                "Invalid --bin value. The numerator cannot exceed the denominator.",
+                self.tracking_client,
+                Tracking.ErrorEvent.USER_ERROR,
+            )
+
+        same_bins = self._read_same_bin_files()
+
+        return {
+            "sliceIndex": slice_index,
+            "sliceCount": slice_count,
+            "sameBins": same_bins,
+        }
+
+    def _read_same_bin_files(self) -> list[list[TestPath]]:
+        if not self.same_bin_files:
+            return []
+
+        formatter = self.same_bin_formatter
+        if formatter is None:
+            print_error_and_die(
+                "--same-bin is not supported for this test runner.",
+                self.tracking_client,
+                Tracking.ErrorEvent.USER_ERROR,
+            )
+
+        same_bins: list[list[TestPath]] = []
+        seen_tests: set[str] = set()
+
+        for same_bin_file in self.same_bin_files:
+            try:
+                with open(same_bin_file, "r", encoding="utf-8") as fp:
+                    tests = [line.strip() for line in fp if line.strip()]
+            except OSError as exc:
+                print_error_and_die(
+                    f"Failed to read --same-bin file '{same_bin_file}': {exc}",
+                    self.tracking_client,
+                    Tracking.ErrorEvent.USER_ERROR,
+                )
+
+            unique_tests = list(dict.fromkeys(tests))
+
+            group: list[TestPath] = []
+            for test in unique_tests:
+                if test in seen_tests:
+                    print_error_and_die(
+                        f"Error: test '{test}' is listed in multiple --same-bin files.",
+                        self.tracking_client,
+                        Tracking.ErrorEvent.USER_ERROR,
+                    )
+                seen_tests.add(test)
+
+                # For type check
+                assert formatter is not None, "--same -bin is not supported for this test runner"
+                formatted = formatter(test)
+                if not formatted:
+                    print_error_and_die(
+                        f"Failed to parse test '{test}' from --same-bin file {same_bin_file}",
+                        self.tracking_client,
+                        Tracking.ErrorEvent.USER_ERROR,
+                    )
+                group.append(formatted)
+
+            same_bins.append(group)
+
+        return same_bins
+
     def _collect_potential_test_files(self):
         LOOSE_TEST_FILE_PATTERN = r'(\.(test|spec)\.|_test\.|Test\.|Spec\.|test/|tests/|__tests__/|src/test/)'
         EXCLUDE_PATTERN = r'(BUILD|Makefile|Dockerfile|LICENSE|.gitignore|.gitkeep|.keep|id_rsa|rsa|blank|taglib)|\.(xml|json|jsonl|txt|yml|yaml|toml|md|png|jpg|jpeg|gif|svg|sql|html|css|graphql|proto|gz|zip|rz|bzl|conf|config|snap|pem|crt|key|lock|jpi|hpi|jelly|properties|jar|ini|mod|sum|bmp|env|envrc|sh)$'  # noqa E501
@@ -463,13 +582,75 @@ class Subset(TestPathWriter):
                 e, "Warning: the service failed to subset. Falling back to running all tests")
             return SubsetResult.from_test_paths(self.test_paths)
 
+    def _requires_test_input(self) -> bool:
+        return (
+            self.input_snapshot_id is None
+            and not self.is_get_tests_from_previous_sessions  # noqa: W503
+            and len(self.test_paths) == 0  # noqa: W503
+        )
+
+    def _should_skip_stdin(self) -> bool:
+        if self.is_get_tests_from_previous_sessions or self.is_get_tests_from_guess:
+            return True
+
+        if self.input_snapshot_id is not None:
+            if not sys.stdin.isatty():
+                warn_and_exit_if_fail_fast_mode(
+                    "Warning: --input-snapshot-id is set so stdin will be ignored."
+                )
+            return True
+        return False
+
+    def _validate_print_input_snapshot_option(self):
+        if not self.print_input_snapshot_id:
+            return
+
+        conflicts: list[str] = []
+        option_checks = [
+            ("--target", self.target is not None),
+            ("--time", self.time is not None),
+            ("--confidence", self.confidence is not None),
+            ("--goal-spec", self.goal_spec is not None),
+            ("--rest", self.rest is not None),
+            ("--bin", self.bin_target is not None),
+            ("--same-bin", bool(self.same_bin_files)),
+            ("--ignore-new-tests", self.ignore_new_tests),
+            ("--ignore-flaky-tests-above", self.ignore_flaky_tests_above is not None),
+            ("--prioritize-tests-failed-within-hours", self.prioritize_tests_failed_within_hours is not None),
+            ("--prioritized-tests-mapping", self.prioritized_tests_mapping_file is not None),
+            ("--get-tests-from-previous-sessions", self.is_get_tests_from_previous_sessions),
+            ("--get-tests-from-guess", self.is_get_tests_from_guess),
+            ("--output-exclusion-rules", self.is_output_exclusion_rules),
+            ("--non-blocking", self.is_non_blocking),
+        ]
+
+        for option_name, is_set in option_checks:
+            if is_set:
+                conflicts.append(option_name)
+
+        if conflicts:
+            conflict_list = ", ".join(conflicts)
+            print_error_and_die(
+                f"--print-input-snapshot-id cannot be used with {conflict_list}.",
+                self.tracking_client,
+                Tracking.ErrorEvent.USER_ERROR,
+            )
+
+    def _print_input_snapshot_id_value(self, subset_result: SubsetResult):
+        if not subset_result.subset_id:
+            raise click.ClickException(
+                "Subset request did not return an input snapshot ID. Please re-run the command."
+            )
+
+        click.echo(subset_result.subset_id)
+
     def run(self):
         """called after tests are scanned to compute the optimized order"""
 
         if self.is_get_tests_from_guess:
             self._collect_potential_test_files()
 
-        if not self.is_get_tests_from_previous_sessions and len(self.test_paths) == 0:
+        if self._requires_test_input():
             if self.input_given:
                 print_error_and_die("ERROR: Given arguments did not match any tests. They appear to be incorrect/non-existent.", tracking_client, Tracking.ErrorEvent.USER_ERROR)  # noqa E501
             else:
@@ -488,6 +669,12 @@ class Subset(TestPathWriter):
 
         if len(subset_result.subset) == 0:
             warn_and_exit_if_fail_fast_mode("Error: no tests found matching the path.")
+            if self.print_input_snapshot_id:
+                self._print_input_snapshot_id_value(subset_result)
+            return
+
+        if self.print_input_snapshot_id:
+            self._print_input_snapshot_id_value(subset_result)
             return
 
         # TODO(Konboi): split subset isn't provided for smart-tests initial release

{smart_tests_cli-2.2.0 → smart_tests_cli-2.3.0}/smart_tests/commands/test_path_writer.py

@@ -1,5 +1,5 @@
 from os.path import join
-from typing import Callable, Dict, List
+from typing import Callable, List
 
 import click
 
@@ -19,7 +19,7 @@ class TestPathWriter(object):
 
     def __init__(self, app: Application):
         self.formatter = self.default_formatter
-        self._same_bin_formatter: Callable[[str], Dict[str, str]] | None = None
+        self._same_bin_formatter: Callable[[str], TestPath] | None = None
         self.separator = "\n"
         self.app = app
 
@@ -43,9 +43,9 @@ class TestPathWriter(object):
                                        for t in test_paths))
 
     @property
-    def same_bin_formatter(self) -> Callable[[str], Dict[str, str]] | None:
+    def same_bin_formatter(self) -> Callable[[str], TestPath] | None:
         return self._same_bin_formatter
 
     @same_bin_formatter.setter
-    def same_bin_formatter(self, v: Callable[[str], Dict[str, str]]):
+    def same_bin_formatter(self, v: Callable[[str], TestPath]):
         self._same_bin_formatter = v

{smart_tests_cli-2.2.0 → smart_tests_cli-2.3.0}/smart_tests/commands/verify.py

@@ -7,13 +7,13 @@ from typing import List
 import click
 
 import smart_tests.args4p.typer as typer
-from smart_tests.utils.commands import Command
-from smart_tests.utils.env_keys import TOKEN_KEY
 from smart_tests.utils.tracking import Tracking, TrackingClient
 
 from .. import args4p
 from ..app import Application
-from ..utils.authentication import get_org_workspace
+from ..utils.authentication import ensure_org_workspace, get_org_workspace
+from ..utils.commands import Command
+from ..utils.env_keys import TOKEN_KEY
 from ..utils.java import get_java_command
 from ..utils.smart_tests_client import SmartTestsClient
 from ..utils.typer_types import emoji
@@ -82,18 +82,8 @@ def verify(app_instance: Application):
     click.echo("Java command: " + repr(java))
     click.echo("smart-tests version: " + repr(version))
 
-    if org is None or workspace is None:
-        msg = (
-            "Could not identify Smart Tests organization/workspace. "
-            "Please confirm if you set SMART_TESTS_TOKEN or SMART_TESTS_ORGANIZATION and SMART_TESTS_WORKSPACE "
-            "environment variables"
-        )
-        tracking_client.send_error_event(
-            event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR,
-            stack_trace=msg
-        )
-        click.secho(msg, fg='red', err=True)
-        raise typer.Exit(1)
+    # raise an error here after we print out the basic diagnostics if LAUNCHABLE_TOKEN is not set.
+    ensure_org_workspace()
 
     try:
         res = client.request("get", "verification")

{smart_tests_cli-2.2.0 → smart_tests_cli-2.3.0}/smart_tests/test_runners/file.py

@@ -2,19 +2,9 @@
 # The most bare-bone versions of the test runner support
 #
 
-from ..commands.subset import Subset
 from . import smart_tests
 
-
-@smart_tests.subset
-def subset(client: Subset):
-    # read lines as test file names
-    for t in client.stdin():
-        client.test_path(t.rstrip("\n"))
-
-    client.run()
-
-
+subset = smart_tests.CommonSubsetImpls(__name__).scan_stdin()
 record_tests = smart_tests.CommonRecordTestImpls(__name__).file_profile_report_files()
 smart_tests.CommonDetectFlakesImpls(__name__).detect_flakes()
 # split_subset = launchable.CommonSplitSubsetImpls(__name__).split_subset()

{smart_tests_cli-2.2.0 → smart_tests_cli-2.3.0}/smart_tests/test_runners/go_test.py

@@ -41,6 +41,7 @@ def subset(client: Subset):
             test_cases = []
     client.formatter = lambda x: f"^{x[1]['name']}$"
     client.separator = '|'
+    client.same_bin_formatter = format_same_bin
     client.run()
 
 

{smart_tests_cli-2.2.0 → smart_tests_cli-2.3.0}/smart_tests/test_runners/gradle.py

@@ -77,6 +77,8 @@ def subset(
         client.formatter = lambda x: f"--tests {x[0]['name']}"
         client.separator = ' '
 
+    client.same_bin_formatter = lambda s: [{"type": "class", "name": s}]
+
     client.run()
 
 

{smart_tests_cli-2.2.0 → smart_tests_cli-2.3.0}/smart_tests/test_runners/jasmine.py

@@ -27,13 +27,7 @@ def record_tests(
     client.run()
 
 
-@smart_tests.subset
-def subset(client):
-    # read lines as test file names
-    for t in client.stdin():
-        client.test_path(t.rstrip("\n"))
-
-    client.run()
+subset = smart_tests.CommonSubsetImpls(__name__).scan_stdin()
 
 
 class JSONReportParser:

smart_tests_cli-2.3.0/smart_tests/test_runners/karma.py

@@ -0,0 +1,174 @@
+import json
+from typing import Annotated, Dict, Generator, List
+
+import click
+
+from ..args4p import typer
+from ..commands.record.case_event import CaseEvent
+from ..testpath import TestPath
+from . import smart_tests
+
+
+@smart_tests.record.tests
+def record_tests(client,
+                 reports: Annotated[List[str], typer.Argument(multiple=True, help="Test report files to process")],
+                 ):
+    client.parse_func = JSONReportParser(client).parse_func
+
+    for r in reports:
+        client.report(r)
+    client.run()
+
+
+@smart_tests.subset
+def subset(client, _with: Annotated[str | None, typer.Option(
+        '--with', help='Format output for specific test runner (e.g., "ng" for Angular CLI)')] = None, ):
+    """
+    Usage:
+        find src -name "*.spec.ts" -o -name "*.spec.js" > test-list.txt
+        cat test-list.txt | launchable subset --target 10% karma
+
+        # Output in ng test format
+        find src -name "*.spec.ts" | launchable subset --target 10% karma --with ng
+    """
+    for t in client.stdin():
+        path = t.strip()
+        if path:
+            client.test_path(path)
+
+    if _with == 'ng':
+        client.formatter = lambda x: "--include={}".format(x[0]['name'])
+        client.separator = " "
+
+    client.run()
+
+
+class JSONReportParser:
+    """
+    Sample Karma report format:
+    {
+      "browsers": {...},
+      "result": {
+        "24461741": [
+          {
+            "fullName": "path/to/spec.ts should do something",
+            "description": "should do something",
+            "id": "spec0",
+            "log": [],
+            "skipped": false,
+            "disabled": false,
+            "pending": false,
+            "success": true,
+            "suite": [
+              "path/to/spec.ts"
+            ],
+            "time": 92,
+            "executedExpectationsCount": 1,
+            "passedExpectations": [...],
+            "properties": null
+          }
+        ]
+      },
+      "summary": {...}
+    }
+    """
+
+    def __init__(self, client):
+        self.client = client
+
+    def parse_func(self, report_file: str) -> Generator[Dict, None, None]:  # type: ignore
+        data: Dict
+        with open(report_file, 'r') as json_file:
+            try:
+                data = json.load(json_file)
+            except Exception:
+                click.echo(
+                    click.style("Error: Failed to load Json report file: {}".format(report_file), fg='red'), err=True)
+                return
+
+        if not self._validate_report_format(data):
+            click.echo(
+                "Error: {} does not appear to be valid Karma report format. "
+                "Make sure you are using karma-json-reporter or a compatible reporter.".format(
+                    report_file), err=True)
+            return
+
+        results = data.get("result", {})
+        for browser_id, specs in results.items():
+            if isinstance(specs, list):
+                for event in self._parse_specs(specs):
+                    yield event
+
+    def _validate_report_format(self, data: Dict) -> bool:
+        if not isinstance(data, dict):
+            return False
+
+        if "result" not in data:
+            return False
+
+        results = data.get("result", {})
+        if not isinstance(results, dict):
+            return False
+
+        for browser_id, specs in results.items():
+            if not isinstance(specs, list):
+                return False
+
+            for spec in specs:
+                if not isinstance(spec, dict):
+                    return False
+                # Check for required fields
+                if "suite" not in spec or "time" not in spec:
+                    return False
+                # Field suite should have at least one element (filename)
+                suite = spec.get("suite", [])
+                if not isinstance(suite, list) or len(suite) == 0:
+                    return False
+
+        return True
+
+    def _parse_specs(self, specs: List[Dict]) -> List[Dict]:
+        events: List[Dict] = []
+
+        for spec in specs:
+            # TODO:
+            # In NextWorld, test filepaths are included in the suite tag
+            # But generally in a Karma test report, a suite tag can be any string
+            # For the time being let's get filepaths from the suite tag,
+            # until we find a standard way to include filepaths in the test reports
+            suite = spec.get("suite", [])
+            filename = suite[0] if suite else ""
+
+            test_path: TestPath = [
+                self.client.make_file_path_component(filename),
+                {"type": "testcase", "name": spec.get("fullName", spec.get("description", ""))}
+            ]
+
+            duration_msec = spec.get("time", 0)
+            status = self._case_event_status_from_spec(spec)
+            stderr = self._parse_stderr(spec)
+
+            events.append(CaseEvent.create(
+                test_path=test_path,
+                duration_secs=duration_msec / 1000 if duration_msec else 0,
+                status=status,
+                stderr=stderr
+            ))
+
+        return events
+
+    def _case_event_status_from_spec(self, spec: Dict) -> int:
+        if spec.get("skipped", False) or spec.get("disabled", False) or spec.get("pending", False):
+            return CaseEvent.TEST_SKIPPED
+
+        if spec.get("success", False):
+            return CaseEvent.TEST_PASSED
+        else:
+            return CaseEvent.TEST_FAILED
+
+    def _parse_stderr(self, spec: Dict) -> str:
+        log_messages = spec.get("log", [])
+        if not log_messages:
+            return ""
+
+        return "\n".join(str(msg) for msg in log_messages if msg)

{smart_tests_cli-2.2.0 → smart_tests_cli-2.3.0}/smart_tests/test_runners/maven.py

@@ -126,6 +126,8 @@ def subset(
         for root in source_roots:
             client.scan(root, '**/*', file2test)
 
+    client.same_bin_formatter = lambda s: [{"type": "class", "name": s}]
+
     client.run()
 
 

{smart_tests_cli-2.2.0 → smart_tests_cli-2.3.0}/smart_tests/test_runners/playwright.py

@@ -13,7 +13,6 @@ import smart_tests.args4p.typer as typer
 from ..args4p.exceptions import BadCmdLineException
 from ..commands.record.case_event import CaseEvent, CaseEventGenerator
 from ..commands.record.tests import RecordTests
-from ..commands.subset import Subset
 from ..testpath import TestPath
 from . import smart_tests
 
@@ -63,13 +62,7 @@ def record_tests(
     client.run()
 
 
-@smart_tests.subset
-def subset(client: Subset):
-    # read lines as test file names
-    for t in client.stdin():
-        client.test_path(t.rstrip("\n"))
-
-    client.run()
+subset = smart_tests.CommonSubsetImpls(__name__).scan_stdin()
 
 
 class JSONReportParser:

{smart_tests_cli-2.2.0 → smart_tests_cli-2.3.0}/smart_tests/test_runners/prove.py

@@ -7,7 +7,6 @@ import smart_tests.args4p.typer as typer
 
 from ..args4p.exceptions import BadCmdLineException
 from ..commands.record.tests import RecordTests
-from ..commands.subset import Subset
 from ..testpath import TestPath
 from . import smart_tests
 
@@ -19,13 +18,7 @@ def remove_leading_number_and_dash(input_string: str) -> str:
     return result
 
 
-@smart_tests.subset
-def subset(client: Subset):
-    # read lines as test file names
-    for t in client.stdin():
-        client.test_path(t.rstrip("\n"))
-
-    client.run()
+subset = smart_tests.CommonSubsetImpls(__name__).scan_stdin()
 
 
 @smart_tests.record.tests