smart-tests-cli 2.0.0__py3-none-any.whl

smart_tests/args4p/decorators.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+from typing import Any, Callable, Optional, Type
+
+from . import decorator
+from .argument import Argument
+from .command import Command, Group
+from .exceptions import BadConfigException
+from .option import NO_DEFAULT, Option
+from .parameter import Parameter
+
+
+def _command(
+        name: Optional[str] = None,
+        help: Optional[str] = None,
+        cls: Type[Command] = Command,
+):
+    def decorator(f: Callable) -> Command:
+        return cls(name=name, help=help, callback=f)
+
+    return decorator
+
+
+@decorator
+def command(name: Optional[str] = None, help: Optional[str] = None) -> Callable[..., Command]:
+    return _command(name, help, Command)
+
+
+@decorator
+def group(name: Optional[str] = None, help: Optional[str] = None) -> Callable[..., Group]:
+    return _command(name, help, Group)  # type: ignore
+
+
+@decorator
+def option(
+        *param_decls: str,
+        help: str | None = None, type: type | Callable | None = None, default: Any = NO_DEFAULT, required: bool = False,
+        metavar: str | None = None, multiple: bool = False, hidden: bool = False
+) -> Callable:
+    '''
+    See README.md for usage
+    '''
+
+    def decorator(f: Callable) -> Callable:
+        if len(param_decls) == 0:
+            raise BadConfigException("Variable name is required")
+
+        variable_name = param_decls[-1]
+        if len(param_decls) == 1:
+            option_names = [f"--{variable_name.replace('_', '-')}"]
+        else:
+            option_names = list(param_decls[:-1])
+
+        o = Option(
+            name=variable_name,
+            option_names=option_names,
+            help=help,
+            type=type,
+            default=default,
+            required=required,
+            metavar=metavar,
+            multiple=multiple,
+            hidden=hidden)
+
+        return _attach(f, o)
+
+    return decorator
+
+
+@decorator
+def argument(
+    name: str,
+    type: type | Callable = str,
+    multiple: bool = False,
+    required: bool = True,
+    metavar: str | None = None,
+    help: str | None = None,
+    default: Any = NO_DEFAULT
+) -> Callable:
+    '''
+    See README.md for usage
+    '''
+    a = Argument(name=name, type=type, multiple=multiple, required=required, metavar=metavar, help=help, default=default)
+    return lambda f: _attach(f, a)
+
+
+def _attach(f: Callable[..., Any], param: Parameter):
+    # depending on whether a command annotation comes before/after parameter annotations, 'f' might be
+    # a naked user-defined function or a Command instance
+    if isinstance(f, Command):
+        f.add_param(param, True)
+    else:
+        if not hasattr(f, "__args4p_params__"):
+            f.__args4p_params__ = []  # type: ignore
+
+        f.__args4p_params__.append(param)  # type: ignore
+
+    return f

smart_tests/args4p/exceptions.py

@@ -0,0 +1,12 @@
+class BadCmdLineException(Exception):
+    '''
+    Indicates that arguments given by the user are invalid.
+
+    The message gets printed, and the CLI exists with non-zero.
+    '''
+
+
+class BadConfigException(Exception):
+    '''
+    Indicates that the option/command/argument declarations are invalid
+    '''

smart_tests/args4p/option.py

@@ -0,0 +1,85 @@
+from typing import Any, Callable, Optional
+
+from .exceptions import BadCmdLineException
+from .parameter import Parameter, normalize_type
+
+
+class NoDefault:
+    '''
+    If there's no default value configured for option/argument, we use `NO_DEFAULT`.
+    In contrast, `None` is a valid and very typical default value.
+    '''
+    pass
+
+
+NO_DEFAULT = NoDefault()
+
+
+class Option(Parameter):
+    clazz = "option"
+
+    hidden: bool
+
+    def __init__(
+            self, name: Optional[str],
+            option_names: list[str],
+            help: str | None = None,
+            type: type | Callable[..., Any] | None = str,
+            default: Any = NO_DEFAULT,
+            required: bool = False,
+            metavar: str | None = None,
+            multiple: bool = False,
+            hidden: bool = False):
+        self.name = name        # type: ignore[assignment]  # once properly constructed, name is never None
+        self.option_names = option_names
+        self.help = help
+        self.type = normalize_type(type)
+        self.default = default
+        self.required = required
+        self.metavar = metavar
+        self.multiple = multiple
+        self.hidden = hidden
+
+    def append(self, existing: Any, option_name: str, args):  # args is ArgList, but typing it creates a circular import
+        '''
+        Given the current value 'existing' that represents the present value to invoke the user function with,
+        this method is called when this option was specified as 'option_name' on the command line.
+        'args' is pointing at the next argument after 'option_name', which may be the value for this option.
+        '''
+
+        if self.type == bool or self.type == Optional[bool]:
+            v = True
+        else:
+            v = args.eat(option_name)
+            try:
+                v = self.type(v)
+            except ValueError as e:
+                raise BadCmdLineException(f"Invalid value '{v}' for option '{option_name}': {str(e)}") from e
+
+        if self.multiple:
+            if existing is None:
+                existing = []
+            existing.append(v)
+            return existing
+        else:
+            return v
+
+    def attach_to_command(self, command):  # typing command makes reference circular
+        super().attach_to_command(command)
+
+        if self.metavar is None:
+            def type_name(t):
+                if hasattr(t, "__name__"):
+                    return t.__name__
+                else:
+                    return str(t)
+            self.metavar = type_name(self.type).upper()
+        if self.type == bool and self.default is NO_DEFAULT:
+            # if the flag is absent, bind the value to False, or else the function signature requires a defalut value,
+            # which is silly
+            self.default = False
+
+    def __repr__(self):
+        return (f"Option(name={self.name!r}, option_names={self.option_names!r}, help={self.help!r}, "
+                f"type={self.type.__name__!r}, default={self.default!r}, required={self.required!r}, "
+                f"metavar={self.metavar!r}, many={self.multiple!r})")

smart_tests/args4p/parameter.py

@@ -0,0 +1,84 @@
+import inspect
+import types
+from typing import Annotated, Any, Callable, Optional, Union, get_args, get_origin
+
+from smart_tests.args4p.exceptions import BadConfigException
+
+
+def to_type(p: inspect.Parameter) -> Optional[type]:
+    '''
+    Given output from inspect.signature, extract the type annotation.
+    '''
+    annotation = p.annotation
+    if annotation == inspect.Parameter.empty:
+        return None
+
+    # we expect a List[something] and we want to extract 'something'
+
+    origin = get_origin(annotation)
+    if origin is Annotated:
+        return get_args(annotation)[0]
+
+    return annotation
+
+
+def normalize_type(t) -> type:
+    if isinstance(t, types.UnionType) or get_origin(t) is Union:
+        # x|None is a common typing of a parameter that confuses args4p.
+        # we want to normalize it to just x
+        # Not sure when UnionType is used and when Union is used, but they both seem to appear
+        args = get_args(t)
+        if len(args) == 2 and args[1] is type(None):
+            return args[0]
+
+    return t
+
+
+class Parameter:
+    '''
+    Common parts of Argument and Option
+    '''
+
+    # the name of the argument, used as the variable name in the user function
+    # when created from typer.Option or typer.Argument, this is not set until attached to a command
+    name: str
+
+    multiple: bool  # True if this argument can appear multiple times
+    required: bool  # True if this argument is required
+    metavar: str | None  # the name to use in help messages for the argument value
+    help: str | None  # the help message for this argument
+    default: Any  # the default value if the argument/option is not provided
+    clazz: str  # "argument" or "option"
+
+    # convert the string argument to a value.
+    # For multiple=True, this is the type of each individual value.
+    # 'type' object itself, like 'int' is a convenient callable to do just that
+    type: type | Callable
+
+    def attach_to_command(self, command):  # typing command makes reference circular
+        def error(msg: str):
+            raise BadConfigException(
+                f"{msg} in function '{command.callback.__name__}': "
+                f"{inspect.getsourcefile(command.callback)}:{inspect.getsourcelines(command.callback)[1]}")
+
+        for name, param in inspect.signature(command.callback).parameters.items():
+            if name == self.name:
+                # we found the parameter that matches the name
+                if self.type is None:
+                    def infer_type() -> type:
+                        t = normalize_type(to_type(param))
+                        if t is None:
+                            raise error(f"Type annotation is missing on parameter '{name}'")
+                        if self.multiple:
+                            # we expect a List[something] and we want to extract 'something'
+                            if get_origin(t) is list:
+                                return get_args(t)[0]
+                            raise error(f"multiple=True requires a List[T] type annotation with parameter '{name}'")
+                        else:
+                            return t
+
+                    self.type = infer_type()
+
+                return
+
+        raise error(f"No parameter named '{self.name}' found")

smart_tests/args4p/typer/__init__.py

@@ -0,0 +1,42 @@
+# This package defines Typer-style annotation based option/command declarations
+from typing import Any, Callable
+
+from ..argument import Argument as _Argument
+from ..option import NO_DEFAULT
+from ..option import Option as _Option
+
+
+def Option(
+        *option_names: str,
+        help: str | None = None, type: type | Callable | None = None, default: Any = NO_DEFAULT, required: bool = False,
+        metavar: str | None = None, multiple: bool = False, hidden: bool = False
+) -> _Option:
+    '''
+    See README.md for usage
+    '''
+
+    return _Option(name=None, option_names=list(option_names), help=help, type=type,
+                   default=default, required=required, metavar=metavar, multiple=multiple, hidden=hidden)
+
+
+def Argument(
+        type: type | Callable = str,
+        multiple: bool = False,
+        required: bool = True,
+        metavar: str | None = None,
+        help: str | None = None,
+        default: Any = NO_DEFAULT
+) -> _Argument:
+    '''
+    See README.md for usage
+    '''
+    return _Argument(name=None, type=type, multiple=multiple, required=required, metavar=metavar, help=help, default=default)
+
+
+class Exit(Exception):
+    '''
+    Raise this exception to exit the CLI with the given exit code
+    '''
+
+    def __init__(self, code: int):
+        self.code = code

smart_tests/commands/compare/__init__.py

@@ -0,0 +1,11 @@
+from ... import args4p
+from ...app import Application
+from .subsets import subsets
+
+
+@args4p.group()
+def compare(app: Application):
+    return app
+
+
+compare.add_command(subsets)

smart_tests/commands/compare/subsets.py

@@ -0,0 +1,58 @@
+from pathlib import Path
+from typing import Annotated, List, Tuple, Union
+
+import click
+from tabulate import tabulate
+
+import smart_tests.args4p.typer as typer
+from smart_tests import args4p
+from smart_tests.app import Application
+from smart_tests.args4p.converters import path
+
+
+@args4p.command()
+def subsets(
+    app: Application,
+    file_before: Annotated[Path, typer.Argument(type=path(exists=True), help="First subset file to compare")],
+    file_after: Annotated[Path, typer.Argument(type=path(exists=True), help="Second subset file to compare")]
+):
+    """
+    Compare two subset files and display changes in test order positions
+    """
+
+    # Read files and map test paths to their indices
+    with open(file_before, 'r') as f:
+        before_tests = f.read().splitlines()
+    before_index_map = {test: idx for idx, test in enumerate(before_tests)}
+
+    with open(file_after, 'r') as f:
+        after_tests = f.read().splitlines()
+    after_index_map = {test: idx for idx, test in enumerate(after_tests)}
+
+    # List of tuples representing test order changes (before, after, diff, test)
+    rows: List[Tuple[Union[int, str], Union[int, str], Union[int, str], str]] = []
+
+    # Calculate order difference and add each test in file_after to changes
+    for after_idx, test in enumerate(after_tests):
+        if test in before_index_map:
+            before_idx = before_index_map[test]
+            diff = after_idx - before_idx
+            rows.append((before_idx + 1, after_idx + 1, diff, test))
+        else:
+            rows.append(('-', after_idx + 1, 'NEW', test))
+
+    # Add all deleted tests to changes
+    for before_idx, test in enumerate(before_tests):
+        if test not in after_index_map:
+            rows.append((before_idx + 1, '-', 'DELETED', test))
+
+    # Sort changes by the order diff
+    rows.sort(key=lambda x: (0 if isinstance(x[2], str) else 1, x[2]))
+
+    # Display results in a tabular format
+    headers = ["Before", "After", "After - Before", "Test"]
+    tabular_data = [
+        (before, after, f"{diff:+}" if isinstance(diff, int) else diff, test)
+        for before, after, diff, test in rows
+    ]
+    click.echo_via_pager(tabulate(tabular_data, headers=headers, tablefmt="github"))

smart_tests/commands/detect_flakes.py

@@ -0,0 +1,105 @@
+import os
+from enum import Enum
+from typing import Annotated
+
+import click
+
+import smart_tests.args4p.typer as typer
+from smart_tests.app import Application
+from smart_tests.args4p.command import Group
+from smart_tests.commands.test_path_writer import TestPathWriter
+from smart_tests.testpath import unparse_test_path
+from smart_tests.utils.commands import Command
+from smart_tests.utils.env_keys import REPORT_ERROR_KEY
+from smart_tests.utils.exceptions import print_error_and_die
+from smart_tests.utils.session import get_session
+from smart_tests.utils.smart_tests_client import SmartTestsClient
+from smart_tests.utils.tracking import Tracking, TrackingClient
+from smart_tests.utils.typer_types import ignorable_error
+
+
+class DetectFlakesRetryThreshold(str, Enum):
+    LOW = "LOW"
+    MEDIUM = "MEDIUM"
+    HIGH = "HIGH"
+
+    @staticmethod
+    def from_str(value: str) -> "DetectFlakesRetryThreshold":
+        for member in DetectFlakesRetryThreshold:
+            if member.value.lower() == value.lower():
+                return member
+        raise ValueError(f"Invalid value for DetectFlakesRetryThreshold: {value}")
+
+
+class DetectFlakes(TestPathWriter):
+    def __init__(
+            self,
+            app: Application,
+            session: Annotated[str, typer.Option(
+                "--session",
+                help="In the format builds/<build-name>/test_sessions/<test-session-id>",
+                metavar="SESSION",
+                required=True
+            )],
+            retry_threshold: Annotated[DetectFlakesRetryThreshold, typer.Option(
+                "--retry-threshold",
+                help="Thoroughness of how \"flake\" is detected",
+                type=DetectFlakesRetryThreshold.from_str,
+                metavar="low|medium|high"
+            )] = DetectFlakesRetryThreshold.MEDIUM,
+            test_runner: Annotated[str | None, typer.Argument()] = None,
+    ):
+        super().__init__(app)
+
+        app.test_runner = test_runner
+        self.tracking_client = TrackingClient(Command.DETECT_FLAKE, app=app)
+        self.client = SmartTestsClient(app=app, tracking_client=self.tracking_client)
+
+        self.session = session
+        self.test_session = None
+        try:
+            self.test_session = get_session(client=self.client, session=session)
+        except ValueError as e:
+            print_error_and_die(msg=str(e), tracking_client=self.tracking_client, event=Tracking.ErrorEvent.USER_ERROR)
+        except Exception as e:
+            if os.getenv(REPORT_ERROR_KEY):
+                raise e
+            else:
+                click.echo(ignorable_error(e), err=True)
+
+        if self.test_session is None:
+            raise typer.Exit(0)     # bail out
+
+        self.retry_threshold = retry_threshold
+
+    def run(self):
+        test_paths = []
+        try:
+            res = self.client.request(
+                "get",
+                "detect-flake",
+                params={
+                    "confidence": self.retry_threshold.value.upper(),
+                    "session-id": os.path.basename(self.session),
+                    "test-runner": self.app.test_runner,
+                })
+
+            res.raise_for_status()
+            test_paths = res.json().get("testPaths", [])
+            if test_paths:
+                self.print(test_paths)
+                click.echo("Trying to retry the following tests:", err=True)
+                for detail in res.json().get("testDetails", []):
+                    click.echo(f"{detail.get('reason'): {unparse_test_path(detail.get('fullTestPath'))}}", err=True)
+        except Exception as e:
+            self.tracking_client.send_error_event(
+                event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR,
+                stack_trace=str(e),
+            )
+            if os.getenv(REPORT_ERROR_KEY):
+                raise e
+            else:
+                click.echo(ignorable_error(e), err=True)
+
+
+detect_flakes = Group(name="detect-flakes", callback=DetectFlakes, help="Detect flaky tests")

smart_tests/commands/inspect/__init__.py

@@ -0,0 +1,13 @@
+from ... import args4p
+from ...app import Application
+from .model import model
+from .subset import subset
+
+
+@args4p.group(help="Inspect test and subset data")
+def inspect(app: Application):
+    return app
+
+
+inspect.add_command(model)
+inspect.add_command(subset)

smart_tests/commands/inspect/model.py

@@ -0,0 +1,52 @@
+import json
+import sys
+from http import HTTPStatus
+from typing import Annotated
+
+import click
+from requests import Response
+from tabulate import tabulate
+
+from smart_tests import args4p
+from smart_tests.app import Application
+from smart_tests.args4p import typer
+from smart_tests.utils.smart_tests_client import SmartTestsClient
+
+
+@args4p.command()
+def model(
+        app: Application,
+        is_json_format: Annotated[bool, typer.Option(
+            '--json',
+            help="display JSON format"
+        )] = False):
+    client = SmartTestsClient(app=app)
+    try:
+        res: Response = client.request("get", "model-metadata")
+
+        if res.status_code == HTTPStatus.NOT_FOUND:
+            click.echo(click.style(
+                "Model metadata currently not available for this workspace.", 'yellow'), err=True)
+            sys.exit()
+
+        res.raise_for_status()
+
+        if is_json_format:
+            display_as_json(res)
+        else:
+            display_as_table(res)
+
+    except Exception as e:
+        client.print_exception_and_recover(e, "Warning: failed to inspect model")
+
+
+def display_as_json(res: Response):
+    res_json = res.json()
+    click.echo(json.dumps(res_json, indent=2))
+
+
+def display_as_table(res: Response):
+    headers = ["Metadata", "Value"]
+    res_json = res.json()
+    rows = [["Training Cutoff Test Session ID", res_json['training_cutoff_test_session_id']]]
+    click.echo(tabulate(rows, headers, tablefmt="github"))