schemathesis 3.39.15__py3-none-any.whl → 4.0.0__py3-none-any.whl

schemathesis/cli/commands/run/context.py

@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Callable, Generator
+
+from schemathesis.cli.commands.run.events import LoadingFinished
+from schemathesis.config import ProjectConfig
+from schemathesis.core.failures import Failure
+from schemathesis.core.result import Err, Ok
+from schemathesis.core.transforms import UNRESOLVABLE
+from schemathesis.core.transport import Response
+from schemathesis.engine import Status, events
+from schemathesis.engine.recorder import CaseNode, ScenarioRecorder
+from schemathesis.generation.case import Case
+from schemathesis.schemas import APIOperation
+
+if TYPE_CHECKING:
+    from schemathesis.generation.stateful.state_machine import ExtractionFailure
+
+
+@dataclass
+class Statistic:
+    """Running statistics about test execution."""
+
+    failures: dict[str, dict[str, GroupedFailures]]
+    # Track first case_id where each unique failure was found
+    unique_failures_map: dict[Failure, str]
+
+    extraction_failures: set[ExtractionFailure]
+
+    tested_operations: set[str]
+
+    total_cases: int
+    cases_with_failures: int
+    cases_without_checks: int
+
+    __slots__ = (
+        "failures",
+        "unique_failures_map",
+        "extraction_failures",
+        "tested_operations",
+        "total_cases",
+        "cases_with_failures",
+        "cases_without_checks",
+    )
+
+    def __init__(self) -> None:
+        self.failures = {}
+        self.unique_failures_map = {}
+        self.extraction_failures = set()
+        self.tested_operations = set()
+        self.total_cases = 0
+        self.cases_with_failures = 0
+        self.cases_without_checks = 0
+
+    def on_scenario_finished(self, recorder: ScenarioRecorder) -> None:
+        """Update statistics and store failures from a new batch of checks."""
+        from schemathesis.generation.stateful.state_machine import ExtractionFailure
+
+        failures = self.failures.get(recorder.label, {})
+
+        self.total_cases += len(recorder.cases)
+
+        extraction_failures = set()
+
+        def collect_history(node: CaseNode, response: Response) -> list[tuple[Case, Response]]:
+            history = [(node.value, response)]
+            current = node
+            while current.parent_id is not None:
+                current_response = recorder.find_response(case_id=current.parent_id)
+                # We need a response to get there, so it should be present
+                assert current_response is not None
+                current = recorder.cases[current.parent_id]
+                history.append((current.value, current_response))
+            return history
+
+        for case_id, case in recorder.cases.items():
+            checks = recorder.checks.get(case_id, [])
+
+            if not checks:
+                self.cases_without_checks += 1
+                continue
+
+            self.tested_operations.add(case.value.operation.label)
+            has_failures = False
+            current_case_failures = []
+            last_failure_info = None
+
+            for check in checks:
+                if check.failure_info is not None:
+                    failure = check.failure_info.failure
+
+                    # Check if this is a new unique failure
+                    if failure not in self.unique_failures_map:
+                        last_failure_info = check.failure_info
+                        self.unique_failures_map[failure] = case_id
+                        current_case_failures.append(failure)
+                        has_failures = True
+                    else:
+                        # This failure was already seen - skip it
+                        continue
+
+            if current_case_failures:
+                assert last_failure_info is not None
+                failures[case_id] = GroupedFailures(
+                    case_id=case_id,
+                    code_sample=last_failure_info.code_sample,
+                    failures=current_case_failures,
+                    response=recorder.interactions[case_id].response,
+                )
+
+            if has_failures:
+                self.cases_with_failures += 1
+
+            if case.transition is None:
+                continue
+            transition = case.transition
+            parent = recorder.cases[transition.parent_id]
+            response = recorder.find_response(case_id=parent.value.id)
+            # We need a response to get there, so it should be present
+            assert response is not None
+
+            for params in transition.parameters.values():
+                for parameter, extracted in params.items():
+                    if isinstance(extracted.value, Ok) and extracted.value.ok() is UNRESOLVABLE:
+                        history = collect_history(parent, response)
+                        extraction_failures.add(
+                            ExtractionFailure(
+                                id=transition.id,
+                                case_id=case_id,
+                                source=parent.value.operation.label,
+                                target=case.value.operation.label,
+                                parameter_name=parameter,
+                                expression=extracted.definition,
+                                history=history,
+                                response=response,
+                                error=None,
+                            )
+                        )
+                    elif isinstance(extracted.value, Err):
+                        history = collect_history(parent, response)
+                        extraction_failures.add(
+                            ExtractionFailure(
+                                id=transition.id,
+                                case_id=case_id,
+                                source=parent.value.operation.label,
+                                target=case.value.operation.label,
+                                parameter_name=parameter,
+                                expression=extracted.definition,
+                                history=history,
+                                response=response,
+                                error=extracted.value.err(),
+                            )
+                        )
+
+        if failures:
+            for group in failures.values():
+                group.failures = sorted(set(group.failures))
+            self.failures[recorder.label] = failures
+
+        if extraction_failures:
+            self.extraction_failures.update(extraction_failures)
+
+
+@dataclass
+class GroupedFailures:
+    """Represents failures grouped by case ID."""
+
+    case_id: str
+    code_sample: str
+    failures: list[Failure]
+    response: Response | None
+
+    __slots__ = ("case_id", "code_sample", "failures", "response")
+
+
+@dataclass
+class ExecutionContext:
+    """Storage for the current context of the execution."""
+
+    config: ProjectConfig
+    find_operation_by_label: Callable[[str], APIOperation | None] | None = None
+    statistic: Statistic = field(default_factory=Statistic)
+    exit_code: int = 0
+    initialization_lines: list[str | Generator[str, None, None]] = field(default_factory=list)
+    summary_lines: list[str | Generator[str, None, None]] = field(default_factory=list)
+
+    def add_initialization_line(self, line: str | Generator[str, None, None]) -> None:
+        self.initialization_lines.append(line)
+
+    def add_summary_line(self, line: str | Generator[str, None, None]) -> None:
+        self.summary_lines.append(line)
+
+    def on_event(self, event: events.EngineEvent) -> None:
+        if isinstance(event, LoadingFinished):
+            self.find_operation_by_label = event.find_operation_by_label
+        if isinstance(event, events.ScenarioFinished):
+            self.statistic.on_scenario_finished(event.recorder)
+        elif isinstance(event, events.NonFatalError) or (
+            isinstance(event, events.PhaseFinished)
+            and event.phase.is_enabled
+            and event.status in (Status.FAILURE, Status.ERROR)
+        ):
+            self.exit_code = 1

schemathesis/cli/commands/run/events.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import time
+import uuid
+from typing import Callable
+
+from schemathesis.config import ProjectConfig
+from schemathesis.core import Specification
+from schemathesis.engine import events
+from schemathesis.schemas import APIOperation, ApiStatistic
+
+
+class LoadingStarted(events.EngineEvent):
+    __slots__ = ("id", "timestamp", "location")
+
+    def __init__(self, *, location: str) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.location = location
+
+
+class LoadingFinished(events.EngineEvent):
+    __slots__ = (
+        "id",
+        "timestamp",
+        "location",
+        "duration",
+        "base_url",
+        "base_path",
+        "specification",
+        "statistic",
+        "schema",
+        "config",
+        "find_operation_by_label",
+    )
+
+    def __init__(
+        self,
+        *,
+        location: str,
+        start_time: float,
+        base_url: str,
+        base_path: str,
+        specification: Specification,
+        statistic: ApiStatistic,
+        schema: dict,
+        config: ProjectConfig,
+        find_operation_by_label: Callable[[str], APIOperation | None],
+    ) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.location = location
+        self.duration = self.timestamp - start_time
+        self.base_url = base_url
+        self.specification = specification
+        self.statistic = statistic
+        self.schema = schema
+        self.base_path = base_path
+        self.config = config
+        self.find_operation_by_label = find_operation_by_label

schemathesis/cli/commands/run/executor.py

@@ -0,0 +1,157 @@
+from __future__ import annotations
+
+import sys
+from typing import Any, Callable
+
+import click
+
+from schemathesis.cli.commands.run.context import ExecutionContext
+from schemathesis.cli.commands.run.events import LoadingFinished, LoadingStarted
+from schemathesis.cli.commands.run.handlers import display_handler_error
+from schemathesis.cli.commands.run.handlers.base import EventHandler
+from schemathesis.cli.commands.run.handlers.cassettes import CassetteWriter
+from schemathesis.cli.commands.run.handlers.junitxml import JunitXMLHandler
+from schemathesis.cli.commands.run.handlers.output import OutputHandler
+from schemathesis.cli.commands.run.loaders import load_schema
+from schemathesis.cli.ext.fs import open_file
+from schemathesis.config import ProjectConfig, ReportFormat
+from schemathesis.core.errors import LoaderError
+from schemathesis.core.fs import file_exists
+from schemathesis.engine import from_schema
+from schemathesis.engine.events import EventGenerator, FatalError, Interrupted
+
+CUSTOM_HANDLERS: list[type[EventHandler]] = []
+
+
+def handler() -> Callable[[type], None]:
+    """Register a new CLI event handler."""
+
+    def _wrapper(cls: type) -> None:
+        CUSTOM_HANDLERS.append(cls)
+
+    return _wrapper
+
+
+def execute(
+    *,
+    location: str,
+    config: ProjectConfig,
+    filter_set: dict[str, Any],
+    args: list[str],
+    params: dict[str, Any],
+) -> None:
+    event_stream = into_event_stream(location=location, config=config, filter_set=filter_set)
+    _execute(event_stream, config=config, args=args, params=params)
+
+
+MISSING_BASE_URL_MESSAGE = "The `--url` option is required when specifying a schema via a file."
+
+
+def into_event_stream(*, location: str, config: ProjectConfig, filter_set: dict[str, Any]) -> EventGenerator:
+    # The whole engine idea is that it communicates with the outside via events, so handlers can react to them
+    # For this reason, even schema loading is done via a separate set of events.
+    loading_started = LoadingStarted(location=location)
+    yield loading_started
+
+    try:
+        schema = load_schema(location=location, config=config)
+        # Schemas don't (yet?) use configs for deciding what operations should be tested, so
+        # a separate FilterSet passed there. It combines both config file filters + CLI options
+        schema.filter_set = schema.config.operations.create_filter_set(**filter_set)
+        if file_exists(location) and schema.config.base_url is None:
+            raise click.UsageError(MISSING_BASE_URL_MESSAGE)
+    except KeyboardInterrupt:
+        yield Interrupted(phase=None)
+        return
+    except LoaderError as exc:
+        yield FatalError(exception=exc)
+        return
+
+    yield LoadingFinished(
+        location=location,
+        start_time=loading_started.timestamp,
+        base_url=schema.get_base_url(),
+        specification=schema.specification,
+        statistic=schema.statistic,
+        schema=schema.raw_schema,
+        config=schema.config,
+        base_path=schema.base_path,
+        find_operation_by_label=schema.find_operation_by_label,
+    )
+
+    try:
+        yield from from_schema(schema).execute()
+    except Exception as exc:
+        yield FatalError(exception=exc)
+
+
+def initialize_handlers(
+    *,
+    config: ProjectConfig,
+    args: list[str],
+    params: dict[str, Any],
+) -> list[EventHandler]:
+    """Create event handlers based on run configuration."""
+    handlers: list[EventHandler] = []
+
+    if config.reports.junit.enabled:
+        path = config.reports.get_path(ReportFormat.JUNIT)
+        open_file(path)
+        handlers.append(JunitXMLHandler(path))
+    for format, report in (
+        (ReportFormat.VCR, config.reports.vcr),
+        (ReportFormat.HAR, config.reports.har),
+    ):
+        if report.enabled:
+            path = config.reports.get_path(format)
+            open_file(path)
+            handlers.append(CassetteWriter(format=format, path=path, config=config))
+
+    for custom_handler in CUSTOM_HANDLERS:
+        handlers.append(custom_handler(*args, **params))
+
+    handlers.append(OutputHandler(config=config))
+
+    return handlers
+
+
+def _execute(
+    event_stream: EventGenerator,
+    *,
+    config: ProjectConfig,
+    args: list[str],
+    params: dict[str, Any],
+) -> None:
+    handlers: list[EventHandler] = []
+    ctx: ExecutionContext | None = None
+
+    def shutdown() -> None:
+        if ctx is not None:
+            for _handler in handlers:
+                _handler.shutdown(ctx)
+
+    try:
+        handlers = initialize_handlers(config=config, args=args, params=params)
+        ctx = ExecutionContext(config=config)
+
+        for handler in handlers:
+            handler.start(ctx)
+
+        for event in event_stream:
+            ctx.on_event(event)
+            for handler in handlers:
+                try:
+                    handler.handle_event(ctx, event)
+                except Exception as exc:
+                    # `Abort` is used for handled errors
+                    if not isinstance(exc, click.Abort):
+                        display_handler_error(handler, exc)
+                    raise
+    except Exception as exc:
+        if isinstance(exc, click.Abort):
+            # To avoid showing "Aborted!" message, which is the default behavior in Click
+            sys.exit(1)
+        raise
+    finally:
+        shutdown()
+    sys.exit(ctx.exit_code)

schemathesis/cli/commands/run/filters.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from functools import partial
+from typing import Callable, Literal
+
+import click
+
+from schemathesis.cli.ext.groups import grouped_option
+
+
+def _with_filter(*, by: str, mode: Literal["include", "exclude"], modifier: Literal["regex"] | None) -> Callable:
+    """Generate a CLI option for filtering API operations."""
+    param = f"--{mode}-{by}"
+    action = "include in" if mode == "include" else "exclude from"
+    prop = {
+        "operation-id": "ID",
+        "name": "Operation name",
+    }.get(by, by.capitalize())
+    callback = None
+    if modifier:
+        param += f"-{modifier}"
+        prop += " pattern"
+    else:
+        callback = partial(validate_filter, arg_name=param)
+    help_text = f"{prop} to {action} testing."
+    return grouped_option(
+        param,
+        help=help_text,
+        type=str,
+        multiple=modifier is None,
+        callback=callback,
+        hidden=True,
+    )
+
+
+def validate_filter(
+    ctx: click.core.Context, param: click.core.Parameter, raw_value: list[str], arg_name: str
+) -> list[str]:
+    if len(raw_value) != len(set(raw_value)):
+        duplicates = ",".join(sorted({value for value in raw_value if raw_value.count(value) > 1}))
+        raise click.UsageError(f"Duplicate values are not allowed for `{arg_name}`: {duplicates}")
+    return raw_value
+
+
+_BY_VALUES = ("operation-id", "tag", "name", "method", "path")
+
+
+def with_filters(command: Callable) -> Callable:
+    for by in _BY_VALUES:
+        for mode in ("exclude", "include"):
+            for modifier in ("regex", None):
+                command = _with_filter(by=by, mode=mode, modifier=modifier)(command)  # type: ignore[arg-type]
+    return command

schemathesis/cli/commands/run/handlers/__init__.py

@@ -0,0 +1,46 @@
+import click
+
+from schemathesis.cli.commands.run.handlers.base import EventHandler
+from schemathesis.cli.commands.run.handlers.cassettes import CassetteWriter
+from schemathesis.cli.commands.run.handlers.junitxml import JunitXMLHandler
+from schemathesis.cli.commands.run.handlers.output import OutputHandler
+from schemathesis.cli.constants import EXTENSIONS_DOCUMENTATION_URL, ISSUE_TRACKER_URL
+from schemathesis.core.errors import format_exception
+
+__all__ = [
+    "EventHandler",
+    "CassetteWriter",
+    "JunitXMLHandler",
+    "OutputHandler",
+    "display_handler_error",
+]
+
+
+def is_built_in_handler(handler: EventHandler) -> bool:
+    # Look for exact instances, not subclasses
+    return any(type(handler) is class_ for class_ in (CassetteWriter, JunitXMLHandler, OutputHandler))
+
+
+def display_handler_error(handler: EventHandler, exc: Exception) -> None:
+    """Display error that happened within."""
+    is_built_in = is_built_in_handler(handler)
+    if is_built_in:
+        click.secho("Internal Error", fg="red", bold=True)
+        click.secho("\nSchemathesis encountered an unexpected issue.")
+        message = format_exception(exc, with_traceback=True)
+    else:
+        click.secho("CLI Handler Error", fg="red", bold=True)
+        click.echo(
+            f"\nAn error occurred within your custom CLI handler `{click.style(handler.__class__.__name__, bold=True)}`."
+        )
+        message = format_exception(exc, with_traceback=True, skip_frames=1)
+    click.secho(f"\n{message}", fg="red")
+    if is_built_in:
+        click.echo(
+            f"\nWe apologize for the inconvenience. This appears to be an internal issue.\n"
+            f"Please consider reporting this error to our issue tracker:\n\n  {ISSUE_TRACKER_URL}."
+        )
+    else:
+        click.echo(
+            f"\nFor more information on implementing extensions for Schemathesis CLI, visit {EXTENSIONS_DOCUMENTATION_URL}"
+        )

schemathesis/cli/commands/run/handlers/base.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from schemathesis.cli.commands.run.context import ExecutionContext
+    from schemathesis.engine import events
+
+
+class EventHandler:
+    def __init__(self, *args: Any, **params: Any) -> None: ...
+
+    def handle_event(self, ctx: ExecutionContext, event: events.EngineEvent) -> None:
+        raise NotImplementedError
+
+    def start(self, ctx: ExecutionContext) -> None: ...
+
+    def shutdown(self, ctx: ExecutionContext) -> None: ...