schemathesis 3.25.5__py3-none-any.whl → 4.0.0a1__py3-none-any.whl

schemathesis/cli/commands/run/checks.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Sequence
+
+from schemathesis import experimental
+from schemathesis.checks import CHECKS, CheckFunction, ChecksConfig
+
+
+@dataclass
+class CheckArguments:
+    included_check_names: Sequence[str]
+    excluded_check_names: Sequence[str]
+    positive_data_acceptance_allowed_statuses: list[str] | None
+    missing_required_header_allowed_statuses: list[str] | None
+    negative_data_rejection_allowed_statuses: list[str] | None
+    max_response_time: float | None
+
+    __slots__ = (
+        "included_check_names",
+        "excluded_check_names",
+        "positive_data_acceptance_allowed_statuses",
+        "missing_required_header_allowed_statuses",
+        "negative_data_rejection_allowed_statuses",
+        "max_response_time",
+    )
+
+    def into(self) -> tuple[list[CheckFunction], ChecksConfig]:
+        # Determine selected checks
+        if "all" in self.included_check_names:
+            selected_checks = CHECKS.get_all()
+        else:
+            selected_checks = CHECKS.get_by_names(self.included_check_names or [])
+
+        # Prepare checks configuration
+        checks_config: ChecksConfig = {}
+
+        if experimental.POSITIVE_DATA_ACCEPTANCE.is_enabled:
+            from schemathesis.openapi.checks import PositiveDataAcceptanceConfig
+            from schemathesis.specs.openapi.checks import positive_data_acceptance
+
+            selected_checks.append(positive_data_acceptance)
+            if self.positive_data_acceptance_allowed_statuses:
+                checks_config[positive_data_acceptance] = PositiveDataAcceptanceConfig(
+                    allowed_statuses=self.positive_data_acceptance_allowed_statuses
+                )
+
+        if self.missing_required_header_allowed_statuses:
+            from schemathesis.openapi.checks import MissingRequiredHeaderConfig
+            from schemathesis.specs.openapi.checks import missing_required_header
+
+            selected_checks.append(missing_required_header)
+            checks_config[missing_required_header] = MissingRequiredHeaderConfig(
+                allowed_statuses=self.missing_required_header_allowed_statuses
+            )
+
+        if self.negative_data_rejection_allowed_statuses:
+            from schemathesis.openapi.checks import NegativeDataRejectionConfig
+            from schemathesis.specs.openapi.checks import negative_data_rejection
+
+            checks_config[negative_data_rejection] = NegativeDataRejectionConfig(
+                allowed_statuses=self.negative_data_rejection_allowed_statuses
+            )
+
+        if self.max_response_time is not None:
+            from schemathesis.checks import max_response_time as _max_response_time
+            from schemathesis.core.failures import MaxResponseTimeConfig
+
+            checks_config[_max_response_time] = MaxResponseTimeConfig(self.max_response_time)
+            selected_checks.append(_max_response_time)
+
+        if experimental.COVERAGE_PHASE.is_enabled:
+            from schemathesis.specs.openapi.checks import unsupported_method
+
+            selected_checks.append(unsupported_method)
+
+        # Exclude checks based on their names
+        selected_checks = [check for check in selected_checks if check.__name__ not in self.excluded_check_names]
+
+        return selected_checks, checks_config

schemathesis/cli/commands/run/context.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Generator
+
+from schemathesis.core.failures import Failure
+from schemathesis.core.output import OutputConfig
+from schemathesis.core.transport import Response
+from schemathesis.engine import Status, events
+from schemathesis.engine.recorder import ScenarioRecorder
+
+
+@dataclass
+class Statistic:
+    """Running statistics about test execution."""
+
+    outcomes: dict[Status, int]
+    failures: dict[str, dict[str, GroupedFailures]]
+
+    tested_operations: set[str]
+
+    total_cases: int
+    cases_with_failures: int
+    cases_without_checks: int
+
+    __slots__ = (
+        "outcomes",
+        "failures",
+        "tested_operations",
+        "total_cases",
+        "cases_with_failures",
+        "cases_without_checks",
+    )
+
+    def __init__(self) -> None:
+        self.outcomes = {}
+        self.failures = {}
+        self.tested_operations = set()
+        self.total_cases = 0
+        self.cases_with_failures = 0
+        self.cases_without_checks = 0
+
+    def record_checks(self, recorder: ScenarioRecorder) -> None:
+        """Update statistics and store failures from a new batch of checks."""
+        failures = self.failures.get(recorder.label, {})
+
+        self.total_cases += len(recorder.cases)
+
+        for case_id, case in recorder.cases.items():
+            checks = recorder.checks.get(case_id, [])
+
+            if not checks:
+                self.cases_without_checks += 1
+                continue
+
+            self.tested_operations.add(case.value.operation.label)
+            has_failures = False
+            for check in checks:
+                response = recorder.interactions[case_id].response
+
+                # Collect failures
+                if check.failure_info is not None:
+                    has_failures = True
+                    if case_id not in failures:
+                        failures[case_id] = GroupedFailures(
+                            case_id=case_id,
+                            code_sample=check.failure_info.code_sample,
+                            failures=[],
+                            response=response,
+                        )
+                    failures[case_id].failures.append(check.failure_info.failure)
+            if has_failures:
+                self.cases_with_failures += 1
+        if failures:
+            for group in failures.values():
+                group.failures = sorted(set(group.failures))
+            self.failures[recorder.label] = failures
+
+
+@dataclass
+class GroupedFailures:
+    """Represents failures grouped by case ID."""
+
+    case_id: str
+    code_sample: str
+    failures: list[Failure]
+    response: Response | None
+
+    __slots__ = ("case_id", "code_sample", "failures", "response")
+
+
+@dataclass
+class ExecutionContext:
+    """Storage for the current context of the execution."""
+
+    statistic: Statistic = field(default_factory=Statistic)
+    exit_code: int = 0
+    output_config: OutputConfig = field(default_factory=OutputConfig)
+    initialization_lines: list[str | Generator[str, None, None]] = field(default_factory=list)
+    summary_lines: list[str | Generator[str, None, None]] = field(default_factory=list)
+    seed: int | None = None
+
+    def add_initialization_line(self, line: str | Generator[str, None, None]) -> None:
+        self.initialization_lines.append(line)
+
+    def add_summary_line(self, line: str | Generator[str, None, None]) -> None:
+        self.summary_lines.append(line)
+
+    def on_event(self, event: events.EngineEvent) -> None:
+        if isinstance(event, events.ScenarioFinished):
+            self.statistic.record_checks(event.recorder)
+        elif isinstance(event, events.NonFatalError) or (
+            isinstance(event, events.PhaseFinished)
+            and event.phase.is_enabled
+            and event.status in (Status.FAILURE, Status.ERROR)
+        ):
+            self.exit_code = 1

schemathesis/cli/commands/run/events.py

@@ -0,0 +1,35 @@
+import time
+import uuid
+
+from schemathesis.core import Specification
+from schemathesis.engine import events
+from schemathesis.schemas import ApiOperationsCount
+
+
+class LoadingStarted(events.EngineEvent):
+    __slots__ = ("id", "timestamp", "location")
+
+    def __init__(self, *, location: str) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.location = location
+
+
+class LoadingFinished(events.EngineEvent):
+    __slots__ = ("id", "timestamp", "location", "duration", "base_url", "specification", "operations_count")
+
+    def __init__(
+        self,
+        location: str,
+        start_time: float,
+        base_url: str,
+        specification: Specification,
+        operations_count: ApiOperationsCount,
+    ) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.location = location
+        self.duration = self.timestamp - start_time
+        self.base_url = base_url
+        self.specification = specification
+        self.operations_count = operations_count

schemathesis/cli/commands/run/executor.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass
+from typing import Any, Callable
+
+import click
+
+from schemathesis.cli.commands.run.context import ExecutionContext
+from schemathesis.cli.commands.run.events import LoadingFinished, LoadingStarted
+from schemathesis.cli.commands.run.handlers import display_handler_error
+from schemathesis.cli.commands.run.handlers.base import EventHandler
+from schemathesis.cli.commands.run.handlers.cassettes import CassetteConfig, CassetteWriter
+from schemathesis.cli.commands.run.handlers.junitxml import JunitXMLHandler
+from schemathesis.cli.commands.run.handlers.output import OutputHandler
+from schemathesis.cli.commands.run.loaders import AutodetectConfig, load_schema
+from schemathesis.cli.ext.fs import open_file
+from schemathesis.core.errors import LoaderError
+from schemathesis.core.output import OutputConfig
+from schemathesis.engine import from_schema
+from schemathesis.engine.config import EngineConfig
+from schemathesis.engine.events import EventGenerator, FatalError
+from schemathesis.filters import FilterSet
+
+CUSTOM_HANDLERS: list[type[EventHandler]] = []
+
+
+def handler() -> Callable[[type], None]:
+    """Register a new CLI event handler."""
+
+    def _wrapper(cls: type) -> None:
+        CUSTOM_HANDLERS.append(cls)
+
+    return _wrapper
+
+
+@dataclass
+class RunConfig:
+    location: str
+    base_url: str | None
+    filter_set: FilterSet
+    engine: EngineConfig
+    wait_for_schema: float | None
+    rate_limit: str | None
+    output: OutputConfig
+    junit_xml: click.utils.LazyFile | None
+    cassette: CassetteConfig | None
+    args: list[str]
+    params: dict[str, Any]
+
+
+def execute(config: RunConfig) -> None:
+    event_stream = into_event_stream(config)
+    _execute(event_stream, config)
+
+
+def into_event_stream(config: RunConfig) -> EventGenerator:
+    loader_config = AutodetectConfig(
+        location=config.location,
+        network=config.engine.network,
+        wait_for_schema=config.wait_for_schema,
+        base_url=config.base_url,
+        rate_limit=config.rate_limit,
+        output=config.output,
+        generation=config.engine.execution.generation,
+    )
+    loading_started = LoadingStarted(location=config.location)
+    yield loading_started
+
+    try:
+        schema = load_schema(loader_config)
+        schema.filter_set = config.filter_set
+    except LoaderError as exc:
+        yield FatalError(exception=exc)
+        return
+
+    yield LoadingFinished(
+        location=config.location,
+        start_time=loading_started.timestamp,
+        base_url=schema.get_base_url(),
+        specification=schema.specification,
+        operations_count=schema.count_operations(),
+    )
+
+    try:
+        yield from from_schema(schema, config=config.engine).execute()
+    except Exception as exc:
+        yield FatalError(exception=exc)
+
+
+def _execute(event_stream: EventGenerator, config: RunConfig) -> None:
+    handlers: list[EventHandler] = []
+    if config.junit_xml is not None:
+        open_file(config.junit_xml)
+        handlers.append(JunitXMLHandler(config.junit_xml))
+    if config.cassette is not None:
+        open_file(config.cassette.path)
+        handlers.append(CassetteWriter(config=config.cassette))
+    for custom_handler in CUSTOM_HANDLERS:
+        handlers.append(custom_handler(*config.args, **config.params))
+    handlers.append(
+        OutputHandler(
+            workers_num=config.engine.execution.workers_num,
+            rate_limit=config.rate_limit,
+            wait_for_schema=config.wait_for_schema,
+            cassette_config=config.cassette,
+            junit_xml_file=config.junit_xml.name if config.junit_xml is not None else None,
+        )
+    )
+
+    ctx = ExecutionContext(output_config=config.output, seed=config.engine.execution.seed)
+
+    def shutdown() -> None:
+        for _handler in handlers:
+            _handler.shutdown()
+
+    for handler in handlers:
+        handler.start(ctx)
+
+    try:
+        for event in event_stream:
+            ctx.on_event(event)
+            for handler in handlers:
+                try:
+                    handler.handle_event(ctx, event)
+                except Exception as exc:
+                    # `Abort` is used for handled errors
+                    if not isinstance(exc, click.Abort):
+                        display_handler_error(handler, exc)
+                    raise
+    except Exception as exc:
+        if isinstance(exc, click.Abort):
+            # To avoid showing "Aborted!" message, which is the default behavior in Click
+            sys.exit(1)
+        raise
+    finally:
+        shutdown()
+    sys.exit(ctx.exit_code)

schemathesis/cli/commands/run/filters.py

@@ -0,0 +1,194 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable, Literal, Sequence
+
+import click
+
+from schemathesis.cli.ext.groups import grouped_option
+from schemathesis.filters import FilterSet, expression_to_filter_function, is_deprecated
+
+
+def _with_filter(*, by: str, mode: Literal["include", "exclude"], modifier: Literal["regex"] | None) -> Callable:
+    """Generate a CLI option for filtering API operations."""
+    param = f"--{mode}-{by}"
+    action = "include in" if mode == "include" else "exclude from"
+    prop = {
+        "operation-id": "ID",
+        "name": "Operation name",
+    }.get(by, by.capitalize())
+    if modifier:
+        param += f"-{modifier}"
+        prop += " pattern"
+    help_text = f"{prop} to {action} testing."
+    return grouped_option(
+        param,
+        help=help_text,
+        type=str,
+        multiple=modifier is None,
+    )
+
+
+_BY_VALUES = ("operation-id", "tag", "name", "method", "path")
+
+
+def with_filters(command: Callable) -> Callable:
+    for by in _BY_VALUES:
+        for mode in ("exclude", "include"):
+            for modifier in ("regex", None):
+                command = _with_filter(by=by, mode=mode, modifier=modifier)(command)  # type: ignore[arg-type]
+    return command
+
+
+@dataclass
+class FilterArguments:
+    include_path: Sequence[str]
+    include_method: Sequence[str]
+    include_name: Sequence[str]
+    include_tag: Sequence[str]
+    include_operation_id: Sequence[str]
+    include_path_regex: str | None
+    include_method_regex: str | None
+    include_name_regex: str | None
+    include_tag_regex: str | None
+    include_operation_id_regex: str | None
+
+    exclude_path: Sequence[str]
+    exclude_method: Sequence[str]
+    exclude_name: Sequence[str]
+    exclude_tag: Sequence[str]
+    exclude_operation_id: Sequence[str]
+    exclude_path_regex: str | None
+    exclude_method_regex: str | None
+    exclude_name_regex: str | None
+    exclude_tag_regex: str | None
+    exclude_operation_id_regex: str | None
+
+    include_by: str | None
+    exclude_by: str | None
+    exclude_deprecated: bool
+
+    __slots__ = (
+        "include_path",
+        "include_method",
+        "include_name",
+        "include_tag",
+        "include_operation_id",
+        "include_path_regex",
+        "include_method_regex",
+        "include_name_regex",
+        "include_tag_regex",
+        "include_operation_id_regex",
+        "exclude_path",
+        "exclude_method",
+        "exclude_name",
+        "exclude_tag",
+        "exclude_operation_id",
+        "exclude_path_regex",
+        "exclude_method_regex",
+        "exclude_name_regex",
+        "exclude_tag_regex",
+        "exclude_operation_id_regex",
+        "include_by",
+        "exclude_by",
+        "exclude_deprecated",
+    )
+
+    def into(self) -> FilterSet:
+        # Validate unique filter arguments
+        for values, arg_name in (
+            (self.include_path, "--include-path"),
+            (self.include_method, "--include-method"),
+            (self.include_name, "--include-name"),
+            (self.include_tag, "--include-tag"),
+            (self.include_operation_id, "--include-operation-id"),
+            (self.exclude_path, "--exclude-path"),
+            (self.exclude_method, "--exclude-method"),
+            (self.exclude_name, "--exclude-name"),
+            (self.exclude_tag, "--exclude-tag"),
+            (self.exclude_operation_id, "--exclude-operation-id"),
+        ):
+            validate_unique_filter(values, arg_name)
+
+        # Convert include/exclude expressions to functions
+        include_by_function = _filter_by_expression_to_func(self.include_by, "--include-by")
+        exclude_by_function = _filter_by_expression_to_func(self.exclude_by, "--exclude-by")
+
+        filter_set = FilterSet()
+
+        # Apply include filters
+        if include_by_function:
+            filter_set.include(include_by_function)
+        for name_ in self.include_name:
+            filter_set.include(name=name_)
+        for method in self.include_method:
+            filter_set.include(method=method)
+        for path in self.include_path:
+            filter_set.include(path=path)
+        for tag in self.include_tag:
+            filter_set.include(tag=tag)
+        for operation_id in self.include_operation_id:
+            filter_set.include(operation_id=operation_id)
+        if (
+            self.include_name_regex
+            or self.include_method_regex
+            or self.include_path_regex
+            or self.include_tag_regex
+            or self.include_operation_id_regex
+        ):
+            filter_set.include(
+                name_regex=self.include_name_regex,
+                method_regex=self.include_method_regex,
+                path_regex=self.include_path_regex,
+                tag_regex=self.include_tag_regex,
+                operation_id_regex=self.include_operation_id_regex,
+            )
+
+        # Apply exclude filters
+        if exclude_by_function:
+            filter_set.exclude(exclude_by_function)
+        for name_ in self.exclude_name:
+            filter_set.exclude(name=name_)
+        for method in self.exclude_method:
+            filter_set.exclude(method=method)
+        for path in self.exclude_path:
+            filter_set.exclude(path=path)
+        for tag in self.exclude_tag:
+            filter_set.exclude(tag=tag)
+        for operation_id in self.exclude_operation_id:
+            filter_set.exclude(operation_id=operation_id)
+        if (
+            self.exclude_name_regex
+            or self.exclude_method_regex
+            or self.exclude_path_regex
+            or self.exclude_tag_regex
+            or self.exclude_operation_id_regex
+        ):
+            filter_set.exclude(
+                name_regex=self.exclude_name_regex,
+                method_regex=self.exclude_method_regex,
+                path_regex=self.exclude_path_regex,
+                tag_regex=self.exclude_tag_regex,
+                operation_id_regex=self.exclude_operation_id_regex,
+            )
+
+        # Exclude deprecated operations
+        if self.exclude_deprecated:
+            filter_set.exclude(is_deprecated)
+
+        return filter_set
+
+
+def validate_unique_filter(values: Sequence[str], arg_name: str) -> None:
+    if len(values) != len(set(values)):
+        duplicates = ",".join(sorted({value for value in values if values.count(value) > 1}))
+        raise click.UsageError(f"Duplicate values are not allowed for `{arg_name}`: {duplicates}")
+
+
+def _filter_by_expression_to_func(value: str | None, arg_name: str) -> Callable | None:
+    if value:
+        try:
+            return expression_to_filter_function(value)
+        except ValueError:
+            raise click.UsageError(f"Invalid expression for {arg_name}: {value}") from None
+    return None

schemathesis/cli/commands/run/handlers/__init__.py

@@ -0,0 +1,46 @@
+import click
+
+from schemathesis.cli.commands.run.handlers.base import EventHandler
+from schemathesis.cli.commands.run.handlers.cassettes import CassetteWriter
+from schemathesis.cli.commands.run.handlers.junitxml import JunitXMLHandler
+from schemathesis.cli.commands.run.handlers.output import OutputHandler
+from schemathesis.cli.constants import EXTENSIONS_DOCUMENTATION_URL, ISSUE_TRACKER_URL
+from schemathesis.core.errors import format_exception
+
+__all__ = [
+    "EventHandler",
+    "CassetteWriter",
+    "JunitXMLHandler",
+    "OutputHandler",
+    "display_handler_error",
+]
+
+
+def is_built_in_handler(handler: EventHandler) -> bool:
+    # Look for exact instances, not subclasses
+    return any(type(handler) is class_ for class_ in (CassetteWriter, JunitXMLHandler, OutputHandler))
+
+
+def display_handler_error(handler: EventHandler, exc: Exception) -> None:
+    """Display error that happened within."""
+    is_built_in = is_built_in_handler(handler)
+    if is_built_in:
+        click.secho("Internal Error", fg="red", bold=True)
+        click.secho("\nSchemathesis encountered an unexpected issue.")
+        message = format_exception(exc, with_traceback=True)
+    else:
+        click.secho("CLI Handler Error", fg="red", bold=True)
+        click.echo(
+            f"\nAn error occurred within your custom CLI handler `{click.style(handler.__class__.__name__, bold=True)}`."
+        )
+        message = format_exception(exc, with_traceback=True, skip_frames=1)
+    click.secho(f"\n{message}", fg="red")
+    if is_built_in:
+        click.echo(
+            f"\nWe apologize for the inconvenience. This appears to be an internal issue.\n"
+            f"Please consider reporting this error to our issue tracker:\n\n  {ISSUE_TRACKER_URL}."
+        )
+    else:
+        click.echo(
+            f"\nFor more information on implementing extensions for Schemathesis CLI, visit {EXTENSIONS_DOCUMENTATION_URL}"
+        )

schemathesis/cli/commands/run/handlers/base.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from schemathesis.cli.commands.run.context import ExecutionContext
+    from schemathesis.engine import events
+
+
+class EventHandler:
+    def __init__(self, *args: Any, **params: Any) -> None: ...
+
+    def handle_event(self, ctx: ExecutionContext, event: events.EngineEvent) -> None:
+        raise NotImplementedError
+
+    def start(self, ctx: ExecutionContext) -> None: ...
+
+    def shutdown(self) -> None: ...