schemathesis 3.25.6__py3-none-any.whl → 3.39.7__py3-none-any.whl

schemathesis/stateful/__init__.py

@@ -1,21 +1,27 @@
 from __future__ import annotations
+
 import enum
 import json
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any, Callable, Generator
 
-from .. import GenerationConfig
-from ..exceptions import OperationSchemaError
-from ..models import APIOperation, Case
 from ..constants import NOT_SET
 from ..internal.result import Ok, Result
 
 if TYPE_CHECKING:
     import hypothesis
+
+    from .. import GenerationConfig
+    from ..exceptions import OperationSchemaError
+    from ..models import APIOperation, Case
     from ..transports.responses import GenericResponse
     from .state_machine import APIStateMachine
 
 
+class UnresolvableLink(Exception):
+    """Raised when a link cannot be resolved."""
+
+
 @enum.unique
 class Stateful(enum.Enum):
     none = 1
@@ -54,6 +60,9 @@ class StatefulTest:
     def parse(self, case: Case, response: GenericResponse) -> ParsedData:
         raise NotImplementedError
 
+    def is_match(self) -> bool:
+        raise NotImplementedError
+
     def make_operation(self, collected: list[ParsedData]) -> APIOperation:
         raise NotImplementedError
 
@@ -70,8 +79,12 @@ class StatefulData:
 
     def store(self, case: Case, response: GenericResponse) -> None:
         """Parse and store data for a stateful test."""
-        parsed = self.stateful_test.parse(case, response)
-        self.container.append(parsed)
+        try:
+            parsed = self.stateful_test.parse(case, response)
+            self.container.append(parsed)
+        except UnresolvableLink:
+            # For now, ignore if a link cannot be resolved
+            pass
 
 
 @dataclass
@@ -103,22 +116,23 @@ class Feedback:
         from .._hypothesis import create_test
 
         for data in self.stateful_tests.values():
-            operation = data.make_operation()
-            _as_strategy_kwargs: dict[str, Any] | None
-            if callable(as_strategy_kwargs):
-                _as_strategy_kwargs = as_strategy_kwargs(operation)
-            else:
-                _as_strategy_kwargs = as_strategy_kwargs
-            test_function = create_test(
-                operation=operation,
-                test=test,
-                settings=settings,
-                seed=seed,
-                data_generation_methods=operation.schema.data_generation_methods,
-                generation_config=generation_config,
-                as_strategy_kwargs=_as_strategy_kwargs,
-            )
-            yield Ok((operation, test_function))
+            if data.stateful_test.is_match():
+                operation = data.make_operation()
+                _as_strategy_kwargs: dict[str, Any] | None
+                if callable(as_strategy_kwargs):
+                    _as_strategy_kwargs = as_strategy_kwargs(operation)
+                else:
+                    _as_strategy_kwargs = as_strategy_kwargs
+                test_function = create_test(
+                    operation=operation,
+                    test=test,
+                    settings=settings,
+                    seed=seed,
+                    data_generation_methods=operation.schema.data_generation_methods,
+                    generation_config=generation_config,
+                    as_strategy_kwargs=_as_strategy_kwargs,
+                )
+                yield Ok((operation, test_function))
 
 
 def run_state_machine_as_test(

schemathesis/stateful/config.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import timedelta
+from typing import TYPE_CHECKING, Any
+
+from ..constants import DEFAULT_DEADLINE
+
+if TYPE_CHECKING:
+    import hypothesis
+    from requests.auth import HTTPDigestAuth
+
+    from .._override import CaseOverride
+    from ..models import CheckFunction
+    from ..targets import Target
+    from ..transports import RequestConfig
+    from ..types import RawAuth
+
+
+def _default_checks_factory() -> tuple[CheckFunction, ...]:
+    from ..checks import ALL_CHECKS
+    from ..specs.openapi.checks import ensure_resource_availability, use_after_free
+
+    return (*ALL_CHECKS, use_after_free, ensure_resource_availability)
+
+
+def _get_default_hypothesis_settings_kwargs() -> dict[str, Any]:
+    import hypothesis
+
+    return {
+        "phases": (hypothesis.Phase.generate,),
+        "deadline": None,
+        "stateful_step_count": 6,
+        "suppress_health_check": list(hypothesis.HealthCheck),
+    }
+
+
+def _default_hypothesis_settings_factory() -> hypothesis.settings:
+    # To avoid importing hypothesis at the module level
+    import hypothesis
+
+    return hypothesis.settings(**_get_default_hypothesis_settings_kwargs())
+
+
+def _default_request_config_factory() -> RequestConfig:
+    from ..transports import RequestConfig
+
+    return RequestConfig()
+
+
+@dataclass
+class StatefulTestRunnerConfig:
+    """Configuration for the stateful test runner."""
+
+    # Checks to run against each response
+    checks: tuple[CheckFunction, ...] = field(default_factory=_default_checks_factory)
+    # Hypothesis settings for state machine execution
+    hypothesis_settings: hypothesis.settings = field(default_factory=_default_hypothesis_settings_factory)
+    # Request-level configuration
+    request: RequestConfig = field(default_factory=_default_request_config_factory)
+    # Whether to stop the execution after the first failure
+    exit_first: bool = False
+    max_failures: int | None = None
+    # Custom headers sent with each request
+    headers: dict[str, str] = field(default_factory=dict)
+    auth: HTTPDigestAuth | RawAuth | None = None
+    seed: int | None = None
+    override: CaseOverride | None = None
+    max_response_time: int | None = None
+    dry_run: bool = False
+    targets: list[Target] = field(default_factory=list)
+    unique_data: bool = False
+
+    def __post_init__(self) -> None:
+        import hypothesis
+
+        kwargs = _get_hypothesis_settings_kwargs_override(self.hypothesis_settings)
+        if kwargs:
+            self.hypothesis_settings = hypothesis.settings(self.hypothesis_settings, **kwargs)
+
+
+def _get_hypothesis_settings_kwargs_override(settings: hypothesis.settings) -> dict[str, Any]:
+    """Get the settings that should be overridden to match the defaults for API state machines."""
+    import hypothesis
+
+    kwargs = {}
+    hypothesis_default = hypothesis.settings()
+    state_machine_default = _default_hypothesis_settings_factory()
+    if settings.phases == hypothesis_default.phases:
+        kwargs["phases"] = state_machine_default.phases
+    if settings.stateful_step_count == hypothesis_default.stateful_step_count:
+        kwargs["stateful_step_count"] = state_machine_default.stateful_step_count
+    if settings.deadline in (hypothesis_default.deadline, timedelta(milliseconds=DEFAULT_DEADLINE)):
+        kwargs["deadline"] = state_machine_default.deadline
+    if settings.suppress_health_check == hypothesis_default.suppress_health_check:
+        kwargs["suppress_health_check"] = state_machine_default.suppress_health_check
+    return kwargs

schemathesis/stateful/context.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import traceback
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Tuple, Type, Union
+
+from ..constants import NOT_SET
+from ..exceptions import CheckFailed
+from ..targets import TargetMetricCollector
+from . import events
+
+if TYPE_CHECKING:
+    from ..models import Case, Check
+    from ..transports.responses import GenericResponse
+    from ..types import NotSet
+
+FailureKey = Union[Type[CheckFailed], Tuple[str, int]]
+
+
+def _failure_cache_key(exc: CheckFailed | AssertionError) -> FailureKey:
+    """Create a key to identify unique failures."""
+    from hypothesis.internal.escalation import get_trimmed_traceback
+
+    # For CheckFailed, we already have all distinctive information about the failure, which is contained
+    # in the exception type itself.
+    if isinstance(exc, CheckFailed):
+        return exc.__class__
+
+    # Assertion come from the user's code and we may try to group them by location
+    tb = get_trimmed_traceback(exc)
+    filename, lineno, *_ = traceback.extract_tb(tb)[-1]
+    return (filename, lineno)
+
+
+@dataclass
+class RunnerContext:
+    """Mutable context for state machine execution."""
+
+    # All seen failure keys, both grouped and individual ones
+    seen_in_run: set[FailureKey] = field(default_factory=set)
+    # Failures keys seen in the current suite
+    seen_in_suite: set[FailureKey] = field(default_factory=set)
+    # Unique failures collected in the current suite
+    failures_for_suite: list[Check] = field(default_factory=list)
+    # All checks executed in the current run
+    checks_for_step: list[Check] = field(default_factory=list)
+    # Status of the current step
+    current_step_status: events.StepStatus | None = None
+    # The currently processed response
+    current_response: GenericResponse | None = None
+    # Total number of failures
+    failures_count: int = 0
+    # The total number of completed test scenario
+    completed_scenarios: int = 0
+    # Metrics collector for targeted testing
+    metric_collector: TargetMetricCollector = field(default_factory=lambda: TargetMetricCollector(targets=[]))
+    step_outcomes: dict[int, BaseException | None] = field(default_factory=dict)
+
+    @property
+    def current_scenario_status(self) -> events.ScenarioStatus:
+        if self.current_step_status == events.StepStatus.SUCCESS:
+            return events.ScenarioStatus.SUCCESS
+        if self.current_step_status == events.StepStatus.FAILURE:
+            return events.ScenarioStatus.FAILURE
+        if self.current_step_status == events.StepStatus.ERROR:
+            return events.ScenarioStatus.ERROR
+        if self.current_step_status == events.StepStatus.INTERRUPTED:
+            return events.ScenarioStatus.INTERRUPTED
+        return events.ScenarioStatus.REJECTED
+
+    def reset_scenario(self) -> None:
+        self.completed_scenarios += 1
+        self.current_step_status = None
+        self.current_response = None
+        self.step_outcomes.clear()
+
+    def reset_step(self) -> None:
+        self.checks_for_step = []
+
+    def step_succeeded(self) -> None:
+        self.current_step_status = events.StepStatus.SUCCESS
+
+    def step_failed(self) -> None:
+        self.current_step_status = events.StepStatus.FAILURE
+
+    def step_errored(self) -> None:
+        self.current_step_status = events.StepStatus.ERROR
+
+    def step_interrupted(self) -> None:
+        self.current_step_status = events.StepStatus.INTERRUPTED
+
+    def mark_as_seen_in_run(self, exc: CheckFailed) -> None:
+        key = _failure_cache_key(exc)
+        self.seen_in_run.add(key)
+        causes = exc.causes or ()
+        for cause in causes:
+            key = _failure_cache_key(cause)
+            self.seen_in_run.add(key)
+
+    def mark_as_seen_in_suite(self, exc: CheckFailed | AssertionError) -> None:
+        key = _failure_cache_key(exc)
+        self.seen_in_suite.add(key)
+
+    def mark_current_suite_as_seen_in_run(self) -> None:
+        self.seen_in_run.update(self.seen_in_suite)
+
+    def is_seen_in_run(self, exc: CheckFailed | AssertionError) -> bool:
+        key = _failure_cache_key(exc)
+        return key in self.seen_in_run
+
+    def is_seen_in_suite(self, exc: CheckFailed | AssertionError) -> bool:
+        key = _failure_cache_key(exc)
+        return key in self.seen_in_suite
+
+    def add_failed_check(self, check: Check) -> None:
+        self.failures_for_suite.append(check)
+        self.failures_count += 1
+
+    def collect_metric(self, case: Case, response: GenericResponse) -> None:
+        self.metric_collector.store(case, response)
+
+    def maximize_metrics(self) -> None:
+        self.metric_collector.maximize()
+
+    def reset(self) -> None:
+        self.failures_for_suite = []
+        self.seen_in_suite.clear()
+        self.reset_scenario()
+        self.metric_collector.reset()
+
+    def store_step_outcome(self, case: Case, outcome: BaseException | None) -> None:
+        self.step_outcomes[hash(case)] = outcome
+
+    def get_step_outcome(self, case: Case) -> BaseException | None | NotSet:
+        return self.step_outcomes.get(hash(case), NOT_SET)

schemathesis/stateful/events.py

@@ -0,0 +1,274 @@
+from __future__ import annotations
+
+import time
+from dataclasses import asdict as _asdict
+from dataclasses import dataclass
+from enum import Enum
+from typing import TYPE_CHECKING, Any
+
+from ..exceptions import format_exception
+
+if TYPE_CHECKING:
+    from ..models import Case, Check
+    from ..transports.responses import GenericResponse
+    from .state_machine import APIStateMachine
+
+
+class RunStatus(str, Enum):
+    """Status of the state machine run."""
+
+    SUCCESS = "success"
+    FAILURE = "failure"
+    ERROR = "error"
+    INTERRUPTED = "interrupted"
+
+
+@dataclass
+class StatefulEvent:
+    """Basic stateful test event."""
+
+    timestamp: float
+
+    __slots__ = ("timestamp",)
+
+    def asdict(self) -> dict[str, Any]:
+        return _asdict(self)
+
+
+@dataclass
+class RunStarted(StatefulEvent):
+    """Before executing all scenarios."""
+
+    started_at: float
+    state_machine: type[APIStateMachine]
+
+    __slots__ = ("state_machine", "timestamp", "started_at")
+
+    def __init__(self, *, state_machine: type[APIStateMachine]) -> None:
+        self.state_machine = state_machine
+        self.started_at = time.time()
+        self.timestamp = time.monotonic()
+
+    def asdict(self) -> dict[str, Any]:
+        return {
+            "timestamp": self.timestamp,
+            "started_at": self.started_at,
+        }
+
+
+@dataclass
+class RunFinished(StatefulEvent):
+    """After executing all scenarios."""
+
+    status: RunStatus
+
+    __slots__ = ("timestamp", "status")
+
+    def __init__(self, *, status: RunStatus) -> None:
+        self.status = status
+        self.timestamp = time.monotonic()
+
+
+class SuiteStatus(str, Enum):
+    """Status of the suite execution."""
+
+    SUCCESS = "success"
+    FAILURE = "failure"
+    ERROR = "error"
+    INTERRUPTED = "interrupted"
+
+
+@dataclass
+class SuiteStarted(StatefulEvent):
+    """Before executing a set of scenarios."""
+
+    __slots__ = ("timestamp",)
+
+    def __init__(self) -> None:
+        self.timestamp = time.monotonic()
+
+
+@dataclass
+class SuiteFinished(StatefulEvent):
+    """After executing a set of scenarios."""
+
+    status: SuiteStatus
+    failures: list[Check]
+
+    __slots__ = ("timestamp", "status", "failures")
+
+    def __init__(self, *, status: SuiteStatus, failures: list[Check]) -> None:
+        self.status = status
+        self.failures = failures
+        self.timestamp = time.monotonic()
+
+    def asdict(self) -> dict[str, Any]:
+        from ..runner.serialization import SerializedCheck, _serialize_check
+
+        return {
+            "timestamp": self.timestamp,
+            "status": self.status,
+            "failures": [_serialize_check(SerializedCheck.from_check(failure)) for failure in self.failures],
+        }
+
+
+class ScenarioStatus(str, Enum):
+    """Status of a single scenario execution."""
+
+    SUCCESS = "success"
+    FAILURE = "failure"
+    ERROR = "error"
+    # Rejected by Hypothesis
+    REJECTED = "rejected"
+    INTERRUPTED = "interrupted"
+
+
+@dataclass
+class ScenarioStarted(StatefulEvent):
+    """Before a single state machine execution."""
+
+    # Whether this is a scenario that tries to reproduce a failure
+    is_final: bool
+
+    __slots__ = ("timestamp", "is_final")
+
+    def __init__(self, *, is_final: bool) -> None:
+        self.is_final = is_final
+        self.timestamp = time.monotonic()
+
+
+@dataclass
+class ScenarioFinished(StatefulEvent):
+    """After a single state machine execution."""
+
+    status: ScenarioStatus
+    # Whether this is a scenario that tries to reproduce a failure
+    is_final: bool
+
+    __slots__ = ("timestamp", "status", "is_final")
+
+    def __init__(self, *, status: ScenarioStatus, is_final: bool) -> None:
+        self.status = status
+        self.is_final = is_final
+        self.timestamp = time.monotonic()
+
+
+class StepStatus(str, Enum):
+    """Status of a single state machine step."""
+
+    SUCCESS = "success"
+    FAILURE = "failure"
+    ERROR = "error"
+    INTERRUPTED = "interrupted"
+
+
+@dataclass
+class StepStarted(StatefulEvent):
+    """Before a single state machine step."""
+
+    __slots__ = ("timestamp",)
+
+    def __init__(self) -> None:
+        self.timestamp = time.monotonic()
+
+
+@dataclass
+class TransitionId:
+    """Id of the the that was hit."""
+
+    name: str
+    # Status code as defined in the transition, i.e. may be `default`
+    status_code: str
+    source: str
+
+    __slots__ = ("name", "status_code", "source")
+
+
+@dataclass
+class ResponseData:
+    """Common data for responses."""
+
+    status_code: int
+    elapsed: float
+    __slots__ = ("status_code", "elapsed")
+
+
+@dataclass
+class StepFinished(StatefulEvent):
+    """After a single state machine step."""
+
+    status: StepStatus | None
+    transition_id: TransitionId | None
+    target: str
+    case: Case
+    response: GenericResponse | None
+    checks: list[Check]
+
+    __slots__ = ("timestamp", "status", "transition_id", "target", "case", "response", "checks")
+
+    def __init__(
+        self,
+        *,
+        status: StepStatus | None,
+        transition_id: TransitionId | None,
+        target: str,
+        case: Case,
+        response: GenericResponse | None,
+        checks: list[Check],
+    ) -> None:
+        self.status = status
+        self.transition_id = transition_id
+        self.target = target
+        self.case = case
+        self.response = response
+        self.checks = checks
+        self.timestamp = time.monotonic()
+
+    def asdict(self) -> dict[str, Any]:
+        return {
+            "timestamp": self.timestamp,
+            "status": self.status,
+            "transition_id": {
+                "name": self.transition_id.name,
+                "status_code": self.transition_id.status_code,
+                "source": self.transition_id.source,
+            }
+            if self.transition_id is not None
+            else None,
+            "target": self.target,
+            "response": {
+                "status_code": self.response.status_code,
+                "elapsed": self.response.elapsed.total_seconds(),
+            }
+            if self.response is not None
+            else None,
+        }
+
+
+@dataclass
+class Interrupted(StatefulEvent):
+    """The state machine execution was interrupted."""
+
+    __slots__ = ("timestamp",)
+
+    def __init__(self) -> None:
+        self.timestamp = time.monotonic()
+
+
+@dataclass
+class Errored(StatefulEvent):
+    """An error occurred during the state machine execution."""
+
+    exception: Exception
+
+    __slots__ = ("timestamp", "exception")
+
+    def __init__(self, *, exception: Exception) -> None:
+        self.exception = exception
+        self.timestamp = time.monotonic()
+
+    def asdict(self) -> dict[str, Any]:
+        return {
+            "timestamp": self.timestamp,
+            "exception": format_exception(self.exception, True),
+        }