schemathesis 3.25.5__py3-none-any.whl → 3.39.7__py3-none-any.whl

schemathesis/stateful/runner.py

@@ -0,0 +1,309 @@
+from __future__ import annotations
+
+import queue
+import threading
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Generator, Iterator
+
+import hypothesis
+import requests
+from hypothesis.control import current_build_context
+from hypothesis.errors import Flaky, Unsatisfiable
+
+from ..exceptions import CheckFailed
+from ..internal.checks import CheckContext
+from ..targets import TargetMetricCollector
+from . import events
+from .config import StatefulTestRunnerConfig
+from .context import RunnerContext
+from .validation import validate_response
+
+if TYPE_CHECKING:
+    from hypothesis.stateful import Rule
+
+    from ..models import Case, CheckFunction
+    from ..transports.responses import GenericResponse
+    from .state_machine import APIStateMachine, Direction, StepResult
+
+EVENT_QUEUE_TIMEOUT = 0.01
+
+
+@dataclass
+class StatefulTestRunner:
+    """Stateful test runner for the given state machine.
+
+    By default, the test runner executes the state machine in a loop until there are no new failures are found.
+    The loop is executed in a separate thread for better control over the execution and reporting.
+    """
+
+    # State machine class to use
+    state_machine: type[APIStateMachine]
+    # Test runner configuration that defines the runtime behavior
+    config: StatefulTestRunnerConfig = field(default_factory=StatefulTestRunnerConfig)
+    # Event to stop the execution
+    stop_event: threading.Event = field(default_factory=threading.Event)
+    # Queue to communicate with the state machine execution
+    event_queue: queue.Queue = field(default_factory=queue.Queue)
+
+    def execute(self) -> Iterator[events.StatefulEvent]:
+        """Execute a test run for a state machine."""
+        self.stop_event.clear()
+
+        yield events.RunStarted(state_machine=self.state_machine)
+
+        runner_thread = threading.Thread(
+            target=_execute_state_machine_loop,
+            kwargs={
+                "state_machine": self.state_machine,
+                "event_queue": self.event_queue,
+                "config": self.config,
+                "stop_event": self.stop_event,
+            },
+        )
+        run_status = events.RunStatus.SUCCESS
+
+        with thread_manager(runner_thread):
+            try:
+                while True:
+                    try:
+                        event = self.event_queue.get(timeout=EVENT_QUEUE_TIMEOUT)
+                        # Set the run status based on the suite status
+                        # ERROR & INTERRUPTED statuses are terminal, therefore they should not be overridden
+                        if isinstance(event, events.SuiteFinished):
+                            if event.status == events.SuiteStatus.FAILURE:
+                                run_status = events.RunStatus.FAILURE
+                            elif event.status == events.SuiteStatus.ERROR:
+                                run_status = events.RunStatus.ERROR
+                            elif event.status == events.SuiteStatus.INTERRUPTED:
+                                run_status = events.RunStatus.INTERRUPTED
+                        yield event
+                    except queue.Empty:
+                        if not runner_thread.is_alive():
+                            break
+            except KeyboardInterrupt:
+                # Immediately notify the runner thread to stop, even though that the event will be set below in `finally`
+                self.stop()
+                run_status = events.RunStatus.INTERRUPTED
+                yield events.Interrupted()
+            finally:
+                self.stop()
+
+            yield events.RunFinished(status=run_status)
+
+    def stop(self) -> None:
+        """Stop the execution of the state machine."""
+        self.stop_event.set()
+
+
+@contextmanager
+def thread_manager(thread: threading.Thread) -> Generator[None, None, None]:
+    thread.start()
+    try:
+        yield
+    finally:
+        thread.join()
+
+
+def _execute_state_machine_loop(
+    *,
+    state_machine: type[APIStateMachine],
+    event_queue: queue.Queue,
+    config: StatefulTestRunnerConfig,
+    stop_event: threading.Event,
+) -> None:
+    """Execute the state machine testing loop."""
+    from hypothesis import reporting
+    from requests.structures import CaseInsensitiveDict
+
+    from ..transports import RequestsTransport
+
+    ctx = RunnerContext(metric_collector=TargetMetricCollector(targets=config.targets))
+
+    call_kwargs: dict[str, Any] = {"headers": config.headers}
+    if isinstance(state_machine.schema.transport, RequestsTransport):
+        call_kwargs["timeout"] = config.request.prepared_timeout
+        call_kwargs["verify"] = config.request.tls_verify
+        call_kwargs["cert"] = config.request.cert
+        if config.request.proxy is not None:
+            call_kwargs["proxies"] = {"all": config.request.proxy}
+        session = requests.Session()
+        if config.auth is not None:
+            session.auth = config.auth
+        call_kwargs["session"] = session
+    check_ctx = CheckContext(
+        override=config.override,
+        auth=config.auth,
+        headers=CaseInsensitiveDict(config.headers) if config.headers else None,
+    )
+
+    class _InstrumentedStateMachine(state_machine):  # type: ignore[valid-type,misc]
+        """State machine with additional hooks for emitting events."""
+
+        def setup(self) -> None:
+            build_ctx = current_build_context()
+            event_queue.put(events.ScenarioStarted(is_final=build_ctx.is_final))
+            super().setup()
+
+        def get_call_kwargs(self, case: Case) -> dict[str, Any]:
+            return call_kwargs
+
+        def _repr_step(self, rule: Rule, data: dict, result: StepResult) -> str:
+            return ""
+
+        if config.override is not None:
+
+            def before_call(self, case: Case) -> None:
+                for location, entry in config.override.for_operation(case.operation).items():  # type: ignore[union-attr]
+                    if entry:
+                        container = getattr(case, location) or {}
+                        container.update(entry)
+                        setattr(case, location, container)
+                return super().before_call(case)
+
+        def step(self, case: Case, previous: tuple[StepResult, Direction] | None = None) -> StepResult | None:
+            # Checking the stop event once inside `step` is sufficient as it is called frequently
+            # The idea is to stop the execution as soon as possible
+            if stop_event.is_set():
+                raise KeyboardInterrupt
+            event_queue.put(events.StepStarted())
+            try:
+                if config.dry_run:
+                    return None
+                if config.unique_data:
+                    cached = ctx.get_step_outcome(case)
+                    if isinstance(cached, BaseException):
+                        raise cached
+                    elif cached is None:
+                        return None
+                result = super().step(case, previous)
+                ctx.step_succeeded()
+            except CheckFailed as exc:
+                if config.unique_data:
+                    ctx.store_step_outcome(case, exc)
+                ctx.step_failed()
+                raise
+            except Exception as exc:
+                if config.unique_data:
+                    ctx.store_step_outcome(case, exc)
+                ctx.step_errored()
+                raise
+            except KeyboardInterrupt:
+                ctx.step_interrupted()
+                raise
+            except BaseException as exc:
+                if config.unique_data:
+                    ctx.store_step_outcome(case, exc)
+                raise exc
+            else:
+                if config.unique_data:
+                    ctx.store_step_outcome(case, None)
+            finally:
+                transition_id: events.TransitionId | None
+                if previous is not None:
+                    transition = previous[1]
+                    transition_id = events.TransitionId(
+                        name=transition.name,
+                        status_code=transition.status_code,
+                        source=transition.operation.verbose_name,
+                    )
+                else:
+                    transition_id = None
+                event_queue.put(
+                    events.StepFinished(
+                        status=ctx.current_step_status,
+                        transition_id=transition_id,
+                        target=case.operation.verbose_name,
+                        case=case,
+                        response=ctx.current_response,
+                        checks=ctx.checks_for_step,
+                    )
+                )
+                ctx.reset_step()
+            return result
+
+        def validate_response(
+            self, response: GenericResponse, case: Case, additional_checks: tuple[CheckFunction, ...] = ()
+        ) -> None:
+            ctx.collect_metric(case, response)
+            ctx.current_response = response
+            validate_response(
+                response=response,
+                case=case,
+                runner_ctx=ctx,
+                check_ctx=check_ctx,
+                checks=config.checks,
+                additional_checks=additional_checks,
+                max_response_time=config.max_response_time,
+            )
+
+        def teardown(self) -> None:
+            build_ctx = current_build_context()
+            event_queue.put(
+                events.ScenarioFinished(
+                    status=ctx.current_scenario_status,
+                    is_final=build_ctx.is_final,
+                )
+            )
+            ctx.maximize_metrics()
+            ctx.reset_scenario()
+            super().teardown()
+
+    if config.seed is not None:
+        InstrumentedStateMachine = hypothesis.seed(config.seed)(_InstrumentedStateMachine)
+    else:
+        InstrumentedStateMachine = _InstrumentedStateMachine
+
+    def should_stop() -> bool:
+        return config.exit_first or (config.max_failures is not None and ctx.failures_count >= config.max_failures)
+
+    while True:
+        # This loop is running until no new failures are found in a single iteration
+        event_queue.put(events.SuiteStarted())
+        if stop_event.is_set():
+            event_queue.put(events.SuiteFinished(status=events.SuiteStatus.INTERRUPTED, failures=[]))
+            break
+        suite_status = events.SuiteStatus.SUCCESS
+        try:
+            with reporting.with_reporter(lambda _: None):  # type: ignore
+                InstrumentedStateMachine.run(settings=config.hypothesis_settings)
+        except KeyboardInterrupt:
+            # Raised in the state machine when the stop event is set or it is raised by the user's code
+            # that is placed in the base class of the state machine.
+            # Therefore, set the stop event to cover the latter case
+            stop_event.set()
+            suite_status = events.SuiteStatus.INTERRUPTED
+            break
+        except CheckFailed as exc:
+            # When a check fails, the state machine is stopped
+            # The failure is already sent to the queue by the state machine
+            # Here we need to either exit or re-run the state machine with this failure marked as known
+            suite_status = events.SuiteStatus.FAILURE
+            if should_stop():
+                break
+            ctx.mark_as_seen_in_run(exc)
+            continue
+        except Flaky:
+            suite_status = events.SuiteStatus.FAILURE
+            if should_stop():
+                break
+            # Mark all failures in this suite as seen to prevent them being re-discovered
+            ctx.mark_current_suite_as_seen_in_run()
+            continue
+        except Exception as exc:
+            if isinstance(exc, Unsatisfiable) and ctx.completed_scenarios > 0:
+                # Sometimes Hypothesis randomly gives up on generating some complex cases. However, if we know that
+                # values are possible to generate based on the previous observations, we retry the generation
+                if ctx.completed_scenarios >= config.hypothesis_settings.max_examples:
+                    # Avoid infinite restarts
+                    break
+                continue
+            # Any other exception is an inner error and the test run should be stopped
+            suite_status = events.SuiteStatus.ERROR
+            event_queue.put(events.Errored(exception=exc))
+            break
+        finally:
+            event_queue.put(events.SuiteFinished(status=suite_status, failures=ctx.failures_for_suite))
+            ctx.reset()
+        # Exit on the first successful state machine execution
+        break

schemathesis/stateful/sink.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+from . import events
+
+if TYPE_CHECKING:
+    from ..models import Check
+    from .statistic import TransitionStats
+
+
+@dataclass
+class AverageResponseTime:
+    """Average response time for a given status code.
+
+    Stored as a sum of all response times and a count of responses.
+    """
+
+    total: float
+    count: int
+
+    __slots__ = ("total", "count")
+
+    def __init__(self) -> None:
+        self.total = 0.0
+        self.count = 0
+
+
+@dataclass
+class StateMachineSink:
+    """Collects events and stores data about the state machine execution."""
+
+    transitions: TransitionStats
+    response_times: dict[str, dict[int, AverageResponseTime]] = field(default_factory=dict)
+    steps: dict[events.StepStatus, int] = field(default_factory=lambda: {status: 0 for status in events.StepStatus})
+    scenarios: dict[events.ScenarioStatus, int] = field(
+        default_factory=lambda: {status: 0 for status in events.ScenarioStatus}
+    )
+    suites: dict[events.SuiteStatus, int] = field(default_factory=lambda: {status: 0 for status in events.SuiteStatus})
+    failures: list[Check] = field(default_factory=list)
+    start_time: float | None = None
+    end_time: float | None = None
+
+    def consume(self, event: events.StatefulEvent) -> None:
+        self.transitions.consume(event)
+        if isinstance(event, events.RunStarted):
+            self.start_time = event.timestamp
+        elif isinstance(event, events.StepFinished) and event.status is not None:
+            self.steps[event.status] += 1
+            responses = self.response_times.setdefault(event.target, {})
+            if event.response is not None:
+                average = responses.setdefault(event.response.status_code, AverageResponseTime())
+                average.total += event.response.elapsed.total_seconds()
+                average.count += 1
+        elif isinstance(event, events.ScenarioFinished):
+            self.scenarios[event.status] += 1
+        elif isinstance(event, events.SuiteFinished):
+            self.suites[event.status] += 1
+            self.failures.extend(event.failures)
+        elif isinstance(event, events.RunFinished):
+            self.end_time = event.timestamp
+
+    @property
+    def duration(self) -> float | None:
+        if self.start_time is not None and self.end_time is not None:
+            return self.end_time - self.start_time
+        return None

schemathesis/stateful/state_machine.py

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
-import time
 import re
+import time
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, Callable, ClassVar
+from functools import lru_cache
+from typing import TYPE_CHECKING, Any, ClassVar
 
 from hypothesis.errors import InvalidDefinition
 from hypothesis.stateful import RuleBasedStateMachine
@@ -11,7 +12,11 @@ from hypothesis.stateful import RuleBasedStateMachine
 from .._dependency_versions import HYPOTHESIS_HAS_STATEFUL_NAMING_IMPROVEMENTS
 from ..constants import NO_LINKS_ERROR_MESSAGE, NOT_SET
 from ..exceptions import UsageError
-from ..models import APIOperation, Case, CheckFunction
+from ..internal.checks import CheckFunction
+from ..models import APIOperation, Case
+from .config import _default_hypothesis_settings_factory
+from .runner import StatefulTestRunner, StatefulTestRunnerConfig
+from .sink import StateMachineSink
 
 if TYPE_CHECKING:
     import hypothesis
@@ -19,6 +24,7 @@ if TYPE_CHECKING:
 
     from ..schemas import BaseSchema
     from ..transports.responses import GenericResponse
+    from .statistic import TransitionStats
 
 
 @dataclass
@@ -30,7 +36,7 @@ class StepResult:
     elapsed: float
 
 
-def _operation_name_to_identifier(name: str) -> str:
+def _normalize_name(name: str) -> str:
     return re.sub(r"\W|^(?=\d)", "_", name).replace("__", "_")
 
 
@@ -45,6 +51,8 @@ class APIStateMachine(RuleBasedStateMachine):
     # attribute will be renamed in the future
     bundles: ClassVar[dict[str, CaseInsensitiveDict]]  # type: ignore
     schema: BaseSchema
+    # A template for transition statistics that can be filled with data from the state machine during its execution
+    _transition_stats_template: ClassVar[TransitionStats]
 
     def __init__(self) -> None:
         try:
@@ -55,23 +63,50 @@ class APIStateMachine(RuleBasedStateMachine):
             raise
         self.setup()
 
+    @classmethod
+    @lru_cache
+    def _to_test_case(cls) -> type:
+        from . import run_state_machine_as_test
+
+        class StateMachineTestCase(RuleBasedStateMachine.TestCase):
+            settings = _default_hypothesis_settings_factory()
+
+            def runTest(self) -> None:
+                run_state_machine_as_test(cls, settings=self.settings)
+
+            runTest.is_hypothesis_test = True  # type: ignore[attr-defined]
+
+        StateMachineTestCase.__name__ = cls.__name__ + ".TestCase"
+        StateMachineTestCase.__qualname__ = cls.__qualname__ + ".TestCase"
+        return StateMachineTestCase
+
     def _pretty_print(self, value: Any) -> str:
         if isinstance(value, Case):
             # State machines suppose to be reproducible, hence it is OK to get kwargs here
             kwargs = self.get_call_kwargs(value)
             return _print_case(value, kwargs)
-        if isinstance(value, tuple) and len(value) == 2:
-            result, direction = value
-            wrapper = _DirectionWrapper(direction)
-            return super()._pretty_print((result, wrapper))  # type: ignore
         return super()._pretty_print(value)  # type: ignore
 
     if HYPOTHESIS_HAS_STATEFUL_NAMING_IMPROVEMENTS:
 
         def _new_name(self, target: str) -> str:
-            target = _operation_name_to_identifier(target)
+            target = _normalize_name(target)
             return super()._new_name(target)  # type: ignore
 
+    def _get_target_for_result(self, result: StepResult) -> str | None:
+        raise NotImplementedError
+
+    def _add_result_to_targets(self, targets: tuple[str, ...], result: StepResult | None) -> None:
+        if result is None:
+            return
+        target = self._get_target_for_result(result)
+        if target is not None:
+            super()._add_result_to_targets((target,), result)
+
+    @classmethod
+    def format_rules(cls) -> str:
+        raise NotImplementedError
+
     @classmethod
     def run(cls, *, settings: hypothesis.settings | None = None) -> None:
         """Run state machine as a test."""
@@ -79,6 +114,18 @@ class APIStateMachine(RuleBasedStateMachine):
 
         return run_state_machine_as_test(cls, settings=settings)
 
+    @classmethod
+    def runner(cls, *, config: StatefulTestRunnerConfig | None = None) -> StatefulTestRunner:
+        """Create a runner for this state machine."""
+        from .runner import StatefulTestRunnerConfig
+
+        return StatefulTestRunner(cls, config=config or StatefulTestRunnerConfig())
+
+    @classmethod
+    def sink(cls) -> StateMachineSink:
+        """Create a sink to collect events into."""
+        return StateMachineSink(transitions=cls._transition_stats_template.copy())
+
     def setup(self) -> None:
         """Hook method that runs unconditionally in the beginning of each test scenario.
 
@@ -94,14 +141,16 @@ class APIStateMachine(RuleBasedStateMachine):
     def transform(self, result: StepResult, direction: Direction, case: Case) -> Case:
         raise NotImplementedError
 
-    def _step(self, case: Case, previous: tuple[StepResult, Direction] | None = None) -> StepResult:
+    def _step(self, case: Case, previous: StepResult | None = None, link: Direction | None = None) -> StepResult | None:
         # This method is a proxy that is used under the hood during the state machine initialization.
         # The whole point of having it is to make it possible to override `step`; otherwise, custom "step" is ignored.
         # It happens because, at the point of initialization, the final class is not yet created.
         __tracebackhide__ = True
-        return self.step(case, previous)
+        if previous is not None and link is not None:
+            return self.step(case, (previous, link))
+        return self.step(case, None)
 
-    def step(self, case: Case, previous: tuple[StepResult, Direction] | None = None) -> StepResult:
+    def step(self, case: Case, previous: tuple[StepResult, Direction] | None = None) -> StepResult | None:
         """A single state machine step.
 
         :param Case case: Generated test case data that should be sent in an API call to the tested API operation.
@@ -110,6 +159,8 @@ class APIStateMachine(RuleBasedStateMachine):
         Schemathesis prepares data, makes a call and validates the received response.
         It is the most high-level point to extend the testing process. You probably don't need it in most cases.
         """
+        from ..specs.openapi.checks import use_after_free
+
         __tracebackhide__ = True
         if previous is not None:
             result, direction = previous
@@ -120,7 +171,7 @@ class APIStateMachine(RuleBasedStateMachine):
         response = self.call(case, **kwargs)
         elapsed = time.monotonic() - start
         self.after_call(response, case)
-        self.validate_response(response, case)
+        self.validate_response(response, case, additional_checks=(use_after_free,))
         return self.store_result(response, case, elapsed)
 
     def before_call(self, case: Case) -> None:
@@ -189,13 +240,12 @@ class APIStateMachine(RuleBasedStateMachine):
         :return: Response from the application under test.
 
         Note that WSGI/ASGI applications are detected automatically in this method. Depending on the result of this
-        detection the state machine will call ``call``, ``call_wsgi`` or ``call_asgi`` methods.
+        detection the state machine will call the ``call`` method.
 
         Usually, you don't need to override this method unless you are building a different state machine on top of this
         one and want to customize the transport layer itself.
         """
-        method = self._get_call_method(case)
-        return method(**kwargs)
+        return case.call(**kwargs)
 
     def get_call_kwargs(self, case: Case) -> dict[str, Any]:
         """Create custom keyword arguments that will be passed to the :meth:`Case.call` method.
@@ -214,15 +264,6 @@ class APIStateMachine(RuleBasedStateMachine):
         """
         return {}
 
-    def _get_call_method(self, case: Case) -> Callable:
-        if case.app is not None:
-            from starlette.applications import Starlette
-
-            if isinstance(case.app, Starlette):
-                return case.call_asgi
-            return case.call_wsgi
-        return case.call
-
     def validate_response(
         self, response: GenericResponse, case: Case, additional_checks: tuple[CheckFunction, ...] = ()
     ) -> None:
@@ -270,7 +311,7 @@ def _print_case(case: Case, kwargs: dict[str, Any]) -> str:
     headers.update(kwargs.get("headers", {}))
     case.headers = headers
     data = [
-        f"{name}={repr(getattr(case, name))}"
+        f"{name}={getattr(case, name)!r}"
         for name in ("path_parameters", "headers", "cookies", "query", "body", "media_type")
         if getattr(case, name) not in (None, NOT_SET)
     ]
@@ -284,15 +325,3 @@ class Direction:
 
     def set_data(self, case: Case, elapsed: float, **kwargs: Any) -> None:
         raise NotImplementedError
-
-
-@dataclass(repr=False)
-class _DirectionWrapper:
-    """Purely to avoid modification of `Direction.__repr__`."""
-
-    direction: Direction
-
-    def __repr__(self) -> str:
-        path = self.direction.operation.path
-        method = self.direction.operation.method.upper()
-        return f"state.schema['{path}']['{method}'].links['{self.direction.status_code}']['{self.direction.name}']"

schemathesis/stateful/statistic.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from . import events
+
+
+@dataclass
+class TransitionStats:
+    """Statistic for transitions in a state machine."""
+
+    def consume(self, event: events.StatefulEvent) -> None:
+        raise NotImplementedError
+
+    def copy(self) -> TransitionStats:
+        """Create a copy of the statistic."""
+        raise NotImplementedError
+
+    def to_formatted_table(self, width: int) -> str:
+        raise NotImplementedError