schemathesis 3.25.6__py3-none-any.whl → 4.0.0a1__py3-none-any.whl

schemathesis/engine/phases/stateful/_executor.py

@@ -0,0 +1,326 @@
+from __future__ import annotations
+
+import queue
+import time
+import unittest
+from dataclasses import replace
+from typing import Any
+
+import hypothesis
+from hypothesis.control import current_build_context
+from hypothesis.errors import Flaky, Unsatisfiable
+from hypothesis.stateful import Rule
+
+from schemathesis.checks import CheckContext, CheckFunction, run_checks
+from schemathesis.core.failures import Failure, FailureGroup
+from schemathesis.core.transport import Response
+from schemathesis.engine import Status, events
+from schemathesis.engine.context import EngineContext
+from schemathesis.engine.control import ExecutionControl
+from schemathesis.engine.phases import PhaseName
+from schemathesis.engine.phases.stateful.context import StatefulContext
+from schemathesis.engine.recorder import ScenarioRecorder
+from schemathesis.generation.case import Case
+from schemathesis.generation.hypothesis.reporting import ignore_hypothesis_output
+from schemathesis.generation.stateful.state_machine import (
+    DEFAULT_STATE_MACHINE_SETTINGS,
+    APIStateMachine,
+    Direction,
+    StepResult,
+)
+from schemathesis.generation.targets import TargetMetricCollector
+
+
+def _get_hypothesis_settings_kwargs_override(settings: hypothesis.settings) -> dict[str, Any]:
+    """Get the settings that should be overridden to match the defaults for API state machines."""
+    kwargs = {}
+    hypothesis_default = hypothesis.settings()
+    if settings.phases == hypothesis_default.phases:
+        kwargs["phases"] = DEFAULT_STATE_MACHINE_SETTINGS.phases
+    if settings.stateful_step_count == hypothesis_default.stateful_step_count:
+        kwargs["stateful_step_count"] = DEFAULT_STATE_MACHINE_SETTINGS.stateful_step_count
+    if settings.deadline == hypothesis_default.deadline:
+        kwargs["deadline"] = DEFAULT_STATE_MACHINE_SETTINGS.deadline
+    if settings.suppress_health_check == hypothesis_default.suppress_health_check:
+        kwargs["suppress_health_check"] = DEFAULT_STATE_MACHINE_SETTINGS.suppress_health_check
+    return kwargs
+
+
+def execute_state_machine_loop(
+    *,
+    state_machine: type[APIStateMachine],
+    event_queue: queue.Queue,
+    engine: EngineContext,
+) -> None:
+    """Execute the state machine testing loop."""
+    kwargs = _get_hypothesis_settings_kwargs_override(engine.config.execution.hypothesis_settings)
+    if kwargs:
+        config = replace(
+            engine.config,
+            execution=replace(
+                engine.config.execution,
+                hypothesis_settings=hypothesis.settings(engine.config.execution.hypothesis_settings, **kwargs),
+            ),
+        )
+    else:
+        config = engine.config
+
+    ctx = StatefulContext(metric_collector=TargetMetricCollector(targets=config.execution.targets))
+
+    transport_kwargs = engine.transport_kwargs
+
+    class _InstrumentedStateMachine(state_machine):  # type: ignore[valid-type,misc]
+        """State machine with additional hooks for emitting events."""
+
+        def setup(self) -> None:
+            scenario_started = events.ScenarioStarted(label=None, phase=PhaseName.STATEFUL_TESTING, suite_id=suite_id)
+            self._start_time = time.monotonic()
+            self._scenario_id = scenario_started.id
+            event_queue.put(scenario_started)
+            self.recorder = ScenarioRecorder(label="Stateful tests")
+            self._check_ctx = engine.get_check_context(self.recorder)
+
+        def get_call_kwargs(self, case: Case) -> dict[str, Any]:
+            return transport_kwargs
+
+        def _repr_step(self, rule: Rule, data: dict, result: StepResult) -> str:
+            return ""
+
+        if config.override is not None:
+
+            def before_call(self, case: Case) -> None:
+                for location, entry in config.override.for_operation(case.operation).items():  # type: ignore[union-attr]
+                    if entry:
+                        container = getattr(case, location) or {}
+                        container.update(entry)
+                        setattr(case, location, container)
+                return super().before_call(case)
+
+        def step(self, case: Case, previous: tuple[StepResult, Direction] | None = None) -> StepResult | None:
+            # Checking the stop event once inside `step` is sufficient as it is called frequently
+            # The idea is to stop the execution as soon as possible
+            if previous is not None:
+                step_result, _ = previous
+                self.recorder.record_case(parent_id=step_result.case.id, case=case)
+            else:
+                self.recorder.record_case(parent_id=None, case=case)
+            if engine.has_to_stop:
+                raise KeyboardInterrupt
+            step_started = events.StepStarted(
+                phase=PhaseName.STATEFUL_TESTING, suite_id=suite_id, scenario_id=self._scenario_id
+            )
+            event_queue.put(step_started)
+            try:
+                if config.execution.unique_inputs:
+                    cached = ctx.get_step_outcome(case)
+                    if isinstance(cached, BaseException):
+                        raise cached
+                    elif cached is None:
+                        return None
+                result = super().step(case, previous)
+                ctx.step_succeeded()
+            except FailureGroup as exc:
+                if config.execution.unique_inputs:
+                    for failure in exc.exceptions:
+                        ctx.store_step_outcome(case, failure)
+                ctx.step_failed()
+                raise
+            except Exception as exc:
+                if config.execution.unique_inputs:
+                    ctx.store_step_outcome(case, exc)
+                ctx.step_errored()
+                raise
+            except KeyboardInterrupt:
+                ctx.step_interrupted()
+                raise
+            except BaseException as exc:
+                if config.execution.unique_inputs:
+                    ctx.store_step_outcome(case, exc)
+                raise exc
+            else:
+                if config.execution.unique_inputs:
+                    ctx.store_step_outcome(case, None)
+            finally:
+                transition_id: events.TransitionId | None
+                if previous is not None:
+                    transition = previous[1]
+                    transition_id = events.TransitionId(
+                        name=transition.name,
+                        status_code=transition.status_code,
+                        source=transition.operation.label,
+                    )
+                else:
+                    transition_id = None
+                event_queue.put(
+                    events.StepFinished(
+                        id=step_started.id,
+                        suite_id=suite_id,
+                        scenario_id=self._scenario_id,
+                        phase=PhaseName.STATEFUL_TESTING,
+                        status=ctx.current_step_status,
+                        transition_id=transition_id,
+                        target=case.operation.label,
+                        response=ctx.current_response,
+                    )
+                )
+            return result
+
+        def validate_response(
+            self, response: Response, case: Case, additional_checks: tuple[CheckFunction, ...] = ()
+        ) -> None:
+            self.recorder.record_response(case_id=case.id, response=response)
+            ctx.collect_metric(case, response)
+            ctx.current_response = response
+            validate_response(
+                response=response,
+                case=case,
+                stateful_ctx=ctx,
+                check_ctx=self._check_ctx,
+                checks=config.execution.checks,
+                control=engine.control,
+                recorder=self.recorder,
+                additional_checks=additional_checks,
+            )
+
+        def teardown(self) -> None:
+            build_ctx = current_build_context()
+            event_queue.put(
+                events.ScenarioFinished(
+                    id=self._scenario_id,
+                    suite_id=suite_id,
+                    phase=PhaseName.STATEFUL_TESTING,
+                    # With dry run there will be no status
+                    status=ctx.current_scenario_status or Status.SKIP,
+                    recorder=self.recorder,
+                    elapsed_time=time.monotonic() - self._start_time,
+                    skip_reason=None,
+                    is_final=build_ctx.is_final,
+                )
+            )
+            ctx.maximize_metrics()
+            ctx.reset_scenario()
+            super().teardown()
+
+    if config.execution.seed is not None:
+        InstrumentedStateMachine = hypothesis.seed(config.execution.seed)(_InstrumentedStateMachine)
+    else:
+        InstrumentedStateMachine = _InstrumentedStateMachine
+
+    while True:
+        # This loop is running until no new failures are found in a single iteration
+        suite_started = events.SuiteStarted(phase=PhaseName.STATEFUL_TESTING)
+        suite_id = suite_started.id
+        event_queue.put(suite_started)
+        if engine.is_interrupted:
+            event_queue.put(events.Interrupted(phase=PhaseName.STATEFUL_TESTING))
+            event_queue.put(
+                events.SuiteFinished(
+                    id=suite_started.id,
+                    phase=PhaseName.STATEFUL_TESTING,
+                    status=Status.INTERRUPTED,
+                )
+            )
+            break
+        suite_status = Status.SUCCESS
+        try:
+            with ignore_hypothesis_output():  # type: ignore
+                InstrumentedStateMachine.run(settings=config.execution.hypothesis_settings)
+        except KeyboardInterrupt:
+            # Raised in the state machine when the stop event is set or it is raised by the user's code
+            # that is placed in the base class of the state machine.
+            # Therefore, set the stop event to cover the latter case
+            engine.stop()
+            suite_status = Status.INTERRUPTED
+            event_queue.put(events.Interrupted(phase=PhaseName.STATEFUL_TESTING))
+            break
+        except unittest.case.SkipTest:
+            # If `explicit` phase is used and there are not examples
+            suite_status = Status.SKIP
+            break
+        except FailureGroup as exc:
+            # When a check fails, the state machine is stopped
+            # The failure is already sent to the queue by the state machine
+            # Here we need to either exit or re-run the state machine with this failure marked as known
+            suite_status = Status.FAILURE
+            if engine.has_reached_the_failure_limit:
+                break  # type: ignore[unreachable]
+            for failure in exc.exceptions:
+                ctx.mark_as_seen_in_run(failure)
+            continue
+        except Flaky:
+            suite_status = Status.FAILURE
+            if engine.has_reached_the_failure_limit:
+                break  # type: ignore[unreachable]
+            # Mark all failures in this suite as seen to prevent them being re-discovered
+            ctx.mark_current_suite_as_seen_in_run()
+            continue
+        except Exception as exc:
+            if isinstance(exc, Unsatisfiable) and ctx.completed_scenarios > 0:
+                # Sometimes Hypothesis randomly gives up on generating some complex cases. However, if we know that
+                # values are possible to generate based on the previous observations, we retry the generation
+                if ctx.completed_scenarios >= config.execution.hypothesis_settings.max_examples:
+                    # Avoid infinite restarts
+                    break
+                continue
+            # Any other exception is an inner error and the test run should be stopped
+            suite_status = Status.ERROR
+            event_queue.put(
+                events.NonFatalError(
+                    error=exc, phase=PhaseName.STATEFUL_TESTING, label="Stateful tests", related_to_operation=False
+                )
+            )
+            break
+        finally:
+            event_queue.put(
+                events.SuiteFinished(
+                    id=suite_started.id,
+                    phase=PhaseName.STATEFUL_TESTING,
+                    status=suite_status,
+                )
+            )
+            ctx.reset()
+        # Exit on the first successful state machine execution
+        break
+
+
+def validate_response(
+    *,
+    response: Response,
+    case: Case,
+    stateful_ctx: StatefulContext,
+    check_ctx: CheckContext,
+    control: ExecutionControl,
+    checks: list[CheckFunction],
+    recorder: ScenarioRecorder,
+    additional_checks: tuple[CheckFunction, ...] = (),
+) -> None:
+    """Validate the response against the provided checks."""
+
+    def on_failure(name: str, collected: set[Failure], failure: Failure) -> None:
+        if stateful_ctx.is_seen_in_suite(failure) or stateful_ctx.is_seen_in_run(failure):
+            return
+        failure_data = recorder.find_failure_data(parent_id=case.id, failure=failure)
+        recorder.record_check_failure(
+            name=name,
+            case_id=failure_data.case.id,
+            code_sample=failure_data.case.as_curl_command(headers=failure_data.headers, verify=failure_data.verify),
+            failure=failure,
+        )
+        control.count_failure()
+        stateful_ctx.mark_as_seen_in_suite(failure)
+        collected.add(failure)
+
+    def on_success(name: str, case: Case) -> None:
+        recorder.record_check_success(name=name, case_id=case.id)
+
+    failures = run_checks(
+        case=case,
+        response=response,
+        ctx=check_ctx,
+        checks=tuple(checks) + tuple(additional_checks),
+        on_failure=on_failure,
+        on_success=on_success,
+    )
+
+    if failures:
+        raise FailureGroup(list(failures)) from None

schemathesis/engine/phases/stateful/context.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from schemathesis.core import NOT_SET, NotSet
+from schemathesis.core.failures import Failure
+from schemathesis.core.transport import Response
+from schemathesis.engine import Status
+from schemathesis.generation.case import Case
+from schemathesis.generation.targets import TargetMetricCollector
+
+
+@dataclass
+class StatefulContext:
+    """Mutable context for state machine execution."""
+
+    # All seen failure keys, both grouped and individual ones
+    seen_in_run: set[Failure] = field(default_factory=set)
+    # Failures keys seen in the current suite
+    seen_in_suite: set[Failure] = field(default_factory=set)
+    # Status of the current step
+    current_step_status: Status | None = None
+    # The currently processed response
+    current_response: Response | None = None
+    # Total number of failures
+    failures_count: int = 0
+    # The total number of completed test scenario
+    completed_scenarios: int = 0
+    # Metrics collector for targeted testing
+    metric_collector: TargetMetricCollector = field(default_factory=TargetMetricCollector)
+    step_outcomes: dict[int, BaseException | None] = field(default_factory=dict)
+
+    @property
+    def current_scenario_status(self) -> Status | None:
+        return self.current_step_status
+
+    def reset_scenario(self) -> None:
+        self.completed_scenarios += 1
+        self.current_step_status = None
+        self.current_response = None
+        self.step_outcomes.clear()
+
+    def step_succeeded(self) -> None:
+        self.current_step_status = Status.SUCCESS
+
+    def step_failed(self) -> None:
+        self.current_step_status = Status.FAILURE
+
+    def step_errored(self) -> None:
+        self.current_step_status = Status.ERROR
+
+    def step_interrupted(self) -> None:
+        self.current_step_status = Status.INTERRUPTED
+
+    def mark_as_seen_in_run(self, exc: Failure) -> None:
+        self.seen_in_run.add(exc)
+
+    def mark_as_seen_in_suite(self, exc: Failure) -> None:
+        self.seen_in_suite.add(exc)
+
+    def mark_current_suite_as_seen_in_run(self) -> None:
+        self.seen_in_run.update(self.seen_in_suite)
+
+    def is_seen_in_run(self, exc: Failure) -> bool:
+        return exc in self.seen_in_run
+
+    def is_seen_in_suite(self, exc: Failure) -> bool:
+        return exc in self.seen_in_suite
+
+    def collect_metric(self, case: Case, response: Response) -> None:
+        self.metric_collector.store(case, response)
+
+    def maximize_metrics(self) -> None:
+        self.metric_collector.maximize()
+
+    def reset(self) -> None:
+        self.seen_in_suite.clear()
+        self.reset_scenario()
+        self.metric_collector.reset()
+
+    def store_step_outcome(self, case: Case, outcome: BaseException | None) -> None:
+        self.step_outcomes[hash(case)] = outcome
+
+    def get_step_outcome(self, case: Case) -> BaseException | None | NotSet:
+        return self.step_outcomes.get(hash(case), NOT_SET)

schemathesis/engine/phases/unit/__init__.py

@@ -0,0 +1,174 @@
+"""Unit testing by Schemathesis Engine.
+
+This module provides high-level flow for single-, and multi-threaded modes.
+"""
+
+from __future__ import annotations
+
+import queue
+import uuid
+import warnings
+from queue import Queue
+from typing import TYPE_CHECKING, Any
+
+from schemathesis.core.result import Ok
+from schemathesis.engine import Status, events
+from schemathesis.engine.phases import PhaseName, PhaseSkipReason
+from schemathesis.engine.recorder import ScenarioRecorder
+from schemathesis.generation.hypothesis.builder import HypothesisTestConfig
+from schemathesis.generation.hypothesis.reporting import ignore_hypothesis_output
+
+from ._pool import TaskProducer, WorkerPool
+
+if TYPE_CHECKING:
+    from schemathesis.engine.context import EngineContext
+    from schemathesis.engine.phases import Phase
+    from schemathesis.schemas import APIOperation
+
+WORKER_TIMEOUT = 0.1
+
+
+def execute(engine: EngineContext, phase: Phase) -> events.EventGenerator:
+    """Run a set of unit tests.
+
+    Implemented as a producer-consumer pattern via a task queue.
+    The main thread provides an iterator over API operations and worker threads create test functions and run them.
+    """
+    producer = TaskProducer(engine)
+    workers_num = engine.config.execution.workers_num
+
+    suite_started = events.SuiteStarted(phase=phase.name)
+
+    yield suite_started
+
+    status = None
+    is_executed = False
+
+    with WorkerPool(
+        workers_num=workers_num, producer=producer, worker_factory=worker_task, ctx=engine, suite_id=suite_started.id
+    ) as pool:
+        try:
+            while True:
+                try:
+                    event = pool.events_queue.get(timeout=WORKER_TIMEOUT)
+                    is_executed = True
+                    if engine.is_interrupted:
+                        raise KeyboardInterrupt
+                    yield event
+                    if isinstance(event, events.NonFatalError):
+                        status = Status.ERROR
+                    if isinstance(event, events.ScenarioFinished):
+                        if event.status != Status.SKIP and (status is None or status < event.status):
+                            status = event.status
+                        if event.status in (Status.ERROR, Status.FAILURE):
+                            engine.control.count_failure()
+                    if isinstance(event, events.Interrupted) or engine.is_interrupted:
+                        status = Status.INTERRUPTED
+                        engine.stop()
+                    if engine.has_to_stop:
+                        break  # type: ignore[unreachable]
+                except queue.Empty:
+                    if all(not worker.is_alive() for worker in pool.workers):
+                        break
+                    continue
+        except KeyboardInterrupt:
+            engine.stop()
+            status = Status.INTERRUPTED
+            yield events.Interrupted(phase=PhaseName.UNIT_TESTING)
+
+    if not is_executed:
+        phase.skip_reason = PhaseSkipReason.NOTHING_TO_TEST
+        status = Status.SKIP
+    elif status is None:
+        status = Status.SKIP
+    # NOTE: Right now there is just one suite, hence two events go one after another
+    yield events.SuiteFinished(id=suite_started.id, phase=phase.name, status=status)
+    yield events.PhaseFinished(phase=phase, status=status, payload=None)
+
+
+def worker_task(*, events_queue: Queue, producer: TaskProducer, ctx: EngineContext, suite_id: uuid.UUID) -> None:
+    from hypothesis.errors import HypothesisWarning
+
+    from schemathesis.generation.hypothesis.builder import create_test
+
+    from ._executor import run_test, test_func
+
+    warnings.filterwarnings("ignore", message="The recursion limit will not be reset", category=HypothesisWarning)
+    with ignore_hypothesis_output():
+        try:
+            while not ctx.has_to_stop:
+                result = producer.next_operation()
+                if result is None:
+                    break
+
+                if isinstance(result, Ok):
+                    operation = result.ok()
+                    as_strategy_kwargs = get_strategy_kwargs(ctx, operation)
+                    test_function = create_test(
+                        operation=operation,
+                        test_func=test_func,
+                        config=HypothesisTestConfig(
+                            settings=ctx.config.execution.hypothesis_settings,
+                            seed=ctx.config.execution.seed,
+                            generation=ctx.config.execution.generation,
+                            as_strategy_kwargs=as_strategy_kwargs,
+                        ),
+                    )
+
+                    # The test is blocking, meaning that even if CTRL-C comes to the main thread, this tasks will continue
+                    # executing. However, as we set a stop event, it will be checked before the next network request.
+                    # However, this is still suboptimal, as there could be slow requests and they will block for longer
+                    for event in run_test(operation=operation, test_function=test_function, ctx=ctx, suite_id=suite_id):
+                        events_queue.put(event)
+                else:
+                    error = result.err()
+                    if error.method:
+                        label = f"{error.method.upper()} {error.full_path}"
+                        scenario_started = events.ScenarioStarted(
+                            label=label, phase=PhaseName.UNIT_TESTING, suite_id=suite_id
+                        )
+                        events_queue.put(scenario_started)
+
+                        events_queue.put(
+                            events.NonFatalError(
+                                error=error, phase=PhaseName.UNIT_TESTING, label=label, related_to_operation=True
+                            )
+                        )
+
+                        events_queue.put(
+                            events.ScenarioFinished(
+                                id=scenario_started.id,
+                                suite_id=suite_id,
+                                phase=PhaseName.UNIT_TESTING,
+                                status=Status.ERROR,
+                                recorder=ScenarioRecorder(label="Error"),
+                                elapsed_time=0.0,
+                                skip_reason=None,
+                                is_final=True,
+                            )
+                        )
+                    else:
+                        assert error.full_path is not None
+                        events_queue.put(
+                            events.NonFatalError(
+                                error=error,
+                                phase=PhaseName.UNIT_TESTING,
+                                label=error.full_path,
+                                related_to_operation=False,
+                            )
+                        )
+        except KeyboardInterrupt:
+            events_queue.put(events.Interrupted(phase=PhaseName.UNIT_TESTING))
+
+
+def get_strategy_kwargs(ctx: EngineContext, operation: APIOperation) -> dict[str, Any]:
+    kwargs = {}
+    if ctx.config.override is not None:
+        for location, entry in ctx.config.override.for_operation(operation).items():
+            if entry:
+                kwargs[location] = entry
+    if ctx.config.network.headers:
+        kwargs["headers"] = {
+            key: value for key, value in ctx.config.network.headers.items() if key.lower() != "user-agent"
+        }
+    return kwargs