schemathesis 3.39.16__py3-none-any.whl → 4.0.0__py3-none-any.whl

schemathesis/engine/phases/stateful/_executor.py

@@ -0,0 +1,356 @@
+from __future__ import annotations  # noqa: I001
+
+import queue
+import time
+import unittest
+from dataclasses import dataclass
+from typing import Any
+from warnings import catch_warnings
+
+import hypothesis
+import requests
+from hypothesis.control import current_build_context
+from hypothesis.errors import Flaky, Unsatisfiable
+from hypothesis.stateful import Rule
+from requests.exceptions import ChunkedEncodingError
+from requests.structures import CaseInsensitiveDict
+
+from schemathesis.checks import CheckContext, CheckFunction, run_checks
+from schemathesis.core.failures import Failure, FailureGroup
+from schemathesis.core.transport import Response
+from schemathesis.engine import Status, events
+from schemathesis.engine.context import EngineContext
+from schemathesis.engine.control import ExecutionControl
+from schemathesis.engine.errors import (
+    TestingState,
+    UnrecoverableNetworkError,
+    clear_hypothesis_notes,
+    is_unrecoverable_network_error,
+)
+from schemathesis.engine.phases import PhaseName
+from schemathesis.engine.phases.stateful.context import StatefulContext
+from schemathesis.engine.recorder import ScenarioRecorder
+from schemathesis.generation import overrides
+from schemathesis.generation.case import Case
+from schemathesis.generation.hypothesis.reporting import ignore_hypothesis_output
+from schemathesis.generation.stateful import STATEFUL_TESTS_LABEL
+from schemathesis.generation.stateful.state_machine import (
+    DEFAULT_STATE_MACHINE_SETTINGS,
+    APIStateMachine,
+    StepInput,
+    StepOutput,
+)
+from schemathesis.generation.metrics import MetricCollector
+
+
+def _get_hypothesis_settings_kwargs_override(settings: hypothesis.settings) -> dict[str, Any]:
+    """Get the settings that should be overridden to match the defaults for API state machines."""
+    kwargs = {}
+    hypothesis_default = hypothesis.settings()
+    if settings.phases == hypothesis_default.phases:
+        kwargs["phases"] = DEFAULT_STATE_MACHINE_SETTINGS.phases
+    if settings.stateful_step_count == hypothesis_default.stateful_step_count:
+        kwargs["stateful_step_count"] = DEFAULT_STATE_MACHINE_SETTINGS.stateful_step_count
+    if settings.deadline == hypothesis_default.deadline:
+        kwargs["deadline"] = DEFAULT_STATE_MACHINE_SETTINGS.deadline
+    if settings.suppress_health_check == hypothesis_default.suppress_health_check:
+        kwargs["suppress_health_check"] = DEFAULT_STATE_MACHINE_SETTINGS.suppress_health_check
+    return kwargs
+
+
+@dataclass
+class CachedCheckContextData:
+    override: Any
+    auth: Any
+    headers: Any
+    config: Any
+    transport_kwargs: Any
+
+    __slots__ = ("override", "auth", "headers", "config", "transport_kwargs")
+
+
+def execute_state_machine_loop(
+    *,
+    state_machine: type[APIStateMachine],
+    event_queue: queue.Queue,
+    engine: EngineContext,
+) -> None:
+    """Execute the state machine testing loop."""
+    configured_hypothesis_settings = engine.config.get_hypothesis_settings(phase="stateful")
+    kwargs = _get_hypothesis_settings_kwargs_override(configured_hypothesis_settings)
+    hypothesis_settings = hypothesis.settings(configured_hypothesis_settings, **kwargs)
+    generation = engine.config.generation_for(phase="stateful")
+
+    ctx = StatefulContext(metric_collector=MetricCollector(metrics=generation.maximize))
+    state = TestingState()
+
+    # Caches for validate_response to avoid repeated config lookups per operation
+    _check_context_cache: dict[str, CachedCheckContextData] = {}
+
+    class _InstrumentedStateMachine(state_machine):  # type: ignore[valid-type,misc]
+        """State machine with additional hooks for emitting events."""
+
+        def setup(self) -> None:
+            scenario_started = events.ScenarioStarted(label=None, phase=PhaseName.STATEFUL_TESTING, suite_id=suite_id)
+            self._start_time = time.monotonic()
+            self._scenario_id = scenario_started.id
+            event_queue.put(scenario_started)
+
+        def get_call_kwargs(self, case: Case) -> dict[str, Any]:
+            return engine.get_transport_kwargs(operation=case.operation)
+
+        def _repr_step(self, rule: Rule, data: dict, result: StepOutput) -> str:
+            return ""
+
+        def before_call(self, case: Case) -> None:
+            override = overrides.for_operation(engine.config, operation=case.operation)
+            for location in ("query", "headers", "cookies", "path_parameters"):
+                entry = getattr(override, location)
+                if entry:
+                    container = getattr(case, location) or {}
+                    container.update(entry)
+                    setattr(case, location, container)
+            return super().before_call(case)
+
+        def step(self, input: StepInput) -> StepOutput | None:
+            # Checking the stop event once inside `step` is sufficient as it is called frequently
+            # The idea is to stop the execution as soon as possible
+            if engine.has_to_stop:
+                raise KeyboardInterrupt
+            try:
+                if generation.unique_inputs:
+                    cached = ctx.get_step_outcome(input.case)
+                    if isinstance(cached, BaseException):
+                        raise cached
+                    elif cached is None:
+                        return None
+                result = super().step(input)
+                ctx.step_succeeded()
+            except FailureGroup as exc:
+                if generation.unique_inputs:
+                    for failure in exc.exceptions:
+                        ctx.store_step_outcome(input.case, failure)
+                ctx.step_failed()
+                raise
+            except Exception as exc:
+                if isinstance(exc, (requests.ConnectionError, ChunkedEncodingError)) and is_unrecoverable_network_error(
+                    exc
+                ):
+                    transport_kwargs = engine.get_transport_kwargs(operation=input.case.operation)
+                    if exc.request is not None:
+                        headers = {key: value[0] for key, value in exc.request.headers.items()}
+                    else:
+                        headers = {**dict(input.case.headers or {}), **transport_kwargs.get("headers", {})}
+                    verify = transport_kwargs.get("verify", True)
+                    state.unrecoverable_network_error = UnrecoverableNetworkError(
+                        error=exc,
+                        code_sample=input.case.as_curl_command(headers=headers, verify=verify),
+                    )
+
+                if generation.unique_inputs:
+                    ctx.store_step_outcome(input.case, exc)
+                ctx.step_errored()
+                raise
+            except KeyboardInterrupt:
+                ctx.step_interrupted()
+                raise
+            except BaseException as exc:
+                if generation.unique_inputs:
+                    ctx.store_step_outcome(input.case, exc)
+                raise exc
+            else:
+                if generation.unique_inputs:
+                    ctx.store_step_outcome(input.case, None)
+            return result
+
+        def validate_response(
+            self, response: Response, case: Case, additional_checks: tuple[CheckFunction, ...] = (), **kwargs: Any
+        ) -> None:
+            self.recorder.record_response(case_id=case.id, response=response)
+            ctx.collect_metric(case, response)
+            ctx.current_response = response
+
+            label = case.operation.label
+            cached = _check_context_cache.get(label)
+            if cached is None:
+                headers = engine.config.headers_for(operation=case.operation)
+                cached = CachedCheckContextData(
+                    override=overrides.for_operation(engine.config, operation=case.operation),
+                    auth=engine.config.auth_for(operation=case.operation),
+                    headers=CaseInsensitiveDict(headers) if headers else None,
+                    config=engine.config.checks_config_for(operation=case.operation, phase="stateful"),
+                    transport_kwargs=engine.get_transport_kwargs(operation=case.operation),
+                )
+                _check_context_cache[label] = cached
+
+            check_ctx = CheckContext(
+                override=cached.override,
+                auth=cached.auth,
+                headers=cached.headers,
+                config=cached.config,
+                transport_kwargs=cached.transport_kwargs,
+                recorder=self.recorder,
+            )
+            validate_response(
+                response=response,
+                case=case,
+                stateful_ctx=ctx,
+                check_ctx=check_ctx,
+                checks=check_ctx._checks,
+                control=engine.control,
+                recorder=self.recorder,
+                additional_checks=additional_checks,
+            )
+
+        def teardown(self) -> None:
+            build_ctx = current_build_context()
+            event_queue.put(
+                events.ScenarioFinished(
+                    id=self._scenario_id,
+                    suite_id=suite_id,
+                    phase=PhaseName.STATEFUL_TESTING,
+                    label=None,
+                    status=ctx.current_scenario_status or Status.SKIP,
+                    recorder=self.recorder,
+                    elapsed_time=time.monotonic() - self._start_time,
+                    skip_reason=None,
+                    is_final=build_ctx.is_final,
+                )
+            )
+            ctx.maximize_metrics()
+            ctx.reset_scenario()
+            super().teardown()
+
+    seed = engine.config.seed
+
+    while True:
+        # This loop is running until no new failures are found in a single iteration
+        suite_started = events.SuiteStarted(phase=PhaseName.STATEFUL_TESTING)
+        suite_id = suite_started.id
+        event_queue.put(suite_started)
+        if engine.is_interrupted:
+            event_queue.put(events.Interrupted(phase=PhaseName.STATEFUL_TESTING))
+            event_queue.put(
+                events.SuiteFinished(
+                    id=suite_started.id,
+                    phase=PhaseName.STATEFUL_TESTING,
+                    status=Status.INTERRUPTED,
+                )
+            )
+            break
+        suite_status = Status.SUCCESS
+        InstrumentedStateMachine = hypothesis.seed(seed)(_InstrumentedStateMachine)
+        # Predictably change the seed to avoid re-running the same sequences if tests fail
+        # yet have reproducible results
+        seed += 1
+        try:
+            with catch_warnings(), ignore_hypothesis_output():  # type: ignore
+                InstrumentedStateMachine.run(settings=hypothesis_settings)
+        except KeyboardInterrupt:
+            # Raised in the state machine when the stop event is set or it is raised by the user's code
+            # that is placed in the base class of the state machine.
+            # Therefore, set the stop event to cover the latter case
+            engine.stop()
+            suite_status = Status.INTERRUPTED
+            event_queue.put(events.Interrupted(phase=PhaseName.STATEFUL_TESTING))
+            break
+        except unittest.case.SkipTest:
+            # If `explicit` phase is used and there are no examples
+            suite_status = Status.SKIP
+            break
+        except FailureGroup as exc:
+            # When a check fails, the state machine is stopped
+            # The failure is already sent to the queue by the state machine
+            # Here we need to either exit or re-run the state machine with this failure marked as known
+            suite_status = Status.FAILURE
+            if engine.has_reached_the_failure_limit:
+                break  # type: ignore[unreachable]
+            for failure in exc.exceptions:
+                ctx.mark_as_seen_in_run(failure)
+            continue
+        except Flaky:
+            # Ignore flakiness
+            if engine.has_reached_the_failure_limit:
+                break  # type: ignore[unreachable]
+            # Mark all failures in this suite as seen to prevent them being re-discovered
+            ctx.mark_current_suite_as_seen_in_run()
+            continue
+        except Exception as exc:
+            if isinstance(exc, Unsatisfiable) and ctx.completed_scenarios > 0:
+                # Sometimes Hypothesis randomly gives up on generating some complex cases. However, if we know that
+                # values are possible to generate based on the previous observations, we retry the generation
+                if ctx.completed_scenarios >= hypothesis_settings.max_examples:
+                    # Avoid infinite restarts
+                    break
+                continue
+            clear_hypothesis_notes(exc)
+            # Any other exception is an inner error and the test run should be stopped
+            suite_status = Status.ERROR
+            code_sample: str | None = None
+            if state.unrecoverable_network_error is not None:
+                exc = state.unrecoverable_network_error.error
+                code_sample = state.unrecoverable_network_error.code_sample
+            event_queue.put(
+                events.NonFatalError(
+                    error=exc,
+                    phase=PhaseName.STATEFUL_TESTING,
+                    label=STATEFUL_TESTS_LABEL,
+                    related_to_operation=False,
+                    code_sample=code_sample,
+                )
+            )
+            break
+        finally:
+            event_queue.put(
+                events.SuiteFinished(
+                    id=suite_started.id,
+                    phase=PhaseName.STATEFUL_TESTING,
+                    status=suite_status,
+                )
+            )
+            ctx.reset()
+        # Exit on the first successful state machine execution
+        break
+
+
+def validate_response(
+    *,
+    response: Response,
+    case: Case,
+    stateful_ctx: StatefulContext,
+    check_ctx: CheckContext,
+    control: ExecutionControl,
+    checks: list[CheckFunction],
+    recorder: ScenarioRecorder,
+    additional_checks: tuple[CheckFunction, ...] = (),
+) -> None:
+    """Validate the response against the provided checks."""
+
+    def on_failure(name: str, collected: set[Failure], failure: Failure) -> None:
+        if stateful_ctx.is_seen_in_suite(failure) or stateful_ctx.is_seen_in_run(failure):
+            return
+        failure_data = recorder.find_failure_data(parent_id=case.id, failure=failure)
+        recorder.record_check_failure(
+            name=name,
+            case_id=failure_data.case.id,
+            code_sample=failure_data.case.as_curl_command(headers=failure_data.headers, verify=failure_data.verify),
+            failure=failure,
+        )
+        control.count_failure()
+        stateful_ctx.mark_as_seen_in_suite(failure)
+        collected.add(failure)
+
+    def on_success(name: str, case: Case) -> None:
+        recorder.record_check_success(name=name, case_id=case.id)
+
+    failures = run_checks(
+        case=case,
+        response=response,
+        ctx=check_ctx,
+        checks=tuple(checks) + tuple(additional_checks),
+        on_failure=on_failure,
+        on_success=on_success,
+    )
+
+    if failures:
+        raise FailureGroup(list(failures)) from None

schemathesis/engine/phases/stateful/context.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from schemathesis.core import NOT_SET, NotSet
+from schemathesis.core.failures import Failure
+from schemathesis.core.transport import Response
+from schemathesis.engine import Status
+from schemathesis.generation.case import Case
+from schemathesis.generation.metrics import MetricCollector
+
+
+@dataclass
+class StatefulContext:
+    """Mutable context for state machine execution."""
+
+    # All seen failure keys, both grouped and individual ones
+    seen_in_run: set[Failure] = field(default_factory=set)
+    # Failures keys seen in the current suite
+    seen_in_suite: set[Failure] = field(default_factory=set)
+    # Status of the current step
+    current_step_status: Status | None = None
+    # The currently processed response
+    current_response: Response | None = None
+    # Total number of failures
+    failures_count: int = 0
+    # The total number of completed test scenario
+    completed_scenarios: int = 0
+    # Metrics collector for targeted testing
+    metric_collector: MetricCollector = field(default_factory=MetricCollector)
+    step_outcomes: dict[int, BaseException | None] = field(default_factory=dict)
+
+    @property
+    def current_scenario_status(self) -> Status | None:
+        return self.current_step_status
+
+    def reset_scenario(self) -> None:
+        self.completed_scenarios += 1
+        self.current_step_status = None
+        self.current_response = None
+        self.step_outcomes.clear()
+
+    def step_succeeded(self) -> None:
+        self.current_step_status = Status.SUCCESS
+
+    def step_failed(self) -> None:
+        self.current_step_status = Status.FAILURE
+
+    def step_errored(self) -> None:
+        self.current_step_status = Status.ERROR
+
+    def step_interrupted(self) -> None:
+        self.current_step_status = Status.INTERRUPTED
+
+    def mark_as_seen_in_run(self, exc: Failure) -> None:
+        self.seen_in_run.add(exc)
+
+    def mark_as_seen_in_suite(self, exc: Failure) -> None:
+        self.seen_in_suite.add(exc)
+
+    def mark_current_suite_as_seen_in_run(self) -> None:
+        self.seen_in_run.update(self.seen_in_suite)
+
+    def is_seen_in_run(self, exc: Failure) -> bool:
+        return exc in self.seen_in_run
+
+    def is_seen_in_suite(self, exc: Failure) -> bool:
+        return exc in self.seen_in_suite
+
+    def collect_metric(self, case: Case, response: Response) -> None:
+        self.metric_collector.store(case, response)
+
+    def maximize_metrics(self) -> None:
+        self.metric_collector.maximize()
+
+    def reset(self) -> None:
+        self.seen_in_suite.clear()
+        self.reset_scenario()
+        self.metric_collector.reset()
+
+    def store_step_outcome(self, case: Case, outcome: BaseException | None) -> None:
+        self.step_outcomes[hash(case)] = outcome
+
+    def get_step_outcome(self, case: Case) -> BaseException | None | NotSet:
+        return self.step_outcomes.get(hash(case), NOT_SET)

schemathesis/engine/phases/unit/__init__.py

@@ -0,0 +1,212 @@
+"""Unit testing by Schemathesis Engine.
+
+This module provides high-level flow for single-, and multi-threaded modes.
+"""
+
+from __future__ import annotations
+
+import queue
+import uuid
+import warnings
+from queue import Queue
+from typing import TYPE_CHECKING, Any
+
+from schemathesis.core.errors import InvalidSchema
+from schemathesis.core.result import Ok
+from schemathesis.engine import Status, events
+from schemathesis.engine.phases import PhaseName, PhaseSkipReason
+from schemathesis.engine.recorder import ScenarioRecorder
+from schemathesis.generation import overrides
+from schemathesis.generation.hypothesis.builder import HypothesisTestConfig, HypothesisTestMode
+from schemathesis.generation.hypothesis.reporting import ignore_hypothesis_output
+
+from ._pool import TaskProducer, WorkerPool
+
+if TYPE_CHECKING:
+    from schemathesis.engine.context import EngineContext
+    from schemathesis.engine.phases import Phase
+    from schemathesis.schemas import APIOperation
+
+WORKER_TIMEOUT = 0.1
+
+
+def execute(engine: EngineContext, phase: Phase) -> events.EventGenerator:
+    """Run a set of unit tests.
+
+    Implemented as a producer-consumer pattern via a task queue.
+    The main thread provides an iterator over API operations and worker threads create test functions and run them.
+    """
+    if phase.name == PhaseName.EXAMPLES:
+        mode = HypothesisTestMode.EXAMPLES
+    elif phase.name == PhaseName.COVERAGE:
+        mode = HypothesisTestMode.COVERAGE
+    else:
+        mode = HypothesisTestMode.FUZZING
+    producer = TaskProducer(engine)
+
+    suite_started = events.SuiteStarted(phase=phase.name)
+
+    yield suite_started
+
+    status = None
+    is_executed = False
+
+    try:
+        with WorkerPool(
+            workers_num=engine.config.workers,
+            producer=producer,
+            worker_factory=worker_task,
+            ctx=engine,
+            mode=mode,
+            phase=phase.name,
+            suite_id=suite_started.id,
+        ) as pool:
+            try:
+                while True:
+                    try:
+                        event = pool.events_queue.get(timeout=WORKER_TIMEOUT)
+                        is_executed = True
+                        if engine.is_interrupted:
+                            raise KeyboardInterrupt
+                        yield event
+                        if isinstance(event, events.NonFatalError):
+                            status = Status.ERROR
+                        if isinstance(event, events.ScenarioFinished):
+                            if event.status != Status.SKIP and (status is None or status < event.status):
+                                status = event.status
+                            if event.status in (Status.ERROR, Status.FAILURE):
+                                engine.control.count_failure()
+                        if isinstance(event, events.Interrupted) or engine.is_interrupted:
+                            status = Status.INTERRUPTED
+                            engine.stop()
+                        if engine.has_to_stop:
+                            break  # type: ignore[unreachable]
+                    except queue.Empty:
+                        if all(not worker.is_alive() for worker in pool.workers):
+                            break
+                        continue
+            except KeyboardInterrupt:
+                # Soft stop, waiting for workers to terminate
+                engine.stop()
+                status = Status.INTERRUPTED
+                yield events.Interrupted(phase=phase.name)
+    except KeyboardInterrupt:
+        # Hard stop, don't wait for worker threads
+        pass
+
+    if not is_executed:
+        phase.skip_reason = PhaseSkipReason.NOTHING_TO_TEST
+        status = Status.SKIP
+    elif status is None:
+        status = Status.SKIP
+    # NOTE: Right now there is just one suite, hence two events go one after another
+    yield events.SuiteFinished(id=suite_started.id, phase=phase.name, status=status)
+    yield events.PhaseFinished(phase=phase, status=status, payload=None)
+
+
+def worker_task(
+    *,
+    events_queue: Queue,
+    producer: TaskProducer,
+    ctx: EngineContext,
+    mode: HypothesisTestMode,
+    phase: PhaseName,
+    suite_id: uuid.UUID,
+) -> None:
+    from hypothesis.errors import HypothesisWarning, InvalidArgument
+
+    from schemathesis.generation.hypothesis.builder import create_test
+
+    from ._executor import run_test, test_func
+
+    def on_error(error: Exception, *, method: str | None = None, path: str | None = None) -> None:
+        if method and path:
+            label = f"{method.upper()} {path}"
+            scenario_started = events.ScenarioStarted(label=label, phase=phase, suite_id=suite_id)
+            events_queue.put(scenario_started)
+
+            events_queue.put(events.NonFatalError(error=error, phase=phase, label=label, related_to_operation=True))
+
+            events_queue.put(
+                events.ScenarioFinished(
+                    id=scenario_started.id,
+                    suite_id=suite_id,
+                    phase=phase,
+                    label=label,
+                    status=Status.ERROR,
+                    recorder=ScenarioRecorder(label="Error"),
+                    elapsed_time=0.0,
+                    skip_reason=None,
+                    is_final=True,
+                )
+            )
+        else:
+            events_queue.put(
+                events.NonFatalError(
+                    error=error,
+                    phase=phase,
+                    label=path or "-",
+                    related_to_operation=False,
+                )
+            )
+
+    warnings.filterwarnings("ignore", message="The recursion limit will not be reset", category=HypothesisWarning)
+    with ignore_hypothesis_output():
+        try:
+            while not ctx.has_to_stop:
+                result = producer.next_operation()
+                if result is None:
+                    break
+
+                if isinstance(result, Ok):
+                    operation = result.ok()
+                    phases = ctx.config.phases_for(operation=operation)
+                    # Skip tests if this phase is disabled
+                    if (
+                        (phase == PhaseName.EXAMPLES and not phases.examples.enabled)
+                        or (phase == PhaseName.FUZZING and not phases.fuzzing.enabled)
+                        or (phase == PhaseName.COVERAGE and not phases.coverage.enabled)
+                    ):
+                        continue
+                    as_strategy_kwargs = get_strategy_kwargs(ctx, operation=operation)
+                    try:
+                        test_function = create_test(
+                            operation=operation,
+                            test_func=test_func,
+                            config=HypothesisTestConfig(
+                                modes=[mode],
+                                settings=ctx.config.get_hypothesis_settings(operation=operation, phase=phase.name),
+                                seed=ctx.config.seed,
+                                project=ctx.config,
+                                as_strategy_kwargs=as_strategy_kwargs,
+                            ),
+                        )
+                    except (InvalidSchema, InvalidArgument) as exc:
+                        on_error(exc, method=operation.method, path=operation.path)
+                        continue
+
+                    # The test is blocking, meaning that even if CTRL-C comes to the main thread, this tasks will continue
+                    # executing. However, as we set a stop event, it will be checked before the next network request.
+                    # However, this is still suboptimal, as there could be slow requests and they will block for longer
+                    for event in run_test(
+                        operation=operation, test_function=test_function, ctx=ctx, phase=phase, suite_id=suite_id
+                    ):
+                        events_queue.put(event)
+                else:
+                    error = result.err()
+                    on_error(error, method=error.method, path=error.path)
+        except KeyboardInterrupt:
+            events_queue.put(events.Interrupted(phase=phase))
+
+
+def get_strategy_kwargs(ctx: EngineContext, *, operation: APIOperation) -> dict[str, Any]:
+    kwargs = {}
+    override = overrides.for_operation(ctx.config, operation=operation)
+    for location in ("query", "headers", "cookies", "path_parameters"):
+        entry = getattr(override, location)
+        if entry:
+            kwargs[location] = entry
+    headers = ctx.config.headers_for(operation=operation)
+    if headers:
+        kwargs["headers"] = {key: value for key, value in headers.items() if key.lower() != "user-agent"}
+    return kwargs