schemathesis 3.39.7__py3-none-any.whl → 4.0.0a2__py3-none-any.whl

schemathesis/{runner → engine/phases}/probes.py

@@ -10,32 +10,56 @@ from __future__ import annotations
 
 import enum
 import warnings
-from dataclasses import asdict, dataclass, field
-from typing import TYPE_CHECKING, Any
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
 
-from ..constants import USER_AGENT
-from ..exceptions import format_exception
-from ..models import Request, Response
-from ..sanitization import sanitize_request, sanitize_response
-from ..transports import RequestConfig
-from ..transports.auth import get_requests_auth
+from schemathesis.core.result import Err, Ok, Result
+from schemathesis.core.transport import USER_AGENT
+from schemathesis.engine import Status, events
 
 if TYPE_CHECKING:
     import requests
 
-    from ..schemas import BaseSchema
+    from schemathesis.engine.config import NetworkConfig
+    from schemathesis.engine.context import EngineContext
+    from schemathesis.engine.events import EventGenerator
+    from schemathesis.engine.phases import Phase
+    from schemathesis.schemas import BaseSchema
 
 
-HEADER_NAME = "X-Schemathesis-Probe"
+@dataclass
+class ProbePayload:
+    probes: list[ProbeRun]
+
+    __slots__ = ("probes",)
+
+
+def execute(ctx: EngineContext, phase: Phase) -> EventGenerator:
+    """Discover capabilities of the tested app."""
+    probes = run(ctx.schema, ctx.session, ctx.config.network)
+    status = Status.SUCCESS
+    payload: Result[ProbePayload, Exception] | None = None
+    for result in probes:
+        if isinstance(result.probe, NullByteInHeader) and result.is_failure:
+            from ...specs.openapi import formats
+            from ...specs.openapi.formats import HEADER_FORMAT, header_values
+
+            formats.register(HEADER_FORMAT, header_values(blacklist_characters="\n\r\x00"))
+        if result.error is not None:
+            status = Status.ERROR
+            payload = Err(result.error)
+        else:
+            status = Status.SUCCESS
+            payload = Ok(ProbePayload(probes=probes))
+    yield events.PhaseFinished(phase=phase, status=status, payload=payload)
 
 
-@dataclass
-class ProbeConfig:
-    base_url: str | None = None
-    request: RequestConfig = field(default_factory=RequestConfig)
-    auth: tuple[str, str] | None = None
-    auth_type: str | None = None
-    headers: dict[str, str] | None = None
+def run(schema: BaseSchema, session: requests.Session, config: NetworkConfig) -> list[ProbeRun]:
+    """Run all probes against the given schema."""
+    return [send(probe(), session, schema, config) for probe in PROBES]
+
+
+HEADER_NAME = "X-Schemathesis-Probe"
 
 
 @dataclass
@@ -45,7 +69,7 @@ class Probe:
     name: str
 
     def prepare_request(
-        self, session: requests.Session, request: requests.Request, schema: BaseSchema, config: ProbeConfig
+        self, session: requests.Session, request: requests.Request, schema: BaseSchema
     ) -> requests.PreparedRequest:
         raise NotImplementedError
 
@@ -70,49 +94,24 @@ class ProbeRun:
     outcome: ProbeOutcome
     request: requests.PreparedRequest | None = None
     response: requests.Response | None = None
-    error: requests.RequestException | None = None
+    error: Exception | None = None
 
     @property
     def is_failure(self) -> bool:
         return self.outcome == ProbeOutcome.FAILURE
 
-    def serialize(self) -> dict[str, Any]:
-        """Serialize probe results so it can be sent over the network."""
-        if self.request:
-            _request = Request.from_prepared_request(self.request)
-            sanitize_request(_request)
-            request = asdict(_request)
-        else:
-            request = None
-        if self.response:
-            sanitize_response(self.response)
-            response = asdict(Response.from_requests(self.response))
-        else:
-            response = None
-        if self.error:
-            error = format_exception(self.error)
-        else:
-            error = None
-        return {
-            "name": self.probe.name,
-            "outcome": self.outcome.value,
-            "request": request,
-            "response": response,
-            "error": error,
-        }
-
 
 @dataclass
 class NullByteInHeader(Probe):
     """Support NULL bytes in headers."""
 
-    name: str = "NULL_BYTE_IN_HEADER"
+    name: str = "Supports NULL byte in headers"
 
     def prepare_request(
-        self, session: requests.Session, request: requests.Request, schema: BaseSchema, config: ProbeConfig
+        self, session: requests.Session, request: requests.Request, schema: BaseSchema
     ) -> requests.PreparedRequest:
         request.method = "GET"
-        request.url = config.base_url or schema.get_base_url()
+        request.url = schema.get_base_url()
         request.headers = {"X-Schemathesis-Probe-Null": "\x00"}
         return session.prepare_request(request)
 
@@ -125,22 +124,19 @@ class NullByteInHeader(Probe):
 PROBES = (NullByteInHeader,)
 
 
-def send(probe: Probe, session: requests.Session, schema: BaseSchema, config: ProbeConfig) -> ProbeRun:
+def send(probe: Probe, session: requests.Session, schema: BaseSchema, config: NetworkConfig) -> ProbeRun:
     """Send the probe to the application."""
     from requests import PreparedRequest, Request, RequestException
     from requests.exceptions import MissingSchema
     from urllib3.exceptions import InsecureRequestWarning
 
     try:
-        request = probe.prepare_request(session, Request(), schema, config)
+        request = probe.prepare_request(session, Request(), schema)
         request.headers[HEADER_NAME] = probe.name
         request.headers["User-Agent"] = USER_AGENT
-        kwargs: dict[str, Any] = {"timeout": config.request.prepared_timeout or 2}
-        if config.request.proxy is not None:
-            kwargs["proxies"] = {"all": config.request.proxy}
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", InsecureRequestWarning)
-            response = session.send(request, **kwargs)
+            response = session.send(request, timeout=config.timeout or 2)
     except MissingSchema:
         # In-process ASGI/WSGI testing will have local URLs and requires extra handling
         # which is not currently implemented
@@ -150,18 +146,3 @@ def send(probe: Probe, session: requests.Session, schema: BaseSchema, config: Pr
         return ProbeRun(probe, ProbeOutcome.ERROR, req, None, exc)
     result_type = probe.analyze_response(response)
     return ProbeRun(probe, result_type, request, response)
-
-
-def run(schema: BaseSchema, config: ProbeConfig) -> list[ProbeRun]:
-    """Run all probes against the given schema."""
-    from requests import Session
-
-    session = Session()
-    session.headers.update(config.headers or {})
-    session.verify = config.request.tls_verify
-    if config.request.cert is not None:
-        session.cert = config.request.cert
-    if config.auth is not None:
-        session.auth = get_requests_auth(config.auth, config.auth_type)
-
-    return [send(probe(), session, schema, config) for probe in PROBES]

schemathesis/engine/phases/stateful/__init__.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import queue
+import threading
+from typing import TYPE_CHECKING
+
+from schemathesis.engine import Status, events
+from schemathesis.engine.phases import Phase, PhaseName, PhaseSkipReason
+
+if TYPE_CHECKING:
+    from schemathesis.engine.context import EngineContext
+
+EVENT_QUEUE_TIMEOUT = 0.01
+
+
+def execute(engine: EngineContext, phase: Phase) -> events.EventGenerator:
+    from schemathesis.engine.phases.stateful._executor import execute_state_machine_loop
+
+    try:
+        state_machine = engine.schema.as_state_machine()
+    except Exception as exc:
+        yield events.NonFatalError(error=exc, phase=phase.name, label="Stateful tests", related_to_operation=False)
+        return
+
+    event_queue: queue.Queue = queue.Queue()
+
+    thread = threading.Thread(
+        target=execute_state_machine_loop,
+        kwargs={"state_machine": state_machine, "event_queue": event_queue, "engine": engine},
+        name="schemathesis_stateful_tests",
+    )
+    status: Status | None = None
+    is_executed = False
+
+    thread.start()
+    try:
+        while True:
+            try:
+                event = event_queue.get(timeout=EVENT_QUEUE_TIMEOUT)
+                is_executed = True
+                # Set the run status based on the suite status
+                # ERROR & INTERRUPTED statuses are terminal, therefore they should not be overridden
+                if (
+                    isinstance(event, events.SuiteFinished)
+                    and event.status != Status.SKIP
+                    and (status is None or status < event.status)
+                ):
+                    status = event.status
+                yield event
+            except queue.Empty:
+                if not thread.is_alive():
+                    break
+    except KeyboardInterrupt:
+        # Immediately notify the engine thread to stop, even though that the event will be set below in `finally`
+        engine.stop()
+        status = Status.INTERRUPTED
+        yield events.Interrupted(phase=PhaseName.STATEFUL_TESTING)
+    finally:
+        thread.join()
+
+    if not is_executed:
+        phase.skip_reason = PhaseSkipReason.NOTHING_TO_TEST
+        status = Status.SKIP
+    elif status is None:
+        status = Status.SKIP
+    yield events.PhaseFinished(phase=phase, status=status, payload=None)

schemathesis/engine/phases/stateful/_executor.py

@@ -0,0 +1,301 @@
+from __future__ import annotations
+
+import queue
+import time
+import unittest
+from dataclasses import replace
+from typing import Any
+from warnings import catch_warnings
+
+import hypothesis
+from hypothesis.control import current_build_context
+from hypothesis.errors import Flaky, Unsatisfiable
+from hypothesis.stateful import Rule
+
+from schemathesis.checks import CheckContext, CheckFunction, run_checks
+from schemathesis.core.failures import Failure, FailureGroup
+from schemathesis.core.transport import Response
+from schemathesis.engine import Status, events
+from schemathesis.engine.context import EngineContext
+from schemathesis.engine.control import ExecutionControl
+from schemathesis.engine.phases import PhaseName
+from schemathesis.engine.phases.stateful.context import StatefulContext
+from schemathesis.engine.recorder import ScenarioRecorder
+from schemathesis.generation.case import Case
+from schemathesis.generation.hypothesis.reporting import ignore_hypothesis_output
+from schemathesis.generation.stateful.state_machine import (
+    DEFAULT_STATE_MACHINE_SETTINGS,
+    APIStateMachine,
+    StepInput,
+    StepOutput,
+)
+from schemathesis.generation.targets import TargetMetricCollector
+
+
+def _get_hypothesis_settings_kwargs_override(settings: hypothesis.settings) -> dict[str, Any]:
+    """Get the settings that should be overridden to match the defaults for API state machines."""
+    kwargs = {}
+    hypothesis_default = hypothesis.settings()
+    if settings.phases == hypothesis_default.phases:
+        kwargs["phases"] = DEFAULT_STATE_MACHINE_SETTINGS.phases
+    if settings.stateful_step_count == hypothesis_default.stateful_step_count:
+        kwargs["stateful_step_count"] = DEFAULT_STATE_MACHINE_SETTINGS.stateful_step_count
+    if settings.deadline == hypothesis_default.deadline:
+        kwargs["deadline"] = DEFAULT_STATE_MACHINE_SETTINGS.deadline
+    if settings.suppress_health_check == hypothesis_default.suppress_health_check:
+        kwargs["suppress_health_check"] = DEFAULT_STATE_MACHINE_SETTINGS.suppress_health_check
+    return kwargs
+
+
+def execute_state_machine_loop(
+    *,
+    state_machine: type[APIStateMachine],
+    event_queue: queue.Queue,
+    engine: EngineContext,
+) -> None:
+    """Execute the state machine testing loop."""
+    kwargs = _get_hypothesis_settings_kwargs_override(engine.config.execution.hypothesis_settings)
+    if kwargs:
+        config = replace(
+            engine.config,
+            execution=replace(
+                engine.config.execution,
+                hypothesis_settings=hypothesis.settings(engine.config.execution.hypothesis_settings, **kwargs),
+            ),
+        )
+    else:
+        config = engine.config
+
+    ctx = StatefulContext(metric_collector=TargetMetricCollector(targets=config.execution.targets))
+
+    transport_kwargs = engine.transport_kwargs
+
+    class _InstrumentedStateMachine(state_machine):  # type: ignore[valid-type,misc]
+        """State machine with additional hooks for emitting events."""
+
+        def setup(self) -> None:
+            scenario_started = events.ScenarioStarted(label=None, phase=PhaseName.STATEFUL_TESTING, suite_id=suite_id)
+            self._start_time = time.monotonic()
+            self._scenario_id = scenario_started.id
+            event_queue.put(scenario_started)
+            self.recorder = ScenarioRecorder(label="Stateful tests")
+            self._check_ctx = engine.get_check_context(self.recorder)
+
+        def get_call_kwargs(self, case: Case) -> dict[str, Any]:
+            return transport_kwargs
+
+        def _repr_step(self, rule: Rule, data: dict, result: StepOutput) -> str:
+            return ""
+
+        if config.override is not None:
+
+            def before_call(self, case: Case) -> None:
+                for location, entry in config.override.for_operation(case.operation).items():  # type: ignore[union-attr]
+                    if entry:
+                        container = getattr(case, location) or {}
+                        container.update(entry)
+                        setattr(case, location, container)
+                return super().before_call(case)
+
+        def step(self, input: StepInput) -> StepOutput | None:
+            # Checking the stop event once inside `step` is sufficient as it is called frequently
+            # The idea is to stop the execution as soon as possible
+            if input.transition is not None:
+                self.recorder.record_case(
+                    parent_id=input.transition.parent_id, transition=input.transition, case=input.case
+                )
+            else:
+                self.recorder.record_case(parent_id=None, transition=None, case=input.case)
+            if engine.has_to_stop:
+                raise KeyboardInterrupt
+            try:
+                if config.execution.unique_inputs:
+                    cached = ctx.get_step_outcome(input.case)
+                    if isinstance(cached, BaseException):
+                        raise cached
+                    elif cached is None:
+                        return None
+                result = super().step(input)
+                ctx.step_succeeded()
+            except FailureGroup as exc:
+                if config.execution.unique_inputs:
+                    for failure in exc.exceptions:
+                        ctx.store_step_outcome(input.case, failure)
+                ctx.step_failed()
+                raise
+            except Exception as exc:
+                if config.execution.unique_inputs:
+                    ctx.store_step_outcome(input.case, exc)
+                ctx.step_errored()
+                raise
+            except KeyboardInterrupt:
+                ctx.step_interrupted()
+                raise
+            except BaseException as exc:
+                if config.execution.unique_inputs:
+                    ctx.store_step_outcome(input.case, exc)
+                raise exc
+            else:
+                if config.execution.unique_inputs:
+                    ctx.store_step_outcome(input.case, None)
+            return result
+
+        def validate_response(
+            self, response: Response, case: Case, additional_checks: tuple[CheckFunction, ...] = ()
+        ) -> None:
+            self.recorder.record_response(case_id=case.id, response=response)
+            ctx.collect_metric(case, response)
+            ctx.current_response = response
+            validate_response(
+                response=response,
+                case=case,
+                stateful_ctx=ctx,
+                check_ctx=self._check_ctx,
+                checks=config.execution.checks,
+                control=engine.control,
+                recorder=self.recorder,
+                additional_checks=additional_checks,
+            )
+
+        def teardown(self) -> None:
+            build_ctx = current_build_context()
+            event_queue.put(
+                events.ScenarioFinished(
+                    id=self._scenario_id,
+                    suite_id=suite_id,
+                    phase=PhaseName.STATEFUL_TESTING,
+                    label=None,
+                    status=ctx.current_scenario_status or Status.SKIP,
+                    recorder=self.recorder,
+                    elapsed_time=time.monotonic() - self._start_time,
+                    skip_reason=None,
+                    is_final=build_ctx.is_final,
+                )
+            )
+            ctx.maximize_metrics()
+            ctx.reset_scenario()
+            super().teardown()
+
+    if config.execution.seed is not None:
+        InstrumentedStateMachine = hypothesis.seed(config.execution.seed)(_InstrumentedStateMachine)
+    else:
+        InstrumentedStateMachine = _InstrumentedStateMachine
+
+    while True:
+        # This loop is running until no new failures are found in a single iteration
+        suite_started = events.SuiteStarted(phase=PhaseName.STATEFUL_TESTING)
+        suite_id = suite_started.id
+        event_queue.put(suite_started)
+        if engine.is_interrupted:
+            event_queue.put(events.Interrupted(phase=PhaseName.STATEFUL_TESTING))
+            event_queue.put(
+                events.SuiteFinished(
+                    id=suite_started.id,
+                    phase=PhaseName.STATEFUL_TESTING,
+                    status=Status.INTERRUPTED,
+                )
+            )
+            break
+        suite_status = Status.SUCCESS
+        try:
+            with catch_warnings(), ignore_hypothesis_output():  # type: ignore
+                InstrumentedStateMachine.run(settings=config.execution.hypothesis_settings)
+        except KeyboardInterrupt:
+            # Raised in the state machine when the stop event is set or it is raised by the user's code
+            # that is placed in the base class of the state machine.
+            # Therefore, set the stop event to cover the latter case
+            engine.stop()
+            suite_status = Status.INTERRUPTED
+            event_queue.put(events.Interrupted(phase=PhaseName.STATEFUL_TESTING))
+            break
+        except unittest.case.SkipTest:
+            # If `explicit` phase is used and there are no examples
+            suite_status = Status.SKIP
+            break
+        except FailureGroup as exc:
+            # When a check fails, the state machine is stopped
+            # The failure is already sent to the queue by the state machine
+            # Here we need to either exit or re-run the state machine with this failure marked as known
+            suite_status = Status.FAILURE
+            if engine.has_reached_the_failure_limit:
+                break  # type: ignore[unreachable]
+            for failure in exc.exceptions:
+                ctx.mark_as_seen_in_run(failure)
+            continue
+        except Flaky:
+            suite_status = Status.FAILURE
+            if engine.has_reached_the_failure_limit:
+                break  # type: ignore[unreachable]
+            # Mark all failures in this suite as seen to prevent them being re-discovered
+            ctx.mark_current_suite_as_seen_in_run()
+            continue
+        except Exception as exc:
+            if isinstance(exc, Unsatisfiable) and ctx.completed_scenarios > 0:
+                # Sometimes Hypothesis randomly gives up on generating some complex cases. However, if we know that
+                # values are possible to generate based on the previous observations, we retry the generation
+                if ctx.completed_scenarios >= config.execution.hypothesis_settings.max_examples:
+                    # Avoid infinite restarts
+                    break
+                continue
+            # Any other exception is an inner error and the test run should be stopped
+            suite_status = Status.ERROR
+            event_queue.put(
+                events.NonFatalError(
+                    error=exc, phase=PhaseName.STATEFUL_TESTING, label="Stateful tests", related_to_operation=False
+                )
+            )
+            break
+        finally:
+            event_queue.put(
+                events.SuiteFinished(
+                    id=suite_started.id,
+                    phase=PhaseName.STATEFUL_TESTING,
+                    status=suite_status,
+                )
+            )
+            ctx.reset()
+        # Exit on the first successful state machine execution
+        break
+
+
+def validate_response(
+    *,
+    response: Response,
+    case: Case,
+    stateful_ctx: StatefulContext,
+    check_ctx: CheckContext,
+    control: ExecutionControl,
+    checks: list[CheckFunction],
+    recorder: ScenarioRecorder,
+    additional_checks: tuple[CheckFunction, ...] = (),
+) -> None:
+    """Validate the response against the provided checks."""
+
+    def on_failure(name: str, collected: set[Failure], failure: Failure) -> None:
+        if stateful_ctx.is_seen_in_suite(failure) or stateful_ctx.is_seen_in_run(failure):
+            return
+        failure_data = recorder.find_failure_data(parent_id=case.id, failure=failure)
+        recorder.record_check_failure(
+            name=name,
+            case_id=failure_data.case.id,
+            code_sample=failure_data.case.as_curl_command(headers=failure_data.headers, verify=failure_data.verify),
+            failure=failure,
+        )
+        control.count_failure()
+        stateful_ctx.mark_as_seen_in_suite(failure)
+        collected.add(failure)
+
+    def on_success(name: str, case: Case) -> None:
+        recorder.record_check_success(name=name, case_id=case.id)
+
+    failures = run_checks(
+        case=case,
+        response=response,
+        ctx=check_ctx,
+        checks=tuple(checks) + tuple(additional_checks),
+        on_failure=on_failure,
+        on_success=on_success,
+    )
+
+    if failures:
+        raise FailureGroup(list(failures)) from None

schemathesis/engine/phases/stateful/context.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from schemathesis.core import NOT_SET, NotSet
+from schemathesis.core.failures import Failure
+from schemathesis.core.transport import Response
+from schemathesis.engine import Status
+from schemathesis.generation.case import Case
+from schemathesis.generation.targets import TargetMetricCollector
+
+
+@dataclass
+class StatefulContext:
+    """Mutable context for state machine execution."""
+
+    # All seen failure keys, both grouped and individual ones
+    seen_in_run: set[Failure] = field(default_factory=set)
+    # Failures keys seen in the current suite
+    seen_in_suite: set[Failure] = field(default_factory=set)
+    # Status of the current step
+    current_step_status: Status | None = None
+    # The currently processed response
+    current_response: Response | None = None
+    # Total number of failures
+    failures_count: int = 0
+    # The total number of completed test scenario
+    completed_scenarios: int = 0
+    # Metrics collector for targeted testing
+    metric_collector: TargetMetricCollector = field(default_factory=TargetMetricCollector)
+    step_outcomes: dict[int, BaseException | None] = field(default_factory=dict)
+
+    @property
+    def current_scenario_status(self) -> Status | None:
+        return self.current_step_status
+
+    def reset_scenario(self) -> None:
+        self.completed_scenarios += 1
+        self.current_step_status = None
+        self.current_response = None
+        self.step_outcomes.clear()
+
+    def step_succeeded(self) -> None:
+        self.current_step_status = Status.SUCCESS
+
+    def step_failed(self) -> None:
+        self.current_step_status = Status.FAILURE
+
+    def step_errored(self) -> None:
+        self.current_step_status = Status.ERROR
+
+    def step_interrupted(self) -> None:
+        self.current_step_status = Status.INTERRUPTED
+
+    def mark_as_seen_in_run(self, exc: Failure) -> None:
+        self.seen_in_run.add(exc)
+
+    def mark_as_seen_in_suite(self, exc: Failure) -> None:
+        self.seen_in_suite.add(exc)
+
+    def mark_current_suite_as_seen_in_run(self) -> None:
+        self.seen_in_run.update(self.seen_in_suite)
+
+    def is_seen_in_run(self, exc: Failure) -> bool:
+        return exc in self.seen_in_run
+
+    def is_seen_in_suite(self, exc: Failure) -> bool:
+        return exc in self.seen_in_suite
+
+    def collect_metric(self, case: Case, response: Response) -> None:
+        self.metric_collector.store(case, response)
+
+    def maximize_metrics(self) -> None:
+        self.metric_collector.maximize()
+
+    def reset(self) -> None:
+        self.seen_in_suite.clear()
+        self.reset_scenario()
+        self.metric_collector.reset()
+
+    def store_step_outcome(self, case: Case, outcome: BaseException | None) -> None:
+        self.step_outcomes[hash(case)] = outcome
+
+    def get_step_outcome(self, case: Case) -> BaseException | None | NotSet:
+        return self.step_outcomes.get(hash(case), NOT_SET)