schemathesis 3.13.0__py3-none-any.whl → 4.4.2__py3-none-any.whl

schemathesis/engine/events.py

@@ -0,0 +1,284 @@
+from __future__ import annotations
+
+import time
+import uuid
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Generator
+
+from schemathesis.core.result import Result
+from schemathesis.core.schema_analysis import SchemaWarning
+from schemathesis.engine.errors import EngineErrorInfo
+from schemathesis.engine.phases import Phase, PhaseName
+from schemathesis.engine.recorder import ScenarioRecorder
+
+if TYPE_CHECKING:
+    from schemathesis.engine import Status
+    from schemathesis.engine.phases.probes import ProbePayload
+
+EventGenerator = Generator["EngineEvent", None, None]
+
+
+@dataclass
+class EngineEvent:
+    """An event within the engine's lifecycle."""
+
+    id: uuid.UUID
+    timestamp: float
+    # Indicates whether this event is the last in the event stream
+    is_terminal = False
+
+
+@dataclass
+class EngineStarted(EngineEvent):
+    """Start of an engine."""
+
+    __slots__ = ("id", "timestamp")
+
+    def __init__(self) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+
+
+@dataclass
+class PhaseEvent(EngineEvent):
+    """Event associated with a specific execution phase."""
+
+    phase: Phase
+
+
+@dataclass
+class StatefulPhasePayload:
+    inferred_links: int
+
+    __slots__ = ("inferred_links",)
+
+
+@dataclass
+class PhaseStarted(PhaseEvent):
+    """Start of an execution phase."""
+
+    payload: StatefulPhasePayload | None
+
+    __slots__ = ("id", "timestamp", "phase", "payload")
+
+    def __init__(self, *, phase: Phase, payload: StatefulPhasePayload | None) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.phase = phase
+        self.payload = payload
+
+
+@dataclass
+class PhaseFinished(PhaseEvent):
+    """End of an execution phase."""
+
+    status: Status
+    payload: Result[ProbePayload, Exception] | None
+
+    __slots__ = ("id", "timestamp", "phase", "status", "payload")
+
+    def __init__(self, *, phase: Phase, status: Status, payload: Result[ProbePayload, Exception] | None) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.phase = phase
+        self.status = status
+        self.payload = payload
+
+
+@dataclass
+class SchemaAnalysisWarnings(PhaseEvent):
+    """Schema analysis discovered warnings."""
+
+    warnings: list[SchemaWarning]
+
+    __slots__ = ("id", "timestamp", "phase", "warnings")
+
+    def __init__(self, *, phase: Phase, warnings: list[SchemaWarning]) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.phase = phase
+        self.warnings = warnings
+
+
+@dataclass
+class TestEvent(EngineEvent):
+    phase: PhaseName
+
+
+@dataclass
+class SuiteStarted(TestEvent):
+    """Before executing a set of scenarios."""
+
+    __slots__ = ("id", "timestamp", "phase")
+
+    def __init__(self, *, phase: PhaseName) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.phase = phase
+
+
+@dataclass
+class SuiteFinished(TestEvent):
+    """After executing a set of test scenarios."""
+
+    status: Status
+
+    __slots__ = ("id", "timestamp", "phase", "status")
+
+    def __init__(self, *, id: uuid.UUID, phase: PhaseName, status: Status) -> None:
+        self.id = id
+        self.timestamp = time.time()
+        self.phase = phase
+        self.status = status
+
+
+@dataclass
+class ScenarioEvent(TestEvent):
+    suite_id: uuid.UUID
+
+
+@dataclass
+class ScenarioStarted(ScenarioEvent):
+    """Before executing a grouped set of test steps."""
+
+    __slots__ = ("id", "timestamp", "phase", "suite_id", "label")
+
+    def __init__(self, *, phase: PhaseName, suite_id: uuid.UUID, label: str | None) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.phase = phase
+        self.suite_id = suite_id
+        self.label = label
+
+
+@dataclass
+class ScenarioFinished(ScenarioEvent):
+    """After executing a grouped set of test steps."""
+
+    status: Status
+    recorder: ScenarioRecorder
+    elapsed_time: float
+    skip_reason: str | None
+    # Whether this is a scenario that tries to reproduce a failure
+    is_final: bool
+
+    __slots__ = (
+        "id",
+        "timestamp",
+        "phase",
+        "suite_id",
+        "label",
+        "status",
+        "recorder",
+        "elapsed_time",
+        "skip_reason",
+        "is_final",
+    )
+
+    def __init__(
+        self,
+        *,
+        id: uuid.UUID,
+        phase: PhaseName,
+        suite_id: uuid.UUID,
+        label: str | None,
+        status: Status,
+        recorder: ScenarioRecorder,
+        elapsed_time: float,
+        skip_reason: str | None,
+        is_final: bool,
+    ) -> None:
+        self.id = id
+        self.timestamp = time.time()
+        self.phase = phase
+        self.suite_id = suite_id
+        self.label = label
+        self.status = status
+        self.recorder = recorder
+        self.elapsed_time = elapsed_time
+        self.skip_reason = skip_reason
+        self.is_final = is_final
+
+
+@dataclass
+class Interrupted(EngineEvent):
+    """If execution was interrupted by Ctrl-C, or a received SIGTERM."""
+
+    phase: PhaseName | None
+
+    __slots__ = ("id", "timestamp", "phase")
+
+    def __init__(self, *, phase: PhaseName | None) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.phase = phase
+
+
+@dataclass
+class NonFatalError(EngineEvent):
+    """Error that doesn't halt execution but should be reported."""
+
+    info: EngineErrorInfo
+    value: Exception
+    phase: PhaseName
+    label: str
+    related_to_operation: bool
+
+    __slots__ = ("id", "timestamp", "info", "value", "phase", "label", "related_to_operation")
+
+    def __init__(
+        self,
+        *,
+        error: Exception,
+        phase: PhaseName,
+        label: str,
+        related_to_operation: bool,
+        code_sample: str | None = None,
+    ) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.info = EngineErrorInfo(error=error, code_sample=code_sample)
+        self.value = error
+        self.phase = phase
+        self.label = label
+        self.related_to_operation = related_to_operation
+
+    def __eq__(self, other: object) -> bool:
+        assert isinstance(other, NonFatalError)
+        return self.label == other.label and type(self.value) is type(other.value)
+
+    def __hash__(self) -> int:
+        return hash((self.label, type(self.value)))
+
+
+@dataclass
+class FatalError(EngineEvent):
+    """Internal error in the engine."""
+
+    exception: Exception
+    is_terminal = True
+
+    __slots__ = ("id", "timestamp", "exception")
+
+    def __init__(self, *, exception: Exception) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.exception = exception
+
+
+@dataclass
+class EngineFinished(EngineEvent):
+    """The final event of the run.
+
+    No more events after this point.
+    """
+
+    is_terminal = True
+    running_time: float
+
+    __slots__ = ("id", "timestamp", "running_time")
+
+    def __init__(self, *, running_time: float) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.running_time = running_time

schemathesis/engine/observations.py

@@ -0,0 +1,42 @@
+from dataclasses import dataclass
+
+from schemathesis.engine.recorder import ScenarioRecorder
+from schemathesis.schemas import APIOperation
+
+
+@dataclass
+class LocationHeaderEntry:
+    """Value of `Location` coming from API response with a given status code."""
+
+    status_code: int
+    value: str
+
+    __slots__ = ("status_code", "value")
+
+
+@dataclass
+class Observations:
+    """Repository for observations collected during test execution."""
+
+    location_headers: dict[APIOperation, list[LocationHeaderEntry]]
+
+    __slots__ = ("location_headers",)
+
+    def __init__(self) -> None:
+        self.location_headers = {}
+
+    def extract_observations_from(self, recorder: ScenarioRecorder) -> None:
+        """Extract observations from completed test scenario."""
+        for id, interaction in recorder.interactions.items():
+            response = interaction.response
+            if response is not None:
+                location = response.headers.get("location")
+                if location:
+                    # Group location headers by the operation that produced them
+                    entries = self.location_headers.setdefault(recorder.cases[id].value.operation, [])
+                    entries.append(
+                        LocationHeaderEntry(
+                            status_code=response.status_code,
+                            value=location[0],
+                        )
+                    )

schemathesis/engine/phases/__init__.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import enum
+import warnings
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from schemathesis.engine.context import EngineContext
+    from schemathesis.engine.events import EventGenerator
+
+
+class PhaseName(str, enum.Enum):
+    """Available execution phases."""
+
+    PROBING = "API probing"
+    SCHEMA_ANALYSIS = "Schema analysis"
+    EXAMPLES = "Examples"
+    COVERAGE = "Coverage"
+    FUZZING = "Fuzzing"
+    STATEFUL_TESTING = "Stateful"
+
+    @classmethod
+    def defaults(cls) -> list[PhaseName]:
+        return [PhaseName.EXAMPLES, PhaseName.COVERAGE, PhaseName.FUZZING, PhaseName.STATEFUL_TESTING]
+
+    @property
+    def name(self) -> str:
+        return {
+            PhaseName.PROBING: "probing",
+            PhaseName.SCHEMA_ANALYSIS: "schema analysis",
+            PhaseName.EXAMPLES: "examples",
+            PhaseName.COVERAGE: "coverage",
+            PhaseName.FUZZING: "fuzzing",
+            PhaseName.STATEFUL_TESTING: "stateful",
+        }[self]
+
+    @classmethod
+    def from_str(cls, value: str) -> PhaseName:
+        return {
+            "probing": cls.PROBING,
+            "schema analysis": cls.SCHEMA_ANALYSIS,
+            "examples": cls.EXAMPLES,
+            "coverage": cls.COVERAGE,
+            "fuzzing": cls.FUZZING,
+            "stateful": cls.STATEFUL_TESTING,
+        }[value.lower()]
+
+
+class PhaseSkipReason(str, enum.Enum):
+    """Reasons why a phase might not be executed."""
+
+    DISABLED = "disabled"  # Explicitly disabled via config
+    NOT_SUPPORTED = "not supported"  # Feature not supported by schema
+    NOT_APPLICABLE = "not applicable"  # No relevant data (e.g., no links for stateful)
+    FAILURE_LIMIT_REACHED = "failure limit reached"
+    NOTHING_TO_TEST = "nothing to test"
+
+
+@dataclass
+class Phase:
+    """A logically separate engine execution phase."""
+
+    name: PhaseName
+    is_supported: bool
+    is_enabled: bool
+    skip_reason: PhaseSkipReason | None
+
+    __slots__ = ("name", "is_supported", "is_enabled", "skip_reason")
+
+    def __init__(
+        self, name: PhaseName, is_supported: bool, is_enabled: bool = True, skip_reason: PhaseSkipReason | None = None
+    ) -> None:
+        self.name = name
+        self.is_supported = is_supported
+        self.is_enabled = is_enabled
+        self.skip_reason = skip_reason
+
+    def should_execute(self, ctx: EngineContext) -> bool:
+        """Determine if phase should run based on context & configuration."""
+        return self.is_enabled and not ctx.has_to_stop
+
+    def enable(self) -> None:
+        """Enable this test phase."""
+        self.is_enabled = True
+        self.skip_reason = None
+
+
+def execute(ctx: EngineContext, phase: Phase) -> EventGenerator:
+    from urllib3.exceptions import InsecureRequestWarning
+
+    from . import analysis, probes, stateful, unit
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", InsecureRequestWarning)
+
+        if phase.name == PhaseName.PROBING:
+            yield from probes.execute(ctx, phase)
+        elif phase.name == PhaseName.SCHEMA_ANALYSIS:
+            yield from analysis.execute(ctx, phase)
+        elif phase.name == PhaseName.EXAMPLES:
+            yield from unit.execute(ctx, phase)
+        elif phase.name == PhaseName.COVERAGE:
+            yield from unit.execute(ctx, phase)
+        elif phase.name == PhaseName.FUZZING:
+            yield from unit.execute(ctx, phase)
+        elif phase.name == PhaseName.STATEFUL_TESTING:
+            yield from stateful.execute(ctx, phase)

schemathesis/engine/phases/analysis.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from schemathesis.engine import Status, events
+
+if TYPE_CHECKING:
+    from schemathesis.core.schema_analysis import SchemaWarning
+    from schemathesis.engine.context import EngineContext
+    from schemathesis.engine.events import EventGenerator
+    from schemathesis.engine.phases import Phase
+
+
+def execute(ctx: EngineContext, phase: Phase) -> EventGenerator:
+    """Evaluate schema-level warnings once per test run."""
+    warnings = _collect_warnings(ctx)
+    if warnings:
+        yield events.SchemaAnalysisWarnings(phase=phase, warnings=warnings)
+    yield events.PhaseFinished(phase=phase, status=Status.SUCCESS, payload=None)
+
+
+def _collect_warnings(ctx: EngineContext) -> list[SchemaWarning]:
+    from schemathesis.specs.openapi.schemas import BaseOpenAPISchema
+
+    schema = ctx.schema
+    if isinstance(schema, BaseOpenAPISchema):
+        return list(schema.analysis.iter_warnings())
+    return []

schemathesis/engine/phases/probes.py

@@ -0,0 +1,172 @@
+"""Detecting capabilities of the application under test.
+
+Schemathesis sends specially crafted requests to the application before running tests in order to detect whether
+the application supports certain inputs. This is done to avoid false positives in the tests.
+For example, certail web servers do not support NULL bytes in headers, in such cases, the generated test case
+will not reach the tested application at all.
+"""
+
+from __future__ import annotations
+
+import enum
+import warnings
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from schemathesis.core.result import Ok, Result
+from schemathesis.core.transport import USER_AGENT
+from schemathesis.engine import Status, events
+from schemathesis.transport.prepare import get_default_headers
+
+if TYPE_CHECKING:
+    import requests
+
+    from schemathesis.engine.context import EngineContext
+    from schemathesis.engine.events import EventGenerator
+    from schemathesis.engine.phases import Phase
+    from schemathesis.schemas import BaseSchema
+
+
+@dataclass
+class ProbePayload:
+    probes: list[ProbeRun]
+
+    __slots__ = ("probes",)
+
+
+def execute(ctx: EngineContext, phase: Phase) -> EventGenerator:
+    """Discover capabilities of the tested app."""
+    probes = run(ctx)
+    status = Status.SUCCESS
+    payload: Result[ProbePayload, Exception] | None = None
+    for result in probes:
+        if isinstance(result.probe, NullByteInHeader) and result.is_failure:
+            from schemathesis.specs.openapi import formats
+            from schemathesis.specs.openapi.formats import (
+                DEFAULT_HEADER_EXCLUDE_CHARACTERS,
+                HEADER_FORMAT,
+                header_values,
+            )
+
+            formats.register(
+                HEADER_FORMAT, header_values(exclude_characters=DEFAULT_HEADER_EXCLUDE_CHARACTERS + "\x00")
+            )
+        payload = Ok(ProbePayload(probes=probes))
+    yield events.PhaseFinished(phase=phase, status=status, payload=payload)
+
+
+def run(ctx: EngineContext) -> list[ProbeRun]:
+    """Run all probes against the given schema."""
+    return [send(probe(), ctx) for probe in PROBES]
+
+
+HEADER_NAME = "X-Schemathesis-Probe"
+
+
+@dataclass
+class Probe:
+    """A request to determine the capabilities of the application under test."""
+
+    name: str
+
+    __slots__ = ("name",)
+
+    def prepare_request(
+        self, session: requests.Session, request: requests.Request, schema: BaseSchema
+    ) -> requests.PreparedRequest:
+        raise NotImplementedError
+
+    def analyze_response(self, response: requests.Response) -> ProbeOutcome:
+        raise NotImplementedError
+
+
+class ProbeOutcome(str, enum.Enum):
+    # Capability is supported
+    SUCCESS = "success"
+    # Capability is not supported
+    FAILURE = "failure"
+    # Probe is not applicable
+    SKIP = "skip"
+
+
+@dataclass
+class ProbeRun:
+    probe: Probe
+    outcome: ProbeOutcome
+    request: requests.PreparedRequest | None
+    response: requests.Response | None
+    error: Exception | None
+
+    __slots__ = ("probe", "outcome", "request", "response", "error")
+
+    def __init__(
+        self,
+        probe: Probe,
+        outcome: ProbeOutcome,
+        request: requests.PreparedRequest | None = None,
+        response: requests.Response | None = None,
+        error: Exception | None = None,
+    ) -> None:
+        self.probe = probe
+        self.outcome = outcome
+        self.request = request
+        self.response = response
+        self.error = error
+
+    @property
+    def is_failure(self) -> bool:
+        return self.outcome == ProbeOutcome.FAILURE
+
+
+@dataclass
+class NullByteInHeader(Probe):
+    """Support NULL bytes in headers."""
+
+    __slots__ = ("name",)
+
+    def __init__(self) -> None:
+        self.name = "Supports NULL byte in headers"
+
+    def prepare_request(
+        self, session: requests.Session, request: requests.Request, schema: BaseSchema
+    ) -> requests.PreparedRequest:
+        request.method = "GET"
+        request.url = schema.get_base_url()
+        request.headers = {"X-Schemathesis-Probe-Null": "\x00"}
+        return session.prepare_request(request)
+
+    def analyze_response(self, response: requests.Response) -> ProbeOutcome:
+        if response.status_code == 400:
+            return ProbeOutcome.FAILURE
+        return ProbeOutcome.SUCCESS
+
+
+PROBES = (NullByteInHeader,)
+
+
+def send(probe: Probe, ctx: EngineContext) -> ProbeRun:
+    """Send the probe to the application."""
+    from requests import PreparedRequest, Request, RequestException
+    from requests.exceptions import MissingSchema
+    from urllib3.exceptions import InsecureRequestWarning
+
+    try:
+        session = ctx.get_session()
+        request = probe.prepare_request(session, Request(), ctx.schema)
+        request.headers[HEADER_NAME] = probe.name
+        request.headers["User-Agent"] = USER_AGENT
+        for header, value in get_default_headers().items():
+            request.headers.setdefault(header, value)
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", InsecureRequestWarning)
+            response = session.send(request, timeout=ctx.config.request_timeout or 2)
+    except MissingSchema:
+        # In-process ASGI/WSGI testing will have local URLs and requires extra handling
+        # which is not currently implemented
+        return ProbeRun(probe, ProbeOutcome.SKIP, None, None, None)
+    except RequestException as exc:
+        # Consider any network errors as a failed probe
+        req = exc.request if isinstance(exc.request, PreparedRequest) else None
+        return ProbeRun(probe, ProbeOutcome.FAILURE, req, None, exc)
+    result_type = probe.analyze_response(response)
+    return ProbeRun(probe, result_type, request, response)

schemathesis/engine/phases/stateful/__init__.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import queue
+import threading
+from typing import TYPE_CHECKING
+
+from schemathesis.engine import Status, events
+from schemathesis.engine.phases import Phase, PhaseName, PhaseSkipReason
+from schemathesis.generation.stateful import STATEFUL_TESTS_LABEL
+
+if TYPE_CHECKING:
+    from schemathesis.engine.context import EngineContext
+
+EVENT_QUEUE_TIMEOUT = 0.01
+
+
+def execute(engine: EngineContext, phase: Phase) -> events.EventGenerator:
+    from schemathesis.engine.phases.stateful._executor import execute_state_machine_loop
+
+    try:
+        state_machine = engine.schema.as_state_machine()
+    except Exception as exc:
+        yield events.NonFatalError(error=exc, phase=phase.name, label=STATEFUL_TESTS_LABEL, related_to_operation=False)
+        yield events.PhaseFinished(phase=phase, status=Status.ERROR, payload=None)
+        return
+
+    event_queue: queue.Queue = queue.Queue()
+
+    thread = threading.Thread(
+        target=execute_state_machine_loop,
+        kwargs={"state_machine": state_machine, "event_queue": event_queue, "engine": engine},
+        name="schemathesis_stateful_tests",
+    )
+    status: Status | None = None
+    is_executed = False
+
+    thread.start()
+    try:
+        while True:
+            try:
+                event = event_queue.get(timeout=EVENT_QUEUE_TIMEOUT)
+                is_executed = True
+                # Set the run status based on the suite status
+                # ERROR & INTERRUPTED statuses are terminal, therefore they should not be overridden
+                if (
+                    isinstance(event, events.SuiteFinished)
+                    and event.status != Status.SKIP
+                    and (status is None or status < event.status)
+                ):
+                    status = event.status
+                yield event
+            except queue.Empty:
+                if not thread.is_alive():
+                    break
+    except KeyboardInterrupt:
+        # Immediately notify the engine thread to stop, even though that the event will be set below in `finally`
+        engine.stop()
+        status = Status.INTERRUPTED
+        yield events.Interrupted(phase=PhaseName.STATEFUL_TESTING)
+    finally:
+        thread.join()
+
+    if not is_executed:
+        phase.skip_reason = PhaseSkipReason.NOTHING_TO_TEST
+        status = Status.SKIP
+    elif status is None:
+        status = Status.SKIP
+    yield events.PhaseFinished(phase=phase, status=status, payload=None)