schemathesis 3.25.5__py3-none-any.whl → 4.0.0a1__py3-none-any.whl

schemathesis/engine/events.py

@@ -0,0 +1,337 @@
+from __future__ import annotations
+
+import time
+import uuid
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Generator
+
+from schemathesis.core.result import Result
+from schemathesis.core.transport import Response
+from schemathesis.engine.errors import EngineErrorInfo
+from schemathesis.engine.phases import Phase, PhaseName
+from schemathesis.engine.recorder import ScenarioRecorder
+
+if TYPE_CHECKING:
+    from schemathesis.engine import Status
+    from schemathesis.engine.phases.probes import ProbePayload
+
+EventGenerator = Generator["EngineEvent", None, None]
+
+
+@dataclass
+class EngineEvent:
+    """An event within the engine's lifecycle."""
+
+    id: uuid.UUID
+    timestamp: float
+    # Indicates whether this event is the last in the event stream
+    is_terminal = False
+
+
+@dataclass
+class EngineStarted(EngineEvent):
+    """Start of an engine."""
+
+    __slots__ = ("id", "timestamp")
+
+    def __init__(self) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+
+
+@dataclass
+class PhaseEvent(EngineEvent):
+    """Event associated with a specific execution phase."""
+
+    phase: Phase
+
+
+@dataclass
+class PhaseStarted(PhaseEvent):
+    """Start of an execution phase."""
+
+    __slots__ = ("id", "timestamp", "phase")
+
+    def __init__(self, *, phase: Phase) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.phase = phase
+
+
+@dataclass
+class PhaseFinished(PhaseEvent):
+    """End of an execution phase."""
+
+    status: Status
+    payload: Result[ProbePayload, Exception] | None
+
+    __slots__ = ("id", "timestamp", "phase", "status", "payload")
+
+    def __init__(self, *, phase: Phase, status: Status, payload: Result[ProbePayload, Exception] | None) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.phase = phase
+        self.status = status
+        self.payload = payload
+
+
+@dataclass
+class TestEvent(EngineEvent):
+    phase: PhaseName
+
+
+@dataclass
+class SuiteStarted(TestEvent):
+    """Before executing a set of scenarios."""
+
+    __slots__ = ("id", "timestamp", "phase")
+
+    def __init__(self, *, phase: PhaseName) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.phase = phase
+
+
+@dataclass
+class SuiteFinished(TestEvent):
+    """After executing a set of test scenarios."""
+
+    status: Status
+
+    __slots__ = ("id", "timestamp", "phase", "status")
+
+    def __init__(self, *, id: uuid.UUID, phase: PhaseName, status: Status) -> None:
+        self.id = id
+        self.timestamp = time.time()
+        self.phase = phase
+        self.status = status
+
+
+@dataclass
+class ScenarioEvent(TestEvent):
+    suite_id: uuid.UUID
+
+
+@dataclass
+class ScenarioStarted(ScenarioEvent):
+    """Before executing a grouped set of test steps."""
+
+    __slots__ = ("id", "timestamp", "phase", "suite_id", "label")
+
+    def __init__(self, *, phase: PhaseName, suite_id: uuid.UUID, label: str | None) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.phase = phase
+        self.suite_id = suite_id
+        self.label = label
+
+
+@dataclass
+class ScenarioFinished(ScenarioEvent):
+    """After executing a grouped set of test steps."""
+
+    status: Status
+    recorder: ScenarioRecorder
+    elapsed_time: float
+    skip_reason: str | None
+    # Whether this is a scenario that tries to reproduce a failure
+    is_final: bool
+
+    __slots__ = (
+        "id",
+        "timestamp",
+        "phase",
+        "suite_id",
+        "status",
+        "recorder",
+        "elapsed_time",
+        "skip_reason",
+        "is_final",
+    )
+
+    def __init__(
+        self,
+        *,
+        id: uuid.UUID,
+        phase: PhaseName,
+        suite_id: uuid.UUID,
+        status: Status,
+        recorder: ScenarioRecorder,
+        elapsed_time: float,
+        skip_reason: str | None,
+        is_final: bool,
+    ) -> None:
+        self.id = id
+        self.timestamp = time.time()
+        self.phase = phase
+        self.suite_id = suite_id
+        self.status = status
+        self.recorder = recorder
+        self.elapsed_time = elapsed_time
+        self.skip_reason = skip_reason
+        self.is_final = is_final
+
+
+@dataclass
+class StepEvent(ScenarioEvent):
+    scenario_id: uuid.UUID
+
+
+@dataclass
+class StepStarted(StepEvent):
+    """Before executing a test case."""
+
+    __slots__ = (
+        "id",
+        "timestamp",
+        "phase",
+        "suite_id",
+        "scenario_id",
+    )
+
+    def __init__(
+        self,
+        *,
+        phase: PhaseName,
+        suite_id: uuid.UUID,
+        scenario_id: uuid.UUID,
+    ) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.phase = phase
+        self.suite_id = suite_id
+        self.scenario_id = scenario_id
+
+
+@dataclass
+class TransitionId:
+    """Id of the the that was hit."""
+
+    name: str
+    # Status code as defined in the transition, i.e. may be `default`
+    status_code: str
+    source: str
+
+    __slots__ = ("name", "status_code", "source")
+
+
+@dataclass
+class ResponseData:
+    """Common data for responses."""
+
+    status_code: int
+    elapsed: float
+    __slots__ = ("status_code", "elapsed")
+
+
+@dataclass
+class StepFinished(StepEvent):
+    """After executing a test case."""
+
+    status: Status | None
+    transition_id: TransitionId | None
+    target: str
+    response: Response | None
+
+    __slots__ = (
+        "id",
+        "timestamp",
+        "phase",
+        "status",
+        "suite_id",
+        "scenario_id",
+        "transition_id",
+        "target",
+        "response",
+    )
+
+    def __init__(
+        self,
+        *,
+        phase: PhaseName,
+        id: uuid.UUID,
+        suite_id: uuid.UUID,
+        scenario_id: uuid.UUID,
+        status: Status | None,
+        transition_id: TransitionId | None,
+        target: str,
+        response: Response | None,
+    ) -> None:
+        self.id = id
+        self.timestamp = time.time()
+        self.phase = phase
+        self.status = status
+        self.suite_id = suite_id
+        self.scenario_id = scenario_id
+        self.transition_id = transition_id
+        self.target = target
+        self.response = response
+
+
+@dataclass
+class Interrupted(EngineEvent):
+    """If execution was interrupted by Ctrl-C, or a received SIGTERM."""
+
+    phase: PhaseName | None
+
+    __slots__ = ("id", "timestamp", "phase")
+
+    def __init__(self, *, phase: PhaseName | None) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.phase = phase
+
+
+@dataclass
+class NonFatalError(EngineEvent):
+    """Error that doesn't halt execution but should be reported."""
+
+    info: EngineErrorInfo
+    value: Exception
+    phase: PhaseName
+    label: str
+    related_to_operation: bool
+
+    __slots__ = ("id", "timestamp", "info", "value", "phase", "label", "related_to_operation")
+
+    def __init__(self, *, error: Exception, phase: PhaseName, label: str, related_to_operation: bool) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.info = EngineErrorInfo(error=error)
+        self.value = error
+        self.phase = phase
+        self.label = label
+        self.related_to_operation = related_to_operation
+
+
+@dataclass
+class FatalError(EngineEvent):
+    """Internal error in the engine."""
+
+    exception: Exception
+    is_terminal = True
+
+    __slots__ = ("id", "timestamp", "exception")
+
+    def __init__(self, *, exception: Exception) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.exception = exception
+
+
+@dataclass
+class EngineFinished(EngineEvent):
+    """The final event of the run.
+
+    No more events after this point.
+    """
+
+    is_terminal = True
+    running_time: float
+
+    __slots__ = ("id", "timestamp", "running_time")
+
+    def __init__(self, *, running_time: float) -> None:
+        self.id = uuid.uuid4()
+        self.timestamp = time.time()
+        self.running_time = running_time

schemathesis/engine/phases/__init__.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import enum
+import warnings
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from schemathesis.engine.context import EngineContext
+    from schemathesis.engine.events import EventGenerator
+
+
+class PhaseName(enum.Enum):
+    """Available execution phases."""
+
+    PROBING = "API probing"
+    UNIT_TESTING = "Unit testing"
+    STATEFUL_TESTING = "Stateful testing"
+
+    @classmethod
+    def from_str(cls, value: str) -> PhaseName:
+        return {
+            "probing": cls.PROBING,
+            "unit": cls.UNIT_TESTING,
+            "stateful": cls.STATEFUL_TESTING,
+        }[value.lower()]
+
+
+class PhaseSkipReason(str, enum.Enum):
+    """Reasons why a phase might not be executed."""
+
+    DISABLED = "disabled"  # Explicitly disabled via config
+    NOT_SUPPORTED = "not supported"  # Feature not supported by schema
+    NOT_APPLICABLE = "not applicable"  # No relevant data (e.g., no links for stateful)
+    FAILURE_LIMIT_REACHED = "failure limit reached"
+    NOTHING_TO_TEST = "nothing to test"
+
+
+@dataclass
+class Phase:
+    """A logically separate engine execution phase."""
+
+    name: PhaseName
+    is_supported: bool
+    is_enabled: bool = True
+    skip_reason: PhaseSkipReason | None = None
+
+    def should_execute(self, ctx: EngineContext) -> bool:
+        """Determine if phase should run based on context & configuration."""
+        return self.is_enabled and not ctx.has_to_stop
+
+
+def execute(ctx: EngineContext, phase: Phase) -> EventGenerator:
+    from urllib3.exceptions import InsecureRequestWarning
+
+    from . import probes, stateful, unit
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", InsecureRequestWarning)
+
+        if phase.name == PhaseName.PROBING:
+            yield from probes.execute(ctx, phase)
+        elif phase.name == PhaseName.UNIT_TESTING:
+            yield from unit.execute(ctx, phase)
+        elif phase.name == PhaseName.STATEFUL_TESTING:
+            yield from stateful.execute(ctx, phase)

schemathesis/{cli → engine/phases}/probes.py

@@ -5,24 +5,58 @@ the application supports certain inputs. This is done to avoid false positives i
 For example, certail web servers do not support NULL bytes in headers, in such cases, the generated test case
 will not reach the tested application at all.
 """
+
 from __future__ import annotations
 
 import enum
 import warnings
-from dataclasses import asdict, dataclass
-from typing import TYPE_CHECKING, Any
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
 
-from ..constants import USER_AGENT
-from ..exceptions import format_exception
-from ..models import Request, Response
-from ..sanitization import sanitize_request, sanitize_response
-from ..transports.auth import get_requests_auth
+from schemathesis.core.result import Err, Ok, Result
+from schemathesis.core.transport import USER_AGENT
+from schemathesis.engine import Status, events
 
 if TYPE_CHECKING:
     import requests
 
-    from ..schemas import BaseSchema
-    from . import LoaderConfig
+    from schemathesis.engine.config import NetworkConfig
+    from schemathesis.engine.context import EngineContext
+    from schemathesis.engine.events import EventGenerator
+    from schemathesis.engine.phases import Phase
+    from schemathesis.schemas import BaseSchema
+
+
+@dataclass
+class ProbePayload:
+    probes: list[ProbeRun]
+
+    __slots__ = ("probes",)
+
+
+def execute(ctx: EngineContext, phase: Phase) -> EventGenerator:
+    """Discover capabilities of the tested app."""
+    probes = run(ctx.schema, ctx.session, ctx.config.network)
+    status = Status.SUCCESS
+    payload: Result[ProbePayload, Exception] | None = None
+    for result in probes:
+        if isinstance(result.probe, NullByteInHeader) and result.is_failure:
+            from ...specs.openapi import formats
+            from ...specs.openapi.formats import HEADER_FORMAT, header_values
+
+            formats.register(HEADER_FORMAT, header_values(blacklist_characters="\n\r\x00"))
+        if result.error is not None:
+            status = Status.ERROR
+            payload = Err(result.error)
+        else:
+            status = Status.SUCCESS
+            payload = Ok(ProbePayload(probes=probes))
+    yield events.PhaseFinished(phase=phase, status=status, payload=payload)
+
+
+def run(schema: BaseSchema, session: requests.Session, config: NetworkConfig) -> list[ProbeRun]:
+    """Run all probes against the given schema."""
+    return [send(probe(), session, schema, config) for probe in PROBES]
 
 
 HEADER_NAME = "X-Schemathesis-Probe"
@@ -35,15 +69,15 @@ class Probe:
     name: str
 
     def prepare_request(
-        self, session: requests.Session, request: requests.Request, schema: BaseSchema, config: LoaderConfig
+        self, session: requests.Session, request: requests.Request, schema: BaseSchema
     ) -> requests.PreparedRequest:
         raise NotImplementedError
 
-    def analyze_response(self, response: requests.Response) -> ProbeResultType:
+    def analyze_response(self, response: requests.Response) -> ProbeOutcome:
         raise NotImplementedError
 
 
-class ProbeResultType(str, enum.Enum):
+class ProbeOutcome(str, enum.Enum):
     # Capability is supported
     SUCCESS = "success"
     # Capability is not supported
@@ -55,101 +89,60 @@ class ProbeResultType(str, enum.Enum):
 
 
 @dataclass
-class ProbeResult:
-    """Result of a probe."""
-
+class ProbeRun:
     probe: Probe
-    type: ProbeResultType
+    outcome: ProbeOutcome
     request: requests.PreparedRequest | None = None
     response: requests.Response | None = None
-    error: requests.RequestException | None = None
+    error: Exception | None = None
 
     @property
     def is_failure(self) -> bool:
-        return self.type == ProbeResultType.FAILURE
-
-    def serialize(self) -> dict[str, Any]:
-        """Serialize probe results so it can be sent over the network."""
-        if self.request:
-            _request = Request.from_prepared_request(self.request)
-            sanitize_request(_request)
-            request = asdict(_request)
-        else:
-            request = None
-        if self.response:
-            sanitize_response(self.response)
-            response = asdict(Response.from_requests(self.response))
-        else:
-            response = None
-        if self.error:
-            error = format_exception(self.error)
-        else:
-            error = None
-        return {
-            "name": self.probe.name,
-            "type": self.type.value,
-            "request": request,
-            "response": response,
-            "error": error,
-        }
+        return self.outcome == ProbeOutcome.FAILURE
 
 
 @dataclass
 class NullByteInHeader(Probe):
     """Support NULL bytes in headers."""
 
-    name: str = "NULL_BYTE_IN_HEADER"
+    name: str = "Supports NULL byte in headers"
 
     def prepare_request(
-        self, session: requests.Session, request: requests.Request, schema: BaseSchema, config: LoaderConfig
+        self, session: requests.Session, request: requests.Request, schema: BaseSchema
     ) -> requests.PreparedRequest:
         request.method = "GET"
-        request.url = config.base_url or schema.get_base_url()
+        request.url = schema.get_base_url()
         request.headers = {"X-Schemathesis-Probe-Null": "\x00"}
         return session.prepare_request(request)
 
-    def analyze_response(self, response: requests.Response) -> ProbeResultType:
+    def analyze_response(self, response: requests.Response) -> ProbeOutcome:
         if response.status_code == 400:
-            return ProbeResultType.FAILURE
-        return ProbeResultType.SUCCESS
+            return ProbeOutcome.FAILURE
+        return ProbeOutcome.SUCCESS
 
 
 PROBES = (NullByteInHeader,)
 
 
-def send(probe: Probe, session: requests.Session, schema: BaseSchema, config: LoaderConfig) -> ProbeResult:
+def send(probe: Probe, session: requests.Session, schema: BaseSchema, config: NetworkConfig) -> ProbeRun:
     """Send the probe to the application."""
-    from requests import Request, RequestException, PreparedRequest
+    from requests import PreparedRequest, Request, RequestException
     from requests.exceptions import MissingSchema
     from urllib3.exceptions import InsecureRequestWarning
 
     try:
-        request = probe.prepare_request(session, Request(), schema, config)
+        request = probe.prepare_request(session, Request(), schema)
         request.headers[HEADER_NAME] = probe.name
         request.headers["User-Agent"] = USER_AGENT
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", InsecureRequestWarning)
-            response = session.send(request)
+            response = session.send(request, timeout=config.timeout or 2)
     except MissingSchema:
         # In-process ASGI/WSGI testing will have local URLs and requires extra handling
         # which is not currently implemented
-        return ProbeResult(probe, ProbeResultType.SKIP, None, None, None)
+        return ProbeRun(probe, ProbeOutcome.SKIP, None, None, None)
     except RequestException as exc:
         req = exc.request if isinstance(exc.request, PreparedRequest) else None
-        return ProbeResult(probe, ProbeResultType.ERROR, req, None, exc)
+        return ProbeRun(probe, ProbeOutcome.ERROR, req, None, exc)
     result_type = probe.analyze_response(response)
-    return ProbeResult(probe, result_type, request, response)
-
-
-def run(schema: BaseSchema, config: LoaderConfig) -> list[ProbeResult]:
-    """Run all probes against the given schema."""
-    from requests import Session
-
-    session = Session()
-    session.verify = config.request_tls_verify
-    if config.request_cert is not None:
-        session.cert = config.request_cert
-    if config.auth is not None:
-        session.auth = get_requests_auth(config.auth, config.auth_type)
-
-    return [send(probe(), session, schema, config) for probe in PROBES]
+    return ProbeRun(probe, result_type, request, response)

schemathesis/engine/phases/stateful/__init__.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+import queue
+import threading
+from typing import TYPE_CHECKING
+
+from schemathesis.engine import Status, events
+from schemathesis.engine.phases import Phase, PhaseName, PhaseSkipReason
+
+if TYPE_CHECKING:
+    from schemathesis.engine.context import EngineContext
+
+EVENT_QUEUE_TIMEOUT = 0.01
+
+
+def execute(engine: EngineContext, phase: Phase) -> events.EventGenerator:
+    from schemathesis.engine.phases.stateful._executor import execute_state_machine_loop
+
+    try:
+        state_machine = engine.schema.as_state_machine()
+    except Exception as exc:
+        yield events.NonFatalError(error=exc, phase=phase.name, label="Stateful tests", related_to_operation=False)
+        return
+
+    event_queue: queue.Queue = queue.Queue()
+
+    thread = threading.Thread(
+        target=execute_state_machine_loop,
+        kwargs={"state_machine": state_machine, "event_queue": event_queue, "engine": engine},
+    )
+    status: Status | None = None
+    is_executed = False
+
+    thread.start()
+    try:
+        while True:
+            try:
+                event = event_queue.get(timeout=EVENT_QUEUE_TIMEOUT)
+                is_executed = True
+                # Set the run status based on the suite status
+                # ERROR & INTERRUPTED statuses are terminal, therefore they should not be overridden
+                if (
+                    isinstance(event, events.SuiteFinished)
+                    and event.status != Status.SKIP
+                    and (status is None or status < event.status)
+                ):
+                    status = event.status
+                yield event
+            except queue.Empty:
+                if not thread.is_alive():
+                    break
+    except KeyboardInterrupt:
+        # Immediately notify the engine thread to stop, even though that the event will be set below in `finally`
+        engine.stop()
+        status = Status.INTERRUPTED
+        yield events.Interrupted(phase=PhaseName.STATEFUL_TESTING)
+    finally:
+        thread.join()
+
+    if not is_executed:
+        phase.skip_reason = PhaseSkipReason.NOTHING_TO_TEST
+        status = Status.SKIP
+    elif status is None:
+        status = Status.SKIP
+    yield events.PhaseFinished(phase=phase, status=status, payload=None)