schemathesis 4.0.0a1__py3-none-any.whl → 4.0.0a3__py3-none-any.whl

schemathesis/cli/commands/run/validation.py

@@ -256,7 +256,7 @@ def convert_experimental(
     ]
 
 
-def convert_checks(ctx: click.core.Context, param: click.core.Parameter, value: tuple[list[str]]) -> list[str]:
+def reduce_list(ctx: click.core.Context, param: click.core.Parameter, value: tuple[list[str]]) -> list[str]:
     return reduce(operator.iadd, value, [])
 
 

schemathesis/cli/ext/options.py

@@ -80,7 +80,10 @@ class RegistryChoice(BaseCsvChoice):
     def convert(  # type: ignore[return]
         self, value: str, param: click.core.Parameter | None, ctx: click.core.Context | None
     ) -> list[str]:
-        return [item for item in value.split(",") if item]
+        selected, invalid_options = self.parse_value(value)
+        if not invalid_options and selected:
+            return selected
+        self.fail_on_invalid_options(invalid_options, selected)
 
 
 class OptionalInt(click.types.IntRange):

schemathesis/core/errors.py

@@ -102,6 +102,10 @@ class InvalidRegexType(InvalidSchema):
     """Raised when an invalid type is used where a regex pattern is expected."""
 
 
+class InvalidLinkDefinition(InvalidSchema):
+    """Raised when an Open API link references a non-existent operation."""
+
+
 class MalformedMediaType(ValueError):
     """Raised on parsing of incorrect media type."""
 
@@ -148,6 +152,10 @@ class IncorrectUsage(SchemathesisError):
     """Indicates incorrect usage of Schemathesis' public API."""
 
 
+class NoLinksFound(IncorrectUsage):
+    """Raised when no valid links are available for stateful testing."""
+
+
 class InvalidRateLimit(IncorrectUsage):
     """Incorrect input for rate limiting."""
 

schemathesis/core/failures.py

@@ -2,11 +2,12 @@ from __future__ import annotations
 
 import http.client
 import textwrap
+import traceback
 from collections.abc import Sequence
 from dataclasses import dataclass
 from enum import Enum, auto
 from json import JSONDecodeError
-from typing import Callable
+from typing import Any, Callable
 
 from schemathesis.core.compat import BaseExceptionGroup
 from schemathesis.core.output import OutputConfig, prepare_response_payload
@@ -32,7 +33,7 @@ class Severity(Enum):
 class Failure(AssertionError):
     """API check failure."""
 
-    __slots__ = ("operation", "title", "message", "code", "case_id", "severity")
+    __slots__ = ("operation", "title", "message", "case_id", "severity")
 
     def __init__(
         self,
@@ -40,14 +41,12 @@ class Failure(AssertionError):
         operation: str,
         title: str,
         message: str,
-        code: str,
         case_id: str | None = None,
         severity: Severity = Severity.MEDIUM,
     ) -> None:
         self.operation = operation
         self.title = title
         self.message = message
-        self.code = code
         self.case_id = case_id
         self.severity = severity
 
@@ -72,20 +71,58 @@ class Failure(AssertionError):
             return NotImplemented
         return type(self) is type(other) and self.operation == other.operation and self._unique_key == other._unique_key
 
-    @classmethod
-    def from_assertion(cls, *, name: str, operation: str, exc: AssertionError) -> Failure:
-        return Failure(
-            operation=operation,
-            title=f"Custom check failed: `{name}`",
-            message=str(exc),
-            code="custom",
-        )
-
     @property
-    def _unique_key(self) -> str:
+    def _unique_key(self) -> Any:
         return self.message
 
 
+def get_origin(exception: BaseException, seen: tuple[BaseException, ...] = ()) -> tuple:
+    filename, lineno = None, None
+    if tb := exception.__traceback__:
+        filename, lineno, *_ = traceback.extract_tb(tb)[-1]
+    seen = (*seen, exception)
+    context = ()
+    if exception.__context__ is not None and exception.__context__ not in seen:
+        context = get_origin(exception.__context__, seen=seen)
+    return (
+        type(exception),
+        filename,
+        lineno,
+        context,
+        (
+            tuple(get_origin(exc, seen=seen) for exc in exception.exceptions if exc not in seen)
+            if isinstance(exception, BaseExceptionGroup)
+            else ()
+        ),
+    )
+
+
+class CustomFailure(Failure):
+    __slots__ = ("operation", "title", "message", "exception", "case_id", "severity", "origin")
+
+    def __init__(
+        self,
+        *,
+        operation: str,
+        title: str,
+        message: str,
+        exception: AssertionError,
+        case_id: str | None = None,
+        severity: Severity = Severity.MEDIUM,
+    ) -> None:
+        self.operation = operation
+        self.title = title
+        self.message = message
+        self.exception = exception
+        self.case_id = case_id
+        self.severity = severity
+        self.origin = get_origin(exception)
+
+    @property
+    def _unique_key(self) -> Any:
+        return self.origin
+
+
 @dataclass
 class MaxResponseTimeConfig:
     limit: float = 10.0
@@ -94,7 +131,7 @@ class MaxResponseTimeConfig:
 class ResponseTimeExceeded(Failure):
     """Response took longer than expected."""
 
-    __slots__ = ("operation", "elapsed", "deadline", "title", "message", "code", "case_id", "severity")
+    __slots__ = ("operation", "elapsed", "deadline", "title", "message", "case_id", "severity")
 
     def __init__(
         self,
@@ -104,7 +141,6 @@ class ResponseTimeExceeded(Failure):
         deadline: int,
         message: str,
         title: str = "Response time limit exceeded",
-        code: str = "response_time_exceeded",
         case_id: str | None = None,
     ) -> None:
         self.operation = operation
@@ -112,7 +148,6 @@ class ResponseTimeExceeded(Failure):
         self.deadline = deadline
         self.title = title
         self.message = message
-        self.code = code
         self.case_id = case_id
         self.severity = Severity.LOW
 
@@ -124,7 +159,7 @@ class ResponseTimeExceeded(Failure):
 class ServerError(Failure):
     """Server responded with an error."""
 
-    __slots__ = ("operation", "status_code", "title", "message", "code", "case_id", "severity")
+    __slots__ = ("operation", "status_code", "title", "message", "case_id", "severity")
 
     def __init__(
         self,
@@ -133,14 +168,12 @@ class ServerError(Failure):
         status_code: int,
         title: str = "Server error",
         message: str = "",
-        code: str = "server_error",
         case_id: str | None = None,
     ) -> None:
         self.operation = operation
         self.status_code = status_code
         self.title = title
         self.message = message
-        self.code = code
         self.case_id = case_id
         self.severity = Severity.CRITICAL
 
@@ -161,7 +194,6 @@ class MalformedJson(Failure):
         "colno",
         "message",
         "title",
-        "code",
         "case_id",
         "severity",
     )
@@ -177,7 +209,6 @@ class MalformedJson(Failure):
         colno: int,
         message: str,
         title: str = "JSON deserialization error",
-        code: str = "malformed_json",
         case_id: str | None = None,
     ) -> None:
         self.operation = operation
@@ -188,12 +219,11 @@ class MalformedJson(Failure):
         self.colno = colno
         self.message = message
         self.title = title
-        self.code = code
         self.case_id = case_id
         self.severity = Severity.MEDIUM
 
     @property
-    def _unique_key(self) -> str:
+    def _unique_key(self) -> Any:
         return self.title
 
     @classmethod

schemathesis/engine/core.py

@@ -68,7 +68,7 @@ class Engine:
                 skip_reason=PhaseSkipReason.DISABLED,
             )
 
-        if requires_links and self.schema.links_count == 0:
+        if requires_links and self.schema.statistic.links.total == 0:
             return Phase(
                 name=phase_name,
                 is_supported=True,

schemathesis/engine/errors.py

@@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Callable, Iterator, Sequence, cast
 from schemathesis import errors
 from schemathesis.core.errors import (
     RECURSIVE_REFERENCE_ERROR_MESSAGE,
+    InvalidLinkDefinition,
     SerializationNotPossible,
     format_exception,
     get_request_error_extras,
@@ -76,6 +77,9 @@ class EngineErrorInfo:
         """A general error description."""
         import requests
 
+        if isinstance(self._error, InvalidLinkDefinition):
+            return "Invalid Link Definition"
+
         if isinstance(self._error, requests.RequestException):
             return "Network Error"
 
@@ -96,6 +100,7 @@ class EngineErrorInfo:
 
         return {
             RuntimeErrorKind.SCHEMA_UNSUPPORTED: "Unsupported Schema",
+            RuntimeErrorKind.SCHEMA_NO_LINKS_FOUND: "Missing Open API links",
             RuntimeErrorKind.HYPOTHESIS_UNSUPPORTED_GRAPHQL_SCALAR: "Unknown GraphQL Scalar",
             RuntimeErrorKind.SERIALIZATION_UNBOUNDED_PREFIX: "XML serialization error",
             RuntimeErrorKind.SERIALIZATION_NOT_POSSIBLE: "Serialization not possible",
@@ -166,6 +171,7 @@ class EngineErrorInfo:
             RuntimeErrorKind.SCHEMA_INVALID_REGULAR_EXPRESSION,
             RuntimeErrorKind.SCHEMA_UNSUPPORTED,
             RuntimeErrorKind.SCHEMA_GENERIC,
+            RuntimeErrorKind.SCHEMA_NO_LINKS_FOUND,
             RuntimeErrorKind.SERIALIZATION_NOT_POSSIBLE,
             RuntimeErrorKind.HYPOTHESIS_UNSUPPORTED_GRAPHQL_SCALAR,
             RuntimeErrorKind.HYPOTHESIS_UNSATISFIABLE,
@@ -184,11 +190,7 @@ class EngineErrorInfo:
         """Format error message with optional styling and traceback."""
         message = []
 
-        # Title
-        if self._kind == RuntimeErrorKind.SCHEMA_GENERIC:
-            title = "Schema Error"
-        else:
-            title = self.title
+        title = self.title
         if title:
             message.append(f"{title}\n")
 
@@ -246,6 +248,7 @@ def get_runtime_error_suggestion(error_type: RuntimeErrorKind, bold: Callable[[s
     return {
         RuntimeErrorKind.CONNECTION_SSL: f"Bypass SSL verification with {bold('`--request-tls-verify=false`')}.",
         RuntimeErrorKind.HYPOTHESIS_UNSATISFIABLE: "Examine the schema for inconsistencies and consider simplifying it.",
+        RuntimeErrorKind.SCHEMA_NO_LINKS_FOUND: "Review your endpoint filters to include linked operations",
         RuntimeErrorKind.SCHEMA_INVALID_REGULAR_EXPRESSION: "Ensure your regex is compatible with Python's syntax.\n"
         "For guidance, visit: https://docs.python.org/3/library/re.html",
         RuntimeErrorKind.HYPOTHESIS_UNSUPPORTED_GRAPHQL_SCALAR: "Define a custom strategy for it.\n"
@@ -299,6 +302,7 @@ class RuntimeErrorKind(str, enum.Enum):
     HYPOTHESIS_HEALTH_CHECK_LARGE_BASE_EXAMPLE = "hypothesis_health_check_large_base_example"
 
     SCHEMA_INVALID_REGULAR_EXPRESSION = "schema_invalid_regular_expression"
+    SCHEMA_NO_LINKS_FOUND = "schema_no_links_found"
     SCHEMA_UNSUPPORTED = "schema_unsupported"
     SCHEMA_GENERIC = "schema_generic"
 
@@ -350,6 +354,8 @@ def _classify(*, error: Exception) -> RuntimeErrorKind:
         if isinstance(error, errors.InvalidRegexPattern):
             return RuntimeErrorKind.SCHEMA_INVALID_REGULAR_EXPRESSION
         return RuntimeErrorKind.SCHEMA_GENERIC
+    if isinstance(error, errors.NoLinksFound):
+        return RuntimeErrorKind.SCHEMA_NO_LINKS_FOUND
     if isinstance(error, UnsupportedRecursiveReference):
         # Recursive references are not supported right now
         return RuntimeErrorKind.SCHEMA_UNSUPPORTED

schemathesis/engine/events.py

@@ -6,7 +6,6 @@ from dataclasses import dataclass
 from typing import TYPE_CHECKING, Generator
 
 from schemathesis.core.result import Result
-from schemathesis.core.transport import Response
 from schemathesis.engine.errors import EngineErrorInfo
 from schemathesis.engine.phases import Phase, PhaseName
 from schemathesis.engine.recorder import ScenarioRecorder
@@ -142,6 +141,7 @@ class ScenarioFinished(ScenarioEvent):
         "timestamp",
         "phase",
         "suite_id",
+        "label",
         "status",
         "recorder",
         "elapsed_time",
@@ -155,6 +155,7 @@ class ScenarioFinished(ScenarioEvent):
         id: uuid.UUID,
         phase: PhaseName,
         suite_id: uuid.UUID,
+        label: str | None,
         status: Status,
         recorder: ScenarioRecorder,
         elapsed_time: float,
@@ -165,6 +166,7 @@ class ScenarioFinished(ScenarioEvent):
         self.timestamp = time.time()
         self.phase = phase
         self.suite_id = suite_id
+        self.label = label
         self.status = status
         self.recorder = recorder
         self.elapsed_time = elapsed_time
@@ -172,102 +174,6 @@ class ScenarioFinished(ScenarioEvent):
         self.is_final = is_final
 
 
-@dataclass
-class StepEvent(ScenarioEvent):
-    scenario_id: uuid.UUID
-
-
-@dataclass
-class StepStarted(StepEvent):
-    """Before executing a test case."""
-
-    __slots__ = (
-        "id",
-        "timestamp",
-        "phase",
-        "suite_id",
-        "scenario_id",
-    )
-
-    def __init__(
-        self,
-        *,
-        phase: PhaseName,
-        suite_id: uuid.UUID,
-        scenario_id: uuid.UUID,
-    ) -> None:
-        self.id = uuid.uuid4()
-        self.timestamp = time.time()
-        self.phase = phase
-        self.suite_id = suite_id
-        self.scenario_id = scenario_id
-
-
-@dataclass
-class TransitionId:
-    """Id of the the that was hit."""
-
-    name: str
-    # Status code as defined in the transition, i.e. may be `default`
-    status_code: str
-    source: str
-
-    __slots__ = ("name", "status_code", "source")
-
-
-@dataclass
-class ResponseData:
-    """Common data for responses."""
-
-    status_code: int
-    elapsed: float
-    __slots__ = ("status_code", "elapsed")
-
-
-@dataclass
-class StepFinished(StepEvent):
-    """After executing a test case."""
-
-    status: Status | None
-    transition_id: TransitionId | None
-    target: str
-    response: Response | None
-
-    __slots__ = (
-        "id",
-        "timestamp",
-        "phase",
-        "status",
-        "suite_id",
-        "scenario_id",
-        "transition_id",
-        "target",
-        "response",
-    )
-
-    def __init__(
-        self,
-        *,
-        phase: PhaseName,
-        id: uuid.UUID,
-        suite_id: uuid.UUID,
-        scenario_id: uuid.UUID,
-        status: Status | None,
-        transition_id: TransitionId | None,
-        target: str,
-        response: Response | None,
-    ) -> None:
-        self.id = id
-        self.timestamp = time.time()
-        self.phase = phase
-        self.status = status
-        self.suite_id = suite_id
-        self.scenario_id = scenario_id
-        self.transition_id = transition_id
-        self.target = target
-        self.response = response
-
-
 @dataclass
 class Interrupted(EngineEvent):
     """If execution was interrupted by Ctrl-C, or a received SIGTERM."""

schemathesis/engine/phases/stateful/__init__.py

@@ -20,6 +20,7 @@ def execute(engine: EngineContext, phase: Phase) -> events.EventGenerator:
         state_machine = engine.schema.as_state_machine()
     except Exception as exc:
         yield events.NonFatalError(error=exc, phase=phase.name, label="Stateful tests", related_to_operation=False)
+        yield events.PhaseFinished(phase=phase, status=Status.ERROR, payload=None)
         return
 
     event_queue: queue.Queue = queue.Queue()
@@ -27,6 +28,7 @@ def execute(engine: EngineContext, phase: Phase) -> events.EventGenerator:
     thread = threading.Thread(
         target=execute_state_machine_loop,
         kwargs={"state_machine": state_machine, "event_queue": event_queue, "engine": engine},
+        name="schemathesis_stateful_tests",
     )
     status: Status | None = None
     is_executed = False

schemathesis/engine/phases/stateful/_executor.py

@@ -5,6 +5,7 @@ import time
 import unittest
 from dataclasses import replace
 from typing import Any
+from warnings import catch_warnings
 
 import hypothesis
 from hypothesis.control import current_build_context
@@ -25,8 +26,8 @@ from schemathesis.generation.hypothesis.reporting import ignore_hypothesis_outpu
 from schemathesis.generation.stateful.state_machine import (
     DEFAULT_STATE_MACHINE_SETTINGS,
     APIStateMachine,
-    Direction,
-    StepResult,
+    StepInput,
+    StepOutput,
 )
 from schemathesis.generation.targets import TargetMetricCollector
 
@@ -77,13 +78,12 @@ def execute_state_machine_loop(
             self._start_time = time.monotonic()
             self._scenario_id = scenario_started.id
             event_queue.put(scenario_started)
-            self.recorder = ScenarioRecorder(label="Stateful tests")
             self._check_ctx = engine.get_check_context(self.recorder)
 
         def get_call_kwargs(self, case: Case) -> dict[str, Any]:
             return transport_kwargs
 
-        def _repr_step(self, rule: Rule, data: dict, result: StepResult) -> str:
+        def _repr_step(self, rule: Rule, data: dict, result: StepOutput) -> str:
             return ""
 
         if config.override is not None:
@@ -96,38 +96,29 @@ def execute_state_machine_loop(
                         setattr(case, location, container)
                 return super().before_call(case)
 
-        def step(self, case: Case, previous: tuple[StepResult, Direction] | None = None) -> StepResult | None:
+        def step(self, input: StepInput) -> StepOutput | None:
             # Checking the stop event once inside `step` is sufficient as it is called frequently
             # The idea is to stop the execution as soon as possible
-            if previous is not None:
-                step_result, _ = previous
-                self.recorder.record_case(parent_id=step_result.case.id, case=case)
-            else:
-                self.recorder.record_case(parent_id=None, case=case)
             if engine.has_to_stop:
                 raise KeyboardInterrupt
-            step_started = events.StepStarted(
-                phase=PhaseName.STATEFUL_TESTING, suite_id=suite_id, scenario_id=self._scenario_id
-            )
-            event_queue.put(step_started)
             try:
                 if config.execution.unique_inputs:
-                    cached = ctx.get_step_outcome(case)
+                    cached = ctx.get_step_outcome(input.case)
                     if isinstance(cached, BaseException):
                         raise cached
                     elif cached is None:
                         return None
-                result = super().step(case, previous)
+                result = super().step(input)
                 ctx.step_succeeded()
             except FailureGroup as exc:
                 if config.execution.unique_inputs:
                     for failure in exc.exceptions:
-                        ctx.store_step_outcome(case, failure)
+                        ctx.store_step_outcome(input.case, failure)
                 ctx.step_failed()
                 raise
             except Exception as exc:
                 if config.execution.unique_inputs:
-                    ctx.store_step_outcome(case, exc)
+                    ctx.store_step_outcome(input.case, exc)
                 ctx.step_errored()
                 raise
             except KeyboardInterrupt:
@@ -135,34 +126,11 @@ def execute_state_machine_loop(
                 raise
             except BaseException as exc:
                 if config.execution.unique_inputs:
-                    ctx.store_step_outcome(case, exc)
+                    ctx.store_step_outcome(input.case, exc)
                 raise exc
             else:
                 if config.execution.unique_inputs:
-                    ctx.store_step_outcome(case, None)
-            finally:
-                transition_id: events.TransitionId | None
-                if previous is not None:
-                    transition = previous[1]
-                    transition_id = events.TransitionId(
-                        name=transition.name,
-                        status_code=transition.status_code,
-                        source=transition.operation.label,
-                    )
-                else:
-                    transition_id = None
-                event_queue.put(
-                    events.StepFinished(
-                        id=step_started.id,
-                        suite_id=suite_id,
-                        scenario_id=self._scenario_id,
-                        phase=PhaseName.STATEFUL_TESTING,
-                        status=ctx.current_step_status,
-                        transition_id=transition_id,
-                        target=case.operation.label,
-                        response=ctx.current_response,
-                    )
-                )
+                    ctx.store_step_outcome(input.case, None)
             return result
 
         def validate_response(
@@ -189,7 +157,7 @@ def execute_state_machine_loop(
                     id=self._scenario_id,
                     suite_id=suite_id,
                     phase=PhaseName.STATEFUL_TESTING,
-                    # With dry run there will be no status
+                    label=None,
                     status=ctx.current_scenario_status or Status.SKIP,
                     recorder=self.recorder,
                     elapsed_time=time.monotonic() - self._start_time,
@@ -201,10 +169,7 @@ def execute_state_machine_loop(
             ctx.reset_scenario()
             super().teardown()
 
-    if config.execution.seed is not None:
-        InstrumentedStateMachine = hypothesis.seed(config.execution.seed)(_InstrumentedStateMachine)
-    else:
-        InstrumentedStateMachine = _InstrumentedStateMachine
+    seed = config.execution.seed
 
     while True:
         # This loop is running until no new failures are found in a single iteration
@@ -222,8 +187,15 @@ def execute_state_machine_loop(
             )
             break
         suite_status = Status.SUCCESS
+        if seed is not None:
+            InstrumentedStateMachine = hypothesis.seed(seed)(_InstrumentedStateMachine)
+            # Predictably change the seed to avoid re-running the same sequences if tests fail
+            # yet have reproducible results
+            seed += 1
+        else:
+            InstrumentedStateMachine = _InstrumentedStateMachine
         try:
-            with ignore_hypothesis_output():  # type: ignore
+            with catch_warnings(), ignore_hypothesis_output():  # type: ignore
                 InstrumentedStateMachine.run(settings=config.execution.hypothesis_settings)
         except KeyboardInterrupt:
             # Raised in the state machine when the stop event is set or it is raised by the user's code
@@ -234,7 +206,7 @@ def execute_state_machine_loop(
             event_queue.put(events.Interrupted(phase=PhaseName.STATEFUL_TESTING))
             break
         except unittest.case.SkipTest:
-            # If `explicit` phase is used and there are not examples
+            # If `explicit` phase is used and there are no examples
             suite_status = Status.SKIP
             break
         except FailureGroup as exc:

schemathesis/engine/phases/unit/__init__.py

@@ -140,6 +140,7 @@ def worker_task(*, events_queue: Queue, producer: TaskProducer, ctx: EngineConte
                                 id=scenario_started.id,
                                 suite_id=suite_id,
                                 phase=PhaseName.UNIT_TESTING,
+                                label=label,
                                 status=Status.ERROR,
                                 recorder=ScenarioRecorder(label="Error"),
                                 elapsed_time=0.0,

schemathesis/engine/phases/unit/_executor.py

@@ -201,6 +201,7 @@ def run_test(
         id=scenario_started.id,
         suite_id=suite_id,
         phase=PhaseName.UNIT_TESTING,
+        label=operation.label,
         recorder=recorder,
         status=status,
         elapsed_time=test_elapsed_time,
@@ -263,7 +264,7 @@ def cached_test_func(f: Callable) -> Callable:
 
 @cached_test_func
 def test_func(*, ctx: EngineContext, case: Case, recorder: ScenarioRecorder) -> None:
-    recorder.record_case(parent_id=None, case=case)
+    recorder.record_case(parent_id=None, transition=None, case=case)
     try:
         response = case.call(**ctx.transport_kwargs)
     except (requests.Timeout, requests.ConnectionError) as error:

schemathesis/engine/phases/unit/_pool.py

@@ -55,7 +55,7 @@ class WorkerPool:
                     "producer": self.producer,
                     "suite_id": self.suite_id,
                 },
-                name=f"schemathesis_{i}",
+                name=f"schemathesis_unit_tests_{i}",
                 daemon=True,
             )
             self.workers.append(worker)