schemathesis 4.0.0a1__py3-none-any.whl → 4.0.0a2__py3-none-any.whl

schemathesis/cli/commands/run/validation.py

@@ -256,7 +256,7 @@ def convert_experimental(
     ]
 
 
-def convert_checks(ctx: click.core.Context, param: click.core.Parameter, value: tuple[list[str]]) -> list[str]:
+def reduce_list(ctx: click.core.Context, param: click.core.Parameter, value: tuple[list[str]]) -> list[str]:
     return reduce(operator.iadd, value, [])
 
 

schemathesis/cli/ext/options.py

@@ -80,7 +80,10 @@ class RegistryChoice(BaseCsvChoice):
     def convert(  # type: ignore[return]
         self, value: str, param: click.core.Parameter | None, ctx: click.core.Context | None
     ) -> list[str]:
-        return [item for item in value.split(",") if item]
+        selected, invalid_options = self.parse_value(value)
+        if not invalid_options and selected:
+            return selected
+        self.fail_on_invalid_options(invalid_options, selected)
 
 
 class OptionalInt(click.types.IntRange):

schemathesis/core/failures.py

@@ -2,11 +2,12 @@ from __future__ import annotations
 
 import http.client
 import textwrap
+import traceback
 from collections.abc import Sequence
 from dataclasses import dataclass
 from enum import Enum, auto
 from json import JSONDecodeError
-from typing import Callable
+from typing import Any, Callable
 
 from schemathesis.core.compat import BaseExceptionGroup
 from schemathesis.core.output import OutputConfig, prepare_response_payload
@@ -32,7 +33,7 @@ class Severity(Enum):
 class Failure(AssertionError):
     """API check failure."""
 
-    __slots__ = ("operation", "title", "message", "code", "case_id", "severity")
+    __slots__ = ("operation", "title", "message", "case_id", "severity")
 
     def __init__(
         self,
@@ -40,14 +41,12 @@ class Failure(AssertionError):
         operation: str,
         title: str,
         message: str,
-        code: str,
         case_id: str | None = None,
         severity: Severity = Severity.MEDIUM,
     ) -> None:
         self.operation = operation
         self.title = title
         self.message = message
-        self.code = code
         self.case_id = case_id
         self.severity = severity
 
@@ -72,20 +71,58 @@ class Failure(AssertionError):
             return NotImplemented
         return type(self) is type(other) and self.operation == other.operation and self._unique_key == other._unique_key
 
-    @classmethod
-    def from_assertion(cls, *, name: str, operation: str, exc: AssertionError) -> Failure:
-        return Failure(
-            operation=operation,
-            title=f"Custom check failed: `{name}`",
-            message=str(exc),
-            code="custom",
-        )
-
     @property
-    def _unique_key(self) -> str:
+    def _unique_key(self) -> Any:
         return self.message
 
 
+def get_origin(exception: BaseException, seen: tuple[BaseException, ...] = ()) -> tuple:
+    filename, lineno = None, None
+    if tb := exception.__traceback__:
+        filename, lineno, *_ = traceback.extract_tb(tb)[-1]
+    seen = (*seen, exception)
+    context = ()
+    if exception.__context__ is not None and exception.__context__ not in seen:
+        context = get_origin(exception.__context__, seen=seen)
+    return (
+        type(exception),
+        filename,
+        lineno,
+        context,
+        (
+            tuple(get_origin(exc, seen=seen) for exc in exception.exceptions if exc not in seen)
+            if isinstance(exception, BaseExceptionGroup)
+            else ()
+        ),
+    )
+
+
+class CustomFailure(Failure):
+    __slots__ = ("operation", "title", "message", "exception", "case_id", "severity", "origin")
+
+    def __init__(
+        self,
+        *,
+        operation: str,
+        title: str,
+        message: str,
+        exception: AssertionError,
+        case_id: str | None = None,
+        severity: Severity = Severity.MEDIUM,
+    ) -> None:
+        self.operation = operation
+        self.title = title
+        self.message = message
+        self.exception = exception
+        self.case_id = case_id
+        self.severity = severity
+        self.origin = get_origin(exception)
+
+    @property
+    def _unique_key(self) -> Any:
+        return self.origin
+
+
 @dataclass
 class MaxResponseTimeConfig:
     limit: float = 10.0
@@ -94,7 +131,7 @@ class MaxResponseTimeConfig:
 class ResponseTimeExceeded(Failure):
     """Response took longer than expected."""
 
-    __slots__ = ("operation", "elapsed", "deadline", "title", "message", "code", "case_id", "severity")
+    __slots__ = ("operation", "elapsed", "deadline", "title", "message", "case_id", "severity")
 
     def __init__(
         self,
@@ -104,7 +141,6 @@ class ResponseTimeExceeded(Failure):
         deadline: int,
         message: str,
         title: str = "Response time limit exceeded",
-        code: str = "response_time_exceeded",
         case_id: str | None = None,
     ) -> None:
         self.operation = operation
@@ -112,7 +148,6 @@ class ResponseTimeExceeded(Failure):
         self.deadline = deadline
         self.title = title
         self.message = message
-        self.code = code
         self.case_id = case_id
         self.severity = Severity.LOW
 
@@ -124,7 +159,7 @@ class ResponseTimeExceeded(Failure):
 class ServerError(Failure):
     """Server responded with an error."""
 
-    __slots__ = ("operation", "status_code", "title", "message", "code", "case_id", "severity")
+    __slots__ = ("operation", "status_code", "title", "message", "case_id", "severity")
 
     def __init__(
         self,
@@ -133,14 +168,12 @@ class ServerError(Failure):
         status_code: int,
         title: str = "Server error",
         message: str = "",
-        code: str = "server_error",
         case_id: str | None = None,
     ) -> None:
         self.operation = operation
         self.status_code = status_code
         self.title = title
         self.message = message
-        self.code = code
         self.case_id = case_id
         self.severity = Severity.CRITICAL
 
@@ -161,7 +194,6 @@ class MalformedJson(Failure):
         "colno",
         "message",
         "title",
-        "code",
         "case_id",
         "severity",
     )
@@ -177,7 +209,6 @@ class MalformedJson(Failure):
         colno: int,
         message: str,
         title: str = "JSON deserialization error",
-        code: str = "malformed_json",
         case_id: str | None = None,
     ) -> None:
         self.operation = operation
@@ -188,12 +219,11 @@ class MalformedJson(Failure):
         self.colno = colno
         self.message = message
         self.title = title
-        self.code = code
         self.case_id = case_id
         self.severity = Severity.MEDIUM
 
     @property
-    def _unique_key(self) -> str:
+    def _unique_key(self) -> Any:
         return self.title
 
     @classmethod

schemathesis/engine/core.py

@@ -68,7 +68,7 @@ class Engine:
                 skip_reason=PhaseSkipReason.DISABLED,
             )
 
-        if requires_links and self.schema.links_count == 0:
+        if requires_links and self.schema.statistic.links.selected == 0:
             return Phase(
                 name=phase_name,
                 is_supported=True,

schemathesis/engine/events.py

@@ -6,7 +6,6 @@ from dataclasses import dataclass
 from typing import TYPE_CHECKING, Generator
 
 from schemathesis.core.result import Result
-from schemathesis.core.transport import Response
 from schemathesis.engine.errors import EngineErrorInfo
 from schemathesis.engine.phases import Phase, PhaseName
 from schemathesis.engine.recorder import ScenarioRecorder
@@ -142,6 +141,7 @@ class ScenarioFinished(ScenarioEvent):
         "timestamp",
         "phase",
         "suite_id",
+        "label",
         "status",
         "recorder",
         "elapsed_time",
@@ -155,6 +155,7 @@ class ScenarioFinished(ScenarioEvent):
         id: uuid.UUID,
         phase: PhaseName,
         suite_id: uuid.UUID,
+        label: str | None,
         status: Status,
         recorder: ScenarioRecorder,
         elapsed_time: float,
@@ -165,6 +166,7 @@ class ScenarioFinished(ScenarioEvent):
         self.timestamp = time.time()
         self.phase = phase
         self.suite_id = suite_id
+        self.label = label
         self.status = status
         self.recorder = recorder
         self.elapsed_time = elapsed_time
@@ -172,102 +174,6 @@ class ScenarioFinished(ScenarioEvent):
         self.is_final = is_final
 
 
-@dataclass
-class StepEvent(ScenarioEvent):
-    scenario_id: uuid.UUID
-
-
-@dataclass
-class StepStarted(StepEvent):
-    """Before executing a test case."""
-
-    __slots__ = (
-        "id",
-        "timestamp",
-        "phase",
-        "suite_id",
-        "scenario_id",
-    )
-
-    def __init__(
-        self,
-        *,
-        phase: PhaseName,
-        suite_id: uuid.UUID,
-        scenario_id: uuid.UUID,
-    ) -> None:
-        self.id = uuid.uuid4()
-        self.timestamp = time.time()
-        self.phase = phase
-        self.suite_id = suite_id
-        self.scenario_id = scenario_id
-
-
-@dataclass
-class TransitionId:
-    """Id of the the that was hit."""
-
-    name: str
-    # Status code as defined in the transition, i.e. may be `default`
-    status_code: str
-    source: str
-
-    __slots__ = ("name", "status_code", "source")
-
-
-@dataclass
-class ResponseData:
-    """Common data for responses."""
-
-    status_code: int
-    elapsed: float
-    __slots__ = ("status_code", "elapsed")
-
-
-@dataclass
-class StepFinished(StepEvent):
-    """After executing a test case."""
-
-    status: Status | None
-    transition_id: TransitionId | None
-    target: str
-    response: Response | None
-
-    __slots__ = (
-        "id",
-        "timestamp",
-        "phase",
-        "status",
-        "suite_id",
-        "scenario_id",
-        "transition_id",
-        "target",
-        "response",
-    )
-
-    def __init__(
-        self,
-        *,
-        phase: PhaseName,
-        id: uuid.UUID,
-        suite_id: uuid.UUID,
-        scenario_id: uuid.UUID,
-        status: Status | None,
-        transition_id: TransitionId | None,
-        target: str,
-        response: Response | None,
-    ) -> None:
-        self.id = id
-        self.timestamp = time.time()
-        self.phase = phase
-        self.status = status
-        self.suite_id = suite_id
-        self.scenario_id = scenario_id
-        self.transition_id = transition_id
-        self.target = target
-        self.response = response
-
-
 @dataclass
 class Interrupted(EngineEvent):
     """If execution was interrupted by Ctrl-C, or a received SIGTERM."""

schemathesis/engine/phases/stateful/__init__.py

@@ -27,6 +27,7 @@ def execute(engine: EngineContext, phase: Phase) -> events.EventGenerator:
     thread = threading.Thread(
         target=execute_state_machine_loop,
         kwargs={"state_machine": state_machine, "event_queue": event_queue, "engine": engine},
+        name="schemathesis_stateful_tests",
     )
     status: Status | None = None
     is_executed = False

schemathesis/engine/phases/stateful/_executor.py

@@ -5,6 +5,7 @@ import time
 import unittest
 from dataclasses import replace
 from typing import Any
+from warnings import catch_warnings
 
 import hypothesis
 from hypothesis.control import current_build_context
@@ -25,8 +26,8 @@ from schemathesis.generation.hypothesis.reporting import ignore_hypothesis_outpu
 from schemathesis.generation.stateful.state_machine import (
     DEFAULT_STATE_MACHINE_SETTINGS,
     APIStateMachine,
-    Direction,
-    StepResult,
+    StepInput,
+    StepOutput,
 )
 from schemathesis.generation.targets import TargetMetricCollector
 
@@ -83,7 +84,7 @@ def execute_state_machine_loop(
         def get_call_kwargs(self, case: Case) -> dict[str, Any]:
             return transport_kwargs
 
-        def _repr_step(self, rule: Rule, data: dict, result: StepResult) -> str:
+        def _repr_step(self, rule: Rule, data: dict, result: StepOutput) -> str:
             return ""
 
         if config.override is not None:
@@ -96,38 +97,35 @@ def execute_state_machine_loop(
                         setattr(case, location, container)
                 return super().before_call(case)
 
-        def step(self, case: Case, previous: tuple[StepResult, Direction] | None = None) -> StepResult | None:
+        def step(self, input: StepInput) -> StepOutput | None:
             # Checking the stop event once inside `step` is sufficient as it is called frequently
             # The idea is to stop the execution as soon as possible
-            if previous is not None:
-                step_result, _ = previous
-                self.recorder.record_case(parent_id=step_result.case.id, case=case)
+            if input.transition is not None:
+                self.recorder.record_case(
+                    parent_id=input.transition.parent_id, transition=input.transition, case=input.case
+                )
             else:
-                self.recorder.record_case(parent_id=None, case=case)
+                self.recorder.record_case(parent_id=None, transition=None, case=input.case)
             if engine.has_to_stop:
                 raise KeyboardInterrupt
-            step_started = events.StepStarted(
-                phase=PhaseName.STATEFUL_TESTING, suite_id=suite_id, scenario_id=self._scenario_id
-            )
-            event_queue.put(step_started)
             try:
                 if config.execution.unique_inputs:
-                    cached = ctx.get_step_outcome(case)
+                    cached = ctx.get_step_outcome(input.case)
                     if isinstance(cached, BaseException):
                         raise cached
                     elif cached is None:
                         return None
-                result = super().step(case, previous)
+                result = super().step(input)
                 ctx.step_succeeded()
             except FailureGroup as exc:
                 if config.execution.unique_inputs:
                     for failure in exc.exceptions:
-                        ctx.store_step_outcome(case, failure)
+                        ctx.store_step_outcome(input.case, failure)
                 ctx.step_failed()
                 raise
             except Exception as exc:
                 if config.execution.unique_inputs:
-                    ctx.store_step_outcome(case, exc)
+                    ctx.store_step_outcome(input.case, exc)
                 ctx.step_errored()
                 raise
             except KeyboardInterrupt:
@@ -135,34 +133,11 @@ def execute_state_machine_loop(
                 raise
             except BaseException as exc:
                 if config.execution.unique_inputs:
-                    ctx.store_step_outcome(case, exc)
+                    ctx.store_step_outcome(input.case, exc)
                 raise exc
             else:
                 if config.execution.unique_inputs:
-                    ctx.store_step_outcome(case, None)
-            finally:
-                transition_id: events.TransitionId | None
-                if previous is not None:
-                    transition = previous[1]
-                    transition_id = events.TransitionId(
-                        name=transition.name,
-                        status_code=transition.status_code,
-                        source=transition.operation.label,
-                    )
-                else:
-                    transition_id = None
-                event_queue.put(
-                    events.StepFinished(
-                        id=step_started.id,
-                        suite_id=suite_id,
-                        scenario_id=self._scenario_id,
-                        phase=PhaseName.STATEFUL_TESTING,
-                        status=ctx.current_step_status,
-                        transition_id=transition_id,
-                        target=case.operation.label,
-                        response=ctx.current_response,
-                    )
-                )
+                    ctx.store_step_outcome(input.case, None)
             return result
 
         def validate_response(
@@ -189,7 +164,7 @@ def execute_state_machine_loop(
                     id=self._scenario_id,
                     suite_id=suite_id,
                     phase=PhaseName.STATEFUL_TESTING,
-                    # With dry run there will be no status
+                    label=None,
                     status=ctx.current_scenario_status or Status.SKIP,
                     recorder=self.recorder,
                     elapsed_time=time.monotonic() - self._start_time,
@@ -223,7 +198,7 @@ def execute_state_machine_loop(
             break
         suite_status = Status.SUCCESS
         try:
-            with ignore_hypothesis_output():  # type: ignore
+            with catch_warnings(), ignore_hypothesis_output():  # type: ignore
                 InstrumentedStateMachine.run(settings=config.execution.hypothesis_settings)
         except KeyboardInterrupt:
             # Raised in the state machine when the stop event is set or it is raised by the user's code
@@ -234,7 +209,7 @@ def execute_state_machine_loop(
             event_queue.put(events.Interrupted(phase=PhaseName.STATEFUL_TESTING))
             break
         except unittest.case.SkipTest:
-            # If `explicit` phase is used and there are not examples
+            # If `explicit` phase is used and there are no examples
             suite_status = Status.SKIP
             break
         except FailureGroup as exc:

schemathesis/engine/phases/unit/__init__.py

@@ -140,6 +140,7 @@ def worker_task(*, events_queue: Queue, producer: TaskProducer, ctx: EngineConte
                                 id=scenario_started.id,
                                 suite_id=suite_id,
                                 phase=PhaseName.UNIT_TESTING,
+                                label=label,
                                 status=Status.ERROR,
                                 recorder=ScenarioRecorder(label="Error"),
                                 elapsed_time=0.0,

schemathesis/engine/phases/unit/_executor.py

@@ -201,6 +201,7 @@ def run_test(
         id=scenario_started.id,
         suite_id=suite_id,
         phase=PhaseName.UNIT_TESTING,
+        label=operation.label,
         recorder=recorder,
         status=status,
         elapsed_time=test_elapsed_time,
@@ -263,7 +264,7 @@ def cached_test_func(f: Callable) -> Callable:
 
 @cached_test_func
 def test_func(*, ctx: EngineContext, case: Case, recorder: ScenarioRecorder) -> None:
-    recorder.record_case(parent_id=None, case=case)
+    recorder.record_case(parent_id=None, transition=None, case=case)
     try:
         response = case.call(**ctx.transport_kwargs)
     except (requests.Timeout, requests.ConnectionError) as error:

schemathesis/engine/phases/unit/_pool.py

@@ -55,7 +55,7 @@ class WorkerPool:
                     "producer": self.producer,
                     "suite_id": self.suite_id,
                 },
-                name=f"schemathesis_{i}",
+                name=f"schemathesis_unit_tests_{i}",
                 daemon=True,
             )
             self.workers.append(worker)

schemathesis/engine/recorder.py

@@ -14,6 +14,8 @@ from schemathesis.generation.case import Case
 if TYPE_CHECKING:
     import requests
 
+    from schemathesis.generation.stateful.state_machine import Transition
+
 
 @dataclass
 class ScenarioRecorder:
@@ -42,9 +44,9 @@ class ScenarioRecorder:
         self.checks = {}
         self.interactions = {}
 
-    def record_case(self, *, parent_id: str | None, case: Case) -> None:
+    def record_case(self, *, parent_id: str | None, transition: Transition | None, case: Case) -> None:
         """Record a test case and its relationship to a parent, if applicable."""
-        self.cases[case.id] = CaseNode(value=case, parent_id=parent_id)
+        self.cases[case.id] = CaseNode(value=case, parent_id=parent_id, transition=transition)
 
     def record_response(self, *, case_id: str, response: Response) -> None:
         """Record the API response for a given test case."""
@@ -133,8 +135,11 @@ class CaseNode:
 
     value: Case
     parent_id: str | None
+    # Transition may be absent if `parent_id` is present for cases when a case is derived inside a check
+    # and outside of the implemented transition logic (e.g. Open API links)
+    transition: Transition | None
 
-    __slots__ = ("value", "parent_id")
+    __slots__ = ("value", "parent_id", "transition")
 
 
 @dataclass