schemathesis 3.29.1__py3-none-any.whl → 3.30.0__py3-none-any.whl

schemathesis/specs/openapi/stateful/statistic.py

@@ -0,0 +1,198 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Iterator, List, Union
+
+from ....internal.copy import fast_deepcopy
+from ....stateful.statistic import TransitionStats
+from .types import AggregatedResponseCounter, LinkName, ResponseCounter, SourceName, StatusCode, TargetName
+
+if TYPE_CHECKING:
+    from ....stateful import events
+
+
+@dataclass
+class LinkSource:
+    name: str
+    responses: dict[StatusCode, dict[TargetName, dict[LinkName, ResponseCounter]]]
+    is_first: bool
+
+    __slots__ = ("name", "responses", "is_first")
+
+
+@dataclass
+class OperationResponse:
+    status_code: str
+    targets: dict[TargetName, dict[LinkName, ResponseCounter]]
+    is_last: bool
+
+    __slots__ = ("status_code", "targets", "is_last")
+
+
+@dataclass
+class Link:
+    name: str
+    target: str
+    responses: ResponseCounter
+    is_last: bool
+    is_single: bool
+
+    __slots__ = ("name", "target", "responses", "is_last", "is_single")
+
+
+StatisticEntry = Union[LinkSource, OperationResponse, Link]
+
+
+@dataclass
+class FormattedStatisticEntry:
+    line: str
+    entry: StatisticEntry
+    __slots__ = ("line", "entry")
+
+
+@dataclass
+class OpenAPILinkStats(TransitionStats):
+    """Statistics about link transitions for a state machine run."""
+
+    transitions: dict[SourceName, dict[StatusCode, dict[TargetName, dict[LinkName, ResponseCounter]]]]
+
+    roots: dict[TargetName, ResponseCounter] = field(default_factory=dict)
+
+    __slots__ = ("transitions",)
+
+    def consume(self, event: events.StatefulEvent) -> None:
+        from ....stateful import events
+
+        if isinstance(event, events.StepFinished):
+            if event.transition_id is not None:
+                transition_id = event.transition_id
+                source = self.transitions[transition_id.source]
+                transition = source[transition_id.status_code][event.target][transition_id.name]
+                if event.response is not None:
+                    key = event.response.status_code
+                else:
+                    key = None
+                counter = transition.setdefault(key, 0)
+                transition[key] = counter + 1
+            else:
+                # A start of a sequence has an empty source and does not belong to any transition
+                target = self.roots.setdefault(event.target, {})
+                if event.response is not None:
+                    key = event.response.status_code
+                else:
+                    key = None
+                counter = target.setdefault(key, 0)
+                target[key] = counter + 1
+
+    def copy(self) -> OpenAPILinkStats:
+        return self.__class__(transitions=fast_deepcopy(self.transitions))
+
+    def iter(self) -> Iterator[StatisticEntry]:
+        for source_idx, (source, responses) in enumerate(self.transitions.items()):
+            yield LinkSource(name=source, responses=responses, is_first=source_idx == 0)
+            for response_idx, (status_code, targets) in enumerate(responses.items()):
+                yield OperationResponse(
+                    status_code=status_code, targets=targets, is_last=response_idx == len(responses) - 1
+                )
+                for target_idx, (target, links) in enumerate(targets.items()):
+                    for link_idx, (link_name, link_responses) in enumerate(links.items()):
+                        yield Link(
+                            name=link_name,
+                            target=target,
+                            responses=link_responses,
+                            is_last=target_idx == len(targets) - 1 and link_idx == len(links) - 1,
+                            is_single=len(links) == 1,
+                        )
+
+    def iter_with_format(self) -> Iterator[FormattedStatisticEntry]:
+        current_response = None
+        for entry in self.iter():
+            if isinstance(entry, LinkSource):
+                if not entry.is_first:
+                    yield FormattedStatisticEntry(line=f"\n{entry.name}", entry=entry)
+                else:
+                    yield FormattedStatisticEntry(line=f"{entry.name}", entry=entry)
+            elif isinstance(entry, OperationResponse):
+                current_response = entry
+                if entry.is_last:
+                    yield FormattedStatisticEntry(line=f"└── {entry.status_code}", entry=entry)
+                else:
+                    yield FormattedStatisticEntry(line=f"├── {entry.status_code}", entry=entry)
+            else:
+                if current_response is not None and current_response.is_last:
+                    line = "    "
+                else:
+                    line = "│   "
+                if entry.is_last:
+                    line += "└"
+                else:
+                    line += "├"
+                if entry.is_single or entry.name == entry.target:
+                    line += f"── {entry.target}"
+                else:
+                    line += f"── {entry.name} -> {entry.target}"
+                yield FormattedStatisticEntry(line=line, entry=entry)
+
+    def to_formatted_table(self, width: int) -> str:
+        """Format the statistic as a table."""
+        entries = list(self.iter_with_format())
+        lines: List[str | list[str]] = [HEADER, ""]
+        column_widths = [len(column) for column in HEADER]
+        for entry in entries:
+            if isinstance(entry.entry, Link):
+                aggregated = _aggregate_responses(entry.entry.responses)
+                values = [
+                    entry.line,
+                    str(aggregated["2xx"]),
+                    str(aggregated["4xx"]),
+                    str(aggregated["5xx"]),
+                    str(aggregated["Total"]),
+                ]
+                column_widths = [max(column_widths[idx], len(column)) for idx, column in enumerate(values)]
+                lines.append(values)
+            else:
+                lines.append(entry.line)
+        used_width = sum(column_widths) + 4 * PADDING
+        max_space = width - used_width if used_width < width else 0
+        formatted_lines = []
+
+        for line in lines:
+            if isinstance(line, list):
+                formatted_line, *counters = line
+                formatted_line = formatted_line.ljust(column_widths[0] + max_space)
+
+                for column, max_width in zip(counters, column_widths[1:]):
+                    formatted_line += f"{column:>{max_width + PADDING}}"
+
+                formatted_lines.append(formatted_line)
+            else:
+                formatted_lines.append(line)
+
+        return "\n".join(formatted_lines)
+
+
+PADDING = 4
+HEADER = ["Links", "2xx", "4xx", "5xx", "Total"]
+
+
+def _aggregate_responses(responses: ResponseCounter) -> AggregatedResponseCounter:
+    """Aggregate responses by status code ranges."""
+    output: AggregatedResponseCounter = {
+        "2xx": 0,
+        # NOTE: 3xx responses are not counted
+        "4xx": 0,
+        "5xx": 0,
+        "Total": 0,
+    }
+    for status_code, count in responses.items():
+        if status_code is not None:
+            if 200 <= status_code < 300:
+                output["2xx"] += count
+                output["Total"] += count
+            elif 400 <= status_code < 500:
+                output["4xx"] += count
+                output["Total"] += count
+            elif 500 <= status_code < 600:
+                output["5xx"] += count
+                output["Total"] += count
+    return output

schemathesis/specs/openapi/stateful/types.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from typing import Callable, Dict, TypedDict, Union
+
+from ....stateful.state_machine import StepResult
+
+StatusCode = str
+LinkName = str
+TargetName = str
+SourceName = str
+ResponseCounter = Dict[Union[int, None], int]
+FilterFunction = Callable[["StepResult"], bool]
+AggregatedResponseCounter = TypedDict("AggregatedResponseCounter", {"2xx": int, "4xx": int, "5xx": int, "Total": int})

schemathesis/specs/openapi/utils.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+
 import string
 from itertools import product
 from typing import Any, Generator

schemathesis/specs/openapi/validation.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+
 from typing import Any
 
 from ...constants import HTTP_METHODS

schemathesis/stateful/__init__.py

@@ -1,17 +1,19 @@
 from __future__ import annotations
+
 import enum
 import json
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any, Callable, Generator
 
 from .. import GenerationConfig
-from ..exceptions import OperationSchemaError
-from ..models import APIOperation, Case
 from ..constants import NOT_SET
+from ..exceptions import OperationSchemaError
 from ..internal.result import Ok, Result
+from ..models import APIOperation, Case
 
 if TYPE_CHECKING:
     import hypothesis
+
     from ..transports.responses import GenericResponse
     from .state_machine import APIStateMachine
 

schemathesis/stateful/config.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    import hypothesis
+
+    from ..models import CheckFunction
+
+
+def _default_checks_factory() -> tuple[CheckFunction, ...]:
+    from ..checks import ALL_CHECKS
+    from ..specs.openapi.checks import use_after_free
+
+    return ALL_CHECKS + (use_after_free,)
+
+
+def _get_default_hypothesis_settings_kwargs() -> dict[str, Any]:
+    import hypothesis
+
+    return {"phases": (hypothesis.Phase.generate,), "deadline": None}
+
+
+def _default_hypothesis_settings_factory() -> hypothesis.settings:
+    # To avoid importing hypothesis at the module level
+    import hypothesis
+
+    return hypothesis.settings(**_get_default_hypothesis_settings_kwargs())
+
+
+@dataclass
+class StatefulTestRunnerConfig:
+    """Configuration for the stateful test runner."""
+
+    # Checks to run against each response
+    checks: tuple[CheckFunction, ...] = field(default_factory=_default_checks_factory)
+    # Hypothesis settings for state machine execution
+    hypothesis_settings: hypothesis.settings = field(default_factory=_default_hypothesis_settings_factory)
+    # Whether to stop the execution after the first failure
+    exit_first: bool = False
+    # Custom headers sent with each request
+    headers: dict[str, str] = field(default_factory=dict)
+    # Timeout for each request in milliseconds
+    request_timeout: int | None = None
+
+    def __post_init__(self) -> None:
+        import hypothesis
+
+        kwargs = _get_hypothesis_settings_kwargs_override(self.hypothesis_settings)
+        if kwargs:
+            self.hypothesis_settings = hypothesis.settings(self.hypothesis_settings, **kwargs)
+
+
+def _get_hypothesis_settings_kwargs_override(settings: hypothesis.settings) -> dict[str, Any]:
+    """Get the settings that should be overridden to match the defaults for API state machines."""
+    import hypothesis
+
+    kwargs = {}
+    hypothesis_default = hypothesis.settings()
+    state_machine_default = _default_hypothesis_settings_factory()
+    if settings.phases == hypothesis_default.phases:
+        kwargs["phases"] = state_machine_default.phases
+    if settings.deadline == hypothesis_default.deadline:
+        kwargs["deadline"] = state_machine_default.deadline
+    return kwargs

schemathesis/stateful/context.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import traceback
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Tuple, Type, Union
+
+from ..exceptions import CheckFailed
+from . import events
+
+if TYPE_CHECKING:
+    from ..models import Check
+    from ..transports.responses import GenericResponse
+
+FailureKey = Union[Type[CheckFailed], Tuple[str, int]]
+
+
+def _failure_cache_key(exc: CheckFailed | AssertionError) -> FailureKey:
+    """Create a ket to identify unique failures."""
+    from hypothesis.internal.escalation import get_trimmed_traceback
+
+    # For CheckFailed, we already have all distinctive information about the failure, which is contained
+    # in the exception type itself.
+    if isinstance(exc, CheckFailed):
+        return exc.__class__
+
+    # Assertion come from the user's code and we may try to group them by location
+    tb = get_trimmed_traceback(exc)
+    filename, lineno, *_ = traceback.extract_tb(tb)[-1]
+    return (filename, lineno)
+
+
+@dataclass
+class RunnerContext:
+    """Mutable context for state machine execution."""
+
+    # All seen failure keys, both grouped and individual ones
+    seen_in_run: set[FailureKey] = field(default_factory=set)
+    # Failures keys seen in the current suite
+    seen_in_suite: set[FailureKey] = field(default_factory=set)
+    # Unique failures collected in the current suite
+    failures_for_suite: list[Check] = field(default_factory=list)
+    # Status of the current step
+    current_step_status: events.StepStatus = events.StepStatus.SUCCESS
+    current_response: GenericResponse | None = None
+
+    @property
+    def current_scenario_status(self) -> events.ScenarioStatus:
+        if self.current_step_status == events.StepStatus.SUCCESS:
+            return events.ScenarioStatus.SUCCESS
+        elif self.current_step_status == events.StepStatus.FAILURE:
+            return events.ScenarioStatus.FAILURE
+        return events.ScenarioStatus.ERROR
+
+    def reset_step(self) -> None:
+        self.current_step_status = events.StepStatus.SUCCESS
+        self.current_response = None
+
+    def step_failed(self) -> None:
+        self.current_step_status = events.StepStatus.FAILURE
+
+    def step_errored(self) -> None:
+        self.current_step_status = events.StepStatus.ERROR
+
+    def mark_as_seen_in_run(self, exc: CheckFailed) -> None:
+        key = _failure_cache_key(exc)
+        self.seen_in_run.add(key)
+        causes = exc.causes or ()
+        for cause in causes:
+            key = _failure_cache_key(cause)
+            self.seen_in_run.add(key)
+
+    def mark_as_seen_in_suite(self, exc: CheckFailed | AssertionError) -> None:
+        key = _failure_cache_key(exc)
+        self.seen_in_suite.add(key)
+
+    def mark_current_suite_as_seen_in_run(self) -> None:
+        self.seen_in_run.update(self.seen_in_suite)
+
+    def is_seen_in_run(self, exc: CheckFailed | AssertionError) -> bool:
+        key = _failure_cache_key(exc)
+        return key in self.seen_in_run
+
+    def is_seen_in_suite(self, exc: CheckFailed | AssertionError) -> bool:
+        key = _failure_cache_key(exc)
+        return key in self.seen_in_suite
+
+    def add_failed_check(self, check: Check) -> None:
+        self.failures_for_suite.append(check)
+
+    def reset(self) -> None:
+        self.failures_for_suite = []
+        self.seen_in_suite.clear()
+        self.reset_step()

schemathesis/stateful/events.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from enum import Enum
+from typing import TYPE_CHECKING, Type
+
+if TYPE_CHECKING:
+    from ..models import Check
+    from .state_machine import APIStateMachine
+
+
+class RunStatus(str, Enum):
+    """Status of the state machine run."""
+
+    SUCCESS = "success"
+    FAILURE = "failure"
+    ERROR = "error"
+    INTERRUPTED = "interrupted"
+
+
+@dataclass
+class StatefulEvent:
+    """Basic stateful test event."""
+
+    __slots__ = ("timestamp",)
+
+
+@dataclass
+class RunStarted(StatefulEvent):
+    """Before executing all scenarios."""
+
+    state_machine: Type[APIStateMachine]
+
+    __slots__ = ("state_machine", "timestamp", "started_at")
+
+    def __init__(self, *, state_machine: Type[APIStateMachine]) -> None:
+        self.state_machine = state_machine
+        self.started_at = time.time()
+        self.timestamp = time.monotonic()
+
+
+@dataclass
+class RunFinished(StatefulEvent):
+    """After executing all scenarios."""
+
+    status: RunStatus
+
+    __slots__ = ("timestamp", "status")
+
+    def __init__(self, *, status: RunStatus) -> None:
+        self.status = status
+        self.timestamp = time.monotonic()
+
+
+class SuiteStatus(str, Enum):
+    """Status of the suite execution."""
+
+    SUCCESS = "success"
+    FAILURE = "failure"
+    ERROR = "error"
+    INTERRUPTED = "interrupted"
+
+
+@dataclass
+class SuiteStarted(StatefulEvent):
+    """Before executing a set of scenarios."""
+
+    __slots__ = ("timestamp",)
+
+    def __init__(self) -> None:
+        self.timestamp = time.monotonic()
+
+
+@dataclass
+class SuiteFinished(StatefulEvent):
+    """After executing a set of scenarios."""
+
+    status: SuiteStatus
+    failures: list[Check]
+
+    __slots__ = ("timestamp", "status", "failures")
+
+    def __init__(self, *, status: SuiteStatus, failures: list[Check]) -> None:
+        self.status = status
+        self.failures = failures
+        self.timestamp = time.monotonic()
+
+
+class ScenarioStatus(str, Enum):
+    """Status of a single scenario execution."""
+
+    SUCCESS = "success"
+    FAILURE = "failure"
+    # TODO: Count for Hypothesis' rejected?
+    ERROR = "error"
+
+
+@dataclass
+class ScenarioStarted(StatefulEvent):
+    """Before a single state machine execution."""
+
+    # Whether this is a scenario that tries to reproduce a failure
+    is_final: bool
+
+    __slots__ = ("timestamp", "is_final")
+
+    def __init__(self, *, is_final: bool) -> None:
+        self.is_final = is_final
+        self.timestamp = time.monotonic()
+
+
+@dataclass
+class ScenarioFinished(StatefulEvent):
+    """After a single state machine execution."""
+
+    status: ScenarioStatus
+    # Whether this is a scenario that tries to reproduce a failure
+    is_final: bool
+
+    __slots__ = ("timestamp", "status", "is_final")
+
+    def __init__(self, *, status: ScenarioStatus, is_final: bool) -> None:
+        self.status = status
+        self.is_final = is_final
+        self.timestamp = time.monotonic()
+
+
+class StepStatus(str, Enum):
+    """Status of a single state machine step."""
+
+    SUCCESS = "success"
+    FAILURE = "failure"
+    ERROR = "error"
+
+
+@dataclass
+class StepStarted(StatefulEvent):
+    """Before a single state machine step."""
+
+    __slots__ = ("timestamp",)
+
+    def __init__(self) -> None:
+        self.timestamp = time.monotonic()
+
+
+@dataclass
+class TransitionId:
+    """Id of the the that was hit."""
+
+    name: str
+    # Status code as defined in the transition, i.e. may be `default`
+    status_code: str
+    source: str
+
+    __slots__ = ("name", "status_code", "source")
+
+
+@dataclass
+class ResponseData:
+    """Common data for responses."""
+
+    status_code: int
+    elapsed: float
+    __slots__ = ("status_code", "elapsed")
+
+
+@dataclass
+class StepFinished(StatefulEvent):
+    """After a single state machine step."""
+
+    status: StepStatus
+    transition_id: TransitionId | None
+    target: str
+    response: ResponseData | None
+
+    __slots__ = ("timestamp", "status", "transition_id", "target", "response")
+
+    def __init__(
+        self, *, status: StepStatus, transition_id: TransitionId | None, target: str, response: ResponseData | None
+    ) -> None:
+        self.status = status
+        self.transition_id = transition_id
+        self.target = target
+        self.response = response
+        self.timestamp = time.monotonic()
+
+
+@dataclass
+class Interrupted(StatefulEvent):
+    """The state machine execution was interrupted."""
+
+    __slots__ = ("timestamp",)
+
+    def __init__(self) -> None:
+        self.timestamp = time.monotonic()
+
+
+@dataclass
+class Errored(StatefulEvent):
+    """An error occurred during the state machine execution."""
+
+    exception: Exception
+
+    __slots__ = ("timestamp", "exception")
+
+    def __init__(self, *, exception: Exception) -> None:
+        self.exception = exception
+        self.timestamp = time.monotonic()