schemathesis 4.0.0a1__py3-none-any.whl → 4.0.0a3__py3-none-any.whl

schemathesis/specs/openapi/stateful/__init__.py

@@ -1,131 +1,159 @@
 from __future__ import annotations
 
-from collections import defaultdict
+from dataclasses import dataclass
 from functools import lru_cache
 from typing import TYPE_CHECKING, Any, Callable, Iterator
 
 from hypothesis import strategies as st
 from hypothesis.stateful import Bundle, Rule, precondition, rule
 
-from schemathesis.core import NOT_SET, NotSet
 from schemathesis.core.result import Ok
+from schemathesis.engine.recorder import ScenarioRecorder
+from schemathesis.generation import GenerationMode
 from schemathesis.generation.case import Case
 from schemathesis.generation.hypothesis import strategies
-from schemathesis.generation.stateful.state_machine import APIStateMachine, Direction, StepResult, _normalize_name
-
-from ....generation import GenerationMode
-from .. import expressions
-from ..links import get_all_links
-from ..utils import expand_status_code
+from schemathesis.generation.stateful.state_machine import APIStateMachine, StepInput, StepOutput, _normalize_name
+from schemathesis.schemas import APIOperation
+from schemathesis.specs.openapi.links import OpenApiLink, get_all_links
+from schemathesis.specs.openapi.stateful.control import TransitionController
+from schemathesis.specs.openapi.utils import expand_status_code
 
 if TYPE_CHECKING:
-    from schemathesis.generation.stateful.state_machine import StepResult
-
-    from ..schemas import BaseOpenAPISchema
+    from schemathesis.generation.stateful.state_machine import StepOutput
+    from schemathesis.specs.openapi.schemas import BaseOpenAPISchema
 
-FilterFunction = Callable[["StepResult"], bool]
+FilterFunction = Callable[["StepOutput"], bool]
 
 
 class OpenAPIStateMachine(APIStateMachine):
-    _response_matchers: dict[str, Callable[[StepResult], str | None]]
+    _response_matchers: dict[str, Callable[[StepOutput], str | None]]
+    _transitions: ApiTransitions
+
+    def __init__(self) -> None:
+        self.recorder = ScenarioRecorder(label="Stateful tests")
+        self.control = TransitionController(self._transitions)
+        super().__init__()
 
-    def _get_target_for_result(self, result: StepResult) -> str | None:
+    def _get_target_for_result(self, result: StepOutput) -> str | None:
         matcher = self._response_matchers.get(result.case.operation.label)
         if matcher is None:
             return None
         return matcher(result)
 
-    def transform(self, result: StepResult, direction: Direction, case: Case) -> Case:
-        context = expressions.ExpressionContext(case=result.case, response=result.response)
-        direction.set_data(case, context=context)
-        return case
-
 
 # The proportion of negative tests generated for "root" transitions
-NEGATIVE_TEST_CASES_THRESHOLD = 20
+NEGATIVE_TEST_CASES_THRESHOLD = 10
 
 
-def create_state_machine(schema: BaseOpenAPISchema) -> type[APIStateMachine]:
-    """Create a state machine class.
+@dataclass
+class OperationTransitions:
+    """Transitions for a single operation."""
 
-    It aims to avoid making calls that are not likely to lead to a stateful call later. For example:
-      1. POST /users/
-      2. GET /users/{id}/
+    __slots__ = ("incoming", "outgoing")
 
-    This state machine won't make calls to (2) without having a proper response from (1) first.
-    """
+    def __init__(self) -> None:
+        self.incoming: list[OpenApiLink] = []
+        self.outgoing: list[OpenApiLink] = []
+
+
+@dataclass
+class ApiTransitions:
+    """Stores all transitions grouped by operation."""
+
+    __slots__ = ("operations",)
+
+    def __init__(self) -> None:
+        # operation label -> its transitions
+        self.operations: dict[str, OperationTransitions] = {}
+
+    def add_outgoing(self, source: str, link: OpenApiLink) -> None:
+        """Record an outgoing transition from source operation."""
+        self.operations.setdefault(source, OperationTransitions()).outgoing.append(link)
+        self.operations.setdefault(link.target.label, OperationTransitions()).incoming.append(link)
+
+
+def collect_transitions(operations: list[APIOperation]) -> ApiTransitions:
+    """Collect all transitions between operations."""
+    transitions = ApiTransitions()
+
+    selected_labels = {operation.label for operation in operations}
+    for operation in operations:
+        for _, link in get_all_links(operation):
+            if link.target.label in selected_labels:
+                transitions.add_outgoing(operation.label, link)
+
+    return transitions
+
+
+def create_state_machine(schema: BaseOpenAPISchema) -> type[APIStateMachine]:
     operations = [result.ok() for result in schema.get_all_operations() if isinstance(result, Ok)]
     bundles = {}
-    incoming_transitions = defaultdict(list)
-    _response_matchers: dict[str, Callable[[StepResult], str | None]] = {}
-    # Statistic structure follows the links and count for each response status code
+    transitions = collect_transitions(operations)
+    _response_matchers: dict[str, Callable[[StepOutput], str | None]] = {}
+
+    # Create bundles and matchers
     for operation in operations:
         all_status_codes = tuple(operation.definition.raw["responses"])
         bundle_matchers = []
-        for _, link in get_all_links(operation):
-            bundle_name = f"{operation.label} -> {link.status_code}"
-            bundles[bundle_name] = Bundle(bundle_name)
-            target_operation = link.get_target_operation()
-            incoming_transitions[target_operation.label].append(link)
-            bundle_matchers.append((bundle_name, make_response_filter(link.status_code, all_status_codes)))
+
+        if operation.label in transitions.operations:
+            # Use outgoing transitions
+            for link in transitions.operations[operation.label].outgoing:
+                bundle_name = f"{operation.label} -> {link.status_code}"
+                bundles[bundle_name] = Bundle(bundle_name)
+                bundle_matchers.append((bundle_name, make_response_filter(link.status_code, all_status_codes)))
+
         if bundle_matchers:
             _response_matchers[operation.label] = make_response_matcher(bundle_matchers)
+
     rules = {}
     catch_all = Bundle("catch_all")
 
     for target in operations:
-        incoming = incoming_transitions.get(target.label)
-        if incoming is not None:
-            for link in incoming:
-                source = link.operation
-                bundle_name = f"{source.label} -> {link.status_code}"
-                name = _normalize_name(f"{target.label} -> {link.status_code}")
-                case_strategy = strategies.combine(
-                    [target.as_strategy(generation_mode=mode) for mode in schema.generation_config.modes]
-                )
-                bundle = bundles[bundle_name]
-                rules[name] = transition(
-                    name=name,
-                    target=catch_all,
-                    previous=bundle,
-                    case=case_strategy,
-                    link=st.just(link),
+        if target.label in transitions.operations:
+            incoming = transitions.operations[target.label].incoming
+            if incoming:
+                for link in incoming:
+                    bundle_name = f"{link.source.label} -> {link.status_code}"
+                    name = _normalize_name(f"{link.status_code} -> {target.label}")
+                    name = _normalize_name(f"{link.source.label} -> {link.status_code} -> {target.label}")
+                    assert name not in rules
+                    rules[name] = precondition(is_transition_allowed(bundle_name, link.source.label, target.label))(
+                        transition(
+                            name=name,
+                            target=catch_all,
+                            input=bundles[bundle_name].flatmap(
+                                into_step_input(target=target, link=link, modes=schema.generation_config.modes)
+                            ),
+                        )
+                    )
+            if transitions.operations[target.label].outgoing and target.method == "post":
+                # Allow POST methods for operations with outgoing transitions.
+                # This approach also includes cases when there is an incoming transition back to POST
+                # For example, POST /users/ -> GET /users/{id}/
+                # The source operation has no prerequisite, but we need to allow this rule to be executed
+                # in order to reach other transitions
+                name = _normalize_name(f"{target.label} -> X")
+                if len(schema.generation_config.modes) == 1:
+                    case_strategy = target.as_strategy(generation_mode=schema.generation_config.modes[0])
+                else:
+                    _strategies = {
+                        method: target.as_strategy(generation_mode=method) for method in schema.generation_config.modes
+                    }
+
+                    @st.composite  # type: ignore[misc]
+                    def case_strategy_factory(
+                        draw: st.DrawFn, strategies: dict[GenerationMode, st.SearchStrategy] = _strategies
+                    ) -> Case:
+                        if draw(st.integers(min_value=0, max_value=99)) < NEGATIVE_TEST_CASES_THRESHOLD:
+                            return draw(strategies[GenerationMode.NEGATIVE])
+                        return draw(strategies[GenerationMode.POSITIVE])
+
+                    case_strategy = case_strategy_factory()
+
+                rules[name] = precondition(is_root_allowed(target.label))(
+                    transition(name=name, target=catch_all, input=case_strategy.map(StepInput.initial))
                 )
-        elif any(
-            incoming.operation.label == target.label
-            for transitions in incoming_transitions.values()
-            for incoming in transitions
-        ):
-            # No incoming transitions, but has at least one outgoing transition
-            # For example, POST /users/ -> GET /users/{id}/
-            # The source operation has no prerequisite, but we need to allow this rule to be executed
-            # in order to reach other transitions
-            name = _normalize_name(f"{target.label} -> X")
-            if len(schema.generation_config.modes) == 1:
-                case_strategy = target.as_strategy(generation_mode=schema.generation_config.modes[0])
-            else:
-                _strategies = {
-                    method: target.as_strategy(generation_mode=method) for method in schema.generation_config.modes
-                }
-
-                @st.composite  # type: ignore[misc]
-                def case_strategy_factory(
-                    draw: st.DrawFn, strategies: dict[GenerationMode, st.SearchStrategy] = _strategies
-                ) -> Case:
-                    if draw(st.integers(min_value=0, max_value=99)) < NEGATIVE_TEST_CASES_THRESHOLD:
-                        return draw(strategies[GenerationMode.NEGATIVE])
-                    return draw(strategies[GenerationMode.POSITIVE])
-
-                case_strategy = case_strategy_factory()
-
-            rules[name] = precondition(ensure_links_followed)(
-                transition(
-                    name=name,
-                    target=catch_all,
-                    previous=st.none(),
-                    case=case_strategy,
-                )
-            )
 
     return type(
         "APIWorkflow",
@@ -134,41 +162,85 @@ def create_state_machine(schema: BaseOpenAPISchema) -> type[APIStateMachine]:
             "schema": schema,
             "bundles": bundles,
             "_response_matchers": _response_matchers,
+            "_transitions": transitions,
             **rules,
         },
     )
 
 
-def ensure_links_followed(machine: APIStateMachine) -> bool:
-    # If there are responses that have links to follow, reject any rule without incoming transitions
-    for bundle in machine.bundles.values():
-        if bundle:
-            return False
-    return True
+def into_step_input(
+    target: APIOperation, link: OpenApiLink, modes: list[GenerationMode]
+) -> Callable[[StepOutput], st.SearchStrategy[StepInput]]:
+    def builder(_output: StepOutput) -> st.SearchStrategy[StepInput]:
+        @st.composite  # type: ignore[misc]
+        def inner(draw: st.DrawFn, output: StepOutput) -> StepInput:
+            transition_data = link.extract(output)
 
-
-def transition(
-    *,
-    name: str,
-    target: Bundle,
-    previous: Bundle | st.SearchStrategy,
-    case: st.SearchStrategy,
-    link: st.SearchStrategy | NotSet = NOT_SET,
-) -> Callable[[Callable], Rule]:
-    def step_function(*args_: Any, **kwargs_: Any) -> StepResult | None:
-        return APIStateMachine._step(*args_, **kwargs_)
+            kwargs: dict[str, Any] = {
+                container: {
+                    name: extracted.value.ok()
+                    for name, extracted in data.items()
+                    if isinstance(extracted.value, Ok) and extracted.value.ok() is not None
+                }
+                for container, data in transition_data.parameters.items()
+            }
+            if (
+                transition_data.request_body is not None
+                and isinstance(transition_data.request_body.value, Ok)
+                and not link.merge_body
+            ):
+                kwargs["body"] = transition_data.request_body.value.ok()
+            cases = strategies.combine([target.as_strategy(generation_mode=mode, **kwargs) for mode in modes])
+            case = draw(cases)
+            if (
+                transition_data.request_body is not None
+                and isinstance(transition_data.request_body.value, Ok)
+                and link.merge_body
+            ):
+                new = transition_data.request_body.value.ok()
+                if isinstance(case.body, dict) and isinstance(new, dict):
+                    case.body = {**case.body, **new}
+                else:
+                    case.body = new
+            return StepInput(case=case, transition=transition_data)
+
+        return inner(output=_output)
+
+    return builder
+
+
+def is_transition_allowed(bundle_name: str, source: str, target: str) -> Callable[[OpenAPIStateMachine], bool]:
+    def inner(machine: OpenAPIStateMachine) -> bool:
+        return bool(machine.bundles.get(bundle_name)) and machine.control.allow_transition(source, target)
+
+    return inner
+
+
+def is_root_allowed(label: str) -> Callable[[OpenAPIStateMachine], bool]:
+    def inner(machine: OpenAPIStateMachine) -> bool:
+        return machine.control.allow_root_transition(label, machine.bundles)
+
+    return inner
+
+
+def transition(*, name: str, target: Bundle, input: st.SearchStrategy[StepInput]) -> Callable[[Callable], Rule]:
+    def step_function(self: OpenAPIStateMachine, input: StepInput) -> StepOutput | None:
+        if input.transition is not None:
+            self.recorder.record_case(
+                parent_id=input.transition.parent_id, transition=input.transition, case=input.case
+            )
+        else:
+            self.recorder.record_case(parent_id=None, transition=None, case=input.case)
+        self.control.record_step(input, self.recorder)
+        return APIStateMachine._step(self, input=input)
 
     step_function.__name__ = name
 
-    kwargs = {"target": target, "previous": previous, "case": case}
-    if not isinstance(link, NotSet):
-        kwargs["link"] = link
-
-    return rule(**kwargs)(step_function)
+    return rule(target=target, input=input)(step_function)
 
 
-def make_response_matcher(matchers: list[tuple[str, FilterFunction]]) -> Callable[[StepResult], str | None]:
-    def compare(result: StepResult) -> str | None:
+def make_response_matcher(matchers: list[tuple[str, FilterFunction]]) -> Callable[[StepOutput], str | None]:
+    def compare(result: StepOutput) -> str | None:
         for bundle_name, response_filter in matchers:
             if response_filter(result):
                 return bundle_name
@@ -196,7 +268,7 @@ def match_status_code(status_code: str) -> FilterFunction:
     """
     status_codes = set(expand_status_code(status_code))
 
-    def compare(result: StepResult) -> bool:
+    def compare(result: StepOutput) -> bool:
         return result.response.status_code in status_codes
 
     compare.__name__ = f"match_{status_code}_response"
@@ -214,7 +286,7 @@ def default_status_code(status_codes: Iterator[str]) -> FilterFunction:
         status_code for value in status_codes if value != "default" for status_code in expand_status_code(value)
     }
 
-    def match_default_response(result: StepResult) -> bool:
+    def match_default_response(result: StepOutput) -> bool:
         return result.response.status_code not in expanded_status_codes
 
     return match_default_response

schemathesis/specs/openapi/stateful/control.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+from collections import Counter
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from schemathesis.engine.recorder import ScenarioRecorder
+from schemathesis.generation.stateful.state_machine import DEFAULT_STATEFUL_STEP_COUNT
+
+if TYPE_CHECKING:
+    from requests.structures import CaseInsensitiveDict
+
+    from schemathesis.generation.stateful.state_machine import StepInput
+    from schemathesis.specs.openapi.stateful import ApiTransitions
+
+
+# It is enough to be able to catch double-click type of issues
+MAX_OPERATIONS_PER_SOURCE_CAP = 2
+# Maximum number of concurrent root sources (e.g., active users in the system)
+MAX_ROOT_SOURCES = 2
+
+
+def _get_max_operations_per_source(transitions: ApiTransitions) -> int:
+    """Calculate global limit based on number of sources to maximize diversity of used API calls."""
+    sources = len(transitions.operations)
+
+    if sources == 0:
+        return MAX_OPERATIONS_PER_SOURCE_CAP
+
+    # Total steps divided by number of sources, but never below the cap
+    return max(MAX_OPERATIONS_PER_SOURCE_CAP, DEFAULT_STATEFUL_STEP_COUNT // sources)
+
+
+@dataclass
+class TransitionController:
+    """Controls which transitions can be executed in a state machine."""
+
+    __slots__ = ("transitions", "max_operations_per_source", "statistic")
+
+    def __init__(self, transitions: ApiTransitions) -> None:
+        # Incoming & outgoing transitions available in the state machine
+        self.transitions = transitions
+        self.max_operations_per_source = _get_max_operations_per_source(transitions)
+        # source -> derived API calls
+        self.statistic: dict[str, dict[str, Counter[str]]] = {}
+
+    def record_step(self, input: StepInput, recorder: ScenarioRecorder) -> None:
+        """Record API call input."""
+        case = input.case
+
+        if (
+            case.operation.label in self.transitions.operations
+            and self.transitions.operations[case.operation.label].outgoing
+        ):
+            # This API operation has outgoing transitions, hence record it as a source
+            entry = self.statistic.setdefault(input.case.operation.label, {})
+            entry[input.case.id] = Counter()
+
+        if input.transition is not None:
+            # Find immediate parent and record as derived operation
+            parent = recorder.cases[input.transition.parent_id]
+            source = parent.value.operation.label
+            case_id = parent.value.id
+
+            if source in self.statistic and case_id in self.statistic[source]:
+                self.statistic[source][case_id][case.operation.label] += 1
+
+    def allow_root_transition(self, source: str, bundles: dict[str, CaseInsensitiveDict]) -> bool:
+        """Decide if this root transition should be allowed now."""
+        if len(self.statistic.get(source, {})) < MAX_ROOT_SOURCES:
+            return True
+
+        # If all non-root operations are blocked, then allow root ones to make progress
+        history = {name.split("->")[0].strip() for name, values in bundles.items() if values}
+        return all(
+            incoming.source.label not in history
+            or not self.allow_transition(incoming.source.label, incoming.target.label)
+            for transitions in self.transitions.operations.values()
+            for incoming in transitions.incoming
+            if transitions.incoming
+        )
+
+    def allow_transition(self, source: str, target: str) -> bool:
+        """Decide if this transition should be allowed now."""
+        existing = self.statistic.get(source, {})
+        total = sum(metric.get(target, 0) for metric in existing.values())
+        return total < self.max_operations_per_source

{schemathesis-4.0.0a1.dist-info → schemathesis-4.0.0a3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: schemathesis
-Version: 4.0.0a1
+Version: 4.0.0a3
 Summary: Property-based testing framework for Open API and GraphQL based apps
 Project-URL: Documentation, https://schemathesis.readthedocs.io/en/stable/
 Project-URL: Changelog, https://schemathesis.readthedocs.io/en/stable/changelog.html