schemathesis 3.39.7__py3-none-any.whl → 4.0.0a2__py3-none-any.whl

schemathesis/generation/hypothesis/strategies.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from functools import reduce
+from operator import or_
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from hypothesis import strategies as st
+
+
+def combine(strategies: list[st.SearchStrategy] | tuple[st.SearchStrategy]) -> st.SearchStrategy:
+    """Combine a list of strategies into a single one.
+
+    If the input is `[a, b, c]`, then the result is equivalent to `a | b | c`.
+    """
+    return reduce(or_, strategies[1:], strategies[0])

schemathesis/generation/meta.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+from schemathesis.generation import GenerationMode
+
+
+class TestPhase(str, Enum):
+    __test__ = False
+
+    EXPLICIT = "explicit"
+    COVERAGE = "coverage"
+    GENERATE = "generate"
+
+
+class ComponentKind(str, Enum):
+    """Components that can be generated."""
+
+    QUERY = "query"
+    PATH_PARAMETERS = "path_parameters"
+    HEADERS = "headers"
+    COOKIES = "cookies"
+    BODY = "body"
+
+
+@dataclass
+class ComponentInfo:
+    """Information about how a specific component was generated."""
+
+    mode: GenerationMode
+
+    __slots__ = ("mode",)
+
+
+@dataclass
+class GeneratePhaseData:
+    """Metadata specific to generate phase."""
+
+
+@dataclass
+class ExplicitPhaseData:
+    """Metadata specific to explicit phase."""
+
+
+@dataclass
+class CoveragePhaseData:
+    """Metadata specific to coverage phase."""
+
+    description: str
+    location: str | None
+    parameter: str | None
+    parameter_location: str | None
+
+    __slots__ = ("description", "location", "parameter", "parameter_location")
+
+
+@dataclass
+class PhaseInfo:
+    """Phase-specific information."""
+
+    name: TestPhase
+    data: CoveragePhaseData | ExplicitPhaseData | GeneratePhaseData
+
+    __slots__ = ("name", "data")
+
+    @classmethod
+    def coverage(
+        cls,
+        description: str,
+        location: str | None = None,
+        parameter: str | None = None,
+        parameter_location: str | None = None,
+    ) -> PhaseInfo:
+        return cls(
+            name=TestPhase.COVERAGE,
+            data=CoveragePhaseData(
+                description=description, location=location, parameter=parameter, parameter_location=parameter_location
+            ),
+        )
+
+    @classmethod
+    def generate(cls) -> PhaseInfo:
+        return cls(name=TestPhase.GENERATE, data=GeneratePhaseData())
+
+
+@dataclass
+class GenerationInfo:
+    """Information about test case generation."""
+
+    time: float
+    mode: GenerationMode
+
+    __slots__ = ("time", "mode")
+
+
+@dataclass
+class CaseMetadata:
+    """Complete metadata for generated cases."""
+
+    generation: GenerationInfo
+    components: dict[ComponentKind, ComponentInfo]
+    phase: PhaseInfo
+
+    __slots__ = ("generation", "components", "phase")
+
+    def __init__(
+        self,
+        generation: GenerationInfo,
+        components: dict[ComponentKind, ComponentInfo],
+        phase: PhaseInfo,
+    ) -> None:
+        self.generation = generation
+        self.components = components
+        self.phase = phase

schemathesis/generation/modes.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from enum import Enum
+
+
+class GenerationMode(str, Enum):
+    """Defines what data Schemathesis generates for tests."""
+
+    # Generate data, that fits the API schema
+    POSITIVE = "positive"
+    # Doesn't fit the API schema
+    NEGATIVE = "negative"
+
+    @classmethod
+    def default(cls) -> GenerationMode:
+        return cls.POSITIVE
+
+    @classmethod
+    def all(cls) -> list[GenerationMode]:
+        return list(GenerationMode)
+
+    @property
+    def is_positive(self) -> bool:
+        return self == GenerationMode.POSITIVE
+
+    @property
+    def is_negative(self) -> bool:
+        return self == GenerationMode.NEGATIVE

schemathesis/generation/overrides.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Callable
+
+from schemathesis.core.errors import IncorrectUsage
+from schemathesis.core.marks import Mark
+from schemathesis.core.transforms import diff
+from schemathesis.generation.meta import ComponentKind
+
+if TYPE_CHECKING:
+    from schemathesis.generation.case import Case
+    from schemathesis.schemas import APIOperation, ParameterSet
+
+
+@dataclass
+class Override:
+    """Overrides for various parts of a test case."""
+
+    query: dict[str, str]
+    headers: dict[str, str]
+    cookies: dict[str, str]
+    path_parameters: dict[str, str]
+
+    def for_operation(self, operation: APIOperation) -> dict[str, dict[str, str]]:
+        return {
+            "query": (_for_parameters(self.query, operation.query)),
+            "headers": (_for_parameters(self.headers, operation.headers)),
+            "cookies": (_for_parameters(self.cookies, operation.cookies)),
+            "path_parameters": (_for_parameters(self.path_parameters, operation.path_parameters)),
+        }
+
+    @classmethod
+    def from_components(cls, components: dict[ComponentKind, StoredValue], case: Case) -> Override:
+        return Override(
+            **{
+                kind.value: get_component_diff(stored=stored, current=getattr(case, kind.value))
+                for kind, stored in components.items()
+            }
+        )
+
+
+def _for_parameters(overridden: dict[str, str], defined: ParameterSet) -> dict[str, str]:
+    output = {}
+    for param in defined:
+        if param.name in overridden:
+            output[param.name] = overridden[param.name]
+    return output
+
+
+@dataclass
+class StoredValue:
+    value: dict[str, Any] | None
+    is_generated: bool
+
+    __slots__ = ("value", "is_generated")
+
+
+def store_original_state(value: dict[str, Any] | None) -> dict[str, Any] | None:
+    if isinstance(value, Mapping):
+        return value.copy()
+    return value
+
+
+def get_component_diff(stored: StoredValue, current: dict[str, Any] | None) -> dict[str, Any]:
+    """Calculate difference between stored and current components."""
+    if not (current and stored.value):
+        return {}
+    if stored.is_generated:
+        return diff(stored.value, current)
+    return current
+
+
+def store_components(case: Case) -> dict[ComponentKind, StoredValue]:
+    """Store original component states for a test case."""
+    return {
+        kind: StoredValue(
+            value=store_original_state(getattr(case, kind.value)),
+            is_generated=bool(case.meta and kind in case.meta.components),
+        )
+        for kind in [
+            ComponentKind.QUERY,
+            ComponentKind.HEADERS,
+            ComponentKind.COOKIES,
+            ComponentKind.PATH_PARAMETERS,
+        ]
+    }
+
+
+OverrideMark = Mark[Override](attr_name="override")
+
+
+def check_no_override_mark(test: Callable) -> None:
+    if OverrideMark.is_set(test):
+        raise IncorrectUsage(f"`{test.__name__}` has already been decorated with `override`.")

schemathesis/generation/stateful/__init__.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import hypothesis
+
+    from schemathesis.generation.stateful.state_machine import APIStateMachine
+
+
+def run_state_machine_as_test(
+    state_machine_factory: type[APIStateMachine], *, settings: hypothesis.settings | None = None
+) -> None:
+    """Run a state machine as a test.
+
+    It automatically adds the `_min_steps` argument if ``Hypothesis`` is recent enough.
+    """
+    from hypothesis.stateful import run_state_machine_as_test as _run_state_machine_as_test
+
+    return _run_state_machine_as_test(state_machine_factory, settings=settings, _min_steps=2)

schemathesis/{stateful → generation/stateful}/state_machine.py

@@ -1,39 +1,86 @@
 from __future__ import annotations
 
 import re
-import time
 from dataclasses import dataclass
 from functools import lru_cache
 from typing import TYPE_CHECKING, Any, ClassVar
 
+import hypothesis
 from hypothesis.errors import InvalidDefinition
 from hypothesis.stateful import RuleBasedStateMachine
 
-from .._dependency_versions import HYPOTHESIS_HAS_STATEFUL_NAMING_IMPROVEMENTS
-from ..constants import NO_LINKS_ERROR_MESSAGE, NOT_SET
-from ..exceptions import UsageError
-from ..internal.checks import CheckFunction
-from ..models import APIOperation, Case
-from .config import _default_hypothesis_settings_factory
-from .runner import StatefulTestRunner, StatefulTestRunnerConfig
-from .sink import StateMachineSink
+from schemathesis.checks import CheckFunction
+from schemathesis.core.errors import IncorrectUsage
+from schemathesis.core.result import Result
+from schemathesis.core.transport import Response
+from schemathesis.generation.case import Case
 
 if TYPE_CHECKING:
     import hypothesis
     from requests.structures import CaseInsensitiveDict
 
-    from ..schemas import BaseSchema
-    from ..transports.responses import GenericResponse
-    from .statistic import TransitionStats
+    from schemathesis.schemas import BaseSchema
+
+
+NO_LINKS_ERROR_MESSAGE = (
+    "Stateful testing requires at least one OpenAPI link in the schema, but no links detected. "
+    "Please add OpenAPI links to enable stateful testing or use stateless tests instead. \n"
+    "See https://schemathesis.readthedocs.io/en/stable/stateful.html#how-to-specify-connections for more information."
+)
+
+DEFAULT_STATE_MACHINE_SETTINGS = hypothesis.settings(
+    phases=[hypothesis.Phase.generate],
+    deadline=None,
+    stateful_step_count=6,
+    suppress_health_check=list(hypothesis.HealthCheck),
+)
+
+
+@dataclass
+class StepInput:
+    """Input for a single state machine step."""
+
+    case: Case
+    transition: Transition | None  # None for initial steps
+
+    __slots__ = ("case", "transition")
+
+    @classmethod
+    def initial(cls, case: Case) -> StepInput:
+        return cls(case=case, transition=None)
+
+
+@dataclass
+class Transition:
+    """Data about transition execution."""
+
+    # ID of the transition (e.g. link name)
+    id: str
+    parent_id: str
+    parameters: dict[str, dict[str, ExtractedParam]]
+    request_body: ExtractedParam | None
+
+    __slots__ = ("id", "parent_id", "parameters", "request_body")
+
+
+@dataclass
+class ExtractedParam:
+    """Result of parameter extraction."""
+
+    definition: Any
+    value: Result[Any, Exception]
+
+    __slots__ = ("definition", "value")
 
 
 @dataclass
-class StepResult:
+class StepOutput:
     """Output from a single transition of a state machine."""
 
-    response: GenericResponse
+    response: Response
     case: Case
-    elapsed: float
+
+    __slots__ = ("response", "case")
 
 
 def _normalize_name(name: str) -> str:
@@ -51,25 +98,23 @@ class APIStateMachine(RuleBasedStateMachine):
     # attribute will be renamed in the future
     bundles: ClassVar[dict[str, CaseInsensitiveDict]]  # type: ignore
     schema: BaseSchema
-    # A template for transition statistics that can be filled with data from the state machine during its execution
-    _transition_stats_template: ClassVar[TransitionStats]
 
     def __init__(self) -> None:
         try:
             super().__init__()  # type: ignore
         except InvalidDefinition as exc:
             if "defines no rules" in str(exc):
-                raise UsageError(NO_LINKS_ERROR_MESSAGE) from None
+                raise IncorrectUsage(NO_LINKS_ERROR_MESSAGE) from None
             raise
         self.setup()
 
     @classmethod
     @lru_cache
     def _to_test_case(cls) -> type:
-        from . import run_state_machine_as_test
+        from schemathesis.generation.stateful import run_state_machine_as_test
 
         class StateMachineTestCase(RuleBasedStateMachine.TestCase):
-            settings = _default_hypothesis_settings_factory()
+            settings = DEFAULT_STATE_MACHINE_SETTINGS
 
             def runTest(self) -> None:
                 run_state_machine_as_test(cls, settings=self.settings)
@@ -80,33 +125,20 @@ class APIStateMachine(RuleBasedStateMachine):
         StateMachineTestCase.__qualname__ = cls.__qualname__ + ".TestCase"
         return StateMachineTestCase
 
-    def _pretty_print(self, value: Any) -> str:
-        if isinstance(value, Case):
-            # State machines suppose to be reproducible, hence it is OK to get kwargs here
-            kwargs = self.get_call_kwargs(value)
-            return _print_case(value, kwargs)
-        return super()._pretty_print(value)  # type: ignore
-
-    if HYPOTHESIS_HAS_STATEFUL_NAMING_IMPROVEMENTS:
+    def _new_name(self, target: str) -> str:
+        target = _normalize_name(target)
+        return super()._new_name(target)  # type: ignore
 
-        def _new_name(self, target: str) -> str:
-            target = _normalize_name(target)
-            return super()._new_name(target)  # type: ignore
-
-    def _get_target_for_result(self, result: StepResult) -> str | None:
+    def _get_target_for_result(self, result: StepOutput) -> str | None:
         raise NotImplementedError
 
-    def _add_result_to_targets(self, targets: tuple[str, ...], result: StepResult | None) -> None:
+    def _add_result_to_targets(self, targets: tuple[str, ...], result: StepOutput | None) -> None:
         if result is None:
             return
         target = self._get_target_for_result(result)
         if target is not None:
             super()._add_result_to_targets((target,), result)
 
-    @classmethod
-    def format_rules(cls) -> str:
-        raise NotImplementedError
-
     @classmethod
     def run(cls, *, settings: hypothesis.settings | None = None) -> None:
         """Run state machine as a test."""
@@ -114,23 +146,8 @@ class APIStateMachine(RuleBasedStateMachine):
 
         return run_state_machine_as_test(cls, settings=settings)
 
-    @classmethod
-    def runner(cls, *, config: StatefulTestRunnerConfig | None = None) -> StatefulTestRunner:
-        """Create a runner for this state machine."""
-        from .runner import StatefulTestRunnerConfig
-
-        return StatefulTestRunner(cls, config=config or StatefulTestRunnerConfig())
-
-    @classmethod
-    def sink(cls) -> StateMachineSink:
-        """Create a sink to collect events into."""
-        return StateMachineSink(transitions=cls._transition_stats_template.copy())
-
     def setup(self) -> None:
-        """Hook method that runs unconditionally in the beginning of each test scenario.
-
-        Does nothing by default.
-        """
+        """Hook method that runs unconditionally in the beginning of each test scenario."""
 
     def teardown(self) -> None:
         pass
@@ -138,19 +155,11 @@ class APIStateMachine(RuleBasedStateMachine):
     # To provide the return type in the rendered documentation
     teardown.__doc__ = RuleBasedStateMachine.teardown.__doc__
 
-    def transform(self, result: StepResult, direction: Direction, case: Case) -> Case:
-        raise NotImplementedError
-
-    def _step(self, case: Case, previous: StepResult | None = None, link: Direction | None = None) -> StepResult | None:
-        # This method is a proxy that is used under the hood during the state machine initialization.
-        # The whole point of having it is to make it possible to override `step`; otherwise, custom "step" is ignored.
-        # It happens because, at the point of initialization, the final class is not yet created.
+    def _step(self, input: StepInput) -> StepOutput | None:
         __tracebackhide__ = True
-        if previous is not None and link is not None:
-            return self.step(case, (previous, link))
-        return self.step(case, None)
+        return self.step(input)
 
-    def step(self, case: Case, previous: tuple[StepResult, Direction] | None = None) -> StepResult | None:
+    def step(self, input: StepInput) -> StepOutput:
         """A single state machine step.
 
         :param Case case: Generated test case data that should be sent in an API call to the tested API operation.
@@ -159,20 +168,13 @@ class APIStateMachine(RuleBasedStateMachine):
         Schemathesis prepares data, makes a call and validates the received response.
         It is the most high-level point to extend the testing process. You probably don't need it in most cases.
         """
-        from ..specs.openapi.checks import use_after_free
-
         __tracebackhide__ = True
-        if previous is not None:
-            result, direction = previous
-            case = self.transform(result, direction, case)
-        self.before_call(case)
-        kwargs = self.get_call_kwargs(case)
-        start = time.monotonic()
-        response = self.call(case, **kwargs)
-        elapsed = time.monotonic() - start
-        self.after_call(response, case)
-        self.validate_response(response, case, additional_checks=(use_after_free,))
-        return self.store_result(response, case, elapsed)
+        self.before_call(input.case)
+        kwargs = self.get_call_kwargs(input.case)
+        response = self.call(input.case, **kwargs)
+        self.after_call(response, input.case)
+        self.validate_response(response, input.case)
+        return StepOutput(response, input.case)
 
     def before_call(self, case: Case) -> None:
         """Hook method for modifying the case data before making a request.
@@ -199,7 +201,7 @@ class APIStateMachine(RuleBasedStateMachine):
                         case.body["is_fake"] = True
         """
 
-    def after_call(self, response: GenericResponse, case: Case) -> None:
+    def after_call(self, response: Response, case: Case) -> None:
         """Hook method for additional actions with case or response instances.
 
         :param response: Response from the application under test.
@@ -232,7 +234,7 @@ class APIStateMachine(RuleBasedStateMachine):
             # PATCH /users/{user_id} -> 500
         """
 
-    def call(self, case: Case, **kwargs: Any) -> GenericResponse:
+    def call(self, case: Case, **kwargs: Any) -> Response:
         """Make a request to the API.
 
         :param Case case: Generated test case data that should be sent in an API call to the tested API operation.
@@ -265,14 +267,14 @@ class APIStateMachine(RuleBasedStateMachine):
         return {}
 
     def validate_response(
-        self, response: GenericResponse, case: Case, additional_checks: tuple[CheckFunction, ...] = ()
+        self, response: Response, case: Case, additional_checks: list[CheckFunction] | None = None
     ) -> None:
         """Validate an API response.
 
         :param response: Response from the application under test.
         :param Case case: Generated test case data that should be sent in an API call to the tested API operation.
         :param additional_checks: A list of checks that will be run together with the default ones.
-        :raises CheckFailed: If any of the supplied checks failed.
+        :raises FailureGroup: If any of the supplied checks failed.
 
         If you need to change the default checks or provide custom validation rules, you can do it here.
 
@@ -298,30 +300,3 @@ class APIStateMachine(RuleBasedStateMachine):
         """
         __tracebackhide__ = True
         case.validate_response(response, additional_checks=additional_checks)
-
-    def store_result(self, response: GenericResponse, case: Case, elapsed: float) -> StepResult:
-        return StepResult(response, case, elapsed)
-
-
-def _print_case(case: Case, kwargs: dict[str, Any]) -> str:
-    from requests.structures import CaseInsensitiveDict
-
-    operation = f"state.schema['{case.operation.path}']['{case.operation.method.upper()}']"
-    headers = case.headers or CaseInsensitiveDict()
-    headers.update(kwargs.get("headers", {}))
-    case.headers = headers
-    data = [
-        f"{name}={getattr(case, name)!r}"
-        for name in ("path_parameters", "headers", "cookies", "query", "body", "media_type")
-        if getattr(case, name) not in (None, NOT_SET)
-    ]
-    return f"{operation}.make_case({', '.join(data)})"
-
-
-class Direction:
-    name: str
-    status_code: str
-    operation: APIOperation
-
-    def set_data(self, case: Case, elapsed: float, **kwargs: Any) -> None:
-        raise NotImplementedError

schemathesis/generation/targets.py

@@ -0,0 +1,69 @@
+"""Support for Targeted Property-Based Testing."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable, Sequence
+
+from schemathesis.core.registries import Registry
+from schemathesis.core.transport import Response
+from schemathesis.generation.case import Case
+
+
+@dataclass
+class TargetContext:
+    case: Case
+    response: Response
+
+    __slots__ = ("case", "response")
+
+
+TargetFunction = Callable[[TargetContext], float]
+
+TARGETS = Registry[TargetFunction]()
+target = TARGETS.register
+
+
+@target
+def response_time(ctx: TargetContext) -> float:
+    """Response time as a metric to maximize."""
+    return ctx.response.elapsed
+
+
+class TargetMetricCollector:
+    """Collect multiple observations for target metrics."""
+
+    __slots__ = ("targets", "observations")
+
+    def __init__(self, targets: list[TargetFunction] | None = None) -> None:
+        self.targets = targets or []
+        self.observations: dict[str, list[float]] = {target.__name__: [] for target in self.targets}
+
+    def reset(self) -> None:
+        """Reset all collected observations."""
+        for target in self.targets:
+            self.observations[target.__name__].clear()
+
+    def store(self, case: Case, response: Response) -> None:
+        """Calculate target metrics & store them."""
+        context = TargetContext(case=case, response=response)
+        for target in self.targets:
+            self.observations[target.__name__].append(target(context))
+
+    def maximize(self) -> None:
+        """Give feedback to the Hypothesis engine, so it maximizes the aggregated metrics."""
+        import hypothesis
+
+        for target in self.targets:
+            # Currently aggregation is just a sum
+            metric = sum(self.observations[target.__name__])
+            hypothesis.target(metric, label=target.__name__)
+
+
+def run(targets: Sequence[TargetFunction], case: Case, response: Response) -> None:
+    import hypothesis
+
+    context = TargetContext(case=case, response=response)
+    for target in targets:
+        value = target(context)
+        hypothesis.target(value, label=target.__name__)

schemathesis/graphql/__init__.py

@@ -0,0 +1,15 @@
+from schemathesis.graphql.loaders import from_asgi, from_dict, from_file, from_path, from_url, from_wsgi
+
+from ..specs.graphql import nodes
+from ..specs.graphql.scalars import scalar
+
+__all__ = [
+    "from_url",
+    "from_asgi",
+    "from_wsgi",
+    "from_file",
+    "from_path",
+    "from_dict",
+    "nodes",
+    "scalar",
+]