schemathesis 3.13.0__py3-none-any.whl → 4.4.2__py3-none-any.whl

schemathesis/generation/metrics.py

@@ -0,0 +1,93 @@
+"""Support for Targeted Property-Based Testing."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable, Sequence
+
+from schemathesis.core.registries import Registry
+from schemathesis.core.transport import Response
+from schemathesis.generation.case import Case
+
+
+@dataclass
+class MetricContext:
+    """Context for evaluating a metric on a single test execution.
+
+    This object bundles together the test `case` that was sent and
+    the corresponding HTTP `response`. Metric functions receive an
+    instance of `MetricContext` to compute a numeric score.
+    """
+
+    case: Case
+    """Generated test case."""
+    response: Response
+    """The HTTP response returned by the server for this test case."""
+
+    __slots__ = ("case", "response")
+
+
+MetricFunction = Callable[[MetricContext], float]
+
+METRICS = Registry[MetricFunction]()
+
+
+def metric(func: MetricFunction) -> MetricFunction:
+    """Decorator to register a custom metric for targeted property-based testing.
+
+    Example:
+        ```python
+        import schemathesis
+
+        @schemathesis.metric
+        def response_size(ctx: schemathesis.MetricContext) -> float:
+            return float(len(ctx.response.content))
+        ```
+
+    """
+    return METRICS.register(func)
+
+
+@metric
+def response_time(ctx: MetricContext) -> float:
+    """Response time as a metric to maximize."""
+    return ctx.response.elapsed
+
+
+class MetricCollector:
+    """Collect multiple observations for metrics."""
+
+    __slots__ = ("metrics", "observations")
+
+    def __init__(self, metrics: list[MetricFunction] | None = None) -> None:
+        self.metrics = metrics or []
+        self.observations: dict[str, list[float]] = {metric.__name__: [] for metric in self.metrics}
+
+    def reset(self) -> None:
+        """Reset all collected observations."""
+        for metric in self.metrics:
+            self.observations[metric.__name__].clear()
+
+    def store(self, case: Case, response: Response) -> None:
+        """Calculate metrics & store them."""
+        ctx = MetricContext(case=case, response=response)
+        for metric in self.metrics:
+            self.observations[metric.__name__].append(metric(ctx))
+
+    def maximize(self) -> None:
+        """Give feedback to the Hypothesis engine, so it maximizes the aggregated metrics."""
+        import hypothesis
+
+        for metric in self.metrics:
+            # Currently aggregation is just a sum
+            value = sum(self.observations[metric.__name__])
+            hypothesis.target(value, label=metric.__name__)
+
+
+def maximize(metrics: Sequence[MetricFunction], case: Case, response: Response) -> None:
+    import hypothesis
+
+    ctx = MetricContext(case=case, response=response)
+    for metric in metrics:
+        value = metric(ctx)
+        hypothesis.target(value, label=metric.__name__)

schemathesis/generation/modes.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from enum import Enum
+
+
+class GenerationMode(str, Enum):
+    """Defines what data Schemathesis generates for tests."""
+
+    # Generate data, that fits the API schema
+    POSITIVE = "positive"
+    # Doesn't fit the API schema
+    NEGATIVE = "negative"
+
+    @property
+    def is_positive(self) -> bool:
+        return self == GenerationMode.POSITIVE
+
+    @property
+    def is_negative(self) -> bool:
+        return self == GenerationMode.NEGATIVE

schemathesis/generation/overrides.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Iterator
+
+from schemathesis.config import ProjectConfig
+from schemathesis.core.parameters import ParameterLocation
+from schemathesis.core.transforms import diff
+
+if TYPE_CHECKING:
+    from schemathesis.generation.case import Case
+    from schemathesis.schemas import APIOperation, OperationParameter
+
+
+@dataclass
+class Override:
+    """Overrides for various parts of a test case."""
+
+    query: dict[str, str]
+    headers: dict[str, str]
+    cookies: dict[str, str]
+    path_parameters: dict[str, str]
+    body: dict[str, str]
+
+    __slots__ = ("query", "headers", "cookies", "path_parameters", "body")
+
+    def items(self) -> Iterator[tuple[ParameterLocation, dict[str, str]]]:
+        for key, value in (
+            (ParameterLocation.QUERY, self.query),
+            (ParameterLocation.HEADER, self.headers),
+            (ParameterLocation.COOKIE, self.cookies),
+            (ParameterLocation.PATH, self.path_parameters),
+        ):
+            if value:
+                yield key, value
+
+    @classmethod
+    def from_components(cls, components: dict[ParameterLocation, StoredValue], case: Case) -> Override:
+        return Override(
+            **{
+                kind.container_name: get_component_diff(stored=stored, current=getattr(case, kind.container_name))
+                for kind, stored in components.items()
+            }
+        )
+
+
+def for_operation(config: ProjectConfig, *, operation: APIOperation) -> Override:
+    operation_config = config.operations.get_for_operation(operation)
+
+    output = Override(query={}, headers={}, cookies={}, path_parameters={}, body={})
+    groups = [
+        (output.query, operation.query),
+        (output.headers, operation.headers),
+        (output.cookies, operation.cookies),
+        (output.path_parameters, operation.path_parameters),
+    ]
+    for container, params in groups:
+        for param in params:
+            # Attempt to get the override from the operation-specific configuration.
+            value = None
+            if operation_config:
+                value = _get_override_value(param, operation_config.parameters)
+            # Fallback to the global project configuration.
+            if value is None:
+                value = _get_override_value(param, config.parameters)
+            if value is not None:
+                container[param.name] = value
+
+    return output
+
+
+def _get_override_value(param: OperationParameter, parameters: dict[str, Any]) -> Any:
+    key = param.name
+    full_key = f"{param.location.value}.{param.name}"
+    if key in parameters:
+        return parameters[key]
+    elif full_key in parameters:
+        return parameters[full_key]
+    return None
+
+
+@dataclass
+class StoredValue:
+    value: Any
+    is_generated: bool
+
+    __slots__ = ("value", "is_generated")
+
+
+def store_original_state(value: Any) -> Any:
+    if isinstance(value, Mapping):
+        return dict(value)
+    return value
+
+
+def get_component_diff(stored: StoredValue, current: Any) -> dict[str, Any]:
+    """Calculate difference between stored and current components."""
+    if not (current and stored.value):
+        return {}
+    if stored.is_generated:
+        # Only compute diff for mapping types (dicts)
+        # Non-mapping bodies (e.g., GraphQL strings) are not tracked
+        if isinstance(stored.value, Mapping) and isinstance(current, Mapping):
+            return diff(stored.value, current)
+        return {}
+    # For non-generated components, return current if it's a dict, otherwise empty
+    if isinstance(current, Mapping):
+        return dict(current)
+    return {}
+
+
+def store_components(case: Case) -> dict[ParameterLocation, StoredValue]:
+    """Store original component states for a test case."""
+    return {
+        kind: StoredValue(
+            value=store_original_state(getattr(case, kind.container_name)),
+            is_generated=bool(case._meta and kind in case._meta.components),
+        )
+        for kind in [
+            ParameterLocation.QUERY,
+            ParameterLocation.HEADER,
+            ParameterLocation.COOKIE,
+            ParameterLocation.PATH,
+            ParameterLocation.BODY,
+        ]
+    }

schemathesis/generation/stateful/__init__.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import hypothesis
+
+    from schemathesis.generation.stateful.state_machine import APIStateMachine
+
+__all__ = [
+    "APIStateMachine",
+]
+
+
+def __getattr__(name: str) -> type[APIStateMachine]:
+    if name == "APIStateMachine":
+        from schemathesis.generation.stateful.state_machine import APIStateMachine
+
+        return APIStateMachine
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+STATEFUL_TESTS_LABEL = "Stateful tests"
+
+
+def run_state_machine_as_test(
+    state_machine_factory: type[APIStateMachine], *, settings: hypothesis.settings | None = None
+) -> None:
+    """Run a state machine as a test.
+
+    It automatically adds the `_min_steps` argument if ``Hypothesis`` is recent enough.
+    """
+    from hypothesis.stateful import run_state_machine_as_test as _run_state_machine_as_test
+
+    __tracebackhide__ = True
+
+    return _run_state_machine_as_test(state_machine_factory, settings=settings, _min_steps=2)

schemathesis/generation/stateful/state_machine.py

@@ -0,0 +1,294 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import TYPE_CHECKING, Any, ClassVar
+
+import hypothesis
+from hypothesis.errors import InvalidDefinition
+from hypothesis.stateful import RuleBasedStateMachine
+
+from schemathesis.checks import CheckFunction
+from schemathesis.core import DEFAULT_STATEFUL_STEP_COUNT
+from schemathesis.core.errors import STATEFUL_TESTING_GUIDE_URL, NoLinksFound
+from schemathesis.core.result import Result
+from schemathesis.core.transport import Response
+from schemathesis.generation.case import Case
+
+if TYPE_CHECKING:
+    import hypothesis
+    from requests.structures import CaseInsensitiveDict
+
+    from schemathesis.schemas import BaseSchema
+
+
+DEFAULT_STATE_MACHINE_SETTINGS = hypothesis.settings(
+    phases=[hypothesis.Phase.generate],
+    deadline=None,
+    stateful_step_count=DEFAULT_STATEFUL_STEP_COUNT,
+    suppress_health_check=list(hypothesis.HealthCheck),
+)
+
+
+@dataclass
+class StepInput:
+    """Input for a single state machine step."""
+
+    case: Case
+    transition: Transition | None  # None for initial steps
+    # What parameters were actually applied
+    # Data extraction failures can prevent it, as well as transitions can be skipped in some cases
+    # to improve discovery of bugs triggered by non-stateful inputs during stateful testing
+    applied_parameters: list[str]
+
+    __slots__ = ("case", "transition", "applied_parameters")
+
+    @classmethod
+    def initial(cls, case: Case) -> StepInput:
+        return cls(case=case, transition=None, applied_parameters=[])
+
+    @property
+    def is_applied(self) -> bool:
+        # If the transition has no parameters or body, count it as applied
+        if self.transition is not None and not self.transition.parameters and self.transition.request_body is None:
+            return True
+        return bool(self.applied_parameters)
+
+
+@dataclass
+class Transition:
+    """Data about transition execution."""
+
+    # ID of the transition (e.g. link name)
+    id: str
+    parent_id: str
+    is_inferred: bool
+    parameters: dict[str, dict[str, ExtractedParam]]
+    request_body: ExtractedParam | None
+
+    __slots__ = ("id", "parent_id", "is_inferred", "parameters", "request_body")
+
+
+@dataclass
+class ExtractedParam:
+    """Result of parameter extraction."""
+
+    definition: Any
+    value: Result[Any, Exception]
+    is_required: bool
+
+    __slots__ = ("definition", "value", "is_required")
+
+
+@dataclass
+class ExtractionFailure:
+    """Represents a failure to extract data from a transition."""
+
+    # e.g., "GetUser"
+    id: str
+    case_id: str
+    # e.g., "POST /users"
+    source: str
+    # e.g., "GET /users/{userId}"
+    target: str
+    # e.g., "userId"
+    parameter_name: str
+    # e.g., "$response.body#/id"
+    expression: str
+    # Previous test cases in the chain, from newest to oldest
+    # Stored as a case + response pair
+    history: list[tuple[Case, Response]]
+    # The actual response that caused the failure
+    response: Response
+    error: Exception | None
+
+    __slots__ = ("id", "case_id", "source", "target", "parameter_name", "expression", "history", "response", "error")
+
+    def __eq__(self, other: object) -> bool:
+        assert isinstance(other, ExtractionFailure)
+        return (
+            self.source == other.source
+            and self.target == other.target
+            and self.id == other.id
+            and self.parameter_name == other.parameter_name
+            and self.expression == other.expression
+        )
+
+    def __hash__(self) -> int:
+        return hash(
+            (
+                self.source,
+                self.target,
+                self.id,
+                self.parameter_name,
+                self.expression,
+            )
+        )
+
+
+@dataclass
+class StepOutput:
+    """Output from a single transition of a state machine."""
+
+    response: Response
+    case: Case
+
+    __slots__ = ("response", "case")
+
+
+def _normalize_name(name: str) -> str:
+    return re.sub(r"\W|^(?=\d)", "_", name).replace("__", "_")
+
+
+class APIStateMachine(RuleBasedStateMachine):
+    """State machine for executing API operation sequences based on OpenAPI links.
+
+    Automatically generates test scenarios by chaining API operations according
+    to their defined relationships in the schema.
+    """
+
+    # This is a convenience attribute, which happened to clash with `RuleBasedStateMachine` instance level attribute
+    # They don't interfere, since it is properly overridden on the Hypothesis side, but it is likely that this
+    # attribute will be renamed in the future
+    bundles: ClassVar[dict[str, CaseInsensitiveDict]]
+    schema: BaseSchema
+
+    def __init__(self) -> None:
+        try:
+            super().__init__()
+        except InvalidDefinition as exc:
+            if "defines no rules" in str(exc):
+                if not self.schema.statistic.links.total:
+                    message = "Schema contains no link definitions required for stateful testing"
+                else:
+                    message = "All link definitions required for stateful testing are excluded by filters"
+                message += f"\n\nLearn how to define links: {STATEFUL_TESTING_GUIDE_URL}"
+                raise NoLinksFound(message) from None
+            raise
+        self.setup()
+
+    @classmethod
+    @lru_cache
+    def _to_test_case(cls) -> type:
+        from schemathesis.generation.stateful import run_state_machine_as_test
+
+        class StateMachineTestCase(RuleBasedStateMachine.TestCase):
+            settings = DEFAULT_STATE_MACHINE_SETTINGS
+
+            def runTest(self) -> None:
+                run_state_machine_as_test(cls, settings=self.settings)
+
+            runTest.is_hypothesis_test = True  # type: ignore[attr-defined]
+
+        StateMachineTestCase.__name__ = cls.__name__ + ".TestCase"
+        StateMachineTestCase.__qualname__ = cls.__qualname__ + ".TestCase"
+        return StateMachineTestCase
+
+    def _new_name(self, target: str) -> str:
+        target = _normalize_name(target)
+        return super()._new_name(target)
+
+    def _get_target_for_result(self, result: StepOutput) -> str | None:
+        raise NotImplementedError
+
+    def _add_result_to_targets(self, targets: tuple[str, ...], result: StepOutput | None) -> None:
+        if result is None:
+            return
+        target = self._get_target_for_result(result)
+        if target is not None:
+            super()._add_result_to_targets((target,), result)
+
+    def _add_results_to_targets(self, targets: tuple[str, ...], results: list[StepOutput | None]) -> None:
+        # Hypothesis >6.131.15
+        for result in results:
+            if result is None:
+                continue
+            target = self._get_target_for_result(result)
+            if target is not None:
+                super()._add_results_to_targets((target,), [result])
+
+    @classmethod
+    def run(cls, *, settings: hypothesis.settings | None = None) -> None:
+        """Execute the state machine test scenarios.
+
+        Args:
+            settings: Hypothesis settings for test execution.
+
+        """
+        from . import run_state_machine_as_test
+
+        __tracebackhide__ = True
+        return run_state_machine_as_test(cls, settings=settings)
+
+    def setup(self) -> None:
+        """Called once at the beginning of each test scenario."""
+
+    def teardown(self) -> None:
+        """Called once at the end of each test scenario."""
+
+    # To provide the return type in the rendered documentation
+    teardown.__doc__ = RuleBasedStateMachine.teardown.__doc__
+
+    def _step(self, input: StepInput) -> StepOutput | None:
+        __tracebackhide__ = True
+        return self.step(input)
+
+    def step(self, input: StepInput) -> StepOutput:
+        __tracebackhide__ = True
+        self.before_call(input.case)
+        kwargs = self.get_call_kwargs(input.case)
+        response = self.call(input.case, **kwargs)
+        self.after_call(response, input.case)
+        self.validate_response(response, input.case, **kwargs)
+        return StepOutput(response, input.case)
+
+    def before_call(self, case: Case) -> None:
+        """Called before each API operation in the scenario.
+
+        Args:
+            case: Test case data for the operation.
+
+        """
+
+    def after_call(self, response: Response, case: Case) -> None:
+        """Called after each API operation in the scenario.
+
+        Args:
+            response: HTTP response from the operation.
+            case: Test case data that was executed.
+
+        """
+
+    def call(self, case: Case, **kwargs: Any) -> Response:
+        return case.call(**kwargs)
+
+    def get_call_kwargs(self, case: Case) -> dict[str, Any]:
+        """Returns keyword arguments for the API call.
+
+        Args:
+            case: Test case being executed.
+
+        Returns:
+            Dictionary passed to the `case.call()` method.
+
+        """
+        return {}
+
+    def validate_response(
+        self, response: Response, case: Case, additional_checks: list[CheckFunction] | None = None, **kwargs: Any
+    ) -> None:
+        """Validates the API response using configured checks.
+
+        Args:
+            response: HTTP response to validate.
+            case: Test case that generated the response.
+            additional_checks: Extra validation functions to run.
+            kwargs: Transport-level keyword arguments.
+
+        Raises:
+            FailureGroup: When validation checks fail.
+
+        """
+        __tracebackhide__ = True
+        case.validate_response(response, additional_checks=additional_checks, transport_kwargs=kwargs)

schemathesis/graphql/__init__.py

@@ -0,0 +1,15 @@
+from schemathesis.graphql.loaders import from_asgi, from_dict, from_file, from_path, from_url, from_wsgi
+
+from ..specs.graphql import nodes
+from ..specs.graphql.scalars import scalar
+
+__all__ = [
+    "from_url",
+    "from_asgi",
+    "from_wsgi",
+    "from_file",
+    "from_path",
+    "from_dict",
+    "nodes",
+    "scalar",
+]

schemathesis/graphql/checks.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from schemathesis.core.failures import Failure, Severity
+
+if TYPE_CHECKING:
+    from graphql.error import GraphQLFormattedError
+
+
+class UnexpectedGraphQLResponse(Failure):
+    """GraphQL response is not a JSON object."""
+
+    __slots__ = ("operation", "type_name", "title", "message", "case_id", "severity")
+
+    def __init__(
+        self,
+        *,
+        operation: str,
+        type_name: str,
+        title: str = "Unexpected GraphQL Response",
+        message: str,
+        case_id: str | None = None,
+    ) -> None:
+        self.operation = operation
+        self.type_name = type_name
+        self.title = title
+        self.message = message
+        self.case_id = case_id
+        self.severity = Severity.MEDIUM
+
+    @property
+    def _unique_key(self) -> str:
+        return self.type_name
+
+
+class GraphQLClientError(Failure):
+    """GraphQL query has not been executed."""
+
+    __slots__ = ("operation", "errors", "title", "message", "case_id", "_unique_key_cache", "severity")
+
+    def __init__(
+        self,
+        *,
+        operation: str,
+        message: str,
+        errors: list[GraphQLFormattedError],
+        title: str = "GraphQL client error",
+        case_id: str | None = None,
+    ) -> None:
+        self.operation = operation
+        self.errors = errors
+        self.title = title
+        self.message = message
+        self.case_id = case_id
+        self._unique_key_cache: str | None = None
+        self.severity = Severity.MEDIUM
+
+    @property
+    def _unique_key(self) -> str:
+        if self._unique_key_cache is None:
+            self._unique_key_cache = _group_graphql_errors(self.errors)
+        return self._unique_key_cache
+
+
+class GraphQLServerError(Failure):
+    """GraphQL response indicates at least one server error."""
+
+    __slots__ = ("operation", "errors", "title", "message", "case_id", "_unique_key_cache", "severity")
+
+    def __init__(
+        self,
+        *,
+        operation: str,
+        message: str,
+        errors: list[GraphQLFormattedError],
+        title: str = "GraphQL server error",
+        case_id: str | None = None,
+    ) -> None:
+        self.operation = operation
+        self.errors = errors
+        self.title = title
+        self.message = message
+        self.case_id = case_id
+        self._unique_key_cache: str | None = None
+        self.severity = Severity.CRITICAL
+
+    @property
+    def _unique_key(self) -> str:
+        if self._unique_key_cache is None:
+            self._unique_key_cache = _group_graphql_errors(self.errors)
+        return self._unique_key_cache
+
+
+def _group_graphql_errors(errors: list[GraphQLFormattedError]) -> str:
+    entries = []
+    for error in errors:
+        message = error["message"]
+        if "locations" in error:
+            message += ";locations:"
+            for location in sorted(error["locations"]):
+                message += f"({location['line'], location['column']})"
+        if "path" in error:
+            message += ";path:"
+            for chunk in error["path"]:
+                message += str(chunk)
+        entries.append(message)
+    entries.sort()
+    return "".join(entries)