schemathesis 4.0.0a11__py3-none-any.whl → 4.0.0b1__py3-none-any.whl

schemathesis/generation/case.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Mapping
 
+from schemathesis import transport
 from schemathesis.checks import CHECKS, CheckContext, CheckFunction, run_checks
 from schemathesis.core import NOT_SET, SCHEMATHESIS_TEST_CASE_HEADER, NotSet, curl
 from schemathesis.core.failures import FailureGroup, failure_report_title, format_failures
@@ -11,42 +12,121 @@ from schemathesis.generation import generate_random_case_id
 from schemathesis.generation.meta import CaseMetadata
 from schemathesis.generation.overrides import Override, store_components
 from schemathesis.hooks import HookContext, dispatch
-from schemathesis.transport.prepare import prepare_request
+from schemathesis.transport.prepare import prepare_path, prepare_request
 
 if TYPE_CHECKING:
+    import httpx
+    import requests
     import requests.auth
     from requests.structures import CaseInsensitiveDict
+    from werkzeug.test import TestResponse
 
     from schemathesis.schemas import APIOperation
 
 
+def _default_headers() -> CaseInsensitiveDict:
+    from requests.structures import CaseInsensitiveDict
+
+    return CaseInsensitiveDict()
+
+
 @dataclass
 class Case:
-    """A single test case parameters."""
+    """Generated test case data for a single API operation."""
 
     operation: APIOperation
     method: str
+    """HTTP verb (`GET`, `POST`, etc.)"""
     path: str
-    # Unique test case identifier
-    id: str = field(default_factory=generate_random_case_id, compare=False)
-    path_parameters: dict[str, Any] | None = None
-    headers: CaseInsensitiveDict | None = None
-    cookies: dict[str, Any] | None = None
-    query: dict[str, Any] | None = None
+    """Path template from schema (e.g., `/users/{user_id}`)"""
+    id: str
+    """Random ID sent in headers for log correlation"""
+    path_parameters: dict[str, Any]
+    """Generated path variables (e.g., `{"user_id": "123"}`)"""
+    headers: CaseInsensitiveDict
+    """Generated HTTP headers"""
+    cookies: dict[str, Any]
+    """Generated cookies"""
+    query: dict[str, Any]
+    """Generated query parameters"""
     # By default, there is no body, but we can't use `None` as the default value because it clashes with `null`
     # which is a valid payload.
-    body: list | dict[str, Any] | str | int | float | bool | bytes | NotSet = NOT_SET
-    # The media type for cases with a payload. For example, "application/json"
-    media_type: str | None = None
+    body: list | dict[str, Any] | str | int | float | bool | bytes | NotSet
+    """Generated request body"""
+    media_type: str | None
+    """Media type from OpenAPI schema (e.g., "multipart/form-data")"""
 
-    meta: CaseMetadata | None = field(compare=False, default=None)
+    meta: CaseMetadata | None
 
-    _auth: requests.auth.AuthBase | None = None
-    _has_explicit_auth: bool = False
+    _auth: requests.auth.AuthBase | None
+    _has_explicit_auth: bool
 
-    def __post_init__(self) -> None:
+    __slots__ = (
+        "operation",
+        "method",
+        "path",
+        "id",
+        "path_parameters",
+        "headers",
+        "cookies",
+        "query",
+        "body",
+        "media_type",
+        "meta",
+        "_auth",
+        "_has_explicit_auth",
+        "_components",
+    )
+
+    def __init__(
+        self,
+        operation: APIOperation,
+        method: str,
+        path: str,
+        *,
+        id: str | None = None,
+        path_parameters: dict[str, Any] | None = None,
+        headers: CaseInsensitiveDict | None = None,
+        cookies: dict[str, Any] | None = None,
+        query: dict[str, Any] | None = None,
+        body: list | dict[str, Any] | str | int | float | bool | bytes | "NotSet" = NOT_SET,
+        media_type: str | None = None,
+        meta: CaseMetadata | None = None,
+        _auth: requests.auth.AuthBase | None = None,
+        _has_explicit_auth: bool = False,
+    ) -> None:
+        self.operation = operation
+        self.method = method
+        self.path = path
+
+        self.id = id if id is not None else generate_random_case_id()
+        self.path_parameters = path_parameters if path_parameters is not None else {}
+        self.headers = headers if headers is not None else _default_headers()
+        self.cookies = cookies if cookies is not None else {}
+        self.query = query if query is not None else {}
+        self.body = body
+        self.media_type = media_type
+        self.meta = meta
+        self._auth = _auth
+        self._has_explicit_auth = _has_explicit_auth
         self._components = store_components(self)
 
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Case):
+            return NotImplemented
+
+        return (
+            self.operation == other.operation
+            and self.method == other.method
+            and self.path == other.path
+            and self.path_parameters == other.path_parameters
+            and self.headers == other.headers
+            and self.cookies == other.cookies
+            and self.query == other.query
+            and self.body == other.body
+            and self.media_type == other.media_type
+        )
+
     @property
     def _override(self) -> Override:
         return Override.from_components(self._components, self)
@@ -56,6 +136,8 @@ class Case:
         first = True
         for name in ("path_parameters", "headers", "cookies", "query", "body"):
             value = getattr(self, name)
+            if name != "body" and not value:
+                continue
             if value is not None and not isinstance(value, NotSet):
                 if first:
                     first = False
@@ -69,8 +151,19 @@ class Case:
 
     def _repr_pretty_(self, *args: Any, **kwargs: Any) -> None: ...
 
+    @property
+    def formatted_path(self) -> str:
+        """Path template with variables substituted (e.g., /users/{user_id} → /users/123)."""
+        return prepare_path(self.path, self.path_parameters)
+
     def as_curl_command(self, headers: Mapping[str, Any] | None = None, verify: bool = True) -> str:
-        """Construct a curl command for a given case."""
+        """Generate a curl command that reproduces this test case.
+
+        Args:
+            headers: Additional headers to include in the command.
+            verify: When False, adds `--insecure` flag to curl command.
+
+        """
         request_data = prepare_request(self, headers, config=self.operation.schema.config.output.sanitization)
         return curl.generate(
             method=str(request_data.method),
@@ -82,7 +175,6 @@ class Case:
         )
 
     def as_transport_kwargs(self, base_url: str | None = None, headers: dict[str, str] | None = None) -> dict[str, Any]:
-        """Convert the test case into a dictionary acceptable by the underlying transport call."""
         return self.operation.schema.transport.serialize_case(self, base_url=base_url, headers=headers)
 
     def call(
@@ -94,11 +186,28 @@ class Case:
         cookies: dict[str, Any] | None = None,
         **kwargs: Any,
     ) -> Response:
+        """Make an HTTP request using this test case's data without validation.
+
+        Use when you need to validate response separately
+
+        Args:
+            base_url: Override the schema's base URL.
+            session: Reuse an existing requests session.
+            headers: Additional headers.
+            params: Additional query parameters.
+            cookies: Additional cookies.
+            **kwargs: Additional transport-level arguments.
+
+        """
         hook_context = HookContext(operation=self.operation)
         dispatch("before_call", hook_context, self, **kwargs)
         if self.operation.app is not None:
-            kwargs["app"] = self.operation.app
-        response = self.operation.schema.transport.send(
+            kwargs.setdefault("app", self.operation.app)
+        if "app" in kwargs:
+            transport_ = transport.get(kwargs["app"])
+        else:
+            transport_ = self.operation.schema.transport
+        response = transport_.send(
             self,
             session=session,
             base_url=base_url,
@@ -112,26 +221,29 @@ class Case:
 
     def validate_response(
         self,
-        response: Response,
+        response: Response | httpx.Response | requests.Response | TestResponse,
         checks: list[CheckFunction] | None = None,
         additional_checks: list[CheckFunction] | None = None,
         excluded_checks: list[CheckFunction] | None = None,
         headers: dict[str, Any] | None = None,
         transport_kwargs: dict[str, Any] | None = None,
     ) -> None:
-        """Validate application response.
+        """Validate a response against the API schema and built-in checks.
 
-        By default, all available checks will be applied.
+        Args:
+            response: Response to validate.
+            checks: Explicit set of checks to run.
+            additional_checks: Additional custom checks to run.
+            excluded_checks: Built-in checks to skip.
+            headers: Headers used in the original request.
+            transport_kwargs: Transport arguments used in the original request.
 
-        :param response: Application response.
-        :param checks: A tuple of check functions that accept ``response`` and ``case``.
-        :param additional_checks: A tuple of additional checks that will be executed after ones from the ``checks``
-            argument.
-        :param excluded_checks: Checks excluded from the default ones.
         """
         __tracebackhide__ = True
         from requests.structures import CaseInsensitiveDict
 
+        response = Response.from_any(response)
+
         checks = [
             check
             for check in list(checks or CHECKS.get_all()) + list(additional_checks or [])
@@ -167,6 +279,7 @@ class Case:
                 curl=curl,
                 config=self.operation.schema.config.output,
             )
+            message += "\n\n"
             raise FailureGroup(_failures, message) from None
 
     def call_and_validate(
@@ -179,6 +292,18 @@ class Case:
         excluded_checks: list[CheckFunction] | None = None,
         **kwargs: Any,
     ) -> Response:
+        """Make an HTTP request and validates the response automatically.
+
+        Args:
+            base_url: Override the schema's base URL.
+            session: Reuse an existing requests session.
+            headers: Additional headers to send.
+            checks: Explicit set of checks to run.
+            additional_checks: Additional custom checks to run.
+            excluded_checks: Built-in checks to skip.
+            **kwargs: Additional transport-level arguments.
+
+        """
         __tracebackhide__ = True
         response = self.call(base_url, session, headers, **kwargs)
         self.validate_response(

schemathesis/generation/coverage.py

@@ -121,7 +121,7 @@ class CoverageContext:
         path: list[str | int] | None = None,
     ) -> None:
         self.location = location
-        self.generation_modes = generation_modes if generation_modes is not None else GenerationMode.all()
+        self.generation_modes = generation_modes if generation_modes is not None else list(GenerationMode)
         self.path = path or []
 
     @contextmanager

schemathesis/generation/hypothesis/builder.py

@@ -9,7 +9,7 @@ from time import perf_counter
 from typing import Any, Callable, Generator, Mapping
 
 import hypothesis
-from hypothesis import Phase
+from hypothesis import Phase, Verbosity
 from hypothesis import strategies as st
 from hypothesis._settings import all_settings
 from hypothesis.errors import Unsatisfiable
@@ -27,6 +27,7 @@ from schemathesis.core.validation import has_invalid_characters, is_latin_1_enco
 from schemathesis.generation import GenerationMode, coverage
 from schemathesis.generation.case import Case
 from schemathesis.generation.hypothesis import DEFAULT_DEADLINE, examples, setup, strategies
+from schemathesis.generation.hypothesis.examples import add_single_example
 from schemathesis.generation.hypothesis.given import GivenInput
 from schemathesis.generation.meta import (
     CaseMetadata,
@@ -96,14 +97,17 @@ def create_test(
     if settings.deadline == default.deadline:
         settings = hypothesis.settings(settings, deadline=DEFAULT_DEADLINE)
 
+    if settings.verbosity == default.verbosity:
+        settings = hypothesis.settings(settings, verbosity=Verbosity.quiet)
+
     if config.settings is not None:
         # Merge the user-provided settings with the current ones
         settings = hypothesis.settings(
-            settings,
+            config.settings,
             **{
-                item: getattr(config.settings, item)
+                item: getattr(settings, item)
                 for item in all_settings
-                if getattr(config.settings, item) != getattr(default, item)
+                if getattr(settings, item) != getattr(default, item)
             },
         )
 
@@ -126,7 +130,14 @@ def create_test(
         and Phase.explicit in settings.phases
         and specification.supports_feature(SpecificationFeature.EXAMPLES)
     ):
-        hypothesis_test = add_examples(hypothesis_test, operation, hook_dispatcher=hook_dispatcher, **strategy_kwargs)
+        phases_config = config.project.phases_for(operation=operation)
+        hypothesis_test = add_examples(
+            hypothesis_test,
+            operation,
+            fill_missing=phases_config.examples.fill_missing,
+            hook_dispatcher=hook_dispatcher,
+            **strategy_kwargs,
+        )
 
     if (
         HypothesisTestMode.COVERAGE in config.modes
@@ -142,7 +153,8 @@ def create_test(
             generation.modes,
             auth_storage,
             config.as_strategy_kwargs,
-            phases_config.coverage.unexpected_methods,
+            generate_duplicate_query_parameters=phases_config.coverage.generate_duplicate_query_parameters,
+            unexpected_methods=phases_config.coverage.unexpected_methods,
         )
 
     setattr(hypothesis_test, SETTINGS_ATTRIBUTE_NAME, settings)
@@ -188,7 +200,11 @@ def make_async_test(test: Callable) -> Callable:
 
 
 def add_examples(
-    test: Callable, operation: APIOperation, hook_dispatcher: HookDispatcher | None = None, **kwargs: Any
+    test: Callable,
+    operation: APIOperation,
+    fill_missing: bool,
+    hook_dispatcher: HookDispatcher | None = None,
+    **kwargs: Any,
 ) -> Callable:
     """Add examples to the Hypothesis test, if they are specified in the schema."""
     from hypothesis_jsonschema._canonicalise import HypothesisRefResolutionError
@@ -212,7 +228,11 @@ def add_examples(
         if isinstance(exc, SchemaError):
             InvalidRegexMark.set(test, exc)
 
-    context = HookContext(operation)  # context should be passed here instead
+    if fill_missing and not result:
+        strategy = operation.as_strategy()
+        add_single_example(strategy, result)
+
+    context = HookContext(operation=operation)  # context should be passed here instead
     GLOBAL_HOOK_DISPATCHER.dispatch("before_add_examples", context, result)
     operation.schema.hooks.dispatch("before_add_examples", context, result)
     if hook_dispatcher:
@@ -246,6 +266,7 @@ def add_coverage(
     generation_modes: list[GenerationMode],
     auth_storage: AuthStorage | None,
     as_strategy_kwargs: dict[str, Any],
+    generate_duplicate_query_parameters: bool,
     unexpected_methods: set[str] | None = None,
 ) -> Callable:
     from schemathesis.specs.openapi.constants import LOCATION_TO_CONTAINER
@@ -259,7 +280,9 @@ def add_coverage(
         for container in LOCATION_TO_CONTAINER.values()
         if container in as_strategy_kwargs
     }
-    for case in _iter_coverage_cases(operation, generation_modes, unexpected_methods):
+    for case in _iter_coverage_cases(
+        operation, generation_modes, generate_duplicate_query_parameters, unexpected_methods
+    ):
         if case.media_type and operation.schema.transport.get_first_matching_media_type(case.media_type) is None:
             continue
         adjust_urlencoded_payload(case)
@@ -396,6 +419,7 @@ def _stringify_value(val: Any, container_name: str) -> Any:
 def _iter_coverage_cases(
     operation: APIOperation,
     generation_modes: list[GenerationMode],
+    generate_duplicate_query_parameters: bool,
     unexpected_methods: set[str] | None = None,
 ) -> Generator[Case, None, None]:
     from schemathesis.specs.openapi.constants import LOCATION_TO_CONTAINER
@@ -451,7 +475,7 @@ def _iter_coverage_cases(
             data = template.with_body(value=value, media_type=body.media_type)
             yield operation.Case(
                 **data.kwargs,
-                meta=CaseMetadata(
+                _meta=CaseMetadata(
                     generation=GenerationInfo(
                         time=elapsed,
                         mode=value.generation_mode,
@@ -473,7 +497,7 @@ def _iter_coverage_cases(
                     data = template.with_body(value=next_value, media_type=body.media_type)
                     yield operation.Case(
                         **data.kwargs,
-                        meta=CaseMetadata(
+                        _meta=CaseMetadata(
                             generation=GenerationInfo(
                                 time=instant.elapsed,
                                 mode=next_value.generation_mode,
@@ -494,7 +518,7 @@ def _iter_coverage_cases(
         seen_positive.insert(data.kwargs)
         yield operation.Case(
             **data.kwargs,
-            meta=CaseMetadata(
+            _meta=CaseMetadata(
                 generation=GenerationInfo(
                     time=template_time,
                     mode=GenerationMode.POSITIVE,
@@ -522,7 +546,7 @@ def _iter_coverage_cases(
 
             yield operation.Case(
                 **data.kwargs,
-                meta=CaseMetadata(
+                _meta=CaseMetadata(
                     generation=GenerationInfo(time=instant.elapsed, mode=value.generation_mode),
                     components=data.components,
                     phase=PhaseInfo.coverage(
@@ -542,14 +566,14 @@ def _iter_coverage_cases(
             yield operation.Case(
                 **data.kwargs,
                 method=method.upper(),
-                meta=CaseMetadata(
+                _meta=CaseMetadata(
                     generation=GenerationInfo(time=instant.elapsed, mode=GenerationMode.NEGATIVE),
                     components=data.components,
                     phase=PhaseInfo.coverage(description=f"Unspecified HTTP method: {method.upper()}"),
                 ),
             )
         # Generate duplicate query parameters
-        if operation.query:
+        if generate_duplicate_query_parameters and operation.query:
             container = template["query"]
             for parameter in operation.query:
                 instant = Instant()
@@ -564,7 +588,7 @@ def _iter_coverage_cases(
                     )
                     yield operation.Case(
                         **data.kwargs,
-                        meta=CaseMetadata(
+                        _meta=CaseMetadata(
                             generation=GenerationInfo(time=instant.elapsed, mode=GenerationMode.NEGATIVE),
                             components=data.components,
                             phase=PhaseInfo.coverage(
@@ -589,7 +613,7 @@ def _iter_coverage_cases(
                 )
                 yield operation.Case(
                     **data.kwargs,
-                    meta=CaseMetadata(
+                    _meta=CaseMetadata(
                         generation=GenerationInfo(time=instant.elapsed, mode=GenerationMode.NEGATIVE),
                         components=data.components,
                         phase=PhaseInfo.coverage(
@@ -631,7 +655,7 @@ def _iter_coverage_cases(
             )
             return operation.Case(
                 **data.kwargs,
-                meta=CaseMetadata(
+                _meta=CaseMetadata(
                     generation=GenerationInfo(
                         time=_instant.elapsed,
                         mode=_generation_mode,
@@ -757,7 +781,7 @@ def _case_to_kwargs(case: Case) -> dict:
     kwargs = {}
     for container_name in LOCATION_TO_CONTAINER.values():
         value = getattr(case, container_name)
-        if isinstance(value, CaseInsensitiveDict):
+        if isinstance(value, CaseInsensitiveDict) and value:
             kwargs[container_name] = dict(value)
         elif value and value is not NOT_SET:
             kwargs[container_name] = value

schemathesis/generation/metrics.py

@@ -0,0 +1,93 @@
+"""Support for Targeted Property-Based Testing."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable, Sequence
+
+from schemathesis.core.registries import Registry
+from schemathesis.core.transport import Response
+from schemathesis.generation.case import Case
+
+
+@dataclass
+class MetricContext:
+    """Context for evaluating a metric on a single test execution.
+
+    This object bundles together the test `case` that was sent and
+    the corresponding HTTP `response`. Metric functions receive an
+    instance of `MetricContext` to compute a numeric score.
+    """
+
+    case: Case
+    """Generated test case."""
+    response: Response
+    """The HTTP response returned by the server for this test case."""
+
+    __slots__ = ("case", "response")
+
+
+MetricFunction = Callable[[MetricContext], float]
+
+METRICS = Registry[MetricFunction]()
+
+
+def metric(func: MetricFunction) -> MetricFunction:
+    """Decorator to register a custom metric for targeted property-based testing.
+
+    Example:
+        ```python
+        import schemathesis
+
+        @schemathesis.metric
+        def response_size(ctx: schemathesis.MetricContext) -> float:
+            return float(len(ctx.response.content))
+        ```
+
+    """
+    return METRICS.register(func)
+
+
+@metric
+def response_time(ctx: MetricContext) -> float:
+    """Response time as a metric to maximize."""
+    return ctx.response.elapsed
+
+
+class MetricCollector:
+    """Collect multiple observations for metrics."""
+
+    __slots__ = ("metrics", "observations")
+
+    def __init__(self, metrics: list[MetricFunction] | None = None) -> None:
+        self.metrics = metrics or []
+        self.observations: dict[str, list[float]] = {metric.__name__: [] for metric in self.metrics}
+
+    def reset(self) -> None:
+        """Reset all collected observations."""
+        for metric in self.metrics:
+            self.observations[metric.__name__].clear()
+
+    def store(self, case: Case, response: Response) -> None:
+        """Calculate metrics & store them."""
+        ctx = MetricContext(case=case, response=response)
+        for metric in self.metrics:
+            self.observations[metric.__name__].append(metric(ctx))
+
+    def maximize(self) -> None:
+        """Give feedback to the Hypothesis engine, so it maximizes the aggregated metrics."""
+        import hypothesis
+
+        for metric in self.metrics:
+            # Currently aggregation is just a sum
+            value = sum(self.observations[metric.__name__])
+            hypothesis.target(value, label=metric.__name__)
+
+
+def maximize(metrics: Sequence[MetricFunction], case: Case, response: Response) -> None:
+    import hypothesis
+
+    ctx = MetricContext(case=case, response=response)
+    for metric in metrics:
+        value = metric(ctx)
+        hypothesis.target(value, label=metric.__name__)

schemathesis/generation/modes.py

@@ -11,14 +11,6 @@ class GenerationMode(str, Enum):
     # Doesn't fit the API schema
     NEGATIVE = "negative"
 
-    @classmethod
-    def default(cls) -> GenerationMode:
-        return cls.POSITIVE
-
-    @classmethod
-    def all(cls) -> list[GenerationMode]:
-        return list(GenerationMode)
-
     @property
     def is_positive(self) -> bool:
         return self == GenerationMode.POSITIVE

schemathesis/generation/overrides.py

@@ -2,17 +2,15 @@ from __future__ import annotations
 
 from collections.abc import Mapping
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, Callable
+from typing import TYPE_CHECKING, Any, Iterator
 
 from schemathesis.config import ProjectConfig
-from schemathesis.core.errors import IncorrectUsage
-from schemathesis.core.marks import Mark
 from schemathesis.core.transforms import diff
 from schemathesis.generation.meta import ComponentKind
 
 if TYPE_CHECKING:
     from schemathesis.generation.case import Case
-    from schemathesis.schemas import APIOperation, Parameter, ParameterSet
+    from schemathesis.schemas import APIOperation, Parameter
 
 
 @dataclass
@@ -24,13 +22,15 @@ class Override:
     cookies: dict[str, str]
     path_parameters: dict[str, str]
 
-    def for_operation(self, operation: APIOperation) -> dict[str, dict[str, str]]:
-        return {
-            "query": (_for_parameters(self.query, operation.query)),
-            "headers": (_for_parameters(self.headers, operation.headers)),
-            "cookies": (_for_parameters(self.cookies, operation.cookies)),
-            "path_parameters": (_for_parameters(self.path_parameters, operation.path_parameters)),
-        }
+    def items(self) -> Iterator[tuple[str, dict[str, str]]]:
+        for key, value in (
+            ("query", self.query),
+            ("headers", self.headers),
+            ("cookies", self.cookies),
+            ("path_parameters", self.path_parameters),
+        ):
+            if value:
+                yield key, value
 
     @classmethod
     def from_components(cls, components: dict[ComponentKind, StoredValue], case: Case) -> Override:
@@ -77,14 +77,6 @@ def _get_override_value(param: Parameter, parameters: dict[str, Any]) -> Any:
     return None
 
 
-def _for_parameters(overridden: dict[str, str], defined: ParameterSet) -> dict[str, str]:
-    output = {}
-    for param in defined:
-        if param.name in overridden:
-            output[param.name] = overridden[param.name]
-    return output
-
-
 @dataclass
 class StoredValue:
     value: dict[str, Any] | None
@@ -122,11 +114,3 @@ def store_components(case: Case) -> dict[ComponentKind, StoredValue]:
             ComponentKind.PATH_PARAMETERS,
         ]
     }
-
-
-OverrideMark = Mark[Override](attr_name="override")
-
-
-def check_no_override_mark(test: Callable) -> None:
-    if OverrideMark.is_set(test):
-        raise IncorrectUsage(f"`{test.__name__}` has already been decorated with `override`.")