schemathesis 4.0.0a10__py3-none-any.whl → 4.0.0a12__py3-none-any.whl

schemathesis/generation/hypothesis/builder.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import asyncio
-import os
 from dataclasses import dataclass, field
 from enum import Enum
 from functools import wraps
@@ -10,22 +9,25 @@ from time import perf_counter
 from typing import Any, Callable, Generator, Mapping
 
 import hypothesis
-from hypothesis import Phase
+from hypothesis import Phase, Verbosity
 from hypothesis import strategies as st
 from hypothesis._settings import all_settings
 from hypothesis.errors import Unsatisfiable
 from jsonschema.exceptions import SchemaError
+from requests.models import CaseInsensitiveDict
 
 from schemathesis import auths
 from schemathesis.auths import AuthStorage, AuthStorageMark
-from schemathesis.core import NOT_SET, NotSet, SpecificationFeature, media_types, string_to_boolean
+from schemathesis.config import ProjectConfig
+from schemathesis.core import NOT_SET, NotSet, SpecificationFeature, media_types
 from schemathesis.core.errors import InvalidSchema, SerializationNotPossible
 from schemathesis.core.marks import Mark
 from schemathesis.core.transport import prepare_urlencoded
 from schemathesis.core.validation import has_invalid_characters, is_latin_1_encodable
-from schemathesis.generation import GenerationConfig, GenerationMode, coverage
+from schemathesis.generation import GenerationMode, coverage
 from schemathesis.generation.case import Case
 from schemathesis.generation.hypothesis import DEFAULT_DEADLINE, examples, setup, strategies
+from schemathesis.generation.hypothesis.examples import add_single_example
 from schemathesis.generation.hypothesis.given import GivenInput
 from schemathesis.generation.meta import (
     CaseMetadata,
@@ -49,7 +51,7 @@ class HypothesisTestMode(str, Enum):
 
 @dataclass
 class HypothesisTestConfig:
-    generation: GenerationConfig
+    project: ProjectConfig
     modes: list[HypothesisTestMode]
     settings: hypothesis.settings | None = None
     seed: int | None = None
@@ -71,11 +73,11 @@ def create_test(
     strategy_kwargs = {
         "hooks": hook_dispatcher,
         "auth_storage": auth_storage,
-        "generation_config": config.generation,
         **config.as_strategy_kwargs,
     }
+    generation = config.project.generation_for(operation=operation)
     strategy = strategies.combine(
-        [operation.as_strategy(generation_mode=mode, **strategy_kwargs) for mode in config.generation.modes]
+        [operation.as_strategy(generation_mode=mode, **strategy_kwargs) for mode in generation.modes]
     )
 
     hypothesis_test = create_base_test(
@@ -95,6 +97,9 @@ def create_test(
     if settings.deadline == default.deadline:
         settings = hypothesis.settings(settings, deadline=DEFAULT_DEADLINE)
 
+    if settings.verbosity == default.verbosity:
+        settings = hypothesis.settings(settings, verbosity=Verbosity.quiet)
+
     if config.settings is not None:
         # Merge the user-provided settings with the current ones
         settings = hypothesis.settings(
@@ -125,25 +130,31 @@ def create_test(
         and Phase.explicit in settings.phases
         and specification.supports_feature(SpecificationFeature.EXAMPLES)
     ):
-        hypothesis_test = add_examples(hypothesis_test, operation, hook_dispatcher=hook_dispatcher, **strategy_kwargs)
-
-    disable_coverage = string_to_boolean(os.getenv("SCHEMATHESIS_DISABLE_COVERAGE", ""))
+        phases_config = config.project.phases_for(operation=operation)
+        hypothesis_test = add_examples(
+            hypothesis_test,
+            operation,
+            fill_missing=phases_config.examples.fill_missing,
+            hook_dispatcher=hook_dispatcher,
+            **strategy_kwargs,
+        )
 
     if (
-        not disable_coverage
-        and HypothesisTestMode.COVERAGE in config.modes
+        HypothesisTestMode.COVERAGE in config.modes
         and Phase.explicit in settings.phases
         and specification.supports_feature(SpecificationFeature.COVERAGE)
         and not config.given_args
         and not config.given_kwargs
     ):
+        phases_config = config.project.phases_for(operation=operation)
         hypothesis_test = add_coverage(
             hypothesis_test,
             operation,
-            config.generation.modes,
+            generation.modes,
             auth_storage,
             config.as_strategy_kwargs,
-            config.generation.unexpected_methods,
+            generate_duplicate_query_parameters=phases_config.coverage.generate_duplicate_query_parameters,
+            unexpected_methods=phases_config.coverage.unexpected_methods,
         )
 
     setattr(hypothesis_test, SETTINGS_ATTRIBUTE_NAME, settings)
@@ -189,7 +200,11 @@ def make_async_test(test: Callable) -> Callable:
 
 
 def add_examples(
-    test: Callable, operation: APIOperation, hook_dispatcher: HookDispatcher | None = None, **kwargs: Any
+    test: Callable,
+    operation: APIOperation,
+    fill_missing: bool,
+    hook_dispatcher: HookDispatcher | None = None,
+    **kwargs: Any,
 ) -> Callable:
     """Add examples to the Hypothesis test, if they are specified in the schema."""
     from hypothesis_jsonschema._canonicalise import HypothesisRefResolutionError
@@ -213,7 +228,11 @@ def add_examples(
         if isinstance(exc, SchemaError):
             InvalidRegexMark.set(test, exc)
 
-    context = HookContext(operation)  # context should be passed here instead
+    if fill_missing and not result:
+        strategy = operation.as_strategy()
+        add_single_example(strategy, result)
+
+    context = HookContext(operation=operation)  # context should be passed here instead
     GLOBAL_HOOK_DISPATCHER.dispatch("before_add_examples", context, result)
     operation.schema.hooks.dispatch("before_add_examples", context, result)
     if hook_dispatcher:
@@ -247,6 +266,7 @@ def add_coverage(
     generation_modes: list[GenerationMode],
     auth_storage: AuthStorage | None,
     as_strategy_kwargs: dict[str, Any],
+    generate_duplicate_query_parameters: bool,
     unexpected_methods: set[str] | None = None,
 ) -> Callable:
     from schemathesis.specs.openapi.constants import LOCATION_TO_CONTAINER
@@ -260,7 +280,9 @@ def add_coverage(
         for container in LOCATION_TO_CONTAINER.values()
         if container in as_strategy_kwargs
     }
-    for case in _iter_coverage_cases(operation, generation_modes, unexpected_methods):
+    for case in _iter_coverage_cases(
+        operation, generation_modes, generate_duplicate_query_parameters, unexpected_methods
+    ):
         if case.media_type and operation.schema.transport.get_first_matching_media_type(case.media_type) is None:
             continue
         adjust_urlencoded_payload(case)
@@ -388,7 +410,7 @@ def _stringify_value(val: Any, container_name: str) -> Any:
             # Having a list here ensures there will be multiple query parameters wit the same name
             return [_stringify_value(item, container_name) for item in val]
         # use comma-separated values style for arrays
-        return ",".join(_stringify_value(sub, container_name) for sub in val)
+        return ",".join(str(_stringify_value(sub, container_name)) for sub in val)
     if isinstance(val, dict):
         return {key: _stringify_value(sub, container_name) for key, sub in val.items()}
     return val
@@ -397,6 +419,7 @@ def _stringify_value(val: Any, container_name: str) -> Any:
 def _iter_coverage_cases(
     operation: APIOperation,
     generation_modes: list[GenerationMode],
+    generate_duplicate_query_parameters: bool,
     unexpected_methods: set[str] | None = None,
 ) -> Generator[Case, None, None]:
     from schemathesis.specs.openapi.constants import LOCATION_TO_CONTAINER
@@ -411,6 +434,10 @@ def _iter_coverage_cases(
     responses = find_in_responses(operation)
     # NOTE: The HEAD method is excluded
     unexpected_methods = unexpected_methods or {"get", "put", "post", "delete", "options", "patch", "trace"}
+
+    seen_negative = coverage.HashSet()
+    seen_positive = coverage.HashSet()
+
     for parameter in operation.iter_parameters():
         location = parameter.location
         name = parameter.name
@@ -473,7 +500,7 @@ def _iter_coverage_cases(
                         meta=CaseMetadata(
                             generation=GenerationInfo(
                                 time=instant.elapsed,
-                                mode=value.generation_mode,
+                                mode=next_value.generation_mode,
                             ),
                             components=data.components,
                             phase=PhaseInfo.coverage(
@@ -488,6 +515,7 @@ def _iter_coverage_cases(
                     break
     elif GenerationMode.POSITIVE in generation_modes:
         data = template.unmodified()
+        seen_positive.insert(data.kwargs)
         yield operation.Case(
             **data.kwargs,
             meta=CaseMetadata(
@@ -510,6 +538,12 @@ def _iter_coverage_cases(
             except StopIteration:
                 break
 
+            if value.generation_mode == GenerationMode.NEGATIVE:
+                seen_negative.insert(data.kwargs)
+            elif value.generation_mode == GenerationMode.POSITIVE and not seen_positive.insert(data.kwargs):
+                # Was already generated before
+                continue
+
             yield operation.Case(
                 **data.kwargs,
                 meta=CaseMetadata(
@@ -539,7 +573,7 @@ def _iter_coverage_cases(
                 ),
             )
         # Generate duplicate query parameters
-        if operation.query:
+        if generate_duplicate_query_parameters and operation.query:
             container = template["query"]
             for parameter in operation.query:
                 instant = Instant()
@@ -689,6 +723,9 @@ def _iter_coverage_cases(
             if GenerationMode.NEGATIVE in generation_modes:
                 subschema = _combination_schema(only_required, required, parameter_set)
                 for case in _yield_negative(subschema, location, container_name):
+                    kwargs = _case_to_kwargs(case)
+                    if not seen_negative.insert(kwargs):
+                        continue
                     assert case.meta is not None
                     assert isinstance(case.meta.phase.data, CoveragePhaseData)
                     # Already generated in one of the blocks above
@@ -738,6 +775,19 @@ def _iter_coverage_cases(
                         )
 
 
+def _case_to_kwargs(case: Case) -> dict:
+    from schemathesis.specs.openapi.constants import LOCATION_TO_CONTAINER
+
+    kwargs = {}
+    for container_name in LOCATION_TO_CONTAINER.values():
+        value = getattr(case, container_name)
+        if isinstance(value, CaseInsensitiveDict):
+            kwargs[container_name] = dict(value)
+        elif value and value is not NOT_SET:
+            kwargs[container_name] = value
+    return kwargs
+
+
 def find_invalid_headers(headers: Mapping) -> Generator[tuple[str, str], None, None]:
     for name, value in headers.items():
         if not is_latin_1_encodable(value) or has_invalid_characters(name, value):

schemathesis/generation/meta.py

@@ -9,9 +9,9 @@ from schemathesis.generation import GenerationMode
 class TestPhase(str, Enum):
     __test__ = False
 
-    EXPLICIT = "explicit"
+    EXAMPLES = "examples"
     COVERAGE = "coverage"
-    GENERATE = "generate"
+    FUZZING = "fuzzing"
 
 
 class ComponentKind(str, Enum):
@@ -81,7 +81,7 @@ class PhaseInfo:
 
     @classmethod
     def generate(cls) -> PhaseInfo:
-        return cls(name=TestPhase.GENERATE, data=GeneratePhaseData())
+        return cls(name=TestPhase.FUZZING, data=GeneratePhaseData())
 
 
 @dataclass

schemathesis/generation/metrics.py

@@ -0,0 +1,93 @@
+"""Support for Targeted Property-Based Testing."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable, Sequence
+
+from schemathesis.core.registries import Registry
+from schemathesis.core.transport import Response
+from schemathesis.generation.case import Case
+
+
+@dataclass
+class MetricContext:
+    """Context for evaluating a metric on a single test execution.
+
+    This object bundles together the test `case` that was sent and
+    the corresponding HTTP `response`. Metric functions receive an
+    instance of `MetricContext` to compute a numeric score.
+    """
+
+    case: Case
+    """Generated test case."""
+    response: Response
+    """The HTTP response returned by the server for this test case."""
+
+    __slots__ = ("case", "response")
+
+
+MetricFunction = Callable[[MetricContext], float]
+
+METRICS = Registry[MetricFunction]()
+
+
+def metric(func: MetricFunction) -> MetricFunction:
+    """Decorator to register a custom metric for targeted property-based testing.
+
+    Example:
+        ```python
+        import schemathesis
+
+        @schemathesis.metric
+        def response_size(ctx: schemathesis.MetricContext) -> float:
+            return float(len(ctx.response.content))
+        ```
+
+    """
+    return METRICS.register(func)
+
+
+@metric
+def response_time(ctx: MetricContext) -> float:
+    """Response time as a metric to maximize."""
+    return ctx.response.elapsed
+
+
+class MetricCollector:
+    """Collect multiple observations for metrics."""
+
+    __slots__ = ("metrics", "observations")
+
+    def __init__(self, metrics: list[MetricFunction] | None = None) -> None:
+        self.metrics = metrics or []
+        self.observations: dict[str, list[float]] = {metric.__name__: [] for metric in self.metrics}
+
+    def reset(self) -> None:
+        """Reset all collected observations."""
+        for metric in self.metrics:
+            self.observations[metric.__name__].clear()
+
+    def store(self, case: Case, response: Response) -> None:
+        """Calculate metrics & store them."""
+        ctx = MetricContext(case=case, response=response)
+        for metric in self.metrics:
+            self.observations[metric.__name__].append(metric(ctx))
+
+    def maximize(self) -> None:
+        """Give feedback to the Hypothesis engine, so it maximizes the aggregated metrics."""
+        import hypothesis
+
+        for metric in self.metrics:
+            # Currently aggregation is just a sum
+            value = sum(self.observations[metric.__name__])
+            hypothesis.target(value, label=metric.__name__)
+
+
+def maximize(metrics: Sequence[MetricFunction], case: Case, response: Response) -> None:
+    import hypothesis
+
+    ctx = MetricContext(case=case, response=response)
+    for metric in metrics:
+        value = metric(ctx)
+        hypothesis.target(value, label=metric.__name__)

schemathesis/generation/modes.py

@@ -11,14 +11,6 @@ class GenerationMode(str, Enum):
     # Doesn't fit the API schema
     NEGATIVE = "negative"
 
-    @classmethod
-    def default(cls) -> GenerationMode:
-        return cls.POSITIVE
-
-    @classmethod
-    def all(cls) -> list[GenerationMode]:
-        return list(GenerationMode)
-
     @property
     def is_positive(self) -> bool:
         return self == GenerationMode.POSITIVE

schemathesis/generation/overrides.py

@@ -4,6 +4,7 @@ from collections.abc import Mapping
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Callable
 
+from schemathesis.config import ProjectConfig
 from schemathesis.core.errors import IncorrectUsage
 from schemathesis.core.marks import Mark
 from schemathesis.core.transforms import diff
@@ -11,7 +12,7 @@ from schemathesis.generation.meta import ComponentKind
 
 if TYPE_CHECKING:
     from schemathesis.generation.case import Case
-    from schemathesis.schemas import APIOperation, ParameterSet
+    from schemathesis.schemas import APIOperation, Parameter, ParameterSet
 
 
 @dataclass
@@ -41,6 +42,41 @@ class Override:
         )
 
 
+def for_operation(config: ProjectConfig, *, operation: APIOperation) -> Override:
+    operation_config = config.operations.get_for_operation(operation)
+
+    output = Override(query={}, headers={}, cookies={}, path_parameters={})
+    groups = [
+        (output.query, operation.query),
+        (output.headers, operation.headers),
+        (output.cookies, operation.cookies),
+        (output.path_parameters, operation.path_parameters),
+    ]
+    for container, params in groups:
+        for param in params:
+            # Attempt to get the override from the operation-specific configuration.
+            value = None
+            if operation_config:
+                value = _get_override_value(param, operation_config.parameters)
+            # Fallback to the global project configuration.
+            if value is None:
+                value = _get_override_value(param, config.parameters)
+            if value is not None:
+                container[param.name] = value
+
+    return output
+
+
+def _get_override_value(param: Parameter, parameters: dict[str, Any]) -> Any:
+    key = param.name
+    full_key = f"{param.location}.{param.name}"
+    if key in parameters:
+        return parameters[key]
+    elif full_key in parameters:
+        return parameters[full_key]
+    return None
+
+
 def _for_parameters(overridden: dict[str, str], defined: ParameterSet) -> dict[str, str]:
     output = {}
     for param in defined:

schemathesis/generation/stateful/__init__.py

@@ -7,6 +7,8 @@ if TYPE_CHECKING:
 
     from schemathesis.generation.stateful.state_machine import APIStateMachine
 
+STATEFUL_TESTS_LABEL = "Stateful tests"
+
 
 def run_state_machine_as_test(
     state_machine_factory: type[APIStateMachine], *, settings: hypothesis.settings | None = None
@@ -17,4 +19,6 @@ def run_state_machine_as_test(
     """
     from hypothesis.stateful import run_state_machine_as_test as _run_state_machine_as_test
 
+    __tracebackhide__ = True
+
     return _run_state_machine_as_test(state_machine_factory, settings=settings, _min_steps=2)

schemathesis/generation/stateful/state_machine.py

@@ -10,6 +10,7 @@ from hypothesis.errors import InvalidDefinition
 from hypothesis.stateful import RuleBasedStateMachine
 
 from schemathesis.checks import CheckFunction
+from schemathesis.core import DEFAULT_STATEFUL_STEP_COUNT
 from schemathesis.core.errors import NoLinksFound
 from schemathesis.core.result import Result
 from schemathesis.core.transport import Response
@@ -22,7 +23,6 @@ if TYPE_CHECKING:
     from schemathesis.schemas import BaseSchema
 
 
-DEFAULT_STATEFUL_STEP_COUNT = 6
 DEFAULT_STATE_MACHINE_SETTINGS = hypothesis.settings(
     phases=[hypothesis.Phase.generate],
     deadline=None,
@@ -184,11 +184,19 @@ class APIStateMachine(RuleBasedStateMachine):
         if target is not None:
             super()._add_result_to_targets((target,), result)
 
+    def _add_results_to_targets(self, targets: tuple[str, ...], results: list[StepOutput]) -> None:
+        # Hypothesis >6.131.15
+        for result in results:
+            target = self._get_target_for_result(result)
+            if target is not None:
+                super()._add_results_to_targets((target,), [result])
+
     @classmethod
     def run(cls, *, settings: hypothesis.settings | None = None) -> None:
         """Run state machine as a test."""
         from . import run_state_machine_as_test
 
+        __tracebackhide__ = True
         return run_state_machine_as_test(cls, settings=settings)
 
     def setup(self) -> None: