schemathesis 4.0.0a11__py3-none-any.whl → 4.0.0a12__py3-none-any.whl

schemathesis/engine/phases/stateful/_executor.py

@@ -1,4 +1,4 @@
-from __future__ import annotations
+from __future__ import annotations  # noqa: I001
 
 import queue
 import time
@@ -8,9 +8,11 @@ from typing import Any
 from warnings import catch_warnings
 
 import hypothesis
+import requests
 from hypothesis.control import current_build_context
 from hypothesis.errors import Flaky, Unsatisfiable
 from hypothesis.stateful import Rule
+from requests.exceptions import ChunkedEncodingError
 from requests.structures import CaseInsensitiveDict
 
 from schemathesis.checks import CheckContext, CheckFunction, run_checks
@@ -19,19 +21,26 @@ from schemathesis.core.transport import Response
 from schemathesis.engine import Status, events
 from schemathesis.engine.context import EngineContext
 from schemathesis.engine.control import ExecutionControl
+from schemathesis.engine.errors import (
+    TestingState,
+    UnrecoverableNetworkError,
+    clear_hypothesis_notes,
+    is_unrecoverable_network_error,
+)
 from schemathesis.engine.phases import PhaseName
 from schemathesis.engine.phases.stateful.context import StatefulContext
 from schemathesis.engine.recorder import ScenarioRecorder
 from schemathesis.generation import overrides
 from schemathesis.generation.case import Case
 from schemathesis.generation.hypothesis.reporting import ignore_hypothesis_output
+from schemathesis.generation.stateful import STATEFUL_TESTS_LABEL
 from schemathesis.generation.stateful.state_machine import (
     DEFAULT_STATE_MACHINE_SETTINGS,
     APIStateMachine,
     StepInput,
     StepOutput,
 )
-from schemathesis.generation.targets import TargetMetricCollector
+from schemathesis.generation.metrics import MetricCollector
 
 
 def _get_hypothesis_settings_kwargs_override(settings: hypothesis.settings) -> dict[str, Any]:
@@ -72,7 +81,8 @@ def execute_state_machine_loop(
     hypothesis_settings = hypothesis.settings(configured_hypothesis_settings, **kwargs)
     generation = engine.config.generation_for(phase="stateful")
 
-    ctx = StatefulContext(metric_collector=TargetMetricCollector(targets=generation.maximize))
+    ctx = StatefulContext(metric_collector=MetricCollector(metrics=generation.maximize))
+    state = TestingState()
 
     # Caches for validate_response to avoid repeated config lookups per operation
     _check_context_cache: dict[str, CachedCheckContextData] = {}
@@ -123,6 +133,20 @@ def execute_state_machine_loop(
                 ctx.step_failed()
                 raise
             except Exception as exc:
+                if isinstance(exc, (requests.ConnectionError, ChunkedEncodingError)) and is_unrecoverable_network_error(
+                    exc
+                ):
+                    transport_kwargs = engine.get_transport_kwargs(operation=input.case.operation)
+                    if exc.request is not None:
+                        headers = {key: value[0] for key, value in exc.request.headers.items()}
+                    else:
+                        headers = {**dict(input.case.headers or {}), **transport_kwargs.get("headers", {})}
+                    verify = transport_kwargs.get("verify", True)
+                    state.unrecoverable_network_error = UnrecoverableNetworkError(
+                        error=exc,
+                        code_sample=input.case.as_curl_command(headers=headers, verify=verify),
+                    )
+
                 if generation.unique_inputs:
                     ctx.store_step_outcome(input.case, exc)
                 ctx.step_errored()
@@ -172,7 +196,7 @@ def execute_state_machine_loop(
                 case=case,
                 stateful_ctx=ctx,
                 check_ctx=check_ctx,
-                checks=check_ctx.checks,
+                checks=check_ctx._checks,
                 control=engine.control,
                 recorder=self.recorder,
                 additional_checks=additional_checks,
@@ -259,11 +283,20 @@ def execute_state_machine_loop(
                     # Avoid infinite restarts
                     break
                 continue
+            clear_hypothesis_notes(exc)
             # Any other exception is an inner error and the test run should be stopped
             suite_status = Status.ERROR
+            code_sample: str | None = None
+            if state.unrecoverable_network_error is not None:
+                exc = state.unrecoverable_network_error.error
+                code_sample = state.unrecoverable_network_error.code_sample
             event_queue.put(
                 events.NonFatalError(
-                    error=exc, phase=PhaseName.STATEFUL_TESTING, label="Stateful tests", related_to_operation=False
+                    error=exc,
+                    phase=PhaseName.STATEFUL_TESTING,
+                    label=STATEFUL_TESTS_LABEL,
+                    related_to_operation=False,
+                    code_sample=code_sample,
                 )
             )
             break

schemathesis/engine/phases/stateful/context.py

@@ -7,7 +7,7 @@ from schemathesis.core.failures import Failure
 from schemathesis.core.transport import Response
 from schemathesis.engine import Status
 from schemathesis.generation.case import Case
-from schemathesis.generation.targets import TargetMetricCollector
+from schemathesis.generation.metrics import MetricCollector
 
 
 @dataclass
@@ -27,7 +27,7 @@ class StatefulContext:
     # The total number of completed test scenario
     completed_scenarios: int = 0
     # Metrics collector for targeted testing
-    metric_collector: TargetMetricCollector = field(default_factory=TargetMetricCollector)
+    metric_collector: MetricCollector = field(default_factory=MetricCollector)
     step_outcomes: dict[int, BaseException | None] = field(default_factory=dict)
 
     @property

schemathesis/engine/phases/unit/_executor.py

@@ -11,6 +11,7 @@ from hypothesis.errors import InvalidArgument
 from hypothesis_jsonschema._canonicalise import HypothesisRefResolutionError
 from jsonschema.exceptions import SchemaError as JsonSchemaError
 from jsonschema.exceptions import ValidationError
+from requests.exceptions import ChunkedEncodingError
 from requests.structures import CaseInsensitiveDict
 
 from schemathesis.checks import CheckContext, run_checks
@@ -33,13 +34,17 @@ from schemathesis.engine import Status, events
 from schemathesis.engine.context import EngineContext
 from schemathesis.engine.errors import (
     DeadlineExceeded,
+    TestingState,
     UnexpectedError,
+    UnrecoverableNetworkError,
     UnsupportedRecursiveReference,
+    clear_hypothesis_notes,
     deduplicate_errors,
+    is_unrecoverable_network_error,
 )
 from schemathesis.engine.phases import PhaseName
 from schemathesis.engine.recorder import ScenarioRecorder
-from schemathesis.generation import overrides, targets
+from schemathesis.generation import metrics, overrides
 from schemathesis.generation.case import Case
 from schemathesis.generation.hypothesis.builder import (
     InvalidHeadersExampleMark,
@@ -70,9 +75,12 @@ def run_test(
     skip_reason = None
     test_start_time = time.monotonic()
     recorder = ScenarioRecorder(label=operation.label)
+    state = TestingState()
 
-    def non_fatal_error(error: Exception) -> events.NonFatalError:
-        return events.NonFatalError(error=error, phase=phase, label=operation.label, related_to_operation=True)
+    def non_fatal_error(error: Exception, code_sample: str | None = None) -> events.NonFatalError:
+        return events.NonFatalError(
+            error=error, phase=phase, label=operation.label, related_to_operation=True, code_sample=code_sample
+        )
 
     def scenario_finished(status: Status) -> events.ScenarioFinished:
         return events.ScenarioFinished(
@@ -111,6 +119,7 @@ def run_test(
         with catch_warnings(record=True) as warnings, ignore_hypothesis_output():
             test_function(
                 ctx=ctx,
+                state=state,
                 errors=errors,
                 check_ctx=check_ctx,
                 recorder=recorder,
@@ -198,6 +207,7 @@ def run_test(
         yield non_fatal_error(InvalidRegexPattern.from_schema_error(exc, from_examples=False))
     except Exception as exc:
         status = Status.ERROR
+        clear_hypothesis_notes(exc)
         # Likely a YAML parsing issue. E.g. `00:00:00.00` (without quotes) is parsed as float `0.0`
         if str(exc) == "first argument must be string or compiled pattern":
             yield non_fatal_error(
@@ -207,7 +217,10 @@ def run_test(
                 )
             )
         else:
-            yield non_fatal_error(exc)
+            code_sample: str | None = None
+            if state.unrecoverable_network_error is not None and state.unrecoverable_network_error.error is exc:
+                code_sample = state.unrecoverable_network_error.code_sample
+            yield non_fatal_error(exc, code_sample=code_sample)
     if (
         status == Status.SUCCESS
         and continue_on_failure
@@ -270,6 +283,7 @@ def cached_test_func(f: Callable) -> Callable:
     def wrapped(
         *,
         ctx: EngineContext,
+        state: TestingState,
         case: Case,
         errors: list[Exception],
         check_ctx: CheckContext,
@@ -313,6 +327,21 @@ def cached_test_func(f: Callable) -> Callable:
         except (KeyboardInterrupt, Failure):
             raise
         except Exception as exc:
+            if isinstance(exc, (requests.ConnectionError, ChunkedEncodingError)) and is_unrecoverable_network_error(
+                exc
+            ):
+                # Server likely has crashed and does not accept any connections at all
+                # Don't report these error - only the original crash should be reported
+                if exc.request is not None:
+                    headers = {key: value[0] for key, value in exc.request.headers.items()}
+                else:
+                    headers = {**dict(case.headers or {}), **transport_kwargs.get("headers", {})}
+                verify = transport_kwargs.get("verify", True)
+                state.unrecoverable_network_error = UnrecoverableNetworkError(
+                    error=exc,
+                    code_sample=case.as_curl_command(headers=headers, verify=verify),
+                )
+                raise
             errors.append(exc)
             raise UnexpectedError from None
 
@@ -334,14 +363,14 @@ def test_func(
     recorder.record_case(parent_id=None, transition=None, case=case)
     try:
         response = case.call(**transport_kwargs)
-    except (requests.Timeout, requests.ConnectionError) as error:
+    except (requests.Timeout, requests.ConnectionError, ChunkedEncodingError) as error:
         if isinstance(error.request, requests.Request):
             recorder.record_request(case_id=case.id, request=error.request.prepare())
         elif isinstance(error.request, requests.PreparedRequest):
             recorder.record_request(case_id=case.id, request=error.request)
         raise
     recorder.record_response(case_id=case.id, response=response)
-    targets.run(generation.maximize, case=case, response=response)
+    metrics.maximize(generation.maximize, case=case, response=response)
     validate_response(
         case=case,
         ctx=check_ctx,
@@ -378,7 +407,7 @@ def validate_response(
         case=case,
         response=response,
         ctx=ctx,
-        checks=ctx.checks,
+        checks=ctx._checks,
         on_failure=on_failure,
         on_success=on_success,
     )

schemathesis/generation/__init__.py

@@ -10,9 +10,6 @@ __all__ = [
 ]
 
 
-DEFAULT_GENERATOR_MODES = [GenerationMode.default()]
-
-
 CASE_ID_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
 BASE = len(CASE_ID_ALPHABET)
 # Separate `Random` as Hypothesis might interfere with the default one

schemathesis/generation/case.py

@@ -167,6 +167,7 @@ class Case:
                 curl=curl,
                 config=self.operation.schema.config.output,
             )
+            message += "\n\n"
             raise FailureGroup(_failures, message) from None
 
     def call_and_validate(

schemathesis/generation/coverage.py

@@ -121,7 +121,7 @@ class CoverageContext:
         path: list[str | int] | None = None,
     ) -> None:
         self.location = location
-        self.generation_modes = generation_modes if generation_modes is not None else GenerationMode.all()
+        self.generation_modes = generation_modes if generation_modes is not None else list(GenerationMode)
         self.path = path or []
 
     @contextmanager

schemathesis/generation/hypothesis/builder.py

@@ -9,7 +9,7 @@ from time import perf_counter
 from typing import Any, Callable, Generator, Mapping
 
 import hypothesis
-from hypothesis import Phase
+from hypothesis import Phase, Verbosity
 from hypothesis import strategies as st
 from hypothesis._settings import all_settings
 from hypothesis.errors import Unsatisfiable
@@ -27,6 +27,7 @@ from schemathesis.core.validation import has_invalid_characters, is_latin_1_enco
 from schemathesis.generation import GenerationMode, coverage
 from schemathesis.generation.case import Case
 from schemathesis.generation.hypothesis import DEFAULT_DEADLINE, examples, setup, strategies
+from schemathesis.generation.hypothesis.examples import add_single_example
 from schemathesis.generation.hypothesis.given import GivenInput
 from schemathesis.generation.meta import (
     CaseMetadata,
@@ -96,6 +97,9 @@ def create_test(
     if settings.deadline == default.deadline:
         settings = hypothesis.settings(settings, deadline=DEFAULT_DEADLINE)
 
+    if settings.verbosity == default.verbosity:
+        settings = hypothesis.settings(settings, verbosity=Verbosity.quiet)
+
     if config.settings is not None:
         # Merge the user-provided settings with the current ones
         settings = hypothesis.settings(
@@ -126,7 +130,14 @@ def create_test(
         and Phase.explicit in settings.phases
         and specification.supports_feature(SpecificationFeature.EXAMPLES)
     ):
-        hypothesis_test = add_examples(hypothesis_test, operation, hook_dispatcher=hook_dispatcher, **strategy_kwargs)
+        phases_config = config.project.phases_for(operation=operation)
+        hypothesis_test = add_examples(
+            hypothesis_test,
+            operation,
+            fill_missing=phases_config.examples.fill_missing,
+            hook_dispatcher=hook_dispatcher,
+            **strategy_kwargs,
+        )
 
     if (
         HypothesisTestMode.COVERAGE in config.modes
@@ -142,7 +153,8 @@ def create_test(
             generation.modes,
             auth_storage,
             config.as_strategy_kwargs,
-            phases_config.coverage.unexpected_methods,
+            generate_duplicate_query_parameters=phases_config.coverage.generate_duplicate_query_parameters,
+            unexpected_methods=phases_config.coverage.unexpected_methods,
         )
 
     setattr(hypothesis_test, SETTINGS_ATTRIBUTE_NAME, settings)
@@ -188,7 +200,11 @@ def make_async_test(test: Callable) -> Callable:
 
 
 def add_examples(
-    test: Callable, operation: APIOperation, hook_dispatcher: HookDispatcher | None = None, **kwargs: Any
+    test: Callable,
+    operation: APIOperation,
+    fill_missing: bool,
+    hook_dispatcher: HookDispatcher | None = None,
+    **kwargs: Any,
 ) -> Callable:
     """Add examples to the Hypothesis test, if they are specified in the schema."""
     from hypothesis_jsonschema._canonicalise import HypothesisRefResolutionError
@@ -212,7 +228,11 @@ def add_examples(
         if isinstance(exc, SchemaError):
             InvalidRegexMark.set(test, exc)
 
-    context = HookContext(operation)  # context should be passed here instead
+    if fill_missing and not result:
+        strategy = operation.as_strategy()
+        add_single_example(strategy, result)
+
+    context = HookContext(operation=operation)  # context should be passed here instead
     GLOBAL_HOOK_DISPATCHER.dispatch("before_add_examples", context, result)
     operation.schema.hooks.dispatch("before_add_examples", context, result)
     if hook_dispatcher:
@@ -246,6 +266,7 @@ def add_coverage(
     generation_modes: list[GenerationMode],
     auth_storage: AuthStorage | None,
     as_strategy_kwargs: dict[str, Any],
+    generate_duplicate_query_parameters: bool,
     unexpected_methods: set[str] | None = None,
 ) -> Callable:
     from schemathesis.specs.openapi.constants import LOCATION_TO_CONTAINER
@@ -259,7 +280,9 @@ def add_coverage(
         for container in LOCATION_TO_CONTAINER.values()
         if container in as_strategy_kwargs
     }
-    for case in _iter_coverage_cases(operation, generation_modes, unexpected_methods):
+    for case in _iter_coverage_cases(
+        operation, generation_modes, generate_duplicate_query_parameters, unexpected_methods
+    ):
         if case.media_type and operation.schema.transport.get_first_matching_media_type(case.media_type) is None:
             continue
         adjust_urlencoded_payload(case)
@@ -396,6 +419,7 @@ def _stringify_value(val: Any, container_name: str) -> Any:
 def _iter_coverage_cases(
     operation: APIOperation,
     generation_modes: list[GenerationMode],
+    generate_duplicate_query_parameters: bool,
     unexpected_methods: set[str] | None = None,
 ) -> Generator[Case, None, None]:
     from schemathesis.specs.openapi.constants import LOCATION_TO_CONTAINER
@@ -549,7 +573,7 @@ def _iter_coverage_cases(
                 ),
             )
         # Generate duplicate query parameters
-        if operation.query:
+        if generate_duplicate_query_parameters and operation.query:
             container = template["query"]
             for parameter in operation.query:
                 instant = Instant()

schemathesis/generation/metrics.py

@@ -0,0 +1,93 @@
+"""Support for Targeted Property-Based Testing."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable, Sequence
+
+from schemathesis.core.registries import Registry
+from schemathesis.core.transport import Response
+from schemathesis.generation.case import Case
+
+
+@dataclass
+class MetricContext:
+    """Context for evaluating a metric on a single test execution.
+
+    This object bundles together the test `case` that was sent and
+    the corresponding HTTP `response`. Metric functions receive an
+    instance of `MetricContext` to compute a numeric score.
+    """
+
+    case: Case
+    """Generated test case."""
+    response: Response
+    """The HTTP response returned by the server for this test case."""
+
+    __slots__ = ("case", "response")
+
+
+MetricFunction = Callable[[MetricContext], float]
+
+METRICS = Registry[MetricFunction]()
+
+
+def metric(func: MetricFunction) -> MetricFunction:
+    """Decorator to register a custom metric for targeted property-based testing.
+
+    Example:
+        ```python
+        import schemathesis
+
+        @schemathesis.metric
+        def response_size(ctx: schemathesis.MetricContext) -> float:
+            return float(len(ctx.response.content))
+        ```
+
+    """
+    return METRICS.register(func)
+
+
+@metric
+def response_time(ctx: MetricContext) -> float:
+    """Response time as a metric to maximize."""
+    return ctx.response.elapsed
+
+
+class MetricCollector:
+    """Collect multiple observations for metrics."""
+
+    __slots__ = ("metrics", "observations")
+
+    def __init__(self, metrics: list[MetricFunction] | None = None) -> None:
+        self.metrics = metrics or []
+        self.observations: dict[str, list[float]] = {metric.__name__: [] for metric in self.metrics}
+
+    def reset(self) -> None:
+        """Reset all collected observations."""
+        for metric in self.metrics:
+            self.observations[metric.__name__].clear()
+
+    def store(self, case: Case, response: Response) -> None:
+        """Calculate metrics & store them."""
+        ctx = MetricContext(case=case, response=response)
+        for metric in self.metrics:
+            self.observations[metric.__name__].append(metric(ctx))
+
+    def maximize(self) -> None:
+        """Give feedback to the Hypothesis engine, so it maximizes the aggregated metrics."""
+        import hypothesis
+
+        for metric in self.metrics:
+            # Currently aggregation is just a sum
+            value = sum(self.observations[metric.__name__])
+            hypothesis.target(value, label=metric.__name__)
+
+
+def maximize(metrics: Sequence[MetricFunction], case: Case, response: Response) -> None:
+    import hypothesis
+
+    ctx = MetricContext(case=case, response=response)
+    for metric in metrics:
+        value = metric(ctx)
+        hypothesis.target(value, label=metric.__name__)

schemathesis/generation/modes.py

@@ -11,14 +11,6 @@ class GenerationMode(str, Enum):
     # Doesn't fit the API schema
     NEGATIVE = "negative"
 
-    @classmethod
-    def default(cls) -> GenerationMode:
-        return cls.POSITIVE
-
-    @classmethod
-    def all(cls) -> list[GenerationMode]:
-        return list(GenerationMode)
-
     @property
     def is_positive(self) -> bool:
         return self == GenerationMode.POSITIVE

schemathesis/generation/stateful/__init__.py

@@ -7,6 +7,8 @@ if TYPE_CHECKING:
 
     from schemathesis.generation.stateful.state_machine import APIStateMachine
 
+STATEFUL_TESTS_LABEL = "Stateful tests"
+
 
 def run_state_machine_as_test(
     state_machine_factory: type[APIStateMachine], *, settings: hypothesis.settings | None = None
@@ -17,4 +19,6 @@ def run_state_machine_as_test(
     """
     from hypothesis.stateful import run_state_machine_as_test as _run_state_machine_as_test
 
+    __tracebackhide__ = True
+
     return _run_state_machine_as_test(state_machine_factory, settings=settings, _min_steps=2)

schemathesis/generation/stateful/state_machine.py

@@ -196,6 +196,7 @@ class APIStateMachine(RuleBasedStateMachine):
         """Run state machine as a test."""
         from . import run_state_machine_as_test
 
+        __tracebackhide__ = True
         return run_state_machine_as_test(cls, settings=settings)
 
     def setup(self) -> None: