schemathesis 4.0.0a11__py3-none-any.whl → 4.0.0b1__py3-none-any.whl

schemathesis/engine/errors.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 
 import enum
 import re
+from dataclasses import dataclass
 from functools import cached_property
 from typing import TYPE_CHECKING, Callable, Iterator, Sequence, cast
 
@@ -24,6 +25,8 @@ from schemathesis.core.errors import (
 
 if TYPE_CHECKING:
     import hypothesis.errors
+    import requests
+    from requests.exceptions import ChunkedEncodingError
 
 __all__ = ["EngineErrorInfo", "DeadlineExceeded", "UnsupportedRecursiveReference", "UnexpectedError"]
 
@@ -61,8 +64,9 @@ class EngineErrorInfo:
     It serves as a caching wrapper around exceptions to avoid repeated computations.
     """
 
-    def __init__(self, error: Exception) -> None:
+    def __init__(self, error: Exception, code_sample: str | None = None) -> None:
         self._error = error
+        self._code_sample = code_sample
 
     def __str__(self) -> str:
         return self._error_repr
@@ -212,6 +216,9 @@ class EngineErrorInfo:
             message.append("")  # Empty line before extras
             message.extend(f"{indent}{extra}" for extra in extras)
 
+        if self._code_sample is not None:
+            message.append(f"\nReproduce with: \n\n    {self._code_sample}")
+
         # Suggestion
         suggestion = get_runtime_error_suggestion(self._kind, bold=bold)
         if suggestion is not None:
@@ -254,7 +261,7 @@ def get_runtime_error_suggestion(error_type: RuntimeErrorKind, bold: Callable[[s
         RuntimeErrorKind.SCHEMA_INVALID_REGULAR_EXPRESSION: "Ensure your regex is compatible with Python's syntax.\n"
         "For guidance, visit: https://docs.python.org/3/library/re.html",
         RuntimeErrorKind.HYPOTHESIS_UNSUPPORTED_GRAPHQL_SCALAR: "Define a custom strategy for it.\n"
-        "For guidance, visit: https://schemathesis.readthedocs.io/en/stable/graphql.html#custom-scalars",
+        "For guidance, visit: https://schemathesis.readthedocs.io/en/latest/guides/graphql-custom-scalars/",
         RuntimeErrorKind.HYPOTHESIS_HEALTH_CHECK_DATA_TOO_LARGE: _format_health_check_suggestion("data_too_large"),
         RuntimeErrorKind.HYPOTHESIS_HEALTH_CHECK_FILTER_TOO_MUCH: _format_health_check_suggestion("filter_too_much"),
         RuntimeErrorKind.HYPOTHESIS_HEALTH_CHECK_TOO_SLOW: _format_health_check_suggestion("too_slow"),
@@ -403,3 +410,55 @@ def canonicalize_error_message(error: Exception, with_traceback: bool = True) ->
     message = MEMORY_ADDRESS_RE.sub("0xbaaaaaaaaaad", message)
     # Remove URL information
     return URL_IN_ERROR_MESSAGE_RE.sub("", message)
+
+
+def clear_hypothesis_notes(exc: Exception) -> None:
+    notes = getattr(exc, "__notes__", [])
+    if any("while generating" in note for note in notes):
+        notes.clear()
+
+
+def is_unrecoverable_network_error(exc: Exception) -> bool:
+    from http.client import RemoteDisconnected
+
+    from urllib3.exceptions import ProtocolError
+
+    def has_connection_reset(inner: BaseException) -> bool:
+        exc_str = str(inner)
+        if any(pattern in exc_str for pattern in ["Connection reset by peer", "[Errno 104]", "ECONNRESET"]):
+            return True
+
+        if inner.__context__ is not None:
+            return has_connection_reset(inner.__context__)
+
+        return False
+
+    if isinstance(exc.__context__, ProtocolError):
+        if len(exc.__context__.args) == 2 and isinstance(exc.__context__.args[1], RemoteDisconnected):
+            return True
+        if len(exc.__context__.args) == 1 and exc.__context__.args[0] == "Response ended prematurely":
+            return True
+
+    return has_connection_reset(exc)
+
+
+@dataclass()
+class UnrecoverableNetworkError:
+    error: requests.ConnectionError | ChunkedEncodingError
+    code_sample: str
+
+    __slots__ = ("error", "code_sample")
+
+    def __init__(self, error: requests.ConnectionError | ChunkedEncodingError, code_sample: str) -> None:
+        self.error = error
+        self.code_sample = code_sample
+
+
+@dataclass
+class TestingState:
+    unrecoverable_network_error: UnrecoverableNetworkError | None
+
+    __slots__ = ("unrecoverable_network_error",)
+
+    def __init__(self) -> None:
+        self.unrecoverable_network_error = None

schemathesis/engine/events.py

@@ -200,10 +200,18 @@ class NonFatalError(EngineEvent):
 
     __slots__ = ("id", "timestamp", "info", "value", "phase", "label", "related_to_operation")
 
-    def __init__(self, *, error: Exception, phase: PhaseName, label: str, related_to_operation: bool) -> None:
+    def __init__(
+        self,
+        *,
+        error: Exception,
+        phase: PhaseName,
+        label: str,
+        related_to_operation: bool,
+        code_sample: str | None = None,
+    ) -> None:
         self.id = uuid.uuid4()
         self.timestamp = time.time()
-        self.info = EngineErrorInfo(error=error)
+        self.info = EngineErrorInfo(error=error, code_sample=code_sample)
         self.value = error
         self.phase = phase
         self.label = label

schemathesis/engine/phases/probes.py

@@ -16,6 +16,7 @@ from typing import TYPE_CHECKING
 from schemathesis.core.result import Err, Ok, Result
 from schemathesis.core.transport import USER_AGENT
 from schemathesis.engine import Status, events
+from schemathesis.transport.prepare import get_default_headers
 
 if TYPE_CHECKING:
     import requests
@@ -134,6 +135,8 @@ def send(probe: Probe, ctx: EngineContext) -> ProbeRun:
         request = probe.prepare_request(session, Request(), ctx.schema)
         request.headers[HEADER_NAME] = probe.name
         request.headers["User-Agent"] = USER_AGENT
+        for header, value in get_default_headers().items():
+            request.headers.setdefault(header, value)
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", InsecureRequestWarning)
             response = session.send(request, timeout=ctx.config.request_timeout or 2)

schemathesis/engine/phases/stateful/__init__.py

@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING
 
 from schemathesis.engine import Status, events
 from schemathesis.engine.phases import Phase, PhaseName, PhaseSkipReason
+from schemathesis.generation.stateful import STATEFUL_TESTS_LABEL
 
 if TYPE_CHECKING:
     from schemathesis.engine.context import EngineContext
@@ -19,7 +20,7 @@ def execute(engine: EngineContext, phase: Phase) -> events.EventGenerator:
     try:
         state_machine = engine.schema.as_state_machine()
     except Exception as exc:
-        yield events.NonFatalError(error=exc, phase=phase.name, label="Stateful tests", related_to_operation=False)
+        yield events.NonFatalError(error=exc, phase=phase.name, label=STATEFUL_TESTS_LABEL, related_to_operation=False)
         yield events.PhaseFinished(phase=phase, status=Status.ERROR, payload=None)
         return
 

schemathesis/engine/phases/stateful/_executor.py

@@ -1,4 +1,4 @@
-from __future__ import annotations
+from __future__ import annotations  # noqa: I001
 
 import queue
 import time
@@ -8,9 +8,11 @@ from typing import Any
 from warnings import catch_warnings
 
 import hypothesis
+import requests
 from hypothesis.control import current_build_context
 from hypothesis.errors import Flaky, Unsatisfiable
 from hypothesis.stateful import Rule
+from requests.exceptions import ChunkedEncodingError
 from requests.structures import CaseInsensitiveDict
 
 from schemathesis.checks import CheckContext, CheckFunction, run_checks
@@ -19,19 +21,26 @@ from schemathesis.core.transport import Response
 from schemathesis.engine import Status, events
 from schemathesis.engine.context import EngineContext
 from schemathesis.engine.control import ExecutionControl
+from schemathesis.engine.errors import (
+    TestingState,
+    UnrecoverableNetworkError,
+    clear_hypothesis_notes,
+    is_unrecoverable_network_error,
+)
 from schemathesis.engine.phases import PhaseName
 from schemathesis.engine.phases.stateful.context import StatefulContext
 from schemathesis.engine.recorder import ScenarioRecorder
 from schemathesis.generation import overrides
 from schemathesis.generation.case import Case
 from schemathesis.generation.hypothesis.reporting import ignore_hypothesis_output
+from schemathesis.generation.stateful import STATEFUL_TESTS_LABEL
 from schemathesis.generation.stateful.state_machine import (
     DEFAULT_STATE_MACHINE_SETTINGS,
     APIStateMachine,
     StepInput,
     StepOutput,
 )
-from schemathesis.generation.targets import TargetMetricCollector
+from schemathesis.generation.metrics import MetricCollector
 
 
 def _get_hypothesis_settings_kwargs_override(settings: hypothesis.settings) -> dict[str, Any]:
@@ -72,7 +81,8 @@ def execute_state_machine_loop(
     hypothesis_settings = hypothesis.settings(configured_hypothesis_settings, **kwargs)
     generation = engine.config.generation_for(phase="stateful")
 
-    ctx = StatefulContext(metric_collector=TargetMetricCollector(targets=generation.maximize))
+    ctx = StatefulContext(metric_collector=MetricCollector(metrics=generation.maximize))
+    state = TestingState()
 
     # Caches for validate_response to avoid repeated config lookups per operation
     _check_context_cache: dict[str, CachedCheckContextData] = {}
@@ -123,6 +133,20 @@ def execute_state_machine_loop(
                 ctx.step_failed()
                 raise
             except Exception as exc:
+                if isinstance(exc, (requests.ConnectionError, ChunkedEncodingError)) and is_unrecoverable_network_error(
+                    exc
+                ):
+                    transport_kwargs = engine.get_transport_kwargs(operation=input.case.operation)
+                    if exc.request is not None:
+                        headers = {key: value[0] for key, value in exc.request.headers.items()}
+                    else:
+                        headers = {**dict(input.case.headers or {}), **transport_kwargs.get("headers", {})}
+                    verify = transport_kwargs.get("verify", True)
+                    state.unrecoverable_network_error = UnrecoverableNetworkError(
+                        error=exc,
+                        code_sample=input.case.as_curl_command(headers=headers, verify=verify),
+                    )
+
                 if generation.unique_inputs:
                     ctx.store_step_outcome(input.case, exc)
                 ctx.step_errored()
@@ -172,7 +196,7 @@ def execute_state_machine_loop(
                 case=case,
                 stateful_ctx=ctx,
                 check_ctx=check_ctx,
-                checks=check_ctx.checks,
+                checks=check_ctx._checks,
                 control=engine.control,
                 recorder=self.recorder,
                 additional_checks=additional_checks,
@@ -259,11 +283,20 @@ def execute_state_machine_loop(
                     # Avoid infinite restarts
                     break
                 continue
+            clear_hypothesis_notes(exc)
             # Any other exception is an inner error and the test run should be stopped
             suite_status = Status.ERROR
+            code_sample: str | None = None
+            if state.unrecoverable_network_error is not None:
+                exc = state.unrecoverable_network_error.error
+                code_sample = state.unrecoverable_network_error.code_sample
             event_queue.put(
                 events.NonFatalError(
-                    error=exc, phase=PhaseName.STATEFUL_TESTING, label="Stateful tests", related_to_operation=False
+                    error=exc,
+                    phase=PhaseName.STATEFUL_TESTING,
+                    label=STATEFUL_TESTS_LABEL,
+                    related_to_operation=False,
+                    code_sample=code_sample,
                 )
             )
             break

schemathesis/engine/phases/stateful/context.py

@@ -7,7 +7,7 @@ from schemathesis.core.failures import Failure
 from schemathesis.core.transport import Response
 from schemathesis.engine import Status
 from schemathesis.generation.case import Case
-from schemathesis.generation.targets import TargetMetricCollector
+from schemathesis.generation.metrics import MetricCollector
 
 
 @dataclass
@@ -27,7 +27,7 @@ class StatefulContext:
     # The total number of completed test scenario
     completed_scenarios: int = 0
     # Metrics collector for targeted testing
-    metric_collector: TargetMetricCollector = field(default_factory=TargetMetricCollector)
+    metric_collector: MetricCollector = field(default_factory=MetricCollector)
     step_outcomes: dict[int, BaseException | None] = field(default_factory=dict)
 
     @property

schemathesis/engine/phases/unit/_executor.py

@@ -11,6 +11,7 @@ from hypothesis.errors import InvalidArgument
 from hypothesis_jsonschema._canonicalise import HypothesisRefResolutionError
 from jsonschema.exceptions import SchemaError as JsonSchemaError
 from jsonschema.exceptions import ValidationError
+from requests.exceptions import ChunkedEncodingError
 from requests.structures import CaseInsensitiveDict
 
 from schemathesis.checks import CheckContext, run_checks
@@ -33,13 +34,17 @@ from schemathesis.engine import Status, events
 from schemathesis.engine.context import EngineContext
 from schemathesis.engine.errors import (
     DeadlineExceeded,
+    TestingState,
     UnexpectedError,
+    UnrecoverableNetworkError,
     UnsupportedRecursiveReference,
+    clear_hypothesis_notes,
     deduplicate_errors,
+    is_unrecoverable_network_error,
 )
 from schemathesis.engine.phases import PhaseName
 from schemathesis.engine.recorder import ScenarioRecorder
-from schemathesis.generation import overrides, targets
+from schemathesis.generation import metrics, overrides
 from schemathesis.generation.case import Case
 from schemathesis.generation.hypothesis.builder import (
     InvalidHeadersExampleMark,
@@ -70,9 +75,12 @@ def run_test(
     skip_reason = None
     test_start_time = time.monotonic()
     recorder = ScenarioRecorder(label=operation.label)
+    state = TestingState()
 
-    def non_fatal_error(error: Exception) -> events.NonFatalError:
-        return events.NonFatalError(error=error, phase=phase, label=operation.label, related_to_operation=True)
+    def non_fatal_error(error: Exception, code_sample: str | None = None) -> events.NonFatalError:
+        return events.NonFatalError(
+            error=error, phase=phase, label=operation.label, related_to_operation=True, code_sample=code_sample
+        )
 
     def scenario_finished(status: Status) -> events.ScenarioFinished:
         return events.ScenarioFinished(
@@ -111,6 +119,7 @@ def run_test(
         with catch_warnings(record=True) as warnings, ignore_hypothesis_output():
             test_function(
                 ctx=ctx,
+                state=state,
                 errors=errors,
                 check_ctx=check_ctx,
                 recorder=recorder,
@@ -198,6 +207,7 @@ def run_test(
         yield non_fatal_error(InvalidRegexPattern.from_schema_error(exc, from_examples=False))
     except Exception as exc:
         status = Status.ERROR
+        clear_hypothesis_notes(exc)
         # Likely a YAML parsing issue. E.g. `00:00:00.00` (without quotes) is parsed as float `0.0`
         if str(exc) == "first argument must be string or compiled pattern":
             yield non_fatal_error(
@@ -207,7 +217,10 @@ def run_test(
                 )
             )
         else:
-            yield non_fatal_error(exc)
+            code_sample: str | None = None
+            if state.unrecoverable_network_error is not None and state.unrecoverable_network_error.error is exc:
+                code_sample = state.unrecoverable_network_error.code_sample
+            yield non_fatal_error(exc, code_sample=code_sample)
     if (
         status == Status.SUCCESS
         and continue_on_failure
@@ -270,6 +283,7 @@ def cached_test_func(f: Callable) -> Callable:
     def wrapped(
         *,
         ctx: EngineContext,
+        state: TestingState,
         case: Case,
         errors: list[Exception],
         check_ctx: CheckContext,
@@ -313,6 +327,21 @@ def cached_test_func(f: Callable) -> Callable:
         except (KeyboardInterrupt, Failure):
             raise
         except Exception as exc:
+            if isinstance(exc, (requests.ConnectionError, ChunkedEncodingError)) and is_unrecoverable_network_error(
+                exc
+            ):
+                # Server likely has crashed and does not accept any connections at all
+                # Don't report these error - only the original crash should be reported
+                if exc.request is not None:
+                    headers = {key: value[0] for key, value in exc.request.headers.items()}
+                else:
+                    headers = {**dict(case.headers or {}), **transport_kwargs.get("headers", {})}
+                verify = transport_kwargs.get("verify", True)
+                state.unrecoverable_network_error = UnrecoverableNetworkError(
+                    error=exc,
+                    code_sample=case.as_curl_command(headers=headers, verify=verify),
+                )
+                raise
             errors.append(exc)
             raise UnexpectedError from None
 
@@ -334,14 +363,14 @@ def test_func(
     recorder.record_case(parent_id=None, transition=None, case=case)
     try:
         response = case.call(**transport_kwargs)
-    except (requests.Timeout, requests.ConnectionError) as error:
+    except (requests.Timeout, requests.ConnectionError, ChunkedEncodingError) as error:
         if isinstance(error.request, requests.Request):
             recorder.record_request(case_id=case.id, request=error.request.prepare())
         elif isinstance(error.request, requests.PreparedRequest):
             recorder.record_request(case_id=case.id, request=error.request)
         raise
     recorder.record_response(case_id=case.id, response=response)
-    targets.run(generation.maximize, case=case, response=response)
+    metrics.maximize(generation.maximize, case=case, response=response)
     validate_response(
         case=case,
         ctx=check_ctx,
@@ -378,7 +407,7 @@ def validate_response(
         case=case,
         response=response,
         ctx=ctx,
-        checks=ctx.checks,
+        checks=ctx._checks,
         on_failure=on_failure,
         on_success=on_success,
     )

schemathesis/generation/__init__.py

@@ -10,9 +10,6 @@ __all__ = [
 ]
 
 
-DEFAULT_GENERATOR_MODES = [GenerationMode.default()]
-
-
 CASE_ID_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
 BASE = len(CASE_ID_ALPHABET)
 # Separate `Random` as Hypothesis might interfere with the default one