schemathesis 3.25.5__py3-none-any.whl → 3.39.7__py3-none-any.whl

schemathesis/runner/impl/core.py

@@ -1,14 +1,17 @@
 from __future__ import annotations
+
+import functools
 import logging
+import operator
 import re
 import threading
 import time
 import unittest
 import uuid
+import warnings
 from contextlib import contextmanager
 from dataclasses import dataclass, field
-from types import TracebackType
-from typing import Any, Callable, Generator, Iterable, cast, TYPE_CHECKING, Literal
+from typing import TYPE_CHECKING, Any, Callable, Generator, Iterable, List, Literal, cast
 from warnings import WarningMessage, catch_warnings
 
 import hypothesis
@@ -16,53 +19,74 @@ import requests
 from _pytest.logging import LogCaptureHandler, catching_logs
 from hypothesis.errors import HypothesisException, InvalidArgument
 from hypothesis_jsonschema._canonicalise import HypothesisRefResolutionError
-from jsonschema.exceptions import ValidationError, SchemaError as JsonSchemaError
-from requests.auth import HTTPDigestAuth, _basic_auth_str
+from jsonschema.exceptions import SchemaError as JsonSchemaError
+from jsonschema.exceptions import ValidationError
+from requests.structures import CaseInsensitiveDict
+from urllib3.exceptions import InsecureRequestWarning
 
-from ..._override import CaseOverride
-from ... import failures, hooks
+from ... import experimental, failures, hooks
 from ..._compat import MultipleFailures
 from ..._hypothesis import (
-    has_unsatisfied_example_mark,
-    get_non_serializable_mark,
-    get_invalid_regex_mark,
     get_invalid_example_headers_mark,
+    get_invalid_regex_mark,
+    get_non_serializable_mark,
+    has_unsatisfied_example_mark,
 )
 from ...auths import unregister as unregister_auth
-from ...generation import DataGenerationMethod, GenerationConfig
+from ...checks import _make_max_response_time_failure_message
 from ...constants import (
     DEFAULT_STATEFUL_RECURSION_LIMIT,
     RECURSIVE_REFERENCE_ERROR_MESSAGE,
-    USER_AGENT,
     SERIALIZERS_SUGGESTION_MESSAGE,
+    USER_AGENT,
 )
 from ...exceptions import (
     CheckFailed,
     DeadlineExceeded,
+    InternalError,
+    InvalidHeadersExample,
     InvalidRegularExpression,
     NonCheckError,
     OperationSchemaError,
+    RecursiveReferenceError,
+    SerializationNotPossible,
     SkipTest,
+    format_exception,
     get_grouped_exception,
     maybe_set_assertion_message,
-    format_exception,
-    SerializationNotPossible,
-    InvalidHeadersExample,
 )
+from ...generation import DataGenerationMethod, GenerationConfig
 from ...hooks import HookContext, get_all_by_name
-from ...internal.result import Ok
-from ...models import APIOperation, Case, Check, CheckFunction, Status, TestResult, TestResultSet
-from ...runner import events
+from ...internal.checks import CheckConfig, CheckContext
 from ...internal.datetime import current_datetime
-from ...schemas import BaseSchema
+from ...internal.result import Err, Ok, Result
+from ...models import APIOperation, Case, Check, Status, TestResult
+from ...runner import events
+from ...service import extensions
+from ...service.models import AnalysisResult, AnalysisSuccess
+from ...specs.openapi import formats
 from ...stateful import Feedback, Stateful
+from ...stateful import events as stateful_events
+from ...stateful import runner as stateful_runner
 from ...targets import Target, TargetContext
-from ...types import RawAuth, RequestCert
+from ...transports import RequestConfig, RequestsTransport
+from ...transports.auth import get_requests_auth, prepare_wsgi_headers
 from ...utils import capture_hypothesis_output
+from .. import probes
 from ..serialization import SerializedTestResult
+from .context import RunnerContext
 
 if TYPE_CHECKING:
-    from ...transports.responses import WSGIResponse, GenericResponse
+    from types import TracebackType
+
+    from requests.auth import HTTPDigestAuth
+
+    from ..._override import CaseOverride
+    from ...internal.checks import CheckFunction
+    from ...schemas import BaseSchema
+    from ...service.client import ServiceClient
+    from ...transports.responses import GenericResponse, WSGIResponse
+    from ...types import RawAuth
 
 
 def _should_count_towards_stop(event: events.ExecutionEvent) -> bool:
@@ -76,23 +100,29 @@ class BaseRunner:
     max_response_time: int | None
     targets: Iterable[Target]
     hypothesis_settings: hypothesis.settings
-    generation_config: GenerationConfig
+    generation_config: GenerationConfig | None
+    probe_config: probes.ProbeConfig
+    checks_config: CheckConfig
+    request_config: RequestConfig = field(default_factory=RequestConfig)
     override: CaseOverride | None = None
     auth: RawAuth | None = None
     auth_type: str | None = None
     headers: dict[str, Any] | None = None
-    request_timeout: int | None = None
     store_interactions: bool = False
     seed: int | None = None
     exit_first: bool = False
+    no_failfast: bool = False
     max_failures: int | None = None
     started_at: str = field(default_factory=current_datetime)
+    unique_data: bool = False
     dry_run: bool = False
     stateful: Stateful | None = None
     stateful_recursion_limit: int = DEFAULT_STATEFUL_RECURSION_LIMIT
     count_operations: bool = True
     count_links: bool = True
+    service_client: ServiceClient | None = None
     _failures_counter: int = 0
+    _is_stopping_due_to_failure_limit: bool = False
 
     def execute(self) -> EventStream:
         """Common logic for all runners."""
@@ -103,35 +133,106 @@ class BaseRunner:
         # If auth is explicitly provided, then the global provider is ignored
         if self.auth is not None:
             unregister_auth()
-        results = TestResultSet(seed=self.seed)
-
-        initialized = events.Initialized.from_schema(
-            schema=self.schema, count_operations=self.count_operations, count_links=self.count_links, seed=self.seed
+        ctx = RunnerContext(
+            auth=self.auth,
+            seed=self.seed,
+            stop_event=stop_event,
+            unique_data=self.unique_data,
+            checks_config=self.checks_config,
+            override=self.override,
+            no_failfast=self.no_failfast,
         )
+        start_time = time.monotonic()
+        initialized = None
+        __probes = None
+        __analysis: Result[AnalysisResult, Exception] | None = None
+
+        def _initialize() -> events.Initialized:
+            nonlocal initialized
+            initialized = events.Initialized.from_schema(
+                schema=self.schema,
+                count_operations=self.count_operations,
+                count_links=self.count_links,
+                seed=ctx.seed,
+                start_time=start_time,
+            )
+            return initialized
 
         def _finish() -> events.Finished:
-            if has_all_not_found(results):
-                results.add_warning(ALL_NOT_FOUND_WARNING_MESSAGE)
-            return events.Finished.from_results(results=results, running_time=time.monotonic() - initialized.start_time)
+            if ctx.has_all_not_found:
+                ctx.add_warning(ALL_NOT_FOUND_WARNING_MESSAGE)
+            return events.Finished.from_results(results=ctx.data, running_time=time.monotonic() - start_time)
 
-        if stop_event.is_set():
-            yield _finish()
-            return
+        def _before_probes() -> events.BeforeProbing:
+            return events.BeforeProbing()
+
+        def _run_probes() -> None:
+            if not self.dry_run:
+                nonlocal __probes
+
+                __probes = run_probes(self.schema, self.probe_config)
 
-        yield initialized
+        def _after_probes() -> events.AfterProbing:
+            _probes = cast(List[probes.ProbeRun], __probes)
+            return events.AfterProbing(probes=_probes)
 
-        if stop_event.is_set():
+        def _before_analysis() -> events.BeforeAnalysis:
+            return events.BeforeAnalysis()
+
+        def _run_analysis() -> None:
+            nonlocal __analysis, __probes
+
+            if self.service_client is not None:
+                try:
+                    _probes = cast(List[probes.ProbeRun], __probes)
+                    result = self.service_client.analyze_schema(_probes, self.schema.raw_schema)
+                    if isinstance(result, AnalysisSuccess):
+                        extensions.apply(result.extensions, self.schema)
+                    __analysis = Ok(result)
+                except Exception as exc:
+                    __analysis = Err(exc)
+
+        def _after_analysis() -> events.AfterAnalysis:
+            return events.AfterAnalysis(analysis=__analysis)
+
+        if ctx.is_stopped:
             yield _finish()
             return
 
+        for event_factory in (
+            _initialize,
+            _before_probes,
+            _run_probes,
+            _after_probes,
+            _before_analysis,
+            _run_analysis,
+            _after_analysis,
+        ):
+            event = event_factory()
+            if event is not None:
+                yield event
+            if ctx.is_stopped:
+                yield _finish()  # type: ignore[unreachable]
+                return
+
         try:
-            yield from self._execute(results, stop_event)
+            warnings.simplefilter("ignore", InsecureRequestWarning)
+            if not experimental.STATEFUL_ONLY.is_enabled:
+                yield from self._execute(ctx)
+            if not self._is_stopping_due_to_failure_limit:
+                yield from self._run_stateful_tests(ctx)
         except KeyboardInterrupt:
             yield events.Interrupted()
 
         yield _finish()
 
     def _should_stop(self, event: events.ExecutionEvent) -> bool:
+        result = self.__should_stop(event)
+        if result:
+            self._is_stopping_due_to_failure_limit = True
+        return result
+
+    def __should_stop(self, event: events.ExecutionEvent) -> bool:
         if _should_count_towards_stop(event):
             if self.exit_first:
                 return True
@@ -140,19 +241,116 @@ class BaseRunner:
                 return self._failures_counter >= self.max_failures
         return False
 
-    def _execute(
-        self, results: TestResultSet, stop_event: threading.Event
-    ) -> Generator[events.ExecutionEvent, None, None]:
+    def _execute(self, ctx: RunnerContext) -> Generator[events.ExecutionEvent, None, None]:
         raise NotImplementedError
 
+    def _run_stateful_tests(self, ctx: RunnerContext) -> Generator[events.ExecutionEvent, None, None]:
+        # Run new-style stateful tests
+        if self.stateful is not None and experimental.STATEFUL_TEST_RUNNER.is_enabled and self.schema.links_count > 0:
+            result = TestResult(
+                method="",
+                path="",
+                verbose_name="Stateful tests",
+                seed=ctx.seed,
+                data_generation_method=self.schema.data_generation_methods,
+            )
+            headers = self.headers or {}
+            if isinstance(self.schema.transport, RequestsTransport):
+                auth = get_requests_auth(self.auth, self.auth_type)
+            else:
+                auth = None
+                headers = prepare_wsgi_headers(headers, self.auth, self.auth_type)
+            config = stateful_runner.StatefulTestRunnerConfig(
+                checks=tuple(self.checks),
+                headers=headers,
+                hypothesis_settings=self.hypothesis_settings,
+                exit_first=self.exit_first,
+                max_failures=None if self.max_failures is None else self.max_failures - self._failures_counter,
+                request=self.request_config,
+                auth=auth,
+                seed=ctx.seed,
+                override=self.override,
+            )
+            state_machine = self.schema.as_state_machine()
+            runner = state_machine.runner(config=config)
+            status = Status.success
+
+            def from_step_status(step_status: stateful_events.StepStatus) -> Status:
+                return {
+                    stateful_events.StepStatus.SUCCESS: Status.success,
+                    stateful_events.StepStatus.FAILURE: Status.failure,
+                    stateful_events.StepStatus.ERROR: Status.error,
+                    stateful_events.StepStatus.INTERRUPTED: Status.error,
+                }[step_status]
+
+            if self.store_interactions:
+                if isinstance(state_machine.schema.transport, RequestsTransport):
+
+                    def on_step_finished(event: stateful_events.StepFinished) -> None:
+                        if event.response is not None and event.status is not None:
+                            response = cast(requests.Response, event.response)
+                            result.store_requests_response(
+                                status=from_step_status(event.status),
+                                case=event.case,
+                                response=response,
+                                checks=event.checks,
+                                headers=headers,
+                                session=None,
+                            )
+
+                else:
+
+                    def on_step_finished(event: stateful_events.StepFinished) -> None:
+                        from ...transports.responses import WSGIResponse
+
+                        if event.response is not None and event.status is not None:
+                            response = cast(WSGIResponse, event.response)
+                            result.store_wsgi_response(
+                                status=from_step_status(event.status),
+                                case=event.case,
+                                response=response,
+                                headers=headers,
+                                elapsed=response.elapsed.total_seconds(),
+                                checks=event.checks,
+                            )
+            else:
+
+                def on_step_finished(event: stateful_events.StepFinished) -> None:
+                    return None
+
+            test_start_time: float | None = None
+            test_elapsed_time: float | None = None
+
+            for stateful_event in runner.execute():
+                if isinstance(stateful_event, stateful_events.SuiteFinished):
+                    if stateful_event.failures and status != Status.error:
+                        status = Status.failure
+                elif isinstance(stateful_event, stateful_events.RunStarted):
+                    test_start_time = stateful_event.timestamp
+                elif isinstance(stateful_event, stateful_events.RunFinished):
+                    test_elapsed_time = stateful_event.timestamp - cast(float, test_start_time)
+                elif isinstance(stateful_event, stateful_events.StepFinished):
+                    result.checks.extend(stateful_event.checks)
+                    on_step_finished(stateful_event)
+                elif isinstance(stateful_event, stateful_events.Errored):
+                    status = Status.error
+                    result.add_error(stateful_event.exception)
+                yield events.StatefulEvent(data=stateful_event)
+            ctx.add_result(result)
+            yield events.AfterStatefulExecution(
+                status=status,
+                result=SerializedTestResult.from_test_result(result),
+                elapsed_time=cast(float, test_elapsed_time),
+                data_generation_method=self.schema.data_generation_methods,
+            )
+
     def _run_tests(
         self,
         maker: Callable,
-        template: Callable,
+        test_func: Callable,
         settings: hypothesis.settings,
-        generation_config: GenerationConfig,
-        seed: int | None,
-        results: TestResultSet,
+        generation_config: GenerationConfig | None,
+        ctx: RunnerContext,
         recursion_level: int = 0,
         headers: dict[str, Any] | None = None,
         **kwargs: Any,
@@ -172,15 +370,18 @@ class BaseRunner:
             return kw
 
         for result in maker(
-            template,
+            test_func,
             settings=settings,
             generation_config=generation_config,
-            seed=seed,
+            seed=ctx.seed,
             as_strategy_kwargs=as_strategy_kwargs,
         ):
             if isinstance(result, Ok):
                 operation, test = result.ok()
-                feedback = Feedback(self.stateful, operation)
+                if self.stateful is not None and not experimental.STATEFUL_TEST_RUNNER.is_enabled:
+                    feedback = Feedback(self.stateful, operation)
+                else:
+                    feedback = None
                 # Track whether `BeforeExecution` was already emitted.
                 # Schema error may happen before / after `BeforeExecution`, but it should be emitted only once
                 # and the `AfterExecution` event should have the same correlation id as previous `BeforeExecution`
@@ -189,7 +390,7 @@ class BaseRunner:
                     for event in run_test(
                         operation,
                         test,
-                        results=results,
+                        ctx=ctx,
                         feedback=feedback,
                         recursion_level=recursion_level,
                         data_generation_methods=self.schema.data_generation_methods,
@@ -202,30 +403,39 @@ class BaseRunner:
                         if isinstance(event, events.Interrupted):
                             return
                     # Additional tests, generated via the `feedback` instance
-                    yield from self._run_tests(
-                        feedback.get_stateful_tests,
-                        template,
-                        settings=settings,
-                        generation_config=generation_config,
-                        seed=seed,
-                        recursion_level=recursion_level + 1,
-                        results=results,
-                        headers=headers,
-                        **kwargs,
-                    )
+                    if feedback is not None:
+                        yield from self._run_tests(
+                            feedback.get_stateful_tests,
+                            test_func,
+                            settings=settings,
+                            generation_config=generation_config,
+                            recursion_level=recursion_level + 1,
+                            ctx=ctx,
+                            headers=headers,
+                            **kwargs,
+                        )
                 except OperationSchemaError as exc:
                     yield from handle_schema_error(
                         exc,
-                        results,
+                        ctx,
                         self.schema.data_generation_methods,
                         recursion_level,
                         before_execution_correlation_id=before_execution_correlation_id,
                     )
             else:
                 # Schema errors
-                yield from handle_schema_error(
-                    result.err(), results, self.schema.data_generation_methods, recursion_level
-                )
+                yield from handle_schema_error(result.err(), ctx, self.schema.data_generation_methods, recursion_level)
+
+
+def run_probes(schema: BaseSchema, config: probes.ProbeConfig) -> list[probes.ProbeRun]:
+    """Discover capabilities of the tested app."""
+    results = probes.run(schema, config)
+    for result in results:
+        if isinstance(result.probe, probes.NullByteInHeader) and result.is_failure:
+            from ...specs.openapi.formats import HEADER_FORMAT, header_values
+
+            formats.register(HEADER_FORMAT, header_values(blacklist_characters="\n\r\x00"))
+    return results
 
 
 @dataclass
@@ -259,7 +469,7 @@ class EventStream:
 
 def handle_schema_error(
     error: OperationSchemaError,
-    results: TestResultSet,
+    ctx: RunnerContext,
     data_generation_methods: Iterable[DataGenerationMethod],
     recursion_level: int,
     *,
@@ -304,11 +514,11 @@ def handle_schema_error(
             hypothesis_output=[],
             correlation_id=correlation_id,
         )
-        results.append(result)
+        ctx.add_result(result)
     else:
         # When there is no `method`, then the schema error may cover multiple operations, and we can't display it in
         # the progress bar
-        results.generic_errors.append(error)
+        ctx.add_generic_error(error)
 
 
 def run_test(
@@ -317,7 +527,7 @@ def run_test(
     checks: Iterable[CheckFunction],
     data_generation_methods: Iterable[DataGenerationMethod],
     targets: Iterable[Target],
-    results: TestResultSet,
+    ctx: RunnerContext,
     headers: dict[str, Any] | None,
     recursion_level: int,
     **kwargs: Any,
@@ -342,12 +552,35 @@ def run_test(
     errors: list[Exception] = []
     test_start_time = time.monotonic()
     setup_hypothesis_database_key(test, operation)
+
+    def _on_flaky(exc: Exception) -> Status:
+        if isinstance(exc.__cause__, hypothesis.errors.DeadlineExceeded):
+            status = Status.error
+            result.add_error(DeadlineExceeded.from_exc(exc.__cause__))
+        elif (
+            hasattr(hypothesis.errors, "FlakyFailure")
+            and isinstance(exc, hypothesis.errors.FlakyFailure)
+            and any(isinstance(subexc, hypothesis.errors.DeadlineExceeded) for subexc in exc.exceptions)
+        ):
+            for sub_exc in exc.exceptions:
+                if isinstance(sub_exc, hypothesis.errors.DeadlineExceeded):
+                    result.add_error(DeadlineExceeded.from_exc(sub_exc))
+            status = Status.error
+        elif errors:
+            status = Status.error
+            add_errors(result, errors)
+        else:
+            status = Status.failure
+            result.mark_flaky()
+        return status
+
     try:
         with catch_warnings(record=True) as warnings, capture_hypothesis_output() as hypothesis_output:
             test(
-                checks,
-                targets,
-                result,
+                ctx=ctx,
+                checks=checks,
+                targets=targets,
+                result=result,
                 errors=errors,
                 headers=headers,
                 data_generation_methods=data_generation_methods,
@@ -371,6 +604,8 @@ def run_test(
         result.mark_errored()
         for error in deduplicate_errors(errors):
             result.add_error(error)
+    except hypothesis.errors.Flaky as exc:
+        status = _on_flaky(exc)
     except MultipleFailures:
         # Schemathesis may detect multiple errors that come from different check results
         # They raise different "grouped" exceptions
@@ -379,16 +614,6 @@ def run_test(
             add_errors(result, errors)
         else:
             status = Status.failure
-    except hypothesis.errors.Flaky as exc:
-        if isinstance(exc.__cause__, hypothesis.errors.DeadlineExceeded):
-            status = Status.error
-            result.add_error(DeadlineExceeded.from_exc(exc.__cause__))
-        elif errors:
-            status = Status.error
-            add_errors(result, errors)
-        else:
-            status = Status.failure
-            result.mark_flaky()
     except hypothesis.errors.Unsatisfiable:
         # We need more clear error message here
         status = Status.error
@@ -399,13 +624,29 @@ def run_test(
     except SkipTest as exc:
         status = Status.skip
         result.mark_skipped(exc)
-    except AssertionError:  # comes from `hypothesis-jsonschema`
-        error = reraise(operation)
+    except AssertionError as exc:  # May come from `hypothesis-jsonschema` or `hypothesis`
         status = Status.error
+        try:
+            operation.schema.validate()
+            msg = "Unexpected error during testing of this API operation"
+            exc_msg = str(exc)
+            if exc_msg:
+                msg += f": {exc_msg}"
+            try:
+                raise InternalError(msg) from exc
+            except InternalError as exc:
+                error = exc
+        except ValidationError as exc:
+            error = OperationSchemaError.from_jsonschema_error(
+                exc,
+                path=operation.path,
+                method=operation.method,
+                full_path=operation.schema.get_full_path(operation.path),
+            )
         result.add_error(error)
     except HypothesisRefResolutionError:
         status = Status.error
-        result.add_error(hypothesis.errors.Unsatisfiable(RECURSIVE_REFERENCE_ERROR_MESSAGE))
+        result.add_error(RecursiveReferenceError(RECURSIVE_REFERENCE_ERROR_MESSAGE))
     except InvalidArgument as error:
         status = Status.error
         message = get_invalid_regular_expression_message(warnings)
@@ -433,6 +674,8 @@ def run_test(
             )
         else:
             result.add_error(error)
+    if status == Status.success and ctx.no_failfast and any(check.value == Status.failure for check in result.checks):
+        status = Status.failure
     if has_unsatisfied_example_mark(test):
         status = Status.error
         result.add_error(
@@ -464,10 +707,10 @@ def run_test(
     result.seed = getattr(test, "_hypothesis_internal_use_seed", None) or getattr(
         test, "_hypothesis_internal_use_generated_seed", None
     )
-    results.append(result)
+    ctx.add_result(result)
     for status_code in (401, 403):
         if has_too_many_responses_with_status(result, status_code):
-            results.add_warning(TOO_MANY_RESPONSES_WARNING_TEMPLATE.format(f"`{operation.verbose_name}`", status_code))
+            ctx.add_warning(TOO_MANY_RESPONSES_WARNING_TEMPLATE.format(f"`{operation.verbose_name}`", status_code))
     yield events.AfterExecution.from_result(
         result=result,
         status=status,
@@ -502,22 +745,6 @@ def has_too_many_responses_with_status(result: TestResult, status_code: int) ->
 ALL_NOT_FOUND_WARNING_MESSAGE = "All API responses have a 404 status code. Did you specify the proper API location?"
 
 
-def has_all_not_found(results: TestResultSet) -> bool:
-    """Check if all responses are 404."""
-    has_not_found = False
-    for result in results.results:
-        for check in result.checks:
-            if check.response is not None:
-                if check.response.status_code == 404:
-                    has_not_found = True
-                else:
-                    # There are non-404 responses, no reason to check any other response
-                    return False
-    # Only happens if all responses are 404, or there are no responses at all.
-    # In the first case, it returns True, for the latter - False
-    return has_not_found
-
-
 def setup_hypothesis_database_key(test: Callable, operation: APIOperation) -> None:
     """Make Hypothesis use separate database entries for every API operation.
 
@@ -526,7 +753,7 @@ def setup_hypothesis_database_key(test: Callable, operation: APIOperation) -> No
     # Hypothesis's function digest depends on the test function signature. To reflect it for the web API case,
     # we use all API operation parameters in the digest.
     extra = operation.verbose_name.encode("utf8")
-    for parameter in operation.definition.parameters:
+    for parameter in operation.iter_parameters():
         extra += parameter.serialize(operation).encode("utf8")
     test.hypothesis.inner_test._hypothesis_internal_add_digest = extra  # type: ignore
 
@@ -539,16 +766,6 @@ def get_invalid_regular_expression_message(warnings: list[WarningMessage]) -> st
     return None
 
 
-def reraise(operation: APIOperation) -> OperationSchemaError:
-    try:
-        operation.schema.validate()
-    except ValidationError as exc:
-        return OperationSchemaError.from_jsonschema_error(
-            exc, path=operation.path, method=operation.method, full_path=operation.schema.get_full_path(operation.path)
-        )
-    return OperationSchemaError("Unknown schema error")
-
-
 MEMORY_ADDRESS_RE = re.compile("0x[0-9a-fA-F]+")
 URL_IN_ERROR_MESSAGE_RE = re.compile(r"Max retries exceeded with url: .*? \(Caused by")
 
@@ -564,7 +781,9 @@ def group_errors(errors: list[Exception]) -> None:
     serialization_errors = [error for error in errors if isinstance(error, SerializationNotPossible)]
     if len(serialization_errors) > 1:
         errors[:] = [error for error in errors if not isinstance(error, SerializationNotPossible)]
-        media_types = sum((entry.media_types for entry in serialization_errors), [])
+        media_types: list[str] = functools.reduce(
+            operator.iadd, (entry.media_types for entry in serialization_errors), []
+        )
         errors.append(SerializationNotPossible.from_media_types(*media_types))
 
 
@@ -589,12 +808,14 @@ def deduplicate_errors(errors: list[Exception]) -> Generator[Exception, None, No
 def run_checks(
     *,
     case: Case,
+    ctx: CheckContext,
     checks: Iterable[CheckFunction],
     check_results: list[Check],
     result: TestResult,
     response: GenericResponse,
     elapsed_time: float,
     max_response_time: int | None = None,
+    no_failfast: bool,
 ) -> None:
     errors = []
 
@@ -611,7 +832,7 @@ def run_checks(
         check_name = check.__name__
         copied_case = case.partial_deepcopy()
         try:
-            skip_check = check(response, copied_case)
+            skip_check = check(ctx, response, copied_case)
             if not skip_check:
                 check_result = result.add_success(check_name, copied_case, response, elapsed_time)
                 check_results.append(check_result)
@@ -623,7 +844,7 @@ def run_checks(
 
     if max_response_time:
         if elapsed_time > max_response_time:
-            message = f"Actual: {elapsed_time:.2f}ms\nLimit: {max_response_time}.00ms"
+            message = _make_max_response_time_failure_message(elapsed_time, max_response_time)
             errors.append(AssertionError(message))
             result.add_failure(
                 "max_response_time",
@@ -636,7 +857,7 @@ def run_checks(
         else:
             result.add_success("max_response_time", case, response, elapsed_time)
 
-    if errors:
+    if errors and not no_failfast:
         raise get_grouped_exception(case.operation.verbose_name, *errors)(causes=tuple(errors))
 
 
@@ -697,19 +918,42 @@ def _force_data_generation_method(values: list[DataGenerationMethod], case: Case
     values[:] = [data_generation_method]
 
 
+def cached_test_func(f: Callable) -> Callable:
+    def wrapped(*, ctx: RunnerContext, case: Case, **kwargs: Any) -> None:
+        if ctx.unique_data:
+            cached = ctx.get_cached_outcome(case)
+            if isinstance(cached, BaseException):
+                raise cached
+            elif cached is None:
+                return None
+            try:
+                f(ctx=ctx, case=case, **kwargs)
+            except BaseException as exc:
+                ctx.cache_outcome(case, exc)
+                raise
+            else:
+                ctx.cache_outcome(case, None)
+        else:
+            f(ctx=ctx, case=case, **kwargs)
+
+    wrapped.__name__ = f.__name__
+
+    return wrapped
+
+
+@cached_test_func
 def network_test(
+    *,
+    ctx: RunnerContext,
     case: Case,
     checks: Iterable[CheckFunction],
     targets: Iterable[Target],
     result: TestResult,
     session: requests.Session,
-    request_timeout: int | None,
-    request_tls_verify: bool,
-    request_proxy: str | None,
-    request_cert: RequestCert | None,
+    request_config: RequestConfig,
     store_interactions: bool,
     headers: dict[str, Any] | None,
-    feedback: Feedback,
+    feedback: Feedback | None,
     max_response_time: int | None,
     data_generation_methods: list[DataGenerationMethod],
     dry_run: bool,
@@ -722,85 +966,97 @@ def network_test(
         headers = headers or {}
         if "user-agent" not in {header.lower() for header in headers}:
             headers["User-Agent"] = USER_AGENT
-        timeout = prepare_timeout(request_timeout)
         if not dry_run:
             args = (
+                ctx,
                 checks,
                 targets,
                 result,
                 session,
-                timeout,
+                request_config,
                 store_interactions,
                 headers,
                 feedback,
-                request_tls_verify,
-                request_proxy,
-                request_cert,
                 max_response_time,
             )
             response = _network_test(case, *args)
             add_cases(case, response, _network_test, *args)
+        elif store_interactions:
+            result.store_requests_response(case, None, Status.skip, [], headers=headers, session=session)
 
 
 def _network_test(
     case: Case,
+    ctx: RunnerContext,
     checks: Iterable[CheckFunction],
     targets: Iterable[Target],
     result: TestResult,
     session: requests.Session,
-    timeout: float | None,
+    request_config: RequestConfig,
     store_interactions: bool,
     headers: dict[str, Any] | None,
-    feedback: Feedback,
-    request_tls_verify: bool,
-    request_proxy: str | None,
-    request_cert: RequestCert | None,
+    feedback: Feedback | None,
     max_response_time: int | None,
 ) -> requests.Response:
     check_results: list[Check] = []
+    hook_context = HookContext(operation=case.operation)
+    kwargs: dict[str, Any] = {
+        "session": session,
+        "headers": headers,
+        "timeout": request_config.prepared_timeout,
+        "verify": request_config.tls_verify,
+        "cert": request_config.cert,
+    }
+    if request_config.proxy is not None:
+        kwargs["proxies"] = {"all": request_config.proxy}
+    hooks.dispatch("process_call_kwargs", hook_context, case, kwargs)
     try:
-        hook_context = HookContext(operation=case.operation)
-        kwargs: dict[str, Any] = {
-            "session": session,
-            "headers": headers,
-            "timeout": timeout,
-            "verify": request_tls_verify,
-            "cert": request_cert,
-        }
-        if request_proxy is not None:
-            kwargs["proxies"] = {"all": request_proxy}
-        hooks.dispatch("process_call_kwargs", hook_context, case, kwargs)
         response = case.call(**kwargs)
     except CheckFailed as exc:
         check_name = "request_timeout"
-        requests_kwargs = case.as_requests_kwargs(base_url=case.get_full_base_url(), headers=headers)
+        requests_kwargs = RequestsTransport().serialize_case(case, base_url=case.get_full_base_url(), headers=headers)
         request = requests.Request(**requests_kwargs).prepare()
-        elapsed = cast(float, timeout)  # It is defined and not empty, since the exception happened
+        elapsed = cast(
+            float, request_config.prepared_timeout
+        )  # It is defined and not empty, since the exception happened
         check_result = result.add_failure(
             check_name, case, None, elapsed, f"Response timed out after {1000 * elapsed:.2f}ms", exc.context, request
         )
         check_results.append(check_result)
+        if store_interactions:
+            result.store_requests_response(case, None, Status.failure, [check_result], headers=headers, session=session)
         raise exc
     context = TargetContext(case=case, response=response, response_time=response.elapsed.total_seconds())
     run_targets(targets, context)
     status = Status.success
+
+    check_ctx = CheckContext(
+        override=ctx.override,
+        auth=ctx.auth,
+        headers=CaseInsensitiveDict(headers) if headers else None,
+        config=ctx.checks_config,
+        transport_kwargs=kwargs,
+    )
     try:
         run_checks(
             case=case,
+            ctx=check_ctx,
             checks=checks,
             check_results=check_results,
             result=result,
             response=response,
             elapsed_time=context.response_time * 1000,
             max_response_time=max_response_time,
+            no_failfast=ctx.no_failfast,
         )
     except CheckFailed:
         status = Status.failure
         raise
     finally:
-        feedback.add_test_case(case, response)
+        if feedback is not None:
+            feedback.add_test_case(case, response)
         if store_interactions:
-            result.store_requests_response(case, response, status, check_results)
+            result.store_requests_response(case, response, status, check_results, headers=headers, session=session)
     return response
 
 
@@ -812,15 +1068,9 @@ def get_session(auth: HTTPDigestAuth | RawAuth | None = None) -> Generator[reque
         yield session
 
 
-def prepare_timeout(timeout: int | None) -> float | None:
-    """Request timeout is in milliseconds, but `requests` uses seconds."""
-    output: int | float | None = timeout
-    if timeout is not None:
-        output = timeout / 1000
-    return output
-
-
+@cached_test_func
 def wsgi_test(
+    ctx: RunnerContext,
     case: Case,
     checks: Iterable[CheckFunction],
     targets: Iterable[Target],
@@ -829,7 +1079,7 @@ def wsgi_test(
     auth_type: str | None,
     headers: dict[str, Any] | None,
     store_interactions: bool,
-    feedback: Feedback,
+    feedback: Feedback | None,
     max_response_time: int | None,
     data_generation_methods: list[DataGenerationMethod],
     dry_run: bool,
@@ -838,9 +1088,10 @@ def wsgi_test(
     with ErrorCollector(errors):
         _force_data_generation_method(data_generation_methods, case)
         result.mark_executed()
-        headers = _prepare_wsgi_headers(headers, auth, auth_type)
+        headers = prepare_wsgi_headers(headers, auth, auth_type)
         if not dry_run:
             args = (
+                ctx,
                 checks,
                 targets,
                 result,
@@ -851,78 +1102,73 @@ def wsgi_test(
             )
             response = _wsgi_test(case, *args)
             add_cases(case, response, _wsgi_test, *args)
+        elif store_interactions:
+            result.store_wsgi_response(case, None, headers, None, Status.skip, [])
 
 
 def _wsgi_test(
     case: Case,
+    ctx: RunnerContext,
     checks: Iterable[CheckFunction],
     targets: Iterable[Target],
     result: TestResult,
     headers: dict[str, Any],
     store_interactions: bool,
-    feedback: Feedback,
+    feedback: Feedback | None,
     max_response_time: int | None,
 ) -> WSGIResponse:
+    from ...transports.responses import WSGIResponse
+
     with catching_logs(LogCaptureHandler(), level=logging.DEBUG) as recorded:
-        start = time.monotonic()
         hook_context = HookContext(operation=case.operation)
-        kwargs = {"headers": headers}
+        kwargs: dict[str, Any] = {"headers": headers}
         hooks.dispatch("process_call_kwargs", hook_context, case, kwargs)
-        response = case.call_wsgi(**kwargs)
-        elapsed = time.monotonic() - start
-    context = TargetContext(case=case, response=response, response_time=elapsed)
+        response = cast(WSGIResponse, case.call(**kwargs))
+    context = TargetContext(case=case, response=response, response_time=response.elapsed.total_seconds())
     run_targets(targets, context)
     result.logs.extend(recorded.records)
     status = Status.success
     check_results: list[Check] = []
+    check_ctx = CheckContext(
+        override=ctx.override,
+        auth=ctx.auth,
+        headers=CaseInsensitiveDict(headers) if headers else None,
+        config=ctx.checks_config,
+        transport_kwargs=kwargs,
+    )
     try:
         run_checks(
             case=case,
+            ctx=check_ctx,
             checks=checks,
             check_results=check_results,
             result=result,
             response=response,
             elapsed_time=context.response_time * 1000,
             max_response_time=max_response_time,
+            no_failfast=ctx.no_failfast,
         )
     except CheckFailed:
         status = Status.failure
         raise
     finally:
-        feedback.add_test_case(case, response)
+        if feedback is not None:
+            feedback.add_test_case(case, response)
         if store_interactions:
-            result.store_wsgi_response(case, response, headers, elapsed, status, check_results)
+            result.store_wsgi_response(case, response, headers, response.elapsed.total_seconds(), status, check_results)
     return response
 
 
-def _prepare_wsgi_headers(
-    headers: dict[str, Any] | None, auth: RawAuth | None, auth_type: str | None
-) -> dict[str, Any]:
-    headers = headers or {}
-    if "user-agent" not in {header.lower() for header in headers}:
-        headers["User-Agent"] = USER_AGENT
-    wsgi_auth = get_wsgi_auth(auth, auth_type)
-    if wsgi_auth:
-        headers["Authorization"] = wsgi_auth
-    return headers
-
-
-def get_wsgi_auth(auth: RawAuth | None, auth_type: str | None) -> str | None:
-    if auth:
-        if auth_type == "digest":
-            raise ValueError("Digest auth is not supported for WSGI apps")
-        return _basic_auth_str(*auth)
-    return None
-
-
+@cached_test_func
 def asgi_test(
+    ctx: RunnerContext,
     case: Case,
     checks: Iterable[CheckFunction],
     targets: Iterable[Target],
     result: TestResult,
     store_interactions: bool,
     headers: dict[str, Any] | None,
-    feedback: Feedback,
+    feedback: Feedback | None,
     max_response_time: int | None,
     data_generation_methods: list[DataGenerationMethod],
     dry_run: bool,
@@ -936,6 +1182,7 @@ def asgi_test(
 
         if not dry_run:
             args = (
+                ctx,
                 checks,
                 targets,
                 result,
@@ -946,41 +1193,54 @@ def asgi_test(
             )
             response = _asgi_test(case, *args)
             add_cases(case, response, _asgi_test, *args)
+        elif store_interactions:
+            result.store_requests_response(case, None, Status.skip, [], headers=headers, session=None)
 
 
 def _asgi_test(
     case: Case,
+    ctx: RunnerContext,
     checks: Iterable[CheckFunction],
     targets: Iterable[Target],
     result: TestResult,
     store_interactions: bool,
     headers: dict[str, Any] | None,
-    feedback: Feedback,
+    feedback: Feedback | None,
     max_response_time: int | None,
 ) -> requests.Response:
     hook_context = HookContext(operation=case.operation)
     kwargs: dict[str, Any] = {"headers": headers}
     hooks.dispatch("process_call_kwargs", hook_context, case, kwargs)
-    response = case.call_asgi(**kwargs)
+    response = case.call(**kwargs)
     context = TargetContext(case=case, response=response, response_time=response.elapsed.total_seconds())
     run_targets(targets, context)
     status = Status.success
     check_results: list[Check] = []
+    check_ctx = CheckContext(
+        override=ctx.override,
+        auth=ctx.auth,
+        headers=CaseInsensitiveDict(headers) if headers else None,
+        config=ctx.checks_config,
+        transport_kwargs=kwargs,
+    )
     try:
         run_checks(
             case=case,
+            ctx=check_ctx,
             checks=checks,
             check_results=check_results,
             result=result,
             response=response,
             elapsed_time=context.response_time * 1000,
             max_response_time=max_response_time,
+            no_failfast=ctx.no_failfast,
         )
     except CheckFailed:
         status = Status.failure
         raise
     finally:
-        feedback.add_test_case(case, response)
+        if feedback is not None:
+            feedback.add_test_case(case, response)
         if store_interactions:
-            result.store_requests_response(case, response, status, check_results)
+            result.store_requests_response(case, response, status, check_results, headers, session=None)
     return response