schemathesis 3.35.4__py3-none-any.whl → 3.36.0__py3-none-any.whl

schemathesis/runner/impl/core.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
+import functools
 import logging
+import operator
 import re
 import threading
 import time
@@ -9,7 +11,6 @@ import uuid
 import warnings
 from contextlib import contextmanager
 from dataclasses import dataclass, field
-from types import TracebackType
 from typing import TYPE_CHECKING, Any, Callable, Generator, Iterable, List, Literal, cast
 from warnings import WarningMessage, catch_warnings
 
@@ -20,7 +21,7 @@ from hypothesis.errors import HypothesisException, InvalidArgument
 from hypothesis_jsonschema._canonicalise import HypothesisRefResolutionError
 from jsonschema.exceptions import SchemaError as JsonSchemaError
 from jsonschema.exceptions import ValidationError
-from requests.auth import HTTPDigestAuth
+from requests.structures import CaseInsensitiveDict
 from urllib3.exceptions import InsecureRequestWarning
 
 from ... import experimental, failures, hooks
@@ -31,7 +32,6 @@ from ..._hypothesis import (
     get_non_serializable_mark,
     has_unsatisfied_example_mark,
 )
-from ..._override import CaseOverride
 from ...auths import unregister as unregister_auth
 from ...checks import _make_max_response_time_failure_message
 from ...constants import (
@@ -57,11 +57,11 @@ from ...exceptions import (
 )
 from ...generation import DataGenerationMethod, GenerationConfig
 from ...hooks import HookContext, get_all_by_name
+from ...internal.checks import CheckContext
 from ...internal.datetime import current_datetime
 from ...internal.result import Err, Ok, Result
-from ...models import APIOperation, Case, Check, CheckFunction, Status, TestResult, TestResultSet
+from ...models import APIOperation, Case, Check, Status, TestResult
 from ...runner import events
-from ...schemas import BaseSchema
 from ...service import extensions
 from ...service.models import AnalysisResult, AnalysisSuccess
 from ...specs.openapi import formats
@@ -71,14 +71,22 @@ from ...stateful import runner as stateful_runner
 from ...targets import Target, TargetContext
 from ...transports import RequestConfig, RequestsTransport
 from ...transports.auth import get_requests_auth, prepare_wsgi_headers
-from ...types import RawAuth
 from ...utils import capture_hypothesis_output
 from .. import probes
 from ..serialization import SerializedTestResult
+from .context import RunnerContext
 
 if TYPE_CHECKING:
+    from types import TracebackType
+
+    from requests.auth import HTTPDigestAuth
+
+    from ..._override import CaseOverride
+    from ...internal.checks import CheckFunction
+    from ...schemas import BaseSchema
     from ...service.client import ServiceClient
     from ...transports.responses import GenericResponse, WSGIResponse
+    from ...types import RawAuth
 
 
 def _should_count_towards_stop(event: events.ExecutionEvent) -> bool:
@@ -92,7 +100,7 @@ class BaseRunner:
     max_response_time: int | None
     targets: Iterable[Target]
     hypothesis_settings: hypothesis.settings
-    generation_config: GenerationConfig
+    generation_config: GenerationConfig | None
     probe_config: probes.ProbeConfig
     request_config: RequestConfig = field(default_factory=RequestConfig)
     override: CaseOverride | None = None
@@ -104,6 +112,7 @@ class BaseRunner:
     exit_first: bool = False
     max_failures: int | None = None
     started_at: str = field(default_factory=current_datetime)
+    unique_data: bool = False
     dry_run: bool = False
     stateful: Stateful | None = None
     stateful_recursion_limit: int = DEFAULT_STATEFUL_RECURSION_LIMIT
@@ -122,7 +131,7 @@ class BaseRunner:
         # If auth is explicitly provided, then the global provider is ignored
         if self.auth is not None:
             unregister_auth()
-        results = TestResultSet(seed=self.seed)
+        ctx = RunnerContext(seed=self.seed, stop_event=stop_event, unique_data=self.unique_data)
         start_time = time.monotonic()
         initialized = None
         __probes = None
@@ -134,15 +143,15 @@ class BaseRunner:
                 schema=self.schema,
                 count_operations=self.count_operations,
                 count_links=self.count_links,
-                seed=self.seed,
+                seed=ctx.seed,
                 start_time=start_time,
             )
             return initialized
 
         def _finish() -> events.Finished:
-            if has_all_not_found(results):
-                results.add_warning(ALL_NOT_FOUND_WARNING_MESSAGE)
-            return events.Finished.from_results(results=results, running_time=time.monotonic() - start_time)
+            if ctx.has_all_not_found:
+                ctx.add_warning(ALL_NOT_FOUND_WARNING_MESSAGE)
+            return events.Finished.from_results(results=ctx.data, running_time=time.monotonic() - start_time)
 
         def _before_probes() -> events.BeforeProbing:
             return events.BeforeProbing()
@@ -176,7 +185,7 @@ class BaseRunner:
         def _after_analysis() -> events.AfterAnalysis:
             return events.AfterAnalysis(analysis=__analysis)
 
-        if stop_event.is_set():
+        if ctx.is_stopped:
             yield _finish()
             return
 
@@ -192,16 +201,16 @@ class BaseRunner:
             event = event_factory()
             if event is not None:
                 yield event
-            if stop_event.is_set():
-                yield _finish()
+            if ctx.is_stopped:
+                yield _finish()  # type: ignore[unreachable]
                 return
 
         try:
             warnings.simplefilter("ignore", InsecureRequestWarning)
             if not experimental.STATEFUL_ONLY.is_enabled:
-                yield from self._execute(results, stop_event)
+                yield from self._execute(ctx)
             if not self._is_stopping_due_to_failure_limit:
-                yield from self._run_stateful_tests(results)
+                yield from self._run_stateful_tests(ctx)
         except KeyboardInterrupt:
             yield events.Interrupted()
 
@@ -222,19 +231,17 @@ class BaseRunner:
                 return self._failures_counter >= self.max_failures
         return False
 
-    def _execute(
-        self, results: TestResultSet, stop_event: threading.Event
-    ) -> Generator[events.ExecutionEvent, None, None]:
+    def _execute(self, ctx: RunnerContext) -> Generator[events.ExecutionEvent, None, None]:
         raise NotImplementedError
 
-    def _run_stateful_tests(self, results: TestResultSet) -> Generator[events.ExecutionEvent, None, None]:
+    def _run_stateful_tests(self, ctx: RunnerContext) -> Generator[events.ExecutionEvent, None, None]:
         # Run new-style stateful tests
         if self.stateful is not None and experimental.STATEFUL_TEST_RUNNER.is_enabled and self.schema.links_count > 0:
             result = TestResult(
                 method="",
                 path="",
                 verbose_name="Stateful tests",
-                seed=self.seed,
+                seed=ctx.seed,
                 data_generation_method=self.schema.data_generation_methods,
             )
             headers = self.headers or {}
@@ -251,7 +258,7 @@ class BaseRunner:
                 max_failures=None if self.max_failures is None else self.max_failures - self._failures_counter,
                 request=self.request_config,
                 auth=auth,
-                seed=self.seed,
+                seed=ctx.seed,
                 override=self.override,
             )
             state_machine = self.schema.as_state_machine()
@@ -277,6 +284,8 @@ class BaseRunner:
                                 case=event.case,
                                 response=response,
                                 checks=event.checks,
+                                headers=headers,
+                                session=None,
                             )
 
                 else:
@@ -317,7 +326,7 @@ class BaseRunner:
                     status = Status.error
                     result.add_error(stateful_event.exception)
                 yield events.StatefulEvent(data=stateful_event)
-            results.append(result)
+            ctx.add_result(result)
             yield events.AfterStatefulExecution(
                 status=status,
                 result=SerializedTestResult.from_test_result(result),
@@ -328,11 +337,10 @@ class BaseRunner:
     def _run_tests(
         self,
         maker: Callable,
-        template: Callable,
+        test_func: Callable,
         settings: hypothesis.settings,
-        generation_config: GenerationConfig,
-        seed: int | None,
-        results: TestResultSet,
+        generation_config: GenerationConfig | None,
+        ctx: RunnerContext,
         recursion_level: int = 0,
         headers: dict[str, Any] | None = None,
         **kwargs: Any,
@@ -352,10 +360,10 @@ class BaseRunner:
             return kw
 
         for result in maker(
-            template,
+            test_func,
             settings=settings,
             generation_config=generation_config,
-            seed=seed,
+            seed=ctx.seed,
             as_strategy_kwargs=as_strategy_kwargs,
         ):
             if isinstance(result, Ok):
@@ -372,7 +380,7 @@ class BaseRunner:
                     for event in run_test(
                         operation,
                         test,
-                        results=results,
+                        ctx=ctx,
                         feedback=feedback,
                         recursion_level=recursion_level,
                         data_generation_methods=self.schema.data_generation_methods,
@@ -388,28 +396,25 @@ class BaseRunner:
                     if feedback is not None:
                         yield from self._run_tests(
                             feedback.get_stateful_tests,
-                            template,
+                            test_func,
                             settings=settings,
                             generation_config=generation_config,
-                            seed=seed,
                             recursion_level=recursion_level + 1,
-                            results=results,
+                            ctx=ctx,
                             headers=headers,
                             **kwargs,
                         )
                 except OperationSchemaError as exc:
                     yield from handle_schema_error(
                         exc,
-                        results,
+                        ctx,
                         self.schema.data_generation_methods,
                         recursion_level,
                         before_execution_correlation_id=before_execution_correlation_id,
                     )
             else:
                 # Schema errors
-                yield from handle_schema_error(
-                    result.err(), results, self.schema.data_generation_methods, recursion_level
-                )
+                yield from handle_schema_error(result.err(), ctx, self.schema.data_generation_methods, recursion_level)
 
 
 def run_probes(schema: BaseSchema, config: probes.ProbeConfig) -> list[probes.ProbeRun]:
@@ -454,7 +459,7 @@ class EventStream:
 
 def handle_schema_error(
     error: OperationSchemaError,
-    results: TestResultSet,
+    ctx: RunnerContext,
     data_generation_methods: Iterable[DataGenerationMethod],
     recursion_level: int,
     *,
@@ -499,11 +504,11 @@ def handle_schema_error(
             hypothesis_output=[],
             correlation_id=correlation_id,
         )
-        results.append(result)
+        ctx.add_result(result)
     else:
         # When there is no `method`, then the schema error may cover multiple operations, and we can't display it in
         # the progress bar
-        results.generic_errors.append(error)
+        ctx.add_generic_error(error)
 
 
 def run_test(
@@ -512,7 +517,7 @@ def run_test(
     checks: Iterable[CheckFunction],
     data_generation_methods: Iterable[DataGenerationMethod],
     targets: Iterable[Target],
-    results: TestResultSet,
+    ctx: RunnerContext,
     headers: dict[str, Any] | None,
     recursion_level: int,
     **kwargs: Any,
@@ -562,9 +567,10 @@ def run_test(
     try:
         with catch_warnings(record=True) as warnings, capture_hypothesis_output() as hypothesis_output:
             test(
-                checks,
-                targets,
-                result,
+                ctx=ctx,
+                checks=checks,
+                targets=targets,
+                result=result,
                 errors=errors,
                 headers=headers,
                 data_generation_methods=data_generation_methods,
@@ -689,10 +695,10 @@ def run_test(
     result.seed = getattr(test, "_hypothesis_internal_use_seed", None) or getattr(
         test, "_hypothesis_internal_use_generated_seed", None
     )
-    results.append(result)
+    ctx.add_result(result)
     for status_code in (401, 403):
         if has_too_many_responses_with_status(result, status_code):
-            results.add_warning(TOO_MANY_RESPONSES_WARNING_TEMPLATE.format(f"`{operation.verbose_name}`", status_code))
+            ctx.add_warning(TOO_MANY_RESPONSES_WARNING_TEMPLATE.format(f"`{operation.verbose_name}`", status_code))
     yield events.AfterExecution.from_result(
         result=result,
         status=status,
@@ -727,22 +733,6 @@ def has_too_many_responses_with_status(result: TestResult, status_code: int) ->
 ALL_NOT_FOUND_WARNING_MESSAGE = "All API responses have a 404 status code. Did you specify the proper API location?"
 
 
-def has_all_not_found(results: TestResultSet) -> bool:
-    """Check if all responses are 404."""
-    has_not_found = False
-    for result in results.results:
-        for check in result.checks:
-            if check.response is not None:
-                if check.response.status_code == 404:
-                    has_not_found = True
-                else:
-                    # There are non-404 responses, no reason to check any other response
-                    return False
-    # Only happens if all responses are 404, or there are no responses at all.
-    # In the first case, it returns True, for the latter - False
-    return has_not_found
-
-
 def setup_hypothesis_database_key(test: Callable, operation: APIOperation) -> None:
     """Make Hypothesis use separate database entries for every API operation.
 
@@ -779,7 +769,9 @@ def group_errors(errors: list[Exception]) -> None:
     serialization_errors = [error for error in errors if isinstance(error, SerializationNotPossible)]
     if len(serialization_errors) > 1:
         errors[:] = [error for error in errors if not isinstance(error, SerializationNotPossible)]
-        media_types = sum((entry.media_types for entry in serialization_errors), [])
+        media_types: list[str] = functools.reduce(
+            operator.iadd, (entry.media_types for entry in serialization_errors), []
+        )
         errors.append(SerializationNotPossible.from_media_types(*media_types))
 
 
@@ -804,6 +796,7 @@ def deduplicate_errors(errors: list[Exception]) -> Generator[Exception, None, No
 def run_checks(
     *,
     case: Case,
+    ctx: CheckContext,
     checks: Iterable[CheckFunction],
     check_results: list[Check],
     result: TestResult,
@@ -826,7 +819,7 @@ def run_checks(
         check_name = check.__name__
         copied_case = case.partial_deepcopy()
         try:
-            skip_check = check(response, copied_case)
+            skip_check = check(ctx, response, copied_case)
             if not skip_check:
                 check_result = result.add_success(check_name, copied_case, response, elapsed_time)
                 check_results.append(check_result)
@@ -912,7 +905,33 @@ def _force_data_generation_method(values: list[DataGenerationMethod], case: Case
     values[:] = [data_generation_method]
 
 
+def cached_test_func(f: Callable) -> Callable:
+    def wrapped(*, ctx: RunnerContext, case: Case, **kwargs: Any) -> None:
+        if ctx.unique_data:
+            cached = ctx.get_cached_outcome(case)
+            if isinstance(cached, BaseException):
+                raise cached
+            elif cached is None:
+                return None
+            try:
+                f(ctx=ctx, case=case, **kwargs)
+            except BaseException as exc:
+                ctx.cache_outcome(case, exc)
+                raise
+            else:
+                ctx.cache_outcome(case, None)
+        else:
+            f(ctx=ctx, case=case, **kwargs)
+
+    wrapped.__name__ = f.__name__
+
+    return wrapped
+
+
+@cached_test_func
 def network_test(
+    *,
+    ctx: RunnerContext,
     case: Case,
     checks: Iterable[CheckFunction],
     targets: Iterable[Target],
@@ -948,6 +967,8 @@ def network_test(
             )
             response = _network_test(case, *args)
             add_cases(case, response, _network_test, *args)
+        elif store_interactions:
+            result.store_requests_response(case, None, Status.skip, [], headers=headers, session=session)
 
 
 def _network_test(
@@ -987,13 +1008,18 @@ def _network_test(
             check_name, case, None, elapsed, f"Response timed out after {1000 * elapsed:.2f}ms", exc.context, request
         )
         check_results.append(check_result)
+        if store_interactions:
+            result.store_requests_response(case, None, Status.failure, [check_result], headers=headers, session=session)
         raise exc
     context = TargetContext(case=case, response=response, response_time=response.elapsed.total_seconds())
     run_targets(targets, context)
     status = Status.success
+
+    ctx = CheckContext(headers=CaseInsensitiveDict(headers) if headers else None)
     try:
         run_checks(
             case=case,
+            ctx=ctx,
             checks=checks,
             check_results=check_results,
             result=result,
@@ -1008,7 +1034,7 @@ def _network_test(
         if feedback is not None:
             feedback.add_test_case(case, response)
         if store_interactions:
-            result.store_requests_response(case, response, status, check_results)
+            result.store_requests_response(case, response, status, check_results, headers=headers, session=session)
     return response
 
 
@@ -1020,7 +1046,9 @@ def get_session(auth: HTTPDigestAuth | RawAuth | None = None) -> Generator[reque
         yield session
 
 
+@cached_test_func
 def wsgi_test(
+    ctx: RunnerContext,
     case: Case,
     checks: Iterable[CheckFunction],
     targets: Iterable[Target],
@@ -1051,6 +1079,8 @@ def wsgi_test(
             )
             response = _wsgi_test(case, *args)
             add_cases(case, response, _wsgi_test, *args)
+        elif store_interactions:
+            result.store_wsgi_response(case, None, headers, None, Status.skip, [])
 
 
 def _wsgi_test(
@@ -1075,9 +1105,11 @@ def _wsgi_test(
     result.logs.extend(recorded.records)
     status = Status.success
     check_results: list[Check] = []
+    ctx = CheckContext(headers=CaseInsensitiveDict(headers) if headers else None)
     try:
         run_checks(
             case=case,
+            ctx=ctx,
             checks=checks,
             check_results=check_results,
             result=result,
@@ -1096,7 +1128,9 @@ def _wsgi_test(
     return response
 
 
+@cached_test_func
 def asgi_test(
+    ctx: RunnerContext,
     case: Case,
     checks: Iterable[CheckFunction],
     targets: Iterable[Target],
@@ -1127,6 +1161,8 @@ def asgi_test(
             )
             response = _asgi_test(case, *args)
             add_cases(case, response, _asgi_test, *args)
+        elif store_interactions:
+            result.store_requests_response(case, None, Status.skip, [], headers=headers, session=None)
 
 
 def _asgi_test(
@@ -1147,9 +1183,11 @@ def _asgi_test(
     run_targets(targets, context)
     status = Status.success
     check_results: list[Check] = []
+    ctx = CheckContext(headers=CaseInsensitiveDict(headers) if headers else None)
     try:
         run_checks(
             case=case,
+            ctx=ctx,
             checks=checks,
             check_results=check_results,
             result=result,
@@ -1164,5 +1202,5 @@ def _asgi_test(
         if feedback is not None:
             feedback.add_test_case(case, response)
         if store_interactions:
-            result.store_requests_response(case, response, status, check_results)
+            result.store_requests_response(case, response, status, check_results, headers, session=None)
     return response

schemathesis/runner/impl/solo.py

@@ -1,40 +1,39 @@
 from __future__ import annotations
 
-import threading
 from dataclasses import dataclass
-from typing import Generator
+from typing import TYPE_CHECKING, Generator
 
-from ...models import TestResultSet
 from ...transports.auth import get_requests_auth
 from .. import events
 from .core import BaseRunner, asgi_test, get_session, network_test, wsgi_test
 
+if TYPE_CHECKING:
+    from .. import events
+    from .context import RunnerContext
+
 
 @dataclass
 class SingleThreadRunner(BaseRunner):
     """Fast runner that runs tests sequentially in the main thread."""
 
-    def _execute(
-        self, results: TestResultSet, stop_event: threading.Event
-    ) -> Generator[events.ExecutionEvent, None, None]:
-        for event in self._execute_impl(results):
+    def _execute(self, ctx: RunnerContext) -> Generator[events.ExecutionEvent, None, None]:
+        for event in self._execute_impl(ctx):
             yield event
-            if stop_event.is_set() or self._should_stop(event):
+            if ctx.is_stopped or self._should_stop(event):
                 break
 
-    def _execute_impl(self, results: TestResultSet) -> Generator[events.ExecutionEvent, None, None]:
+    def _execute_impl(self, ctx: RunnerContext) -> Generator[events.ExecutionEvent, None, None]:
         auth = get_requests_auth(self.auth, self.auth_type)
         with get_session(auth) as session:
             yield from self._run_tests(
                 maker=self.schema.get_all_tests,
-                template=network_test,
+                test_func=network_test,
                 settings=self.hypothesis_settings,
                 generation_config=self.generation_config,
-                seed=self.seed,
                 checks=self.checks,
                 max_response_time=self.max_response_time,
                 targets=self.targets,
-                results=results,
+                ctx=ctx,
                 session=session,
                 headers=self.headers,
                 request_config=self.request_config,
@@ -45,17 +44,16 @@ class SingleThreadRunner(BaseRunner):
 
 @dataclass
 class SingleThreadWSGIRunner(SingleThreadRunner):
-    def _execute_impl(self, results: TestResultSet) -> Generator[events.ExecutionEvent, None, None]:
+    def _execute_impl(self, ctx: RunnerContext) -> Generator[events.ExecutionEvent, None, None]:
         yield from self._run_tests(
             maker=self.schema.get_all_tests,
-            template=wsgi_test,
+            test_func=wsgi_test,
             settings=self.hypothesis_settings,
             generation_config=self.generation_config,
-            seed=self.seed,
             checks=self.checks,
             max_response_time=self.max_response_time,
             targets=self.targets,
-            results=results,
+            ctx=ctx,
             auth=self.auth,
             auth_type=self.auth_type,
             headers=self.headers,
@@ -66,17 +64,16 @@ class SingleThreadWSGIRunner(SingleThreadRunner):
 
 @dataclass
 class SingleThreadASGIRunner(SingleThreadRunner):
-    def _execute_impl(self, results: TestResultSet) -> Generator[events.ExecutionEvent, None, None]:
+    def _execute_impl(self, ctx: RunnerContext) -> Generator[events.ExecutionEvent, None, None]:
         yield from self._run_tests(
             maker=self.schema.get_all_tests,
-            template=asgi_test,
+            test_func=asgi_test,
             settings=self.hypothesis_settings,
             generation_config=self.generation_config,
-            seed=self.seed,
             checks=self.checks,
             max_response_time=self.max_response_time,
             targets=self.targets,
-            results=results,
+            ctx=ctx,
             headers=self.headers,
             store_interactions=self.store_interactions,
             dry_run=self.dry_run,