schemathesis 3.25.5__py3-none-any.whl → 3.26.0__py3-none-any.whl

schemathesis/runner/__init__.py

@@ -1,34 +1,37 @@
 from __future__ import annotations
 
 from random import Random
-from typing import Any, Callable, Generator, Iterable, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Callable, Generator, Iterable
 from urllib.parse import urlparse
 
 from .._override import CaseOverride
-from ..generation import DEFAULT_DATA_GENERATION_METHODS, DataGenerationMethod, GenerationConfig
 from ..constants import (
     DEFAULT_DEADLINE,
     DEFAULT_STATEFUL_RECURSION_LIMIT,
     HYPOTHESIS_IN_MEMORY_DATABASE_IDENTIFIER,
 )
-from ..internal.deprecation import deprecated_function
+from ..exceptions import SchemaError
+from ..generation import DEFAULT_DATA_GENERATION_METHODS, DataGenerationMethod, GenerationConfig
 from ..internal.datetime import current_datetime
+from ..internal.deprecation import deprecated_function
 from ..internal.validation import file_exists
-from ..transports.auth import get_requests_auth
-from ..exceptions import SchemaError
 from ..loaders import load_app
 from ..specs.graphql import loaders as gql_loaders
 from ..specs.openapi import loaders as oas_loaders
 from ..targets import DEFAULT_TARGETS, Target
+from ..transports.auth import get_requests_auth
 from ..types import Filter, NotSet, RawAuth, RequestCert
+from .probes import ProbeConfig
 
 if TYPE_CHECKING:
-    from . import events
+    import hypothesis
+
     from ..models import CheckFunction
     from ..schemas import BaseSchema
-    from .impl import BaseRunner
     from ..stateful import Stateful
-    import hypothesis
+    from . import events
+    from .impl import BaseRunner
+    from ..service.client import ServiceClient
 
 
 @deprecated_function(removed_in="4.0", replacement="schemathesis.runner.from_schema")
@@ -75,6 +78,8 @@ def prepare(
     hypothesis_report_multiple_bugs: bool | None = None,
     hypothesis_suppress_health_check: list[hypothesis.HealthCheck] | None = None,
     hypothesis_verbosity: hypothesis.Verbosity | None = None,
+    probe_config: ProbeConfig | None = None,
+    service_client: ServiceClient | None = None,
 ) -> Generator[events.ExecutionEvent, None, None]:
     """Prepare a generator that will run test cases against the given API definition."""
     from ..checks import DEFAULT_CHECKS
@@ -128,6 +133,8 @@ def prepare(
         stateful_recursion_limit=stateful_recursion_limit,
         count_operations=count_operations,
         count_links=count_links,
+        probe_config=probe_config,
+        service_client=service_client,
     )
 
 
@@ -188,6 +195,8 @@ def execute_from_schema(
     stateful_recursion_limit: int = DEFAULT_STATEFUL_RECURSION_LIMIT,
     count_operations: bool = True,
     count_links: bool = True,
+    probe_config: ProbeConfig | None = None,
+    service_client: ServiceClient | None,
 ) -> Generator[events.ExecutionEvent, None, None]:
     """Execute tests for the given schema.
 
@@ -237,6 +246,8 @@ def execute_from_schema(
             stateful_recursion_limit=stateful_recursion_limit,
             count_operations=count_operations,
             count_links=count_links,
+            probe_config=probe_config,
+            service_client=service_client,
         ).execute()
     except SchemaError as error:
         yield events.InternalError.from_schema_error(error)
@@ -341,9 +352,12 @@ def from_schema(
     stateful_recursion_limit: int = DEFAULT_STATEFUL_RECURSION_LIMIT,
     count_operations: bool = True,
     count_links: bool = True,
+    probe_config: ProbeConfig | None = None,
+    service_client: ServiceClient | None = None,
 ) -> BaseRunner:
-    from starlette.applications import Starlette
     import hypothesis
+    from starlette.applications import Starlette
+
     from ..checks import DEFAULT_CHECKS
     from .impl import (
         SingleThreadASGIRunner,
@@ -355,6 +369,7 @@ def from_schema(
     )
 
     checks = checks or DEFAULT_CHECKS
+    probe_config = probe_config or ProbeConfig()
 
     hypothesis_settings = hypothesis_settings or hypothesis.settings(deadline=DEFAULT_DEADLINE)
     generation_config = generation_config or GenerationConfig()
@@ -392,6 +407,8 @@ def from_schema(
                 stateful_recursion_limit=stateful_recursion_limit,
                 count_operations=count_operations,
                 count_links=count_links,
+                probe_config=probe_config,
+                service_client=service_client,
             )
         if isinstance(schema.app, Starlette):
             return ThreadPoolASGIRunner(
@@ -415,6 +432,8 @@ def from_schema(
                 stateful_recursion_limit=stateful_recursion_limit,
                 count_operations=count_operations,
                 count_links=count_links,
+                probe_config=probe_config,
+                service_client=service_client,
             )
         return ThreadPoolWSGIRunner(
             schema=schema,
@@ -438,6 +457,8 @@ def from_schema(
             stateful_recursion_limit=stateful_recursion_limit,
             count_operations=count_operations,
             count_links=count_links,
+            probe_config=probe_config,
+            service_client=service_client,
         )
     if not schema.app:
         return SingleThreadRunner(
@@ -465,6 +486,8 @@ def from_schema(
             stateful_recursion_limit=stateful_recursion_limit,
             count_operations=count_operations,
             count_links=count_links,
+            probe_config=probe_config,
+            service_client=service_client,
         )
     if isinstance(schema.app, Starlette):
         return SingleThreadASGIRunner(
@@ -488,6 +511,8 @@ def from_schema(
             stateful_recursion_limit=stateful_recursion_limit,
             count_operations=count_operations,
             count_links=count_links,
+            probe_config=probe_config,
+            service_client=service_client,
         )
     return SingleThreadWSGIRunner(
         schema=schema,
@@ -510,6 +535,8 @@ def from_schema(
         stateful_recursion_limit=stateful_recursion_limit,
         count_operations=count_operations,
         count_links=count_links,
+        probe_config=probe_config,
+        service_client=service_client,
     )
 
 

schemathesis/runner/events.py

@@ -6,6 +6,7 @@ from dataclasses import asdict, dataclass, field
 from typing import Any, TYPE_CHECKING
 
 from ..internal.datetime import current_datetime
+from ..internal.result import Result
 from ..generation import DataGenerationMethod
 from ..exceptions import SchemaError, SchemaErrorType, format_exception, RuntimeErrorType
 from .serialization import SerializedError, SerializedTestResult
@@ -14,6 +15,8 @@ from .serialization import SerializedError, SerializedTestResult
 if TYPE_CHECKING:
     from ..models import APIOperation, Status, TestResult, TestResultSet
     from ..schemas import BaseSchema
+    from ..service.models import AnalysisResult
+    from . import probes
 
 
 @dataclass
@@ -60,6 +63,7 @@ class Initialized(ExecutionEvent):
         schema: BaseSchema,
         count_operations: bool = True,
         count_links: bool = True,
+        start_time: float | None = None,
         started_at: str | None = None,
         seed: int | None,
     ) -> Initialized:
@@ -70,12 +74,37 @@ class Initialized(ExecutionEvent):
             links_count=schema.links_count if count_links else None,
             location=schema.location,
             base_url=schema.get_base_url(),
+            start_time=start_time or time.monotonic(),
             started_at=started_at or current_datetime(),
             specification_name=schema.verbose_name,
             seed=seed,
         )
 
 
+@dataclass
+class BeforeProbing(ExecutionEvent):
+    pass
+
+
+@dataclass
+class AfterProbing(ExecutionEvent):
+    probes: list[probes.ProbeRun] | None
+
+    def asdict(self, **kwargs: Any) -> dict[str, Any]:
+        probes = self.probes or []
+        return {"probes": [probe.serialize() for probe in probes], "events_type": self.__class__.__name__}
+
+
+@dataclass
+class BeforeAnalysis(ExecutionEvent):
+    pass
+
+
+@dataclass
+class AfterAnalysis(ExecutionEvent):
+    analysis: Result[AnalysisResult, Exception] | None
+
+
 class CurrentOperationMixin:
     method: str
     path: str
@@ -188,6 +217,9 @@ class InternalErrorType(str, enum.Enum):
     OTHER = "other"
 
 
+DEFAULT_INTERNAL_ERROR_MESSAGE = "An internal error occurred during the test run"
+
+
 @dataclass
 class InternalError(ExecutionEvent):
     """An error that happened inside the runner."""
@@ -226,7 +258,7 @@ class InternalError(ExecutionEvent):
             type_=InternalErrorType.OTHER,
             subtype=None,
             title="Test Execution Error",
-            message="An internal error occurred during the test run",
+            message=DEFAULT_INTERNAL_ERROR_MESSAGE,
             extras=[],
         )
 

schemathesis/runner/impl/core.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+
 import logging
 import re
 import threading
@@ -8,7 +9,7 @@ import uuid
 from contextlib import contextmanager
 from dataclasses import dataclass, field
 from types import TracebackType
-from typing import Any, Callable, Generator, Iterable, cast, TYPE_CHECKING, Literal
+from typing import TYPE_CHECKING, Any, Callable, Generator, Iterable, List, Literal, cast
 from warnings import WarningMessage, catch_warnings
 
 import hypothesis
@@ -16,53 +17,59 @@ import requests
 from _pytest.logging import LogCaptureHandler, catching_logs
 from hypothesis.errors import HypothesisException, InvalidArgument
 from hypothesis_jsonschema._canonicalise import HypothesisRefResolutionError
-from jsonschema.exceptions import ValidationError, SchemaError as JsonSchemaError
+from jsonschema.exceptions import SchemaError as JsonSchemaError
+from jsonschema.exceptions import ValidationError
 from requests.auth import HTTPDigestAuth, _basic_auth_str
 
-from ..._override import CaseOverride
 from ... import failures, hooks
 from ..._compat import MultipleFailures
 from ..._hypothesis import (
-    has_unsatisfied_example_mark,
-    get_non_serializable_mark,
-    get_invalid_regex_mark,
     get_invalid_example_headers_mark,
+    get_invalid_regex_mark,
+    get_non_serializable_mark,
+    has_unsatisfied_example_mark,
 )
+from ..._override import CaseOverride
 from ...auths import unregister as unregister_auth
-from ...generation import DataGenerationMethod, GenerationConfig
 from ...constants import (
     DEFAULT_STATEFUL_RECURSION_LIMIT,
     RECURSIVE_REFERENCE_ERROR_MESSAGE,
-    USER_AGENT,
     SERIALIZERS_SUGGESTION_MESSAGE,
+    USER_AGENT,
 )
 from ...exceptions import (
     CheckFailed,
     DeadlineExceeded,
+    InvalidHeadersExample,
     InvalidRegularExpression,
     NonCheckError,
     OperationSchemaError,
+    SerializationNotPossible,
     SkipTest,
+    format_exception,
     get_grouped_exception,
     maybe_set_assertion_message,
-    format_exception,
-    SerializationNotPossible,
-    InvalidHeadersExample,
 )
+from ...generation import DataGenerationMethod, GenerationConfig
 from ...hooks import HookContext, get_all_by_name
-from ...internal.result import Ok
+from ...internal.datetime import current_datetime
+from ...internal.result import Err, Ok, Result
 from ...models import APIOperation, Case, Check, CheckFunction, Status, TestResult, TestResultSet
 from ...runner import events
-from ...internal.datetime import current_datetime
 from ...schemas import BaseSchema
+from ...service import extensions
+from ...service.models import AnalysisResult, AnalysisSuccess
+from ...specs.openapi import formats
 from ...stateful import Feedback, Stateful
 from ...targets import Target, TargetContext
 from ...types import RawAuth, RequestCert
 from ...utils import capture_hypothesis_output
+from .. import probes
 from ..serialization import SerializedTestResult
 
 if TYPE_CHECKING:
-    from ...transports.responses import WSGIResponse, GenericResponse
+    from ...service.client import ServiceClient
+    from ...transports.responses import GenericResponse, WSGIResponse
 
 
 def _should_count_towards_stop(event: events.ExecutionEvent) -> bool:
@@ -77,6 +84,7 @@ class BaseRunner:
     targets: Iterable[Target]
     hypothesis_settings: hypothesis.settings
     generation_config: GenerationConfig
+    probe_config: probes.ProbeConfig
     override: CaseOverride | None = None
     auth: RawAuth | None = None
     auth_type: str | None = None
@@ -92,6 +100,7 @@ class BaseRunner:
     stateful_recursion_limit: int = DEFAULT_STATEFUL_RECURSION_LIMIT
     count_operations: bool = True
     count_links: bool = True
+    service_client: ServiceClient | None = None
     _failures_counter: int = 0
 
     def execute(self) -> EventStream:
@@ -104,26 +113,79 @@ class BaseRunner:
         if self.auth is not None:
             unregister_auth()
         results = TestResultSet(seed=self.seed)
-
-        initialized = events.Initialized.from_schema(
-            schema=self.schema, count_operations=self.count_operations, count_links=self.count_links, seed=self.seed
-        )
+        start_time = time.monotonic()
+        initialized = None
+        __probes = None
+        __analysis: Result[AnalysisResult, Exception] | None = None
+
+        def _initialize() -> events.Initialized:
+            nonlocal initialized
+            initialized = events.Initialized.from_schema(
+                schema=self.schema,
+                count_operations=self.count_operations,
+                count_links=self.count_links,
+                seed=self.seed,
+                start_time=start_time,
+            )
+            return initialized
 
         def _finish() -> events.Finished:
             if has_all_not_found(results):
                 results.add_warning(ALL_NOT_FOUND_WARNING_MESSAGE)
-            return events.Finished.from_results(results=results, running_time=time.monotonic() - initialized.start_time)
+            return events.Finished.from_results(results=results, running_time=time.monotonic() - start_time)
 
-        if stop_event.is_set():
-            yield _finish()
-            return
+        def _before_probes() -> events.BeforeProbing:
+            return events.BeforeProbing()
+
+        def _run_probes() -> None:
+            if not self.dry_run:
+                nonlocal __probes
+
+                __probes = run_probes(self.schema, self.probe_config)
+
+        def _after_probes() -> events.AfterProbing:
+            _probes = cast(List[probes.ProbeRun], __probes)
+            return events.AfterProbing(probes=_probes)
 
-        yield initialized
+        def _before_analysis() -> events.BeforeAnalysis:
+            return events.BeforeAnalysis()
+
+        def _run_analysis() -> None:
+            nonlocal __analysis, __probes
+
+            if self.service_client is not None:
+                try:
+                    _probes = cast(List[probes.ProbeRun], __probes)
+                    result = self.service_client.analyze_schema(_probes, self.schema.raw_schema)
+                    if isinstance(result, AnalysisSuccess):
+                        extensions.apply(result.extensions, self.schema)
+                    __analysis = Ok(result)
+                except Exception as exc:
+                    __analysis = Err(exc)
+
+        def _after_analysis() -> events.AfterAnalysis:
+            return events.AfterAnalysis(analysis=__analysis)
 
         if stop_event.is_set():
             yield _finish()
             return
 
+        for event_factory in (
+            _initialize,
+            _before_probes,
+            _run_probes,
+            _after_probes,
+            _before_analysis,
+            _run_analysis,
+            _after_analysis,
+        ):
+            event = event_factory()
+            if event is not None:
+                yield event
+            if stop_event.is_set():
+                yield _finish()
+                return
+
         try:
             yield from self._execute(results, stop_event)
         except KeyboardInterrupt:
@@ -228,6 +290,20 @@ class BaseRunner:
                 )
 
 
+def run_probes(schema: BaseSchema, config: probes.ProbeConfig) -> list[probes.ProbeRun]:
+    """Discover capabilities of the tested app."""
+    results = probes.run(schema, config)
+    for result in results:
+        if isinstance(result.probe, probes.NullByteInHeader) and result.is_failure:
+            from ...specs.openapi._hypothesis import HEADER_FORMAT, header_values
+
+            formats.register(
+                HEADER_FORMAT,
+                header_values(blacklist_characters="\n\r\x00").map(str.lstrip),
+            )
+    return results
+
+
 @dataclass
 class EventStream:
     """Schemathesis event stream.

schemathesis/{cli → runner}/probes.py

@@ -5,6 +5,7 @@ the application supports certain inputs. This is done to avoid false positives i
 For example, certail web servers do not support NULL bytes in headers, in such cases, the generated test case
 will not reach the tested application at all.
 """
+
 from __future__ import annotations
 
 import enum
@@ -21,13 +22,24 @@ from ..transports.auth import get_requests_auth
 if TYPE_CHECKING:
     import requests
 
+    from ..types import RequestCert
     from ..schemas import BaseSchema
-    from . import LoaderConfig
 
 
 HEADER_NAME = "X-Schemathesis-Probe"
 
 
+@dataclass
+class ProbeConfig:
+    base_url: str | None = None
+    request_tls_verify: bool | str = True
+    request_proxy: str | None = None
+    request_cert: RequestCert | None = None
+    auth: tuple[str, str] | None = None
+    auth_type: str | None = None
+    headers: dict[str, str] | None = None
+
+
 @dataclass
 class Probe:
     """A request to determine the capabilities of the application under test."""
@@ -35,15 +47,15 @@ class Probe:
     name: str
 
     def prepare_request(
-        self, session: requests.Session, request: requests.Request, schema: BaseSchema, config: LoaderConfig
+        self, session: requests.Session, request: requests.Request, schema: BaseSchema, config: ProbeConfig
     ) -> requests.PreparedRequest:
         raise NotImplementedError
 
-    def analyze_response(self, response: requests.Response) -> ProbeResultType:
+    def analyze_response(self, response: requests.Response) -> ProbeOutcome:
         raise NotImplementedError
 
 
-class ProbeResultType(str, enum.Enum):
+class ProbeOutcome(str, enum.Enum):
     # Capability is supported
     SUCCESS = "success"
     # Capability is not supported
@@ -55,18 +67,16 @@ class ProbeResultType(str, enum.Enum):
 
 
 @dataclass
-class ProbeResult:
-    """Result of a probe."""
-
+class ProbeRun:
     probe: Probe
-    type: ProbeResultType
+    outcome: ProbeOutcome
     request: requests.PreparedRequest | None = None
     response: requests.Response | None = None
     error: requests.RequestException | None = None
 
     @property
     def is_failure(self) -> bool:
-        return self.type == ProbeResultType.FAILURE
+        return self.outcome == ProbeOutcome.FAILURE
 
     def serialize(self) -> dict[str, Any]:
         """Serialize probe results so it can be sent over the network."""
@@ -87,7 +97,7 @@ class ProbeResult:
             error = None
         return {
             "name": self.probe.name,
-            "type": self.type.value,
+            "outcome": self.outcome.value,
             "request": request,
             "response": response,
             "error": error,
@@ -101,25 +111,25 @@ class NullByteInHeader(Probe):
     name: str = "NULL_BYTE_IN_HEADER"
 
     def prepare_request(
-        self, session: requests.Session, request: requests.Request, schema: BaseSchema, config: LoaderConfig
+        self, session: requests.Session, request: requests.Request, schema: BaseSchema, config: ProbeConfig
     ) -> requests.PreparedRequest:
         request.method = "GET"
         request.url = config.base_url or schema.get_base_url()
         request.headers = {"X-Schemathesis-Probe-Null": "\x00"}
         return session.prepare_request(request)
 
-    def analyze_response(self, response: requests.Response) -> ProbeResultType:
+    def analyze_response(self, response: requests.Response) -> ProbeOutcome:
         if response.status_code == 400:
-            return ProbeResultType.FAILURE
-        return ProbeResultType.SUCCESS
+            return ProbeOutcome.FAILURE
+        return ProbeOutcome.SUCCESS
 
 
 PROBES = (NullByteInHeader,)
 
 
-def send(probe: Probe, session: requests.Session, schema: BaseSchema, config: LoaderConfig) -> ProbeResult:
+def send(probe: Probe, session: requests.Session, schema: BaseSchema, config: ProbeConfig) -> ProbeRun:
     """Send the probe to the application."""
-    from requests import Request, RequestException, PreparedRequest
+    from requests import PreparedRequest, Request, RequestException
     from requests.exceptions import MissingSchema
     from urllib3.exceptions import InsecureRequestWarning
 
@@ -129,23 +139,24 @@ def send(probe: Probe, session: requests.Session, schema: BaseSchema, config: Lo
         request.headers["User-Agent"] = USER_AGENT
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", InsecureRequestWarning)
-            response = session.send(request)
+            response = session.send(request, timeout=2)
     except MissingSchema:
         # In-process ASGI/WSGI testing will have local URLs and requires extra handling
         # which is not currently implemented
-        return ProbeResult(probe, ProbeResultType.SKIP, None, None, None)
+        return ProbeRun(probe, ProbeOutcome.SKIP, None, None, None)
     except RequestException as exc:
         req = exc.request if isinstance(exc.request, PreparedRequest) else None
-        return ProbeResult(probe, ProbeResultType.ERROR, req, None, exc)
+        return ProbeRun(probe, ProbeOutcome.ERROR, req, None, exc)
     result_type = probe.analyze_response(response)
-    return ProbeResult(probe, result_type, request, response)
+    return ProbeRun(probe, result_type, request, response)
 
 
-def run(schema: BaseSchema, config: LoaderConfig) -> list[ProbeResult]:
+def run(schema: BaseSchema, config: ProbeConfig) -> list[ProbeRun]:
     """Run all probes against the given schema."""
     from requests import Session
 
     session = Session()
+    session.headers.update(config.headers or {})
     session.verify = config.request_tls_verify
     if config.request_cert is not None:
         session.cert = config.request_cert

schemathesis/runner/serialization.py

@@ -2,6 +2,7 @@
 
 They all consist of primitive types and don't have references to schemas, app, etc.
 """
+
 from __future__ import annotations
 import logging
 import re
@@ -226,8 +227,9 @@ class SerializedError:
             message = f"Scalar type '{scalar_name}' is not recognized"
             extras = []
             title = "Unknown GraphQL Scalar"
-        elif isinstance(exception, hypothesis.errors.InvalidArgument) and str(exception).endswith(
-            "larger than Hypothesis is designed to handle"
+        elif isinstance(exception, hypothesis.errors.InvalidArgument) and (
+            str(exception).endswith("larger than Hypothesis is designed to handle")
+            or "can neber generate an example, because min_size is larger than Hypothesis suports."
         ):
             type_ = RuntimeErrorType.HYPOTHESIS_HEALTH_CHECK_LARGE_BASE_EXAMPLE
             message = HEALTH_CHECK_MESSAGE_LARGE_BASE_EXAMPLE

schemathesis/schemas.py

@@ -6,6 +6,7 @@ Their responsibilities:
 
 They give only static definitions of paths.
 """
+
 from __future__ import annotations
 from collections.abc import Mapping, MutableMapping
 from contextlib import nullcontext

schemathesis/serializers.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+
 import binascii
 import os
 from dataclasses import dataclass
@@ -10,13 +11,14 @@ from typing import (
     Collection,
     Dict,
     Generator,
-    cast,
     Protocol,
+    cast,
     runtime_checkable,
 )
 
-from .internal.copy import fast_deepcopy
 from ._xml import _to_xml
+from .internal.copy import fast_deepcopy
+from .internal.jsonschema import traverse_schema
 from .transports.content_types import (
     is_json_media_type,
     is_plain_text_media_type,
@@ -150,6 +152,10 @@ class JSONSerializer:
         return _to_json(value)
 
 
+def _replace_binary(value: dict) -> dict:
+    return {key: value.data if isinstance(value, Binary) else value for key, value in value.items()}
+
+
 def _to_yaml(value: Any) -> dict[str, Any]:
     import yaml
 
@@ -162,10 +168,12 @@ def _to_yaml(value: Any) -> dict[str, Any]:
         return {"data": value}
     if isinstance(value, Binary):
         return {"data": value.data}
+    if isinstance(value, (list, dict)):
+        value = traverse_schema(value, _replace_binary)
     return {"data": yaml.dump(value, Dumper=SafeDumper)}
 
 
-@register("text/yaml", aliases=("text/x-yaml", "application/x-yaml", "text/vnd.yaml"))
+@register("text/yaml", aliases=("text/x-yaml", "text/vnd.yaml", "text/yml", "application/yaml", "application/x-yaml"))
 class YAMLSerializer:
     def as_requests(self, context: SerializerContext, value: Any) -> dict[str, Any]:
         return _to_yaml(value)