schemathesis 3.35.5__py3-none-any.whl → 3.36.0__py3-none-any.whl

schemathesis/runner/impl/threadpool.py

@@ -13,7 +13,6 @@ from hypothesis.errors import HypothesisWarning
 
 from ..._hypothesis import create_test
 from ...internal.result import Ok
-from ...models import CheckFunction
 from ...stateful import Feedback, Stateful
 from ...transports.auth import get_requests_auth
 from ...utils import capture_hypothesis_output
@@ -21,13 +20,13 @@ from .. import events
 from .core import BaseRunner, asgi_test, get_session, handle_schema_error, network_test, run_test, wsgi_test
 
 if TYPE_CHECKING:
-    from .context import RunnerContext
     import hypothesis
 
     from ...generation import DataGenerationMethod, GenerationConfig
-    from ...models import CheckFunction
+    from ...internal.checks import CheckFunction
     from ...targets import Target
     from ...types import RawAuth
+    from .context import RunnerContext
 
 
 def _run_task(
@@ -235,7 +234,7 @@ class ThreadPoolRunner(BaseRunner):
         # It would be better to have a separate producer thread and communicate via threading events.
         # Though it is a bit more complex, so the current solution is suboptimal in terms of resources utilization,
         # but good enough and easy enough to implement.
-        tasks_generator = iter(self.schema.get_all_operations())
+        tasks_generator = iter(self.schema.get_all_operations(generation_config=self.generation_config))
         generator_done = threading.Event()
         tasks_queue: Queue = Queue()
         # Add at least `workers_num` tasks first, so all workers are busy

schemathesis/schemas.py

@@ -241,7 +241,7 @@ class BaseSchema(Mapping):
         raise NotImplementedError
 
     def get_all_operations(
-        self, hooks: HookDispatcher | None = None
+        self, hooks: HookDispatcher | None = None, generation_config: GenerationConfig | None = None
     ) -> Generator[Result[APIOperation, OperationSchemaError], None, None]:
         raise NotImplementedError
 
@@ -276,7 +276,7 @@ class BaseSchema(Mapping):
         _given_kwargs: dict[str, GivenInput] | None = None,
     ) -> Generator[Result[tuple[APIOperation, Callable], OperationSchemaError], None, None]:
         """Generate all operations and Hypothesis tests for them."""
-        for result in self.get_all_operations(hooks=hooks):
+        for result in self.get_all_operations(hooks=hooks, generation_config=generation_config):
             if isinstance(result, Ok):
                 operation = result.ok()
                 _as_strategy_kwargs: dict[str, Any] | None

schemathesis/specs/graphql/loaders.py

@@ -139,12 +139,12 @@ def from_url(
             interval=WAIT_FOR_SCHEMA_INTERVAL,
         )
         def _load_schema(_uri: str, **_kwargs: Any) -> requests.Response:
-            _kwargs.setdefault("timeout", DEFAULT_RESPONSE_TIMEOUT / 1000)
-            return requests.post(_uri, **kwargs)
+            return requests.post(_uri, **_kwargs)
 
     else:
         _load_schema = requests.post
 
+    kwargs.setdefault("timeout", DEFAULT_RESPONSE_TIMEOUT / 1000)
     response = load_schema_from_url(lambda: _load_schema(url, **kwargs))
     raw_schema = extract_schema_from_response(response)
     return from_dict(

schemathesis/specs/graphql/schemas.py

@@ -27,7 +27,7 @@ from requests.structures import CaseInsensitiveDict
 
 from ... import auths
 from ...checks import not_a_server_error
-from ...constants import NOT_SET
+from ...constants import NOT_SET, SCHEMATHESIS_TEST_CASE_HEADER
 from ...exceptions import OperationNotFound, OperationSchemaError
 from ...generation import DataGenerationMethod, GenerationConfig
 from ...hooks import (
@@ -38,7 +38,7 @@ from ...hooks import (
     should_skip_operation,
 )
 from ...internal.result import Ok, Result
-from ...models import APIOperation, Case, CheckFunction, OperationDefinition
+from ...models import APIOperation, Case, OperationDefinition
 from ...schemas import APIOperationMap, BaseSchema
 from ...types import Body, Cookies, Headers, NotSet, PathParameters, Query
 from ..openapi.constants import LOCATION_TO_CONTAINER
@@ -49,6 +49,7 @@ if TYPE_CHECKING:
     from hypothesis.strategies import SearchStrategy
 
     from ...auths import AuthStorage
+    from ...internal.checks import CheckFunction
     from ...stateful import Stateful, StatefulTest
     from ...transports.responses import GenericResponse
 
@@ -61,6 +62,9 @@ class RootType(enum.Enum):
 
 @dataclass(repr=False)
 class GraphQLCase(Case):
+    def __hash__(self) -> int:
+        return hash(self.as_curl_command({SCHEMATHESIS_TEST_CASE_HEADER: "0"}))
+
     def _get_url(self, base_url: str | None) -> str:
         base_url = self._get_base_url(base_url)
         # Replace the path, in case if the user provided any path parameters via hooks
@@ -78,11 +82,12 @@ class GraphQLCase(Case):
         additional_checks: tuple[CheckFunction, ...] = (),
         excluded_checks: tuple[CheckFunction, ...] = (),
         code_sample_style: str | None = None,
+        headers: dict[str, Any] | None = None,
     ) -> None:
         checks = checks or (not_a_server_error,)
         checks += additional_checks
         checks = tuple(check for check in checks if check not in excluded_checks)
-        return super().validate_response(response, checks, code_sample_style=code_sample_style)
+        return super().validate_response(response, checks, code_sample_style=code_sample_style, headers=headers)
 
 
 C = TypeVar("C", bound=Case)
@@ -186,8 +191,7 @@ class GraphQLSchema(BaseSchema):
         return 0
 
     def get_all_operations(
-        self,
-        hooks: HookDispatcher | None = None,
+        self, hooks: HookDispatcher | None = None, generation_config: GenerationConfig | None = None
     ) -> Generator[Result[APIOperation, OperationSchemaError], None, None]:
         schema = self.client_schema
         for root_type, operation_type in (

schemathesis/specs/openapi/checks.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
+import enum
 from http.cookies import SimpleCookie
 from typing import TYPE_CHECKING, Any, Dict, Generator, NoReturn, cast
 from urllib.parse import parse_qs, urlparse
@@ -25,11 +26,12 @@ from .utils import expand_status_code
 if TYPE_CHECKING:
     from requests import PreparedRequest
 
+    from ...internal.checks import CheckContext
     from ...models import APIOperation, Case
     from ...transports.responses import GenericResponse
 
 
-def status_code_conformance(response: GenericResponse, case: Case) -> bool | None:
+def status_code_conformance(ctx: CheckContext, response: GenericResponse, case: Case) -> bool | None:
     from .schemas import BaseOpenAPISchema
 
     if not isinstance(case.operation.schema, BaseOpenAPISchema):
@@ -60,7 +62,7 @@ def _expand_responses(responses: dict[str | int, Any]) -> Generator[int, None, N
         yield from expand_status_code(code)
 
 
-def content_type_conformance(response: GenericResponse, case: Case) -> bool | None:
+def content_type_conformance(ctx: CheckContext, response: GenericResponse, case: Case) -> bool | None:
     from .schemas import BaseOpenAPISchema
 
     if not isinstance(case.operation.schema, BaseOpenAPISchema):
@@ -115,7 +117,7 @@ def _reraise_malformed_media_type(case: Case, exc: ValueError, location: str, ac
     ) from exc
 
 
-def response_headers_conformance(response: GenericResponse, case: Case) -> bool | None:
+def response_headers_conformance(ctx: CheckContext, response: GenericResponse, case: Case) -> bool | None:
     import jsonschema
 
     from .parameters import OpenAPI20Parameter, OpenAPI30Parameter
@@ -171,11 +173,11 @@ def response_headers_conformance(response: GenericResponse, case: Case) -> bool
                     )
                 except jsonschema.ValidationError as exc:
                     exc_class = get_schema_validation_error(case.operation.verbose_name, exc)
-                    ctx = failures.ValidationErrorContext.from_exception(
+                    error_ctx = failures.ValidationErrorContext.from_exception(
                         exc, output_config=case.operation.schema.output_config
                     )
                     try:
-                        raise exc_class("Response header does not conform to the schema", context=ctx) from exc
+                        raise exc_class("Response header does not conform to the schema", context=error_ctx) from exc
                     except Exception as exc:
                         errors.append(exc)
     return _maybe_raise_one_or_more(errors)  # type: ignore[func-returns-value]
@@ -203,7 +205,7 @@ def _coerce_header_value(value: str, schema: dict[str, Any]) -> str | int | floa
     return value
 
 
-def response_schema_conformance(response: GenericResponse, case: Case) -> bool | None:
+def response_schema_conformance(ctx: CheckContext, response: GenericResponse, case: Case) -> bool | None:
     from .schemas import BaseOpenAPISchema
 
     if not isinstance(case.operation.schema, BaseOpenAPISchema):
@@ -211,7 +213,7 @@ def response_schema_conformance(response: GenericResponse, case: Case) -> bool |
     return case.operation.validate_response(response)
 
 
-def negative_data_rejection(response: GenericResponse, case: Case) -> bool | None:
+def negative_data_rejection(ctx: CheckContext, response: GenericResponse, case: Case) -> bool | None:
     from .schemas import BaseOpenAPISchema
 
     if not isinstance(case.operation.schema, BaseOpenAPISchema):
@@ -258,7 +260,7 @@ def has_only_additional_properties_in_non_body_parameters(case: Case) -> bool:
     return True
 
 
-def use_after_free(response: GenericResponse, original: Case) -> bool | None:
+def use_after_free(ctx: CheckContext, response: GenericResponse, original: Case) -> bool | None:
     from ...transports.responses import get_reason
     from .schemas import BaseOpenAPISchema
 
@@ -298,7 +300,7 @@ def use_after_free(response: GenericResponse, original: Case) -> bool | None:
     return None
 
 
-def ensure_resource_availability(response: GenericResponse, original: Case) -> bool | None:
+def ensure_resource_availability(ctx: CheckContext, response: GenericResponse, original: Case) -> bool | None:
     from ...transports.responses import get_reason
     from .schemas import BaseOpenAPISchema
 
@@ -332,7 +334,12 @@ def ensure_resource_availability(response: GenericResponse, original: Case) -> b
     return None
 
 
-def ignored_auth(response: GenericResponse, case: Case) -> bool | None:
+class AuthKind(enum.Enum):
+    EXPLICIT = "explicit"
+    GENERATED = "generated"
+
+
+def ignored_auth(ctx: CheckContext, response: GenericResponse, case: Case) -> bool | None:
     """Check if an operation declares authentication as a requirement but does not actually enforce it."""
     from .schemas import BaseOpenAPISchema
 
@@ -340,32 +347,49 @@ def ignored_auth(response: GenericResponse, case: Case) -> bool | None:
         return True
     security_parameters = _get_security_parameters(case.operation)
     # Authentication is required for this API operation and response is successful
-    # Will it still be successful if there is no auth?
     if security_parameters and 200 <= response.status_code < 300:
-        if _contains_auth(response.request, security_parameters):
-            # If there is auth in the request, then drop it and retry the call
+        auth = _contains_auth(ctx, response.request, security_parameters)
+        if auth == AuthKind.EXPLICIT:
+            # Auth is explicitly set, it is expected to be valid
+            # Check if invalid auth will give an error
             _remove_auth_from_case(case, security_parameters)
             new_response = case.operation.schema.transport.send(case)
             if 200 <= new_response.status_code < 300:
-                # Mutate the response object in place on the best effort basis
-                if hasattr(response, "__attrs__"):
-                    for attribute in new_response.__attrs__:
-                        setattr(response, attribute, getattr(new_response, attribute))
-                else:
-                    response.__dict__.update(new_response.__dict__)
-                _raise_auth_error(new_response, case.operation.verbose_name)
+                _update_response(response, new_response)
+                _raise_no_auth_error(new_response, case.operation.verbose_name, "that requires authentication")
+            # Try to set invalid auth and check if it succeeds
+            for parameter in security_parameters:
+                _set_auth_for_case(case, parameter)
+                new_response = case.operation.schema.transport.send(case)
+                if 200 <= new_response.status_code < 300:
+                    _update_response(response, new_response)
+                    _raise_no_auth_error(new_response, case.operation.verbose_name, "with any auth")
+                _remove_auth_from_case(case, security_parameters)
+        elif auth == AuthKind.GENERATED:
+            # If this auth is generated which means it is likely invalid, then
+            # this request should have been an error
+            _raise_no_auth_error(response, case.operation.verbose_name, "with invalid auth")
         else:
             # Successful response when there is no auth
-            _raise_auth_error(response, case.operation.verbose_name)
+            _raise_no_auth_error(response, case.operation.verbose_name, "that requires authentication")
     return None
 
 
-def _raise_auth_error(response: GenericResponse, operation: str) -> NoReturn:
+def _update_response(old: GenericResponse, new: GenericResponse) -> None:
+    # Mutate the response object in place on the best effort basis
+    if hasattr(old, "__attrs__"):
+        for attribute in new.__attrs__:
+            setattr(old, attribute, getattr(new, attribute))
+    else:
+        old.__dict__.update(new.__dict__)
+
+
+def _raise_no_auth_error(response: GenericResponse, operation: str, suffix: str) -> NoReturn:
     from ...transports.responses import get_reason
 
     exc_class = get_ignored_auth_error(operation)
     reason = get_reason(response.status_code)
-    message = f"The API returned `{response.status_code} {reason}` for `{operation}` that requires authentication."
+    message = f"The API returned `{response.status_code} {reason}` for `{operation}` {suffix}."
     raise exc_class(
         failures.IgnoredAuth.title,
         context=failures.IgnoredAuth(message=message),
@@ -387,7 +411,9 @@ def _get_security_parameters(operation: APIOperation) -> list[SecurityParameter]
     ]
 
 
-def _contains_auth(request: PreparedRequest, security_parameters: list[SecurityParameter]) -> bool:
+def _contains_auth(
+    ctx: CheckContext, request: PreparedRequest, security_parameters: list[SecurityParameter]
+) -> AuthKind | None:
     """Whether a request has authentication declared in the schema."""
     from requests.cookies import RequestsCookieJar
 
@@ -410,10 +436,20 @@ def _contains_auth(request: PreparedRequest, security_parameters: list[SecurityP
         return p["in"] == "cookie" and (p["name"] in cookies or p["name"] in header_cookies)
 
     for parameter in security_parameters:
-        if has_header(parameter) or has_query(parameter) or has_cookie(parameter):
-            return True
+        if has_header(parameter):
+            if ctx.headers is not None and parameter["name"] in ctx.headers:
+                return AuthKind.EXPLICIT
+            return AuthKind.GENERATED
+        if has_cookie(parameter):
+            if ctx.headers is not None and "Cookie" in ctx.headers:
+                cookies = cast(RequestsCookieJar, ctx.headers["Cookie"])  # type: ignore
+                if parameter["name"] in cookies:
+                    return AuthKind.EXPLICIT
+            return AuthKind.GENERATED
+        if has_query(parameter):
+            return AuthKind.GENERATED
 
-    return False
+    return None
 
 
 def _remove_auth_from_case(case: Case, security_parameters: list[SecurityParameter]) -> None:
@@ -431,6 +467,19 @@ def _remove_auth_from_case(case: Case, security_parameters: list[SecurityParamet
             case.cookies.pop(name, None)
 
 
+def _set_auth_for_case(case: Case, parameter: SecurityParameter) -> None:
+    name = parameter["name"]
+    for location, attr_name in (
+        ("header", "headers"),
+        ("query", "query"),
+        ("cookie", "cookies"),
+    ):
+        if parameter["in"] == location:
+            container = getattr(case, attr_name, {})
+            container[name] = "SCHEMATHESIS-INVALID-VALUE"
+            setattr(case, attr_name, container)
+
+
 @dataclass
 class ResourcePath:
     """A path to a resource with variables."""

schemathesis/specs/openapi/loaders.py

@@ -163,12 +163,12 @@ def from_uri(
             interval=WAIT_FOR_SCHEMA_INTERVAL,
         )
         def _load_schema(_uri: str, **_kwargs: Any) -> requests.Response:
-            _kwargs.setdefault("timeout", DEFAULT_RESPONSE_TIMEOUT / 1000)
-            return requests.get(_uri, **kwargs)
+            return requests.get(_uri, **_kwargs)
 
     else:
         _load_schema = requests.get
 
+    kwargs.setdefault("timeout", DEFAULT_RESPONSE_TIMEOUT / 1000)
     response = load_schema_from_url(lambda: _load_schema(uri, **kwargs))
     return from_file(
         response.text,

schemathesis/specs/openapi/schemas.py

@@ -253,7 +253,7 @@ class BaseOpenAPISchema(BaseSchema):
         return self.collect_parameters(itertools.chain(parameters, shared_parameters), operation)
 
     def get_all_operations(
-        self, hooks: HookDispatcher | None = None
+        self, hooks: HookDispatcher | None = None, generation_config: GenerationConfig | None = None
     ) -> Generator[Result[APIOperation, OperationSchemaError], None, None]:
         """Iterate over all operations defined in the API.
 
@@ -308,7 +308,17 @@ class BaseOpenAPISchema(BaseSchema):
                                 continue
                             parameters = resolved.get("parameters", ())
                             parameters = collect_parameters(itertools.chain(parameters, shared_parameters), resolved)
-                            operation = make_operation(path, method, parameters, entry, resolved, scope)
+                            operation = make_operation(
+                                path,
+                                method,
+                                parameters,
+                                entry,
+                                resolved,
+                                scope,
+                                with_security_parameters=generation_config.with_security_parameters
+                                if generation_config
+                                else None,
+                            )
                             context = HookContext(operation=operation)
                             if (
                                 should_skip_operation(GLOBAL_HOOK_DISPATCHER, context)
@@ -383,6 +393,7 @@ class BaseOpenAPISchema(BaseSchema):
         raw: dict[str, Any],
         resolved: dict[str, Any],
         scope: str,
+        with_security_parameters: bool | None = None,
     ) -> APIOperation:
         """Create JSON schemas for the query, body, etc from Swagger parameters definitions."""
         __tracebackhide__ = True
@@ -397,7 +408,12 @@ class BaseOpenAPISchema(BaseSchema):
         )
         for parameter in parameters:
             operation.add_parameter(parameter)
-        if self.generation_config.with_security_parameters:
+        with_security_parameters = (
+            with_security_parameters
+            if with_security_parameters is not None
+            else self.generation_config.with_security_parameters
+        )
+        if with_security_parameters:
             self.security.process_definitions(self.raw_schema, operation, self.resolver)
         self.dispatch_hook("before_init_operation", HookContext(operation=operation), operation)
         return operation

schemathesis/stateful/config.py

@@ -69,6 +69,7 @@ class StatefulTestRunnerConfig:
     max_response_time: int | None = None
     dry_run: bool = False
     targets: list[Target] = field(default_factory=list)
+    unique_data: bool = False
 
     def __post_init__(self) -> None:
         import hypothesis

schemathesis/stateful/context.py

@@ -4,6 +4,7 @@ import traceback
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Tuple, Type, Union
 
+from ..constants import NOT_SET
 from ..exceptions import CheckFailed
 from ..targets import TargetMetricCollector
 from . import events
@@ -11,6 +12,7 @@ from . import events
 if TYPE_CHECKING:
     from ..models import Case, Check
     from ..transports.responses import GenericResponse
+    from ..types import NotSet
 
 FailureKey = Union[Type[CheckFailed], Tuple[str, int]]
 
@@ -52,6 +54,7 @@ class RunnerContext:
     completed_scenarios: int = 0
     # Metrics collector for targeted testing
     metric_collector: TargetMetricCollector = field(default_factory=lambda: TargetMetricCollector(targets=[]))
+    step_outcomes: dict[int, BaseException | None] = field(default_factory=dict)
 
     @property
     def current_scenario_status(self) -> events.ScenarioStatus:
@@ -69,6 +72,7 @@ class RunnerContext:
         self.completed_scenarios += 1
         self.current_step_status = None
         self.current_response = None
+        self.step_outcomes.clear()
 
     def reset_step(self) -> None:
         self.checks_for_step = []
@@ -123,3 +127,9 @@ class RunnerContext:
         self.seen_in_suite.clear()
         self.reset_scenario()
         self.metric_collector.reset()
+
+    def store_step_outcome(self, case: Case, outcome: BaseException | None) -> None:
+        self.step_outcomes[hash(case)] = outcome
+
+    def get_step_outcome(self, case: Case) -> BaseException | None | NotSet:
+        return self.step_outcomes.get(hash(case), NOT_SET)

schemathesis/stateful/runner.py

@@ -163,17 +163,34 @@ def _execute_state_machine_loop(
             try:
                 if config.dry_run:
                     return None
+                if config.unique_data:
+                    cached = ctx.get_step_outcome(case)
+                    if isinstance(cached, BaseException):
+                        raise cached
+                    elif cached is None:
+                        return None
                 result = super().step(case, previous)
                 ctx.step_succeeded()
-            except CheckFailed:
+            except CheckFailed as exc:
+                if config.unique_data:
+                    ctx.store_step_outcome(case, exc)
                 ctx.step_failed()
                 raise
-            except Exception:
+            except Exception as exc:
+                if config.unique_data:
+                    ctx.store_step_outcome(case, exc)
                 ctx.step_errored()
                 raise
             except KeyboardInterrupt:
                 ctx.step_interrupted()
                 raise
+            except BaseException as exc:
+                if config.unique_data:
+                    ctx.store_step_outcome(case, exc)
+                raise exc
+            else:
+                if config.unique_data:
+                    ctx.store_step_outcome(case, None)
             finally:
                 transition_id: events.TransitionId | None
                 if previous is not None:

schemathesis/stateful/state_machine.py

@@ -12,7 +12,8 @@ from hypothesis.stateful import RuleBasedStateMachine
 from .._dependency_versions import HYPOTHESIS_HAS_STATEFUL_NAMING_IMPROVEMENTS
 from ..constants import NO_LINKS_ERROR_MESSAGE, NOT_SET
 from ..exceptions import UsageError
-from ..models import APIOperation, Case, CheckFunction
+from ..internal.checks import CheckFunction
+from ..models import APIOperation, Case
 from .config import _default_hypothesis_settings_factory
 from .runner import StatefulTestRunner, StatefulTestRunnerConfig
 from .sink import StateMachineSink

schemathesis/stateful/validation.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from ..exceptions import CheckFailed, get_grouped_exception
+from ..internal.checks import CheckContext
 
 if TYPE_CHECKING:
     from ..failures import FailureContext
-    from ..models import Case, CheckFunction
+    from ..internal.checks import CheckFunction
+    from ..models import Case
     from ..transports.responses import GenericResponse
     from .context import RunnerContext
 
@@ -19,8 +21,11 @@ def validate_response(
     checks: tuple[CheckFunction, ...],
     additional_checks: tuple[CheckFunction, ...] = (),
     max_response_time: int | None = None,
+    headers: dict[str, Any] | None = None,
 ) -> None:
     """Validate the response against the provided checks."""
+    from requests.structures import CaseInsensitiveDict
+
     from .._compat import MultipleFailures
     from ..checks import _make_max_response_time_failure_message
     from ..failures import ResponseTimeExceeded
@@ -28,6 +33,7 @@ def validate_response(
 
     exceptions: list[CheckFailed | AssertionError] = []
     check_results = ctx.checks_for_step
+    check_ctx = CheckContext(headers=CaseInsensitiveDict(headers) if headers else None)
 
     def _on_failure(exc: CheckFailed | AssertionError, message: str, context: FailureContext | None) -> None:
         exceptions.append(exc)
@@ -62,8 +68,7 @@ def validate_response(
         name = check.__name__
         copied_case = case.partial_deepcopy()
         try:
-            check(response, copied_case)
-            skip_check = check(response, copied_case)
+            skip_check = check(check_ctx, response, copied_case)
             if not skip_check:
                 _on_passed(name, copied_case)
         except CheckFailed as exc:

{schemathesis-3.35.5.dist-info → schemathesis-3.36.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: schemathesis
-Version: 3.35.5
+Version: 3.36.0
 Summary: Property-based testing framework for Open API and GraphQL based apps
 Project-URL: Documentation, https://schemathesis.readthedocs.io/en/stable/
 Project-URL: Changelog, https://schemathesis.readthedocs.io/en/stable/changelog.html