schemathesis 3.39.7__py3-none-any.whl → 4.0.0a2__py3-none-any.whl

schemathesis/engine/recorder.py

@@ -0,0 +1,246 @@
+from __future__ import annotations
+
+import base64
+import time
+import uuid
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Iterator, cast
+
+from schemathesis.core.failures import Failure
+from schemathesis.core.transport import Response
+from schemathesis.engine import Status
+from schemathesis.generation.case import Case
+
+if TYPE_CHECKING:
+    import requests
+
+    from schemathesis.generation.stateful.state_machine import Transition
+
+
+@dataclass
+class ScenarioRecorder:
+    """Tracks and organizes all data related to a logical block of testing.
+
+    Records test cases, their hierarchy, API interactions, and results of checks performed during execution.
+    """
+
+    id: uuid.UUID
+    # Human-readable label
+    label: str
+
+    # Recorded test cases
+    cases: dict[str, CaseNode]
+    # Results of checks categorized by test case ID
+    checks: dict[str, list[CheckNode]]
+    # Network interactions by test case ID
+    interactions: dict[str, Interaction]
+
+    __slots__ = ("id", "label", "status", "roots", "cases", "checks", "interactions")
+
+    def __init__(self, *, label: str) -> None:
+        self.id = uuid.uuid4()
+        self.label = label
+        self.cases = {}
+        self.checks = {}
+        self.interactions = {}
+
+    def record_case(self, *, parent_id: str | None, transition: Transition | None, case: Case) -> None:
+        """Record a test case and its relationship to a parent, if applicable."""
+        self.cases[case.id] = CaseNode(value=case, parent_id=parent_id, transition=transition)
+
+    def record_response(self, *, case_id: str, response: Response) -> None:
+        """Record the API response for a given test case."""
+        request = Request.from_prepared_request(response.request)
+        self.interactions[case_id] = Interaction(request=request, response=response)
+
+    def record_request(self, *, case_id: str, request: requests.PreparedRequest) -> None:
+        """Record a network-level error for a given test case."""
+        self.interactions[case_id] = Interaction(request=Request.from_prepared_request(request), response=None)
+
+    def record_check_failure(self, *, name: str, case_id: str, code_sample: str, failure: Failure) -> None:
+        """Record a failure of a check for a given test case."""
+        self.checks.setdefault(case_id, []).append(
+            CheckNode(
+                name=name,
+                status=Status.FAILURE,
+                failure_info=CheckFailureInfo(code_sample=code_sample, failure=failure),
+            )
+        )
+
+    def record_check_success(self, *, name: str, case_id: str) -> None:
+        """Record a successful pass of a check for a given test case."""
+        self.checks.setdefault(case_id, []).append(CheckNode(name=name, status=Status.SUCCESS, failure_info=None))
+
+    def find_failure_data(self, *, parent_id: str, failure: Failure) -> FailureData:
+        """Retrieve the relevant test case & interaction data for a failure.
+
+        It may happen that a failure comes from a different test case if a check generated some additional
+        test cases & interactions.
+        """
+        case_id = failure.case_id or parent_id
+        case = self.cases[case_id].value
+        request = self.interactions[case_id].request
+        response = self.interactions[case_id].response
+        assert isinstance(response, Response)
+        headers = {key: value[0] for key, value in request.headers.items()}
+        return FailureData(case=case, headers=headers, verify=response.verify)
+
+    def find_parent(self, *, case_id: str) -> Case | None:
+        """Find the parent case of a given test case, if it exists."""
+        case = self.cases.get(case_id)
+        if case is not None and case.parent_id is not None:
+            parent = self.cases.get(case.parent_id)
+            # The recorder state should always be consistent
+            assert parent is not None, "Parent does not exist"
+            return parent.value
+        return None
+
+    def find_related(self, *, case_id: str) -> Iterator[Case]:
+        """Iterate over all ancestors and their children for a given case."""
+        current_id = case_id
+        seen = {current_id}
+
+        while True:
+            current_node = self.cases.get(current_id)
+            if current_node is None or current_node.parent_id is None:
+                break
+
+            # Get all children of the parent (siblings of the current case)
+            parent_id = current_node.parent_id
+            for case_id, maybe_child in self.cases.items():
+                # If this case has the same parent and we haven't seen it yet
+                if parent_id == maybe_child.parent_id and case_id not in seen:
+                    seen.add(case_id)
+                    yield maybe_child.value
+
+            # Move up to the parent
+            current_id = parent_id
+            if current_id not in seen:
+                seen.add(current_id)
+                parent_node = self.cases.get(current_id)
+                if parent_node:
+                    yield parent_node.value
+
+    def find_response(self, *, case_id: str) -> Response | None:
+        """Retrieve the API response for a given test case, if available."""
+        interaction = self.interactions.get(case_id)
+        if interaction is None or interaction.response is None:
+            return None
+        return interaction.response
+
+
+@dataclass
+class CaseNode:
+    """Represents a test case and its parent-child relationship."""
+
+    value: Case
+    parent_id: str | None
+    # Transition may be absent if `parent_id` is present for cases when a case is derived inside a check
+    # and outside of the implemented transition logic (e.g. Open API links)
+    transition: Transition | None
+
+    __slots__ = ("value", "parent_id", "transition")
+
+
+@dataclass
+class CheckNode:
+    name: str
+    status: Status
+    failure_info: CheckFailureInfo | None
+
+    __slots__ = ("name", "status", "failure_info")
+
+
+@dataclass
+class CheckFailureInfo:
+    code_sample: str
+    failure: Failure
+
+    __slots__ = ("code_sample", "failure")
+
+
+def serialize_payload(payload: bytes) -> str:
+    return base64.b64encode(payload).decode()
+
+
+@dataclass(repr=False)
+class Request:
+    """Request data extracted from `Case`."""
+
+    method: str
+    uri: str
+    body: bytes | None
+    body_size: int | None
+    headers: dict[str, list[str]]
+
+    __slots__ = ("method", "uri", "body", "body_size", "headers", "_encoded_body_cache")
+
+    def __init__(
+        self,
+        method: str,
+        uri: str,
+        body: bytes | None,
+        body_size: int | None,
+        headers: dict[str, list[str]],
+    ):
+        self.method = method
+        self.uri = uri
+        self.body = body
+        self.body_size = body_size
+        self.headers = headers
+        self._encoded_body_cache: str | None = None
+
+    @classmethod
+    def from_prepared_request(cls, prepared: requests.PreparedRequest) -> Request:
+        """A prepared request version is already stored in `requests.Response`."""
+        body = prepared.body
+
+        if isinstance(body, str):
+            # can be a string for `application/x-www-form-urlencoded`
+            body = body.encode("utf-8")
+
+        # these values have `str` type at this point
+        uri = cast(str, prepared.url)
+        method = cast(str, prepared.method)
+        return cls(
+            uri=uri,
+            method=method,
+            headers={key: [value] for (key, value) in prepared.headers.items()},
+            body=body,
+            body_size=len(body) if body is not None else None,
+        )
+
+    @property
+    def encoded_body(self) -> str | None:
+        if self.body is not None:
+            if self._encoded_body_cache is None:
+                self._encoded_body_cache = serialize_payload(self.body)
+            return self._encoded_body_cache
+        return None
+
+
+@dataclass
+class Interaction:
+    """Represents a single interaction with the tested application."""
+
+    request: Request
+    response: Response | None
+    timestamp: float
+
+    __slots__ = ("request", "response", "timestamp")
+
+    def __init__(self, request: Request, response: Response | None) -> None:
+        self.request = request
+        self.response = response
+        self.timestamp = time.time()
+
+
+@dataclass
+class FailureData:
+    """Details about a test failure, including the case and its context."""
+
+    case: Case
+    headers: dict[str, str]
+    verify: bool
+
+    __slots__ = ("case", "headers", "verify")

schemathesis/errors.py

@@ -0,0 +1,31 @@
+"""Public Schemathesis errors."""
+
+from schemathesis.core.errors import IncorrectUsage as IncorrectUsage
+from schemathesis.core.errors import InternalError as InternalError
+from schemathesis.core.errors import InvalidHeadersExample as InvalidHeadersExample
+from schemathesis.core.errors import InvalidRateLimit as InvalidRateLimit
+from schemathesis.core.errors import InvalidRegexPattern as InvalidRegexPattern
+from schemathesis.core.errors import InvalidRegexType as InvalidRegexType
+from schemathesis.core.errors import InvalidSchema as InvalidSchema
+from schemathesis.core.errors import LoaderError as LoaderError
+from schemathesis.core.errors import OperationNotFound as OperationNotFound
+from schemathesis.core.errors import SchemathesisError as SchemathesisError
+from schemathesis.core.errors import SerializationError as SerializationError
+from schemathesis.core.errors import SerializationNotPossible as SerializationNotPossible
+from schemathesis.core.errors import UnboundPrefix as UnboundPrefix
+
+__all__ = [
+    "IncorrectUsage",
+    "InternalError",
+    "InvalidHeadersExample",
+    "InvalidRateLimit",
+    "InvalidRegexPattern",
+    "InvalidRegexType",
+    "InvalidSchema",
+    "LoaderError",
+    "OperationNotFound",
+    "SchemathesisError",
+    "SerializationError",
+    "SerializationNotPossible",
+    "UnboundPrefix",
+]

schemathesis/experimental/__init__.py

@@ -1,18 +1,21 @@
 import os
 from dataclasses import dataclass, field
 
-from ..constants import TRUE_VALUES
+from schemathesis.core import string_to_boolean
 
 
 @dataclass(eq=False)
 class Experiment:
     name: str
-    verbose_name: str
     env_var: str
     description: str
     discussion_url: str
     _storage: "ExperimentSet" = field(repr=False)
 
+    @property
+    def label(self) -> str:
+        return self.name.lower().replace(" ", "-")
+
     def enable(self) -> None:
         self._storage.enable(self)
 
@@ -25,7 +28,7 @@ class Experiment:
 
     @property
     def is_env_var_set(self) -> bool:
-        return os.getenv(self.env_var, "").lower() in TRUE_VALUES
+        return string_to_boolean(os.getenv(self.env_var, "")) is True
 
 
 @dataclass
@@ -33,12 +36,9 @@ class ExperimentSet:
     available: set = field(default_factory=set)
     enabled: set = field(default_factory=set)
 
-    def create_experiment(
-        self, name: str, verbose_name: str, env_var: str, description: str, discussion_url: str
-    ) -> Experiment:
+    def create_experiment(self, name: str, env_var: str, description: str, discussion_url: str) -> Experiment:
         instance = Experiment(
             name=name,
-            verbose_name=verbose_name,
             env_var=f"{ENV_PREFIX}_{env_var}",
             description=description,
             discussion_url=discussion_url,
@@ -64,45 +64,14 @@ class ExperimentSet:
 
 ENV_PREFIX = "SCHEMATHESIS_EXPERIMENTAL"
 GLOBAL_EXPERIMENTS = ExperimentSet()
-
-OPEN_API_3_1 = GLOBAL_EXPERIMENTS.create_experiment(
-    name="openapi-3.1",
-    verbose_name="OpenAPI 3.1",
-    env_var="OPENAPI_3_1",
-    description="Support for response validation",
-    discussion_url="https://github.com/schemathesis/schemathesis/discussions/1822",
-)
-SCHEMA_ANALYSIS = GLOBAL_EXPERIMENTS.create_experiment(
-    name="schema-analysis",
-    verbose_name="Schema Analysis",
-    env_var="SCHEMA_ANALYSIS",
-    description="Analyzing API schemas via Schemathesis.io",
-    discussion_url="https://github.com/schemathesis/schemathesis/discussions/2056",
-)
-STATEFUL_TEST_RUNNER = GLOBAL_EXPERIMENTS.create_experiment(
-    name="stateful-test-runner",
-    verbose_name="New Stateful Test Runner",
-    env_var="STATEFUL_TEST_RUNNER",
-    description="State machine-based runner for stateful tests in CLI",
-    discussion_url="https://github.com/schemathesis/schemathesis/discussions/2262",
-)
-STATEFUL_ONLY = GLOBAL_EXPERIMENTS.create_experiment(
-    name="stateful-only",
-    verbose_name="Stateful Only",
-    env_var="STATEFUL_ONLY",
-    description="Run only stateful tests",
-    discussion_url="https://github.com/schemathesis/schemathesis/discussions/2262",
-)
 COVERAGE_PHASE = GLOBAL_EXPERIMENTS.create_experiment(
-    name="coverage-phase",
-    verbose_name="Coverage phase",
+    name="Coverage phase",
     env_var="COVERAGE_PHASE",
     description="Generate covering test cases",
     discussion_url="https://github.com/schemathesis/schemathesis/discussions/2418",
 )
 POSITIVE_DATA_ACCEPTANCE = GLOBAL_EXPERIMENTS.create_experiment(
-    name="positive_data_acceptance",
-    verbose_name="Positive Data Acceptance",
+    name="Positive Data Acceptance",
     env_var="POSITIVE_DATA_ACCEPTANCE",
     description="Verifying schema-conformant data is accepted",
     discussion_url="https://github.com/schemathesis/schemathesis/discussions/2499",

schemathesis/filters.py

@@ -9,12 +9,11 @@ from functools import partial
 from types import SimpleNamespace
 from typing import TYPE_CHECKING, Any, Callable, List, Protocol, Union
 
-from .exceptions import UsageError
-from .types import Filter as FilterType
-from .types import NotSet
+from schemathesis.core.errors import IncorrectUsage
+from schemathesis.core.transforms import resolve_pointer
 
 if TYPE_CHECKING:
-    from .models import APIOperation
+    from schemathesis.schemas import APIOperation
 
 
 class HasAPIOperation(Protocol):
@@ -151,23 +150,6 @@ class FilterSet:
     def clone(self) -> FilterSet:
         return FilterSet(_includes=self._includes.copy(), _excludes=self._excludes.copy())
 
-    def merge(self, other: FilterSet) -> FilterSet:
-        def _merge(lhs: set[Filter], rhs: set[Filter]) -> set[Filter]:
-            result = lhs.copy()
-            for new in rhs:
-                for old in lhs:
-                    for new_matcher in new.matchers:
-                        for old_matcher in old.matchers:
-                            if "=" in new_matcher.label and "=" in old_matcher.label:
-                                if new_matcher.label.split("=")[0] == old_matcher.label.split("=")[0]:
-                                    result.remove(old)
-                result.add(new)
-            return result
-
-        return FilterSet(
-            _includes=_merge(self._includes, other._includes), _excludes=_merge(self._excludes, other._excludes)
-        )
-
     def apply_to(self, operations: list[APIOperation]) -> list[APIOperation]:
         """Get a filtered list of the given operations that match the filters."""
         return [operation for operation in operations if self.match(SimpleNamespace(operation=operation))]
@@ -276,7 +258,7 @@ class FilterSet:
         if func is not None:
             matchers.append(Matcher.for_function(func))
         for attribute, expected, regex in (
-            ("verbose_name", name, name_regex),
+            ("label", name, name_regex),
             ("method", method, method_regex),
             ("path", path, path_regex),
             ("tag", tag, tag_regex),
@@ -284,17 +266,17 @@ class FilterSet:
         ):
             if expected is not None and regex is not None:
                 # To match anything the regex should match the expected value, hence passing them together is useless
-                raise UsageError(ERROR_EXPECTED_AND_REGEX)
+                raise IncorrectUsage(ERROR_EXPECTED_AND_REGEX)
             if expected is not None:
                 matchers.append(Matcher.for_value(attribute, expected))
             if regex is not None:
                 matchers.append(Matcher.for_regex(attribute, regex))
 
         if not matchers:
-            raise UsageError(ERROR_EMPTY_FILTER)
+            raise IncorrectUsage(ERROR_EMPTY_FILTER)
         filter_ = Filter(matchers=tuple(matchers))
         if filter_ in self._includes or filter_ in self._excludes:
-            raise UsageError(ERROR_FILTER_EXISTS)
+            raise IncorrectUsage(ERROR_FILTER_EXISTS)
         if include:
             self._includes.add(filter_)
         else:
@@ -358,74 +340,6 @@ def is_deprecated(ctx: HasAPIOperation) -> bool:
     return ctx.operation.definition.raw.get("deprecated") is True
 
 
-def filter_set_from_components(
-    *,
-    include: bool,
-    method: FilterType | None = None,
-    endpoint: FilterType | None = None,
-    tag: FilterType | None = None,
-    operation_id: FilterType | None = None,
-    skip_deprecated_operations: bool | None | NotSet = None,
-    parent: FilterSet | None = None,
-) -> FilterSet:
-    def _is_defined(x: FilterType | None) -> bool:
-        return x is not None and not isinstance(x, NotSet)
-
-    def _prepare_filter(filter_: FilterType | None) -> RegexValue | None:
-        if filter_ is None or isinstance(filter_, NotSet):
-            return None
-        if isinstance(filter_, str):
-            return filter_
-        return "|".join(f"({f})" for f in filter_)
-
-    new = FilterSet()
-
-    if _is_defined(method) or _is_defined(endpoint) or _is_defined(tag) or _is_defined(operation_id):
-        new._add_filter(
-            include,
-            method_regex=_prepare_filter(method),
-            path_regex=_prepare_filter(endpoint),
-            tag_regex=_prepare_filter(tag),
-            operation_id_regex=_prepare_filter(operation_id),
-        )
-    if skip_deprecated_operations is True and not any(
-        matcher.label == is_deprecated.__name__ for exclude_ in new._excludes for matcher in exclude_.matchers
-    ):
-        new.exclude(func=is_deprecated)
-    # Merge with the parent filter set
-    if parent is not None:
-        for include_ in parent._includes:
-            matchers = include_.matchers
-            ids = []
-            for idx, matcher in enumerate(matchers):
-                label = matcher.label
-                if (
-                    (not isinstance(method, NotSet) and label.startswith("method_regex="))
-                    or (not isinstance(endpoint, NotSet) and label.startswith("path_regex="))
-                    or (not isinstance(tag, NotSet) and matcher.label.startswith("tag_regex="))
-                    or (not isinstance(operation_id, NotSet) and matcher.label.startswith("operation_id_regex="))
-                ):
-                    ids.append(idx)
-            if ids:
-                matchers = tuple(matcher for idx, matcher in enumerate(matchers) if idx not in ids)
-            if matchers:
-                if new._includes:
-                    existing = new._includes.pop()
-                    matchers = existing.matchers + matchers
-                new._includes.add(Filter(matchers=matchers))
-        for exclude_ in parent._excludes:
-            matchers = exclude_.matchers
-            ids = []
-            for idx, matcher in enumerate(exclude_.matchers):
-                if skip_deprecated_operations is False and matcher.label == is_deprecated.__name__:
-                    ids.append(idx)
-            if ids:
-                matchers = tuple(matcher for idx, matcher in enumerate(matchers) if idx not in ids)
-            if matchers:
-                new._excludes.add(exclude_)
-    return new
-
-
 def parse_expression(expression: str) -> tuple[str, str, Any]:
     expression = expression.strip()
 
@@ -452,8 +366,6 @@ def parse_expression(expression: str) -> tuple[str, str, Any]:
 
 
 def expression_to_filter_function(expression: str) -> Callable[[HasAPIOperation], bool]:
-    from .specs.openapi.references import resolve_pointer
-
     pointer, op, value = parse_expression(expression)
 
     if op == "==":

schemathesis/generation/__init__.py

@@ -4,14 +4,13 @@ import random
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING
 
-from ._hypothesis import add_single_example, combine_strategies, get_single_example
-from ._methods import DataGenerationMethod, DataGenerationMethodInput
+from schemathesis.generation.modes import GenerationMode as GenerationMode
 
 if TYPE_CHECKING:
     from hypothesis.strategies import SearchStrategy
 
 
-DEFAULT_DATA_GENERATION_METHODS = (DataGenerationMethod.default(),)
+DEFAULT_GENERATOR_MODES = (GenerationMode.default(),)
 
 
 CASE_ID_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
@@ -40,6 +39,7 @@ class HeaderConfig:
 class GenerationConfig:
     """Holds various configuration options relevant for data generation."""
 
+    modes: list[GenerationMode] = field(default_factory=lambda: [GenerationMode.default()])
     # Allow generating `\x00` bytes in strings
     allow_x00: bool = True
     # Allowing using `null` for optional arguments in GraphQL queries