schemathesis 3.25.5__py3-none-any.whl → 4.0.0a1__py3-none-any.whl

schemathesis/specs/openapi/expressions/nodes.py

@@ -1,13 +1,19 @@
 """Expression nodes description and evaluation logic."""
+
 from __future__ import annotations
+
 from dataclasses import dataclass
 from enum import Enum, unique
-from typing import Any
+from typing import TYPE_CHECKING, Any, cast
 
 from requests.structures import CaseInsensitiveDict
 
-from .. import references
-from .context import ExpressionContext
+from schemathesis.core.transforms import UNRESOLVABLE, resolve_pointer
+from schemathesis.transport.requests import REQUESTS_TRANSPORT
+
+if TYPE_CHECKING:
+    from .context import ExpressionContext
+    from .extractors import Extractor
 
 
 @dataclass
@@ -48,7 +54,12 @@ class URL(Node):
     """A node for `$url` expression."""
 
     def evaluate(self, context: ExpressionContext) -> str:
-        return context.case.get_full_url()
+        import requests
+
+        base_url = context.case.operation.base_url or "http://127.0.0.1"
+        kwargs = REQUESTS_TRANSPORT.serialize_case(context.case, base_url=base_url)
+        prepared = requests.Request(**kwargs).prepare()
+        return cast(str, prepared.url)
 
 
 @dataclass
@@ -73,6 +84,7 @@ class NonBodyRequest(Node):
 
     location: str
     parameter: str
+    extractor: Extractor | None = None
 
     def evaluate(self, context: ExpressionContext) -> str:
         container: dict | CaseInsensitiveDict = {
@@ -82,7 +94,12 @@ class NonBodyRequest(Node):
         }[self.location] or {}
         if self.location == "header":
             container = CaseInsensitiveDict(container)
-        return container[self.parameter]
+        value = container.get(self.parameter)
+        if value is None:
+            return ""
+        if self.extractor is not None:
+            return self.extractor.extract(value) or ""
+        return value
 
 
 @dataclass
@@ -95,7 +112,10 @@ class BodyRequest(Node):
         document = context.case.body
         if self.pointer is None:
             return document
-        return references.resolve_pointer(document, self.pointer[1:])
+        resolved = resolve_pointer(document, self.pointer[1:])
+        if resolved is UNRESOLVABLE:
+            return None
+        return resolved
 
 
 @dataclass
@@ -103,9 +123,15 @@ class HeaderResponse(Node):
     """A node for `$response.header` expressions."""
 
     parameter: str
+    extractor: Extractor | None = None
 
     def evaluate(self, context: ExpressionContext) -> str:
-        return context.response.headers[self.parameter]
+        value = context.response.headers.get(self.parameter.lower())
+        if value is None:
+            return ""
+        if self.extractor is not None:
+            return self.extractor.extract(value[0]) or ""
+        return value[0]
 
 
 @dataclass
@@ -115,13 +141,11 @@ class BodyResponse(Node):
     pointer: str | None = None
 
     def evaluate(self, context: ExpressionContext) -> Any:
-        from ....transports.responses import WSGIResponse
-
-        if isinstance(context.response, WSGIResponse):
-            document = context.response.json
-        else:
-            document = context.response.json()
+        document = context.response.json()
         if self.pointer is None:
             # We need the parsed document - data will be serialized before sending to the application
             return document
-        return references.resolve_pointer(document, self.pointer[1:])
+        resolved = resolve_pointer(document, self.pointer[1:])
+        if resolved is UNRESOLVABLE:
+            return None
+        return resolved

schemathesis/specs/openapi/expressions/parser.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
+
+import re
 from functools import lru_cache
 from typing import Generator
 
-from . import lexer, nodes
+from . import extractors, lexer, nodes
 from .errors import RuntimeExpressionError, UnknownToken
 
 
@@ -53,7 +55,8 @@ def _parse_request(tokens: lexer.TokenGenerator, expr: str) -> nodes.BodyRequest
     if location.value in ("query", "path", "header"):
         skip_dot(tokens, f"$request.{location.value}")
         parameter = take_string(tokens, expr)
-        return nodes.NonBodyRequest(location.value, parameter)
+        extractor = take_extractor(tokens, expr, parameter.end)
+        return nodes.NonBodyRequest(location.value, parameter.value, extractor)
     if location.value == "body":
         try:
             token = next(tokens)
@@ -70,7 +73,8 @@ def _parse_response(tokens: lexer.TokenGenerator, expr: str) -> nodes.HeaderResp
     if location.value == "header":
         skip_dot(tokens, f"$response.{location.value}")
         parameter = take_string(tokens, expr)
-        return nodes.HeaderResponse(parameter)
+        extractor = take_extractor(tokens, expr, parameter.end)
+        return nodes.HeaderResponse(parameter.value, extractor=extractor)
     if location.value == "body":
         try:
             token = next(tokens)
@@ -87,8 +91,25 @@ def skip_dot(tokens: lexer.TokenGenerator, name: str) -> None:
         raise RuntimeExpressionError(f"`{name}` expression should be followed by a dot (`.`). Got: {token.value}")
 
 
-def take_string(tokens: lexer.TokenGenerator, expr: str) -> str:
+def take_string(tokens: lexer.TokenGenerator, expr: str) -> lexer.Token:
     parameter = next(tokens)
     if not parameter.is_string:
         raise RuntimeExpressionError(f"Invalid expression: {expr}")
-    return parameter.value
+    return parameter
+
+
+def take_extractor(tokens: lexer.TokenGenerator, expr: str, current_end: int) -> extractors.Extractor | None:
+    rest = expr[current_end + 1 :]
+    if not rest or rest.startswith("}"):
+        return None
+    extractor = next(tokens)
+    if not extractor.value.startswith("#regex:"):
+        raise RuntimeExpressionError(f"Invalid extractor: {expr}")
+    pattern = extractor.value[len("#regex:") :]
+    try:
+        compiled = re.compile(pattern)
+    except re.error as exc:
+        raise RuntimeExpressionError(f"Invalid regex extractor: {exc}") from None
+    if compiled.groups != 1:
+        raise RuntimeExpressionError("Regex extractor should have exactly one capturing group")
+    return extractors.RegexExtractor(compiled)

schemathesis/specs/openapi/formats.py

@@ -1,7 +1,12 @@
 from __future__ import annotations
 
+import string
+from base64 import b64encode
+from functools import lru_cache
 from typing import TYPE_CHECKING
 
+from schemathesis.transport.serialization import Binary
+
 if TYPE_CHECKING:
     from hypothesis import strategies as st
 
@@ -33,5 +38,45 @@ def unregister_string_format(name: str) -> None:
         raise ValueError(f"Unknown Open API format: {name}") from exc
 
 
+def header_values(blacklist_characters: str = "\n\r") -> st.SearchStrategy[str]:
+    from hypothesis import strategies as st
+
+    return st.text(
+        alphabet=st.characters(min_codepoint=0, max_codepoint=255, blacklist_characters=blacklist_characters)
+        # Header values with leading non-visible chars can't be sent with `requests`
+    ).map(str.lstrip)
+
+
+HEADER_FORMAT = "_header_value"
+
+
+@lru_cache
+def get_default_format_strategies() -> dict[str, st.SearchStrategy]:
+    """Get all default "format" strategies."""
+    from hypothesis import strategies as st
+    from requests.auth import _basic_auth_str
+
+    def make_basic_auth_str(item: tuple[str, str]) -> str:
+        return _basic_auth_str(*item)
+
+    latin1_text = st.text(alphabet=st.characters(min_codepoint=0, max_codepoint=255))
+
+    # Define valid characters here to avoid filtering them out in `is_valid_header` later
+    header_value = header_values()
+
+    return {
+        "binary": st.binary().map(Binary),
+        "byte": st.binary().map(lambda x: b64encode(x).decode()),
+        "uuid": st.uuids().map(str),
+        # RFC 7230, Section 3.2.6
+        "_header_name": st.text(
+            min_size=1, alphabet=st.sampled_from("!#$%&'*+-.^_`|~" + string.digits + string.ascii_letters)
+        ),
+        HEADER_FORMAT: header_value,
+        "_basic_auth": st.tuples(latin1_text, latin1_text).map(make_basic_auth_str),
+        "_bearer_auth": header_value.map("Bearer {}".format),
+    }
+
+
 register = register_string_format
 unregister = unregister_string_format

schemathesis/specs/openapi/links.py

@@ -2,165 +2,31 @@
 
 Based on https://swagger.io/docs/specification/links/
 """
+
 from __future__ import annotations
+
 from dataclasses import dataclass, field
 from difflib import get_close_matches
-from typing import Any, Generator, NoReturn, Sequence, Union, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Generator, Literal, TypedDict, Union, cast
 
-from ...models import APIOperation, Case
-from ...parameters import ParameterSet
-from ...stateful import ParsedData, StatefulTest
-from ...stateful.state_machine import Direction
-from ...types import NotSet
+from schemathesis.core import NOT_SET, NotSet
+from schemathesis.generation.case import Case
+from schemathesis.generation.stateful.state_machine import Direction
+from schemathesis.schemas import APIOperation
 
-from ...constants import NOT_SET
-from ...internal.copy import fast_deepcopy
 from . import expressions
 from .constants import LOCATION_TO_CONTAINER
-from .parameters import OpenAPI20Body, OpenAPI30Body, OpenAPIParameter
-
+from .references import RECURSION_DEPTH_LIMIT
 
 if TYPE_CHECKING:
-    from ...transports.responses import GenericResponse
+    from jsonschema import RefResolver
 
 
-@dataclass(repr=False)
-class Link(StatefulTest):
-    operation: APIOperation
-    parameters: dict[str, Any]
-    request_body: Any = NOT_SET
+SCHEMATHESIS_LINK_EXTENSION = "x-schemathesis"
 
-    def __post_init__(self) -> None:
-        if self.request_body is not NOT_SET and not self.operation.body:
-            # Link defines `requestBody` for a parameter that does not accept one
-            raise ValueError(
-                f"Request body is not defined in API operation {self.operation.method.upper()} {self.operation.path}"
-            )
-
-    @classmethod
-    def from_definition(cls, name: str, definition: dict[str, dict[str, Any]], source_operation: APIOperation) -> Link:
-        # Links can be behind a reference
-        _, definition = source_operation.schema.resolver.resolve_in_scope(  # type: ignore
-            definition, source_operation.definition.scope
-        )
-        if "operationId" in definition:
-            # source_operation.schema is `BaseOpenAPISchema` and has this method
-            operation = source_operation.schema.get_operation_by_id(definition["operationId"])  # type: ignore
-        else:
-            operation = source_operation.schema.get_operation_by_reference(definition["operationRef"])  # type: ignore
-        return cls(
-            # Pylint can't detect that the API operation is always defined at this point
-            # E.g. if there is no matching operation or no operations at all, then a ValueError will be risen
-            name=name,
-            operation=operation,
-            parameters=definition.get("parameters", {}),
-            request_body=definition.get("requestBody", NOT_SET),  # `None` might be a valid value - `null`
-        )
-
-    def parse(self, case: Case, response: GenericResponse) -> ParsedData:
-        """Parse data into a structure expected by links definition."""
-        context = expressions.ExpressionContext(case=case, response=response)
-        parameters = {
-            parameter: expressions.evaluate(expression, context) for parameter, expression in self.parameters.items()
-        }
-        return ParsedData(
-            parameters=parameters,
-            # https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.3.md#link-object
-            # > A literal value or {expression} to use as a request body when calling the target operation.
-            # In this case all literals will be passed as is, and expressions will be evaluated
-            body=expressions.evaluate(self.request_body, context),
-        )
-
-    def make_operation(self, collected: list[ParsedData]) -> APIOperation:
-        """Create a modified version of the original API operation with additional data merged in."""
-        # We split the gathered data among all locations & store the original parameter
-        containers = {
-            location: {
-                parameter.name: {"options": [], "parameter": parameter}
-                for parameter in getattr(self.operation, container_name)
-            }
-            for location, container_name in LOCATION_TO_CONTAINER.items()
-        }
-        # There might be duplicates in the data
-        for item in set(collected):
-            for name, value in item.parameters.items():
-                container = self._get_container_by_parameter_name(name, containers)
-                container.append(value)
-            if "body" in containers["body"] and item.body is not NOT_SET:
-                containers["body"]["body"]["options"].append(item.body)
-        # These are the final `path_parameters`, `query`, and other API operation components
-        components: dict[str, ParameterSet] = {
-            container_name: getattr(self.operation, container_name).__class__()
-            for location, container_name in LOCATION_TO_CONTAINER.items()
-        }
-        # Here are all components that are filled with parameters
-        for location, parameters in containers.items():
-            for parameter_data in parameters.values():
-                parameter = parameter_data["parameter"]
-                if parameter_data["options"]:
-                    definition = fast_deepcopy(parameter.definition)
-                    if "schema" in definition:
-                        # The actual schema doesn't matter since we have a list of allowed values
-                        definition["schema"] = {"enum": parameter_data["options"]}
-                    else:
-                        # Other schema-related keywords will be ignored later, during the canonicalisation step
-                        # inside `hypothesis-jsonschema`
-                        definition["enum"] = parameter_data["options"]
-                    new_parameter: OpenAPIParameter
-                    if isinstance(parameter, OpenAPI30Body):
-                        new_parameter = parameter.__class__(
-                            definition, media_type=parameter.media_type, required=parameter.required
-                        )
-                    elif isinstance(parameter, OpenAPI20Body):
-                        new_parameter = parameter.__class__(definition, media_type=parameter.media_type)
-                    else:
-                        new_parameter = parameter.__class__(definition)
-                    components[LOCATION_TO_CONTAINER[location]].add(new_parameter)
-                else:
-                    # No options were gathered for this parameter - use the original one
-                    components[LOCATION_TO_CONTAINER[location]].add(parameter)
-        return self.operation.clone(**components)
-
-    def _get_container_by_parameter_name(self, full_name: str, templates: dict[str, dict[str, dict[str, Any]]]) -> list:
-        """Detect in what request part the parameters is defined."""
-        location: str | None
-        try:
-            # The parameter name is prefixed with its location. Example: `path.id`
-            location, name = full_name.split(".")
-        except ValueError:
-            location, name = None, full_name
-        if location:
-            try:
-                parameters = templates[location]
-            except KeyError:
-                self._unknown_parameter(full_name)
-        else:
-            for parameters in templates.values():
-                if name in parameters:
-                    break
-            else:
-                self._unknown_parameter(full_name)
-        if not parameters:
-            self._unknown_parameter(full_name)
-        return parameters[name]["options"]
-
-    def _unknown_parameter(self, name: str) -> NoReturn:
-        raise ValueError(
-            f"Parameter `{name}` is not defined in API operation {self.operation.method.upper()} {self.operation.path}"
-        )
-
-
-def get_links(response: GenericResponse, operation: APIOperation, field: str) -> Sequence[Link]:
-    """Get `x-links` / `links` definitions from the schema."""
-    responses = operation.definition.resolved["responses"]
-    if str(response.status_code) in responses:
-        response_definition = responses[str(response.status_code)]
-    elif response.status_code in responses:
-        response_definition = responses[response.status_code]
-    else:
-        response_definition = responses.get("default", {})
-    links = response_definition.get(field, {})
-    return [Link.from_definition(name, definition, operation) for name, definition in links.items()]
+
+class SchemathesisLink(TypedDict):
+    merge_body: bool
 
 
 @dataclass(repr=False)
@@ -174,41 +40,59 @@ class OpenAPILink(Direction):
     status_code: str
     definition: dict[str, Any]
     operation: APIOperation
-    parameters: list[tuple[str | None, str, str]] = field(init=False)
+    parameters: list[tuple[Literal["path", "query", "header", "cookie", "body"] | None, str, str]] = field(init=False)
     body: dict[str, Any] | NotSet = field(init=False)
+    merge_body: bool = True
+
+    def __repr__(self) -> str:
+        path = self.operation.path
+        method = self.operation.method
+        return f"state.schema['{path}']['{method}'].links['{self.status_code}']['{self.name}']"
 
     def __post_init__(self) -> None:
+        extension = self.definition.get(SCHEMATHESIS_LINK_EXTENSION)
         self.parameters = [
             normalize_parameter(parameter, expression)
             for parameter, expression in self.definition.get("parameters", {}).items()
         ]
         self.body = self.definition.get("requestBody", NOT_SET)
+        if extension is not None:
+            self.merge_body = extension.get("merge_body", True)
 
-    def set_data(self, case: Case, elapsed: float, **kwargs: Any) -> None:
+    def set_data(self, case: Case, **kwargs: Any) -> None:
         """Assign all linked definitions to the new case instance."""
         context = kwargs["context"]
         self.set_parameters(case, context)
         self.set_body(case, context)
-        case.set_source(context.response, context.case, elapsed)
 
     def set_parameters(self, case: Case, context: expressions.ExpressionContext) -> None:
         for location, name, expression in self.parameters:
-            container = get_container(case, location, name)
+            location, container = get_container(case, location, name)
             # Might happen if there is directly specified container,
             # but the schema has no parameters of such type at all.
             # Therefore the container is empty, otherwise it will be at least an empty object
             if container is None:
                 message = f"No such parameter in `{case.operation.method.upper()} {case.operation.path}`: `{name}`."
-                possibilities = [param.name for param in case.operation.definition.parameters]
+                possibilities = [param.name for param in case.operation.iter_parameters()]
                 matches = get_close_matches(name, possibilities)
                 if matches:
                     message += f" Did you mean `{matches[0]}`?"
                 raise ValueError(message)
-            container[name] = expressions.evaluate(expression, context)
-
-    def set_body(self, case: Case, context: expressions.ExpressionContext) -> None:
+            value = expressions.evaluate(expression, context)
+            if value is not None:
+                container[name] = value
+
+    def set_body(
+        self,
+        case: Case,
+        context: expressions.ExpressionContext,
+    ) -> None:
         if self.body is not NOT_SET:
-            case.body = expressions.evaluate(self.body, context)
+            evaluated = expressions.evaluate(self.body, context, evaluate_nested=True)
+            if self.merge_body:
+                case.body = merge_body(case.body, evaluated)
+            else:
+                case.body = evaluated
 
     def get_target_operation(self) -> APIOperation:
         if "operationId" in self.definition:
@@ -216,21 +100,32 @@ class OpenAPILink(Direction):
         return self.operation.schema.get_operation_by_reference(self.definition["operationRef"])  # type: ignore
 
 
-def get_container(case: Case, location: str | None, name: str) -> dict[str, Any] | None:
+def merge_body(old: Any, new: Any) -> Any:
+    if isinstance(old, dict) and isinstance(new, dict):
+        return {**old, **new}
+    return new
+
+
+def get_container(
+    case: Case, location: Literal["path", "query", "header", "cookie", "body"] | None, name: str
+) -> tuple[Literal["path", "query", "header", "cookie", "body"], dict[str, Any] | None]:
     """Get a container that suppose to store the given parameter."""
     if location:
         container_name = LOCATION_TO_CONTAINER[location]
     else:
-        for param in case.operation.definition.parameters:
+        for param in case.operation.iter_parameters():
             if param.name == name:
+                location = param.location
                 container_name = LOCATION_TO_CONTAINER[param.location]
                 break
         else:
-            raise ValueError(f"Parameter `{name}` is not defined in API operation `{case.operation.verbose_name}`")
-    return getattr(case, container_name)
+            raise ValueError(f"Parameter `{name}` is not defined in API operation `{case.operation.label}`")
+    return location, getattr(case, container_name)
 
 
-def normalize_parameter(parameter: str, expression: str) -> tuple[str | None, str, str]:
+def normalize_parameter(
+    parameter: str, expression: str
+) -> tuple[Literal["path", "query", "header", "cookie", "body"] | None, str, str]:
     """Normalize runtime expressions.
 
     Runtime expressions may have parameter names prefixed with their location - `path.id`.
@@ -240,13 +135,15 @@ def normalize_parameter(parameter: str, expression: str) -> tuple[str | None, st
     try:
         # The parameter name is prefixed with its location. Example: `path.id`
         location, name = tuple(parameter.split("."))
-        return location, name, expression
+        _location = cast(Literal["path", "query", "header", "cookie", "body"], location)
+        return _location, name, expression
     except ValueError:
         return None, parameter, expression
 
 
 def get_all_links(operation: APIOperation) -> Generator[tuple[str, OpenAPILink], None, None]:
-    for status_code, definition in operation.definition.resolved["responses"].items():
+    for status_code, definition in operation.definition.raw["responses"].items():
+        definition = operation.schema.resolver.resolve_all(definition, RECURSION_DEPTH_LIMIT - 8)  # type: ignore[attr-defined]
         for name, link_definition in definition.get(operation.schema.links_field, {}).items():  # type: ignore
             yield status_code, OpenAPILink(name, status_code, link_definition, operation)
 
@@ -273,6 +170,7 @@ def _get_response_by_status_code(responses: dict[StatusCode, dict[str, Any]], st
 
 
 def add_link(
+    resolver: RefResolver,
     responses: dict[StatusCode, dict[str, Any]],
     links_field: str,
     parameters: dict[str, str] | None,
@@ -282,6 +180,8 @@ def add_link(
     name: str | None = None,
 ) -> None:
     response = _get_response_by_status_code(responses, status_code)
+    if "$ref" in response:
+        _, response = resolver.resolve(response["$ref"])
     links_definition = response.setdefault(links_field, {})
     new_link: dict[str, str | dict[str, str]] = {}
     if parameters is not None:
@@ -295,8 +195,8 @@ def add_link(
         name = name or f"{target.method.upper()} {target.path}"
         # operationId is a dict lookup which is more efficient than using `operationRef`, since it
         # doesn't involve reference resolving when we will look up for this target during testing.
-        if "operationId" in target.definition.resolved:
-            new_link["operationId"] = target.definition.resolved["operationId"]
+        if "operationId" in target.definition.raw:
+            new_link["operationId"] = target.definition.raw["operationId"]
         else:
             new_link["operationRef"] = target.operation_reference
     # The name is arbitrary, so we don't really case what it is,

schemathesis/specs/openapi/media_types.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Collection
+
+from schemathesis.transport import SerializationContext
+from schemathesis.transport.asgi import ASGI_TRANSPORT
+from schemathesis.transport.requests import REQUESTS_TRANSPORT
+from schemathesis.transport.wsgi import WSGI_TRANSPORT
+
+if TYPE_CHECKING:
+    from hypothesis import strategies as st
+
+
+MEDIA_TYPES: dict[str, st.SearchStrategy[bytes]] = {}
+
+
+def register_media_type(name: str, strategy: st.SearchStrategy[bytes], *, aliases: Collection[str] = ()) -> None:
+    """Register a strategy for the given media type."""
+
+    @REQUESTS_TRANSPORT.serializer(name, *aliases)
+    @ASGI_TRANSPORT.serializer(name, *aliases)
+    @WSGI_TRANSPORT.serializer(name, *aliases)
+    def serialize(ctx: SerializationContext, value: Any) -> dict[str, Any]:
+        return {"data": value}
+
+    MEDIA_TYPES[name] = strategy
+    for alias in aliases:
+        MEDIA_TYPES[alias] = strategy
+
+
+def unregister_all() -> None:
+    MEDIA_TYPES.clear()

schemathesis/specs/openapi/negative/__init__.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
+
 from dataclasses import dataclass
 from functools import lru_cache
-from typing import Any
+from typing import TYPE_CHECKING, Any
 from urllib.parse import urlencode
 
 import jsonschema
@@ -10,8 +11,11 @@ from hypothesis_jsonschema import from_schema
 
 from ..constants import ALL_KEYWORDS
 from .mutations import MutationContext
-from .types import Draw, Schema
-from ....generation import GenerationConfig
+
+if TYPE_CHECKING:
+    from schemathesis.generation import GenerationConfig
+
+    from .types import Draw, Schema
 
 
 @dataclass