schemathesis 3.39.7__py3-none-any.whl → 4.0.0a2__py3-none-any.whl

schemathesis/core/loaders.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+import http.client
+from typing import TYPE_CHECKING, Any, Callable, NoReturn
+
+from schemathesis.core.errors import LoaderError, LoaderErrorKind, get_request_error_extras, get_request_error_message
+from schemathesis.core.transport import DEFAULT_RESPONSE_TIMEOUT, USER_AGENT
+
+if TYPE_CHECKING:
+    import requests
+
+
+def prepare_request_kwargs(kwargs: dict[str, Any]) -> None:
+    """Prepare common request kwargs."""
+    headers = kwargs.setdefault("headers", {})
+    if "user-agent" not in {header.lower() for header in headers}:
+        kwargs["headers"]["User-Agent"] = USER_AGENT
+
+
+def handle_request_error(exc: requests.RequestException) -> NoReturn:
+    """Handle request-level errors."""
+    import requests
+
+    url = exc.request.url if exc.request is not None else None
+    if isinstance(exc, requests.exceptions.SSLError):
+        kind = LoaderErrorKind.CONNECTION_SSL
+    elif isinstance(exc, requests.exceptions.ConnectionError):
+        kind = LoaderErrorKind.CONNECTION_OTHER
+    else:
+        kind = LoaderErrorKind.NETWORK_OTHER
+    raise LoaderError(
+        message=get_request_error_message(exc),
+        kind=kind,
+        url=url,
+        extras=get_request_error_extras(exc),
+    ) from exc
+
+
+def raise_for_status(response: requests.Response) -> requests.Response:
+    """Handle response status codes."""
+    status_code = response.status_code
+    if status_code < 400:
+        return response
+
+    reason = http.client.responses.get(status_code, "Unknown")
+    if status_code >= 500:
+        message = f"Failed to load schema due to server error (HTTP {status_code} {reason})"
+        kind = LoaderErrorKind.HTTP_SERVER_ERROR
+    else:
+        message = f"Failed to load schema due to client error (HTTP {status_code} {reason})"
+        kind = (
+            LoaderErrorKind.HTTP_FORBIDDEN
+            if status_code == 403
+            else LoaderErrorKind.HTTP_NOT_FOUND
+            if status_code == 404
+            else LoaderErrorKind.HTTP_CLIENT_ERROR
+        )
+    raise LoaderError(message=message, kind=kind, url=response.request.url, extras=[])
+
+
+def make_request(func: Callable[..., requests.Response], url: str, **kwargs: Any) -> requests.Response:
+    """Make HTTP request with error handling."""
+    import requests
+
+    try:
+        response = func(url, **kwargs)
+        return raise_for_status(response)
+    except requests.RequestException as exc:
+        handle_request_error(exc)
+
+
+WAIT_FOR_SCHEMA_INTERVAL = 0.05
+
+
+def load_from_url(
+    func: Callable[..., requests.Response],
+    *,
+    url: str,
+    wait_for_schema: float | None = None,
+    **kwargs: Any,
+) -> requests.Response:
+    """Load schema from URL with retries."""
+    import backoff
+    import requests
+
+    kwargs.setdefault("timeout", DEFAULT_RESPONSE_TIMEOUT)
+    prepare_request_kwargs(kwargs)
+
+    if wait_for_schema is not None:
+        func = backoff.on_exception(
+            backoff.constant,
+            requests.exceptions.ConnectionError,
+            max_time=wait_for_schema,
+            interval=WAIT_FOR_SCHEMA_INTERVAL,
+        )(func)
+
+    return make_request(func, url, **kwargs)
+
+
+def require_relative_url(url: str) -> None:
+    """Raise an error if the URL is not relative."""
+    # Deliberately simplistic approach
+    if "://" in url or url.startswith("//"):
+        raise ValueError("Schema path should be relative for WSGI/ASGI loaders")

schemathesis/core/marks.py

@@ -0,0 +1,66 @@
+"""A lightweight mechanism to attach Schemathesis-specific metadata to test functions."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable, Generic, TypeVar
+
+from schemathesis.core import NOT_SET, NotSet
+
+METADATA_ATTR = "_schemathesis_metadata"
+
+
+@dataclass
+class SchemathesisMetadata:
+    """Container for all Schemathesis-specific data attached to test functions."""
+
+
+T = TypeVar("T")
+
+
+class Mark(Generic[T]):
+    """Access to specific attributes in SchemathesisMetadata."""
+
+    def __init__(
+        self, *, attr_name: str, default: T | Callable[[], T] | None = None, check: Callable[[T], bool] | None = None
+    ) -> None:
+        self.attr_name = attr_name
+        self._default = default
+        self._check = check
+
+    def _get_default(self) -> T | None:
+        if callable(self._default):
+            return self._default()
+        return self._default
+
+    def _check_value(self, value: T) -> bool:
+        if self._check is not None:
+            return self._check(value)
+        return True
+
+    def get(self, func: Callable) -> T | None:
+        """Get marker value if it's set."""
+        metadata = getattr(func, METADATA_ATTR, None)
+        if metadata is None:
+            return self._get_default()
+        value = getattr(metadata, self.attr_name, NOT_SET)
+        if value is NOT_SET:
+            return self._get_default()
+        assert not isinstance(value, NotSet)
+        if self._check_value(value):
+            return value
+        return self._get_default()
+
+    def set(self, func: Callable, value: T) -> None:
+        """Set marker value, creating metadata if needed."""
+        if not hasattr(func, METADATA_ATTR):
+            setattr(func, METADATA_ATTR, SchemathesisMetadata())
+        metadata = getattr(func, METADATA_ATTR)
+        setattr(metadata, self.attr_name, value)
+
+    def is_set(self, func: Callable) -> bool:
+        """Check if function has metadata with this marker set."""
+        metadata = getattr(func, METADATA_ATTR, None)
+        if metadata is None:
+            return False
+        return hasattr(metadata, self.attr_name)

schemathesis/{transports/content_types.py → core/media_types.py}

@@ -1,6 +1,8 @@
 from functools import lru_cache
 from typing import Generator, Tuple
 
+from schemathesis.core.errors import MalformedMediaType
+
 
 def _parseparam(s: str) -> Generator[str, None, None]:
     while s[:1] == ";":
@@ -15,7 +17,7 @@ def _parseparam(s: str) -> Generator[str, None, None]:
         s = s[end:]
 
 
-def parse_header(line: str) -> Tuple[str, dict]:
+def _parse_header(line: str) -> Tuple[str, dict]:
     parts = _parseparam(";" + line)
     key = parts.__next__()
     pdict = {}
@@ -32,36 +34,36 @@ def parse_header(line: str) -> Tuple[str, dict]:
 
 
 @lru_cache
-def parse_content_type(content_type: str) -> Tuple[str, str]:
+def parse(media_type: str) -> Tuple[str, str]:
     """Parse Content Type and return main type and subtype."""
     try:
-        content_type, _ = parse_header(content_type)
-        main_type, sub_type = content_type.split("/", 1)
+        media_type, _ = _parse_header(media_type)
+        main_type, sub_type = media_type.split("/", 1)
     except ValueError as exc:
-        raise ValueError(f"Malformed media type: `{content_type}`") from exc
+        raise MalformedMediaType(f"Malformed media type: `{media_type}`") from exc
     return main_type.lower(), sub_type.lower()
 
 
-def is_json_media_type(value: str) -> bool:
+def is_json(value: str) -> bool:
     """Detect whether the content type is JSON-compatible.
 
     For example - ``application/problem+json`` matches.
     """
-    main, sub = parse_content_type(value)
+    main, sub = parse(value)
     return main == "application" and (sub == "json" or sub.endswith("+json"))
 
 
-def is_yaml_media_type(value: str) -> bool:
+def is_yaml(value: str) -> bool:
     """Detect whether the content type is YAML-compatible."""
     return value in ("text/yaml", "text/x-yaml", "application/x-yaml", "text/vnd.yaml")
 
 
-def is_plain_text_media_type(value: str) -> bool:
+def is_plain_text(value: str) -> bool:
     """Detect variations of the ``text/plain`` media type."""
-    return parse_content_type(value) == ("text", "plain")
+    return parse(value) == ("text", "plain")
 
 
-def is_xml_media_type(value: str) -> bool:
+def is_xml(value: str) -> bool:
     """Detect variations of the ``application/xml`` media type."""
-    _, sub = parse_content_type(value)
+    _, sub = parse(value)
     return sub == "xml" or sub.endswith("+xml")

schemathesis/{internal/output.py → core/output/__init__.py}

@@ -14,6 +14,7 @@ MAX_WIDTH = 80
 class OutputConfig:
     """Options for configuring various aspects of Schemathesis output."""
 
+    sanitize: bool = True
     truncate: bool = True
     max_payload_size: int = MAX_PAYLOAD_SIZE
     max_lines: int = MAX_LINES

schemathesis/core/output/sanitization.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+from collections.abc import MutableMapping, MutableSequence
+from dataclasses import dataclass, replace
+from typing import Any
+from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit
+
+from schemathesis.core import NOT_SET, NotSet
+
+# Exact keys to sanitize
+DEFAULT_KEYS_TO_SANITIZE = frozenset(
+    (
+        "phpsessid",
+        "xsrf-token",
+        "_csrf",
+        "_csrf_token",
+        "_session",
+        "_xsrf",
+        "aiohttp_session",
+        "api_key",
+        "api-key",
+        "apikey",
+        "auth",
+        "authorization",
+        "connect.sid",
+        "cookie",
+        "credentials",
+        "csrf",
+        "csrf_token",
+        "csrf-token",
+        "csrftoken",
+        "ip_address",
+        "mysql_pwd",
+        "passwd",
+        "password",
+        "private_key",
+        "private-key",
+        "privatekey",
+        "remote_addr",
+        "remote-addr",
+        "secret",
+        "session",
+        "sessionid",
+        "set_cookie",
+        "set-cookie",
+        "token",
+        "x_api_key",
+        "x-api-key",
+        "x_csrftoken",
+        "x-csrftoken",
+        "x_forwarded_for",
+        "x-forwarded-for",
+        "x_real_ip",
+        "x-real-ip",
+    )
+)
+
+# Markers indicating potentially sensitive keys
+DEFAULT_SENSITIVE_MARKERS = frozenset(
+    (
+        "token",
+        "key",
+        "secret",
+        "password",
+        "auth",
+        "session",
+        "passwd",
+        "credential",
+    )
+)
+
+DEFAULT_REPLACEMENT = "[Filtered]"
+
+
+@dataclass
+class SanitizationConfig:
+    """Configuration class for sanitizing sensitive data."""
+
+    keys_to_sanitize: frozenset[str] = DEFAULT_KEYS_TO_SANITIZE
+    sensitive_markers: frozenset[str] = DEFAULT_SENSITIVE_MARKERS
+    replacement: str = DEFAULT_REPLACEMENT
+
+    @classmethod
+    def from_config(
+        cls,
+        base_config: SanitizationConfig,
+        *,
+        replacement: str | NotSet = NOT_SET,
+        keys_to_sanitize: list[str] | NotSet = NOT_SET,
+        sensitive_markers: list[str] | NotSet = NOT_SET,
+    ) -> SanitizationConfig:
+        """Create a new config by replacing specified values."""
+        kwargs: dict[str, Any] = {}
+        if not isinstance(replacement, NotSet):
+            kwargs["replacement"] = replacement
+        if not isinstance(keys_to_sanitize, NotSet):
+            kwargs["keys_to_sanitize"] = frozenset(key.lower() for key in keys_to_sanitize)
+        if not isinstance(sensitive_markers, NotSet):
+            kwargs["sensitive_markers"] = frozenset(marker.lower() for marker in sensitive_markers)
+        return replace(base_config, **kwargs)
+
+    def extend(
+        self,
+        *,
+        keys_to_sanitize: list[str] | NotSet = NOT_SET,
+        sensitive_markers: list[str] | NotSet = NOT_SET,
+    ) -> SanitizationConfig:
+        """Create a new config by extending current sets."""
+        config = self
+        if not isinstance(keys_to_sanitize, NotSet):
+            new_keys = config.keys_to_sanitize.union(key.lower() for key in keys_to_sanitize)
+            config = replace(config, keys_to_sanitize=new_keys)
+
+        if not isinstance(sensitive_markers, NotSet):
+            new_markers = config.sensitive_markers.union(marker.lower() for marker in sensitive_markers)
+            config = replace(config, sensitive_markers=new_markers)
+
+        return config
+
+
+_DEFAULT_SANITIZATION_CONFIG = SanitizationConfig()
+
+
+def configure(
+    replacement: str | NotSet = NOT_SET,
+    keys_to_sanitize: list[str] | NotSet = NOT_SET,
+    sensitive_markers: list[str] | NotSet = NOT_SET,
+) -> None:
+    """Replace current sanitization configuration."""
+    global _DEFAULT_SANITIZATION_CONFIG
+    _DEFAULT_SANITIZATION_CONFIG = SanitizationConfig.from_config(
+        _DEFAULT_SANITIZATION_CONFIG,
+        replacement=replacement,
+        keys_to_sanitize=keys_to_sanitize,
+        sensitive_markers=sensitive_markers,
+    )
+
+
+def extend(
+    keys_to_sanitize: list[str] | NotSet = NOT_SET,
+    sensitive_markers: list[str] | NotSet = NOT_SET,
+) -> None:
+    """Extend current sanitization configuration."""
+    global _DEFAULT_SANITIZATION_CONFIG
+    _DEFAULT_SANITIZATION_CONFIG = _DEFAULT_SANITIZATION_CONFIG.extend(
+        keys_to_sanitize=keys_to_sanitize,
+        sensitive_markers=sensitive_markers,
+    )
+
+
+def sanitize_value(item: Any, *, config: SanitizationConfig | None = None) -> None:
+    """Sanitize sensitive values within a given item.
+
+    This function is recursive and will sanitize sensitive data within nested
+    dictionaries and lists as well.
+    """
+    config = config or _DEFAULT_SANITIZATION_CONFIG
+    if isinstance(item, MutableMapping):
+        for key in list(item.keys()):
+            lower_key = key.lower()
+            if lower_key in config.keys_to_sanitize or any(marker in lower_key for marker in config.sensitive_markers):
+                if isinstance(item[key], list):
+                    item[key] = [config.replacement]
+                else:
+                    item[key] = config.replacement
+        for value in item.values():
+            if isinstance(value, (MutableMapping, MutableSequence)):
+                sanitize_value(value, config=config)
+    elif isinstance(item, MutableSequence):
+        for value in item:
+            if isinstance(value, (MutableMapping, MutableSequence)):
+                sanitize_value(value, config=config)
+
+
+def sanitize_url(url: str, *, config: SanitizationConfig | None = None) -> str:
+    """Sanitize sensitive parts of a given URL.
+
+    This function will sanitize the authority and query parameters in the URL.
+    """
+    config = config or _DEFAULT_SANITIZATION_CONFIG
+    parsed = urlsplit(url)
+
+    # Sanitize authority
+    netloc_parts = parsed.netloc.split("@")
+    if len(netloc_parts) > 1:
+        netloc = f"{config.replacement}@{netloc_parts[-1]}"
+    else:
+        netloc = parsed.netloc
+
+    # Sanitize query parameters
+    query = parse_qs(parsed.query, keep_blank_values=True)
+    sanitize_value(query, config=config)
+    sanitized_query = urlencode(query, doseq=True)
+
+    # Reconstruct the URL
+    sanitized_url_parts = parsed._replace(netloc=netloc, query=sanitized_query)
+    return urlunsplit(sanitized_url_parts)

schemathesis/{throttling.py → core/rate_limit.py}

@@ -1,14 +1,23 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from contextlib import nullcontext
+from typing import TYPE_CHECKING, ContextManager
+from urllib.parse import urlparse
 
-from ._dependency_versions import IS_PYRATE_LIMITER_ABOVE_3
-from .exceptions import UsageError
+from schemathesis.core.errors import InvalidRateLimit
 
 if TYPE_CHECKING:
     from pyrate_limiter import Duration, Limiter
 
 
+def ratelimit(rate_limiter: Limiter | None, base_url: str | None) -> ContextManager:
+    """Limit the rate of sending generated requests."""
+    label = urlparse(base_url).netloc
+    if rate_limiter is not None:
+        rate_limiter.try_acquire(label)
+    return nullcontext()
+
+
 def parse_units(rate: str) -> tuple[int, int]:
     from pyrate_limiter import Duration
 
@@ -21,17 +30,10 @@ def parse_units(rate: str) -> tuple[int, int]:
             "d": Duration.DAY,
         }.get(interval_text)
         if interval is None:
-            raise invalid_rate(rate)
+            raise InvalidRateLimit(rate)
         return int(limit), interval
     except ValueError as exc:
-        raise invalid_rate(rate) from exc
-
-
-def invalid_rate(value: str) -> UsageError:
-    return UsageError(
-        f"Invalid rate limit value: `{value}`. Should be in form `limit/interval`. "
-        "Example: `10/m` for 10 requests per minute."
-    )
+        raise InvalidRateLimit(rate) from exc
 
 
 def _get_max_delay(value: int, unit: Duration) -> int:
@@ -51,11 +53,8 @@ def _get_max_delay(value: int, unit: Duration) -> int:
 
 
 def build_limiter(rate: str) -> Limiter:
-    from ._rate_limiter import Limiter, Rate
+    from pyrate_limiter import Limiter, Rate
 
     limit, interval = parse_units(rate)
     rate = Rate(limit, interval)
-    kwargs = {}
-    if IS_PYRATE_LIMITER_ABOVE_3:
-        kwargs["max_delay"] = _get_max_delay(limit, interval)
-    return Limiter(rate, **kwargs)
+    return Limiter(rate, max_delay=_get_max_delay(limit, interval))

schemathesis/core/registries.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from typing import Callable, Generic, Sequence, TypeVar, Union
+
+T = TypeVar("T", bound=Union[Callable, type])
+
+
+class Registry(Generic[T]):
+    """Container for Schemathesis extensions."""
+
+    __slots__ = ("_items",)
+
+    def __init__(self) -> None:
+        self._items: dict[str, T] = {}
+
+    def register(self, item: T) -> T:
+        self._items[item.__name__] = item
+        return item
+
+    def unregister(self, name: str) -> None:
+        del self._items[name]
+
+    def get_all_names(self) -> list[str]:
+        return list(self._items)
+
+    def get_all(self) -> list[T]:
+        return list(self._items.values())
+
+    def get_by_names(self, names: Sequence[str]) -> list[T]:
+        """Get items by their names."""
+        return [self._items[name] for name in names]

schemathesis/core/transforms.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, List, Mapping, Union, overload
+
+
+def deepclone(value: Any) -> Any:
+    """A specialized version of `deepcopy` that copies only `dict` and `list` and does unrolling.
+
+    It is on average 3x faster than `deepcopy` and given the amount of calls, it is an important optimization.
+    """
+    if isinstance(value, dict):
+        return {
+            k1: (
+                {k2: deepclone(v2) for k2, v2 in v1.items()}
+                if isinstance(v1, dict)
+                else [deepclone(v2) for v2 in v1]
+                if isinstance(v1, list)
+                else v1
+            )
+            for k1, v1 in value.items()
+        }
+    if isinstance(value, list):
+        return [
+            {k2: deepclone(v2) for k2, v2 in v1.items()}
+            if isinstance(v1, dict)
+            else [deepclone(v2) for v2 in v1]
+            if isinstance(v1, list)
+            else v1
+            for v1 in value
+        ]
+    return value
+
+
+def diff(left: Mapping[str, Any], right: Mapping[str, Any]) -> dict[str, Any]:
+    """Calculate the difference between two dictionaries."""
+    diff = {}
+    for key, value in right.items():
+        if key not in left or left[key] != value:
+            diff[key] = value
+    return diff
+
+
+def merge_at(data: dict[str, Any], data_key: str, new: dict[str, Any]) -> None:
+    original = data[data_key] or {}
+    for key, value in new.items():
+        original[key] = value
+    data[data_key] = original
+
+
+JsonValue = Union[Dict[str, Any], List, str, float, int]
+
+
+@overload
+def transform(schema: dict[str, Any], callback: Callable, *args: Any, **kwargs: Any) -> dict[str, Any]: ...
+
+
+@overload
+def transform(schema: list, callback: Callable, *args: Any, **kwargs: Any) -> list: ...
+
+
+@overload
+def transform(schema: str, callback: Callable, *args: Any, **kwargs: Any) -> str: ...
+
+
+@overload
+def transform(schema: float, callback: Callable, *args: Any, **kwargs: Any) -> float: ...
+
+
+def transform(schema: JsonValue, callback: Callable[..., dict[str, Any]], *args: Any, **kwargs: Any) -> JsonValue:
+    """Apply callback recursively to the given schema."""
+    if isinstance(schema, dict):
+        schema = callback(schema, *args, **kwargs)
+        for key, sub_item in schema.items():
+            schema[key] = transform(sub_item, callback, *args, **kwargs)
+    elif isinstance(schema, list):
+        schema = [transform(sub_item, callback, *args, **kwargs) for sub_item in schema]
+    return schema
+
+
+class Unresolvable: ...
+
+
+UNRESOLVABLE = Unresolvable()
+
+
+def resolve_pointer(document: Any, pointer: str) -> dict | list | str | int | float | None | Unresolvable:
+    """Implementation is adapted from Rust's `serde-json` crate.
+
+    Ref: https://github.com/serde-rs/json/blob/master/src/value/mod.rs#L751
+    """
+    if not pointer:
+        return document
+    if not pointer.startswith("/"):
+        return UNRESOLVABLE
+
+    def replace(value: str) -> str:
+        return value.replace("~1", "/").replace("~0", "~")
+
+    tokens = map(replace, pointer.split("/")[1:])
+    target = document
+    for token in tokens:
+        if isinstance(target, dict):
+            target = target.get(token, UNRESOLVABLE)
+            if target is UNRESOLVABLE:
+                return UNRESOLVABLE
+        elif isinstance(target, list):
+            try:
+                target = target[int(token)]
+            except IndexError:
+                return UNRESOLVABLE
+        else:
+            return UNRESOLVABLE
+    return target