sqlspec 0.47.0__cp314-cp314-win_amd64.whl

sqlspec/utils/serializers/_schema.py

@@ -0,0 +1,285 @@
+"""Schema dumping and cache helpers for ``sqlspec.utils.serializers``."""
+
+import os
+from collections import OrderedDict
+from functools import partial
+from threading import RLock
+from typing import TYPE_CHECKING, Any, Final, cast
+
+from sqlspec.typing import MSGSPEC_INSTALLED, UNSET, ArrowReturnFormat, attrs_asdict, msgspec_fields
+from sqlspec.utils.arrow_helpers import convert_dict_to_arrow
+from sqlspec.utils.type_guards import (
+    dataclass_to_dict,
+    has_dict_attribute,
+    is_attrs_instance,
+    is_dataclass_instance,
+    is_dict,
+    is_msgspec_struct,
+    is_pydantic_model,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable
+
+__all__ = (
+    "SchemaSerializer",
+    "get_collection_serializer",
+    "get_serializer_metrics",
+    "reset_serializer_cache",
+    "schema_dump",
+    "serialize_collection",
+)
+
+
+DEBUG_ENV_FLAG: Final[str] = "SQLSPEC_DEBUG_PIPELINE_CACHE"
+_PRIMITIVE_TYPES: Final[tuple[type[Any], ...]] = (str, bytes, int, float, bool)
+_PRIMITIVE_TYPES_SET: Final[frozenset[type[Any]]] = frozenset(_PRIMITIVE_TYPES)
+_SCHEMA_SERIALIZER_CACHE_MAX_SIZE: Final[int] = 1000
+
+
+def _is_truthy(value: "str | None") -> bool:
+    if value is None:
+        return False
+    normalized = value.strip().lower()
+    return normalized in {"1", "true", "yes", "on"}
+
+
+_METRICS_ENABLED: Final[bool] = _is_truthy(os.getenv(DEBUG_ENV_FLAG))
+
+
+class _SerializerCacheMetrics:
+    __slots__ = ("hits", "max_size", "misses", "size")
+
+    def __init__(self) -> None:
+        self.hits = 0
+        self.misses = 0
+        self.size = 0
+        self.max_size = 0
+
+    def record_hit(self, cache_size: int) -> None:
+        if not _METRICS_ENABLED:
+            return
+        self.hits += 1
+        self.size = cache_size
+        self.max_size = max(self.max_size, cache_size)
+
+    def record_miss(self, cache_size: int) -> None:
+        if not _METRICS_ENABLED:
+            return
+        self.misses += 1
+        self.size = cache_size
+        self.max_size = max(self.max_size, cache_size)
+
+    def reset(self) -> None:
+        self.hits = 0
+        self.misses = 0
+        self.size = 0
+        self.max_size = 0
+
+    def snapshot(self) -> "dict[str, int]":
+        return {
+            "hits": self.hits if _METRICS_ENABLED else 0,
+            "misses": self.misses if _METRICS_ENABLED else 0,
+            "max_size": self.max_size if _METRICS_ENABLED else 0,
+            "size": self.size if _METRICS_ENABLED else 0,
+        }
+
+
+class SchemaSerializer:
+    """Serializer pipeline that caches conversions for repeated schema dumps."""
+
+    __slots__ = ("_dump", "_key")
+
+    def __init__(self, key: "tuple[type[Any] | None, bool, bool]", dump: "Callable[[Any], dict[str, Any]]") -> None:
+        self._key = key
+        self._dump = dump
+
+    @property
+    def key(self) -> "tuple[type[Any] | None, bool, bool]":
+        return self._key
+
+    def dump_one(self, item: Any) -> "dict[str, Any]":
+        return self._dump(item)
+
+    def dump_many(self, items: "Iterable[Any]") -> "list[dict[str, Any]]":
+        return [self._dump(item) for item in items]
+
+    def to_arrow(
+        self, items: "Iterable[Any]", *, return_format: "ArrowReturnFormat" = "table", batch_size: int | None = None
+    ) -> Any:
+        payload = self.dump_many(items)
+        return convert_dict_to_arrow(payload, return_format=return_format, batch_size=batch_size)
+
+
+_SERIALIZER_LOCK: RLock = RLock()
+_SCHEMA_SERIALIZERS: "OrderedDict[tuple[type[Any] | None, bool, bool], SchemaSerializer]" = OrderedDict()
+_SERIALIZER_METRICS = _SerializerCacheMetrics()
+
+
+def _make_serializer_key(sample: Any, exclude_unset: bool, wire_format: bool) -> "tuple[type[Any] | None, bool, bool]":
+    if sample is None or isinstance(sample, dict):
+        return (None, exclude_unset, wire_format)
+    return (type(sample), exclude_unset, wire_format)
+
+
+def _dump_identity_dict(value: Any) -> "dict[str, Any]":
+    return cast("dict[str, Any]", value)
+
+
+def _get_msgspec_field_pairs(schema_type: type[Any], *, wire_format: bool) -> "tuple[tuple[str, str], ...]":
+    if not MSGSPEC_INSTALLED:
+        msg = "msgspec is required to serialize msgspec.Struct values"
+        raise RuntimeError(msg)
+
+    if wire_format:
+        return tuple((field.encode_name, field.name) for field in msgspec_fields(schema_type))
+    return tuple((field.name, field.name) for field in msgspec_fields(schema_type))
+
+
+def _dump_msgspec_struct(
+    value: Any, *, field_pairs: "tuple[tuple[str, str], ...]", exclude_unset: bool
+) -> "dict[str, Any]":
+    if not exclude_unset:
+        return {output_name: value.__getattribute__(field_name) for output_name, field_name in field_pairs}
+    return {
+        output_name: field_value
+        for output_name, field_name in field_pairs
+        if (field_value := value.__getattribute__(field_name)) != UNSET
+    }
+
+
+def _dump_dataclass(value: Any, *, exclude_unset: bool) -> "dict[str, Any]":
+    return dataclass_to_dict(value, exclude_empty=exclude_unset)
+
+
+def _dump_pydantic(value: Any, *, exclude_unset: bool) -> "dict[str, Any]":
+    return cast("dict[str, Any]", value.model_dump(exclude_unset=exclude_unset))
+
+
+def _dump_attrs(value: Any) -> "dict[str, Any]":
+    return attrs_asdict(value, recurse=True)
+
+
+def _dump_dict_attr(value: Any) -> "dict[str, Any]":
+    return dict(value.__dict__)
+
+
+def _dump_mapping(value: Any) -> "dict[str, Any]":
+    return dict(value)
+
+
+def _build_msgspec_dump_function(
+    sample: Any, *, exclude_unset: bool, wire_format: bool
+) -> "Callable[[Any], dict[str, Any]]":
+    field_pairs = _get_msgspec_field_pairs(type(sample), wire_format=wire_format)
+    return cast(
+        "Callable[[Any], dict[str, Any]]",
+        partial(_dump_msgspec_struct, field_pairs=field_pairs, exclude_unset=exclude_unset),
+    )
+
+
+def _build_dump_function(sample: Any, exclude_unset: bool, wire_format: bool) -> "Callable[[Any], dict[str, Any]]":
+    if sample is None or isinstance(sample, dict):
+        return _dump_identity_dict
+    if is_dataclass_instance(sample):
+        return cast("Callable[[Any], dict[str, Any]]", partial(_dump_dataclass, exclude_unset=exclude_unset))
+    if is_pydantic_model(sample):
+        return cast("Callable[[Any], dict[str, Any]]", partial(_dump_pydantic, exclude_unset=exclude_unset))
+    if is_msgspec_struct(sample):
+        return _build_msgspec_dump_function(sample, exclude_unset=exclude_unset, wire_format=wire_format)
+    if is_attrs_instance(sample):
+        return _dump_attrs
+    if has_dict_attribute(sample):
+        return _dump_dict_attr
+    return _dump_mapping
+
+
+def get_collection_serializer(
+    sample: Any, *, exclude_unset: bool = True, wire_format: bool = False
+) -> "SchemaSerializer":
+    """Return cached serializer pipeline for the provided sample object."""
+    key = _make_serializer_key(sample, exclude_unset, wire_format)
+    pipeline = _SCHEMA_SERIALIZERS.get(key)
+    if pipeline is not None:
+        try:
+            _SCHEMA_SERIALIZERS.move_to_end(key)
+        except KeyError:
+            pipeline = None
+        else:
+            _SERIALIZER_METRICS.record_hit(len(_SCHEMA_SERIALIZERS))
+            return pipeline
+
+    with _SERIALIZER_LOCK:
+        pipeline = _SCHEMA_SERIALIZERS.get(key)
+        if pipeline is not None:
+            _SCHEMA_SERIALIZERS.move_to_end(key)
+            _SERIALIZER_METRICS.record_hit(len(_SCHEMA_SERIALIZERS))
+            return pipeline
+
+        dump = _build_dump_function(sample, exclude_unset, wire_format)
+        pipeline = SchemaSerializer(key, dump)
+        _SCHEMA_SERIALIZERS[key] = pipeline
+        if len(_SCHEMA_SERIALIZERS) > _SCHEMA_SERIALIZER_CACHE_MAX_SIZE:
+            _SCHEMA_SERIALIZERS.popitem(last=False)
+        _SERIALIZER_METRICS.record_miss(len(_SCHEMA_SERIALIZERS))
+        return pipeline
+
+
+def serialize_collection(
+    items: "Iterable[Any]", *, exclude_unset: bool = True, wire_format: bool = False
+) -> "list[Any]":
+    """Serialize a collection using cached pipelines keyed by item type."""
+    serialized: list[Any] = []
+    cache: dict[tuple[type[Any] | None, bool, bool], SchemaSerializer] = {}
+
+    for item in items:
+        if type(item) in _PRIMITIVE_TYPES_SET or item is None or isinstance(item, dict):
+            serialized.append(item)
+            continue
+
+        key = _make_serializer_key(item, exclude_unset, wire_format)
+        pipeline = cache.get(key)
+        if pipeline is None:
+            pipeline = get_collection_serializer(item, exclude_unset=exclude_unset, wire_format=wire_format)
+            cache[key] = pipeline
+        serialized.append(pipeline.dump_one(item))
+    return serialized
+
+
+def reset_serializer_cache() -> None:
+    """Clear cached serializer pipelines."""
+    with _SERIALIZER_LOCK:
+        _SCHEMA_SERIALIZERS.clear()
+        _SERIALIZER_METRICS.reset()
+
+
+def get_serializer_metrics() -> "dict[str, int]":
+    """Return cache metrics aligned with the core pipeline counters."""
+    with _SERIALIZER_LOCK:
+        metrics = _SERIALIZER_METRICS.snapshot()
+        metrics["size"] = len(_SCHEMA_SERIALIZERS)
+        return metrics
+
+
+def schema_dump(data: Any, *, exclude_unset: bool = True, wire_format: bool = False) -> Any:
+    """Dump a schema model or dict to a plain representation.
+
+    Args:
+        data: A schema instance (msgspec.Struct, Pydantic BaseModel, dataclass, attrs class)
+            or plain dict / primitive.
+        exclude_unset: If True, exclude fields that were never set (msgspec UNSET, Pydantic
+            model_fields_set semantics). No-op for attrs (attrs has no unset concept).
+        wire_format: msgspec-only knob. Default ``False`` emits Python attribute names
+            (``field.name``) for cross-library consistency — keys match Pydantic, dataclass,
+            and attrs output regardless of the Struct's ``rename=`` meta. Pass
+            ``wire_format=True`` to emit ``field.encode_name`` (honours ``rename=`` on the
+            Struct) for wire-aligned JSON / API payloads. Pydantic, dataclass, and attrs
+            branches always use Python attribute names regardless of this flag.
+    """
+    if is_dict(data):
+        return data
+    if type(data) in _PRIMITIVE_TYPES_SET or data is None:
+        return data
+
+    serializer = get_collection_serializer(data, exclude_unset=exclude_unset, wire_format=wire_format)
+    return serializer.dump_one(data)

sqlspec/utils/singleton.py

@@ -0,0 +1,41 @@
+"""Singleton pattern implementation for SQLSpec.
+
+Provides a thread-safe metaclass for implementing the singleton pattern.
+Used for creating classes that should have only one instance per class type.
+"""
+
+import threading
+from typing import Any, TypeVar
+
+from mypy_extensions import mypyc_attr
+
+__all__ = ("SingletonMeta",)
+
+
+_T = TypeVar("_T")
+
+
+@mypyc_attr(native_class=False)
+class SingletonMeta(type):
+    """Metaclass for singleton pattern."""
+
+    _instances: "dict[type, object]" = {}
+    _lock = threading.Lock()
+
+    def __call__(cls: type[_T], *args: Any, **kwargs: Any) -> _T:
+        """Call method for the singleton metaclass.
+
+        Args:
+            cls: The class being instantiated.
+            *args: Positional arguments for the class constructor.
+            **kwargs: Keyword arguments for the class constructor.
+
+        Returns:
+            The singleton instance of the class.
+        """
+        if cls not in SingletonMeta._instances:  # pyright: ignore[reportUnnecessaryContains]
+            with SingletonMeta._lock:
+                if cls not in SingletonMeta._instances:
+                    instance = super().__call__(*args, **kwargs)  # type: ignore[misc]
+                    SingletonMeta._instances[cls] = instance
+        return SingletonMeta._instances[cls]  # type: ignore[return-value]

sqlspec/utils/sync_tools.py

@@ -0,0 +1,316 @@
+"""Utilities for async/sync interoperability in SQLSpec.
+
+This module provides utilities for converting between async and sync functions,
+managing concurrency limits, and handling context managers. Used primarily
+for adapter implementations that need to support both sync and async patterns.
+"""
+
+import asyncio
+import concurrent.futures
+import functools
+import inspect
+import os
+import sys
+from contextlib import AbstractAsyncContextManager, AbstractContextManager
+from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast
+
+from typing_extensions import ParamSpec
+
+from sqlspec.utils.module_loader import module_available
+from sqlspec.utils.portal import get_global_portal
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable, Coroutine
+    from types import TracebackType
+
+if module_available("uvloop"):
+    import uvloop  # pyright: ignore[reportMissingImports]
+else:
+    uvloop = None  # type: ignore[assignment,unused-ignore]
+
+
+ReturnT = TypeVar("ReturnT")
+ParamSpecT = ParamSpec("ParamSpecT")
+T = TypeVar("T")
+
+
+class NoValue:
+    """Sentinel class for missing values."""
+
+
+NO_VALUE = NoValue()
+
+
+class CapacityLimiter:
+    """Limits the number of concurrent operations using a semaphore."""
+
+    def __init__(self, total_tokens: int) -> None:
+        """Initialize the capacity limiter.
+
+        Args:
+            total_tokens: Maximum number of concurrent operations allowed
+        """
+        self._total_tokens = total_tokens
+        self._semaphore_instance: asyncio.Semaphore | None = None
+        self._pid: int | None = None
+
+    @property
+    def _semaphore(self) -> asyncio.Semaphore:
+        """Lazy initialization of asyncio.Semaphore with per-process tracking.
+
+        Reinitializes the semaphore if running in a new process (detected via PID).
+        This ensures pytest-xdist workers each get their own semaphore bound to
+        their event loop, preventing cross-process deadlocks.
+        """
+        current_pid = os.getpid()
+        if self._semaphore_instance is None or self._pid != current_pid:
+            self._semaphore_instance = asyncio.Semaphore(self._total_tokens)
+            self._pid = current_pid
+        return self._semaphore_instance
+
+    async def acquire(self) -> None:
+        """Acquire a token from the semaphore."""
+        await self._semaphore.acquire()
+
+    def release(self) -> None:
+        """Release a token back to the semaphore."""
+        self._semaphore.release()
+
+    @property
+    def total_tokens(self) -> int:
+        """Get the total number of tokens available."""
+        return self._total_tokens
+
+    @total_tokens.setter
+    def total_tokens(self, value: int) -> None:
+        self._total_tokens = value
+        self._semaphore_instance = None
+        self._pid = None
+
+    async def __aenter__(self) -> None:
+        """Async context manager entry."""
+        await self.acquire()
+
+    async def __aexit__(
+        self, exc_type: "type[BaseException] | None", exc_val: "BaseException | None", exc_tb: "TracebackType | None"
+    ) -> None:
+        """Async context manager exit."""
+        self.release()
+
+
+_default_limiter = CapacityLimiter(1000)
+
+
+def _return_value(value: Any) -> Any:
+    return value
+
+
+class _RunWrapper(Generic[ParamSpecT, ReturnT]):
+    __slots__ = ("__dict__", "_function")
+
+    def __init__(self, async_function: "Callable[ParamSpecT, Coroutine[Any, Any, ReturnT]]") -> None:
+        self._function = async_function
+        functools.update_wrapper(self, async_function)
+
+    def __call__(self, *args: "ParamSpecT.args", **kwargs: "ParamSpecT.kwargs") -> "ReturnT":
+        partial_f = functools.partial(self._function, *args, **kwargs)
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            loop = None
+
+        if loop is not None:
+            if loop.is_running():
+                with concurrent.futures.ThreadPoolExecutor() as executor:
+                    future = executor.submit(asyncio.run, partial_f())
+                    return future.result()
+            return asyncio.run(partial_f())
+        if uvloop and sys.platform != "win32":
+            return uvloop.run(partial_f())  # pyright: ignore[reportUnknownMemberType]
+        return asyncio.run(partial_f())
+
+
+def run_(async_function: "Callable[ParamSpecT, Coroutine[Any, Any, ReturnT]]") -> "Callable[ParamSpecT, ReturnT]":
+    """Convert an async function to a blocking function using asyncio.run().
+
+    Args:
+        async_function: The async function to convert.
+
+    Returns:
+        A blocking function that runs the async function.
+    """
+
+    return _RunWrapper(async_function)
+
+
+def await_(
+    async_function: "Callable[ParamSpecT, Coroutine[Any, Any, ReturnT]]", raise_sync_error: bool = False
+) -> "Callable[ParamSpecT, ReturnT]":
+    """Convert an async function to a blocking one, running in the main async loop.
+
+    When no event loop exists, automatically creates and uses a global portal for
+    async-to-sync bridging via background thread. Set raise_sync_error=True to
+    disable this behavior and raise errors instead.
+
+    Args:
+        async_function: The async function to convert.
+        raise_sync_error: If True, raises RuntimeError when no loop exists.
+                         If False (default), uses portal pattern for automatic bridging.
+
+    Returns:
+        A blocking function that runs the async function.
+    """
+
+    return _AwaitWrapper(async_function, raise_sync_error)
+
+
+def async_(
+    function: "Callable[ParamSpecT, ReturnT]", *, limiter: "CapacityLimiter | None" = None
+) -> "Callable[ParamSpecT, Awaitable[ReturnT]]":
+    """Convert a blocking function to an async one using asyncio.to_thread().
+
+    Args:
+        function: The blocking function to convert.
+        limiter: Limit the total number of threads.
+
+    Returns:
+        An async function that runs the original function in a thread.
+    """
+
+    return _AsyncWrapper(function, limiter)
+
+
+def ensure_async_(
+    function: "Callable[ParamSpecT, Awaitable[ReturnT] | ReturnT]",
+) -> "Callable[ParamSpecT, Awaitable[ReturnT]]":
+    """Convert a function to an async one if it is not already.
+
+    Args:
+        function: The function to convert.
+
+    Returns:
+        An async function that runs the original function.
+    """
+    if inspect.iscoroutinefunction(function):
+        return function
+
+    return _EnsureAsyncWrapper(function)
+
+
+class _AwaitWrapper(Generic[ParamSpecT, ReturnT]):
+    __slots__ = ("__dict__", "_function", "_raise_sync_error")
+
+    def __init__(
+        self, async_function: "Callable[ParamSpecT, Coroutine[Any, Any, ReturnT]]", raise_sync_error: bool
+    ) -> None:
+        self._function = async_function
+        self._raise_sync_error = raise_sync_error
+        functools.update_wrapper(self, async_function)
+
+    def __call__(self, *args: "ParamSpecT.args", **kwargs: "ParamSpecT.kwargs") -> "ReturnT":
+        partial_f = functools.partial(self._function, *args, **kwargs)
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            if self._raise_sync_error:
+                msg = "Cannot run async function"
+                raise RuntimeError(msg) from None
+            portal = get_global_portal()
+            typed_partial = cast("Callable[[], Coroutine[Any, Any, ReturnT]]", partial_f)
+            return portal.call(typed_partial)
+        if loop.is_running():
+            try:
+                current_task = asyncio.current_task(loop=loop)
+            except RuntimeError:
+                current_task = None
+
+            if current_task is not None:
+                if self._raise_sync_error:
+                    msg = "await_ cannot be called from within an async task running on the same event loop. Use 'await' instead."
+                    raise RuntimeError(msg)
+                portal = get_global_portal()
+                typed_partial = cast("Callable[[], Coroutine[Any, Any, ReturnT]]", partial_f)
+                return portal.call(typed_partial)
+            future = asyncio.run_coroutine_threadsafe(partial_f(), loop)
+            return future.result()
+        if self._raise_sync_error:
+            msg = "Cannot run async function"
+            raise RuntimeError(msg)
+        portal = get_global_portal()
+        typed_partial = cast("Callable[[], Coroutine[Any, Any, ReturnT]]", partial_f)
+        return portal.call(typed_partial)
+
+
+class _AsyncWrapper(Generic[ParamSpecT, ReturnT]):
+    __slots__ = ("__dict__", "_function", "_limiter")
+
+    def __init__(self, function: "Callable[ParamSpecT, ReturnT]", limiter: "CapacityLimiter | None") -> None:
+        self._function = function
+        self._limiter = limiter
+        functools.update_wrapper(self, function)
+
+    async def __call__(self, *args: "ParamSpecT.args", **kwargs: "ParamSpecT.kwargs") -> "ReturnT":
+        partial_f = functools.partial(self._function, *args, **kwargs)
+        if self._limiter is not None:
+            async with self._limiter:
+                return await asyncio.to_thread(partial_f)
+        return await asyncio.to_thread(partial_f)
+
+
+class _EnsureAsyncWrapper(Generic[ParamSpecT, ReturnT]):
+    __slots__ = ("__dict__", "_function")
+
+    def __init__(self, function: "Callable[ParamSpecT, Awaitable[ReturnT] | ReturnT]") -> None:
+        self._function = function
+        functools.update_wrapper(self, function)
+
+    async def __call__(self, *args: "ParamSpecT.args", **kwargs: "ParamSpecT.kwargs") -> "ReturnT":
+        result = self._function(*args, **kwargs)
+        if inspect.isawaitable(result):
+            return await cast("Awaitable[ReturnT]", result)
+        return result
+
+
+class _ContextManagerWrapper(Generic[T]):
+    def __init__(self, cm: AbstractContextManager[T]) -> None:
+        self._cm = cm
+
+    async def __aenter__(self) -> T:
+        return self._cm.__enter__()
+
+    async def __aexit__(
+        self, exc_type: "type[BaseException] | None", exc_val: "BaseException | None", exc_tb: "TracebackType | None"
+    ) -> "bool | None":
+        return self._cm.__exit__(exc_type, exc_val, exc_tb)
+
+
+def with_ensure_async_(
+    obj: "AbstractContextManager[T] | AbstractAsyncContextManager[T]",
+) -> "AbstractAsyncContextManager[T]":
+    """Convert a context manager to an async one if it is not already.
+
+    Args:
+        obj: The context manager to convert.
+
+    Returns:
+        An async context manager that runs the original context manager.
+    """
+    if isinstance(obj, AbstractContextManager):
+        return cast("AbstractAsyncContextManager[T]", _ContextManagerWrapper(obj))
+    return obj
+
+
+async def get_next(iterable: Any, default: Any = NO_VALUE, *args: Any) -> Any:  # pragma: no cover
+    """Return the next item from an async iterator.
+
+    Args:
+        iterable: An async iterable.
+        default: An optional default value to return if the iterable is empty.
+        *args: The remaining args
+
+    Returns:
+        The next value of the iterable.
+    """
+    if isinstance(default, NoValue):
+        return await anext(iterable)
+    return await anext(iterable, default)