sqlspec 0.27.0__py3-none-any.whl → 0.28.0__py3-none-any.whl

sqlspec/utils/arrow_helpers.py

@@ -0,0 +1,81 @@
+"""Arrow conversion helpers for dict-to-Arrow transformations.
+
+This module provides utilities for converting Python dictionaries to Apache Arrow
+format, handling empty results, NULL values, and type inference.
+"""
+
+from typing import TYPE_CHECKING, Any, Literal
+
+if TYPE_CHECKING:
+    from sqlspec.typing import ArrowRecordBatch, ArrowTable
+
+__all__ = ("convert_dict_to_arrow",)
+
+
+def convert_dict_to_arrow(
+    data: "list[dict[str, Any]]",
+    return_format: Literal["table", "reader", "batches"] = "table",
+    batch_size: int | None = None,
+) -> "ArrowTable | ArrowRecordBatch":
+    """Convert list of dictionaries to Arrow Table or RecordBatch.
+
+    Handles empty results, NULL values, and automatic type inference.
+    Used by adapters that don't have native Arrow support to convert
+    dict-based results to Arrow format.
+
+    Args:
+        data: List of dictionaries (one per row).
+        return_format: Output format - "table" for Table, "batches" for RecordBatch.
+            "reader" is converted to "table" (streaming handled at driver level).
+        batch_size: Chunk size for batching (used when return_format="batches").
+
+    Returns:
+        ArrowTable or ArrowRecordBatch depending on return_format.
+
+
+    Examples:
+        >>> data = [
+        ...     {"id": 1, "name": "Alice"},
+        ...     {"id": 2, "name": "Bob"},
+        ... ]
+        >>> table = convert_dict_to_arrow(data, return_format="table")
+        >>> print(table.num_rows)
+        2
+
+        >>> batch = convert_dict_to_arrow(data, return_format="batch")
+        >>> print(batch.num_rows)
+        2
+    """
+    from sqlspec.utils.module_loader import ensure_pyarrow
+
+    ensure_pyarrow()
+
+    import pyarrow as pa
+
+    # Handle empty results
+    if not data:
+        empty_schema = pa.schema([])
+        empty_table = pa.Table.from_pydict({}, schema=empty_schema)
+
+        if return_format == "batches":
+            # Create empty RecordBatch
+            batches = empty_table.to_batches()
+            return batches[0] if batches else pa.RecordBatch.from_pydict({})
+
+        return empty_table
+
+    # Convert list of dicts to columnar format
+    # This is more efficient than row-by-row conversion
+    columns: dict[str, list[Any]] = {key: [row.get(key) for row in data] for key in data[0]}
+
+    # Create Arrow Table (auto type inference)
+    arrow_table = pa.Table.from_pydict(columns)
+
+    # Return appropriate format
+    if return_format == "batches":
+        # Convert table to single RecordBatch
+        batches = arrow_table.to_batches()
+        return batches[0] if batches else pa.RecordBatch.from_pydict({})
+
+    # return_format == "table" or "reader" (reader handled at driver level)
+    return arrow_table

sqlspec/utils/module_loader.py

@@ -1,7 +1,8 @@
 """Module loading utilities for SQLSpec.
 
-Provides functions for dynamic module imports and path resolution.
-Used for loading modules from dotted paths and converting module paths to filesystem paths.
+Provides functions for dynamic module imports, path resolution, and dependency
+availability checking. Used for loading modules from dotted paths, converting
+module paths to filesystem paths, and ensuring optional dependencies are installed.
 """
 
 import importlib
@@ -9,7 +10,46 @@ from importlib.util import find_spec
 from pathlib import Path
 from typing import Any
 
-__all__ = ("import_string", "module_to_os_path")
+from sqlspec.exceptions import MissingDependencyError
+from sqlspec.typing import (
+    AIOSQL_INSTALLED,
+    ATTRS_INSTALLED,
+    CATTRS_INSTALLED,
+    FSSPEC_INSTALLED,
+    LITESTAR_INSTALLED,
+    MSGSPEC_INSTALLED,
+    NUMPY_INSTALLED,
+    OBSTORE_INSTALLED,
+    OPENTELEMETRY_INSTALLED,
+    ORJSON_INSTALLED,
+    PANDAS_INSTALLED,
+    PGVECTOR_INSTALLED,
+    POLARS_INSTALLED,
+    PROMETHEUS_INSTALLED,
+    PYARROW_INSTALLED,
+    PYDANTIC_INSTALLED,
+)
+
+__all__ = (
+    "ensure_aiosql",
+    "ensure_attrs",
+    "ensure_cattrs",
+    "ensure_fsspec",
+    "ensure_litestar",
+    "ensure_msgspec",
+    "ensure_numpy",
+    "ensure_obstore",
+    "ensure_opentelemetry",
+    "ensure_orjson",
+    "ensure_pandas",
+    "ensure_pgvector",
+    "ensure_polars",
+    "ensure_prometheus",
+    "ensure_pyarrow",
+    "ensure_pydantic",
+    "import_string",
+    "module_to_os_path",
+)
 
 
 def module_to_os_path(dotted_path: str = "app") -> "Path":
@@ -91,3 +131,163 @@ def import_string(dotted_path: str) -> "Any":
     except Exception as e:  # pylint: disable=broad-exception-caught
         _raise_import_error(f"Could not import '{dotted_path}': {e}", e)
     return obj
+
+
+def ensure_aiosql() -> None:
+    """Ensure aiosql is available.
+
+    Raises:
+        MissingDependencyError: If aiosql is not installed.
+    """
+    if not AIOSQL_INSTALLED:
+        raise MissingDependencyError(package="aiosql", install_package="aiosql")
+
+
+def ensure_attrs() -> None:
+    """Ensure attrs is available.
+
+    Raises:
+        MissingDependencyError: If attrs is not installed.
+    """
+    if not ATTRS_INSTALLED:
+        raise MissingDependencyError(package="attrs", install_package="attrs")
+
+
+def ensure_cattrs() -> None:
+    """Ensure cattrs is available.
+
+    Raises:
+        MissingDependencyError: If cattrs is not installed.
+    """
+    if not CATTRS_INSTALLED:
+        raise MissingDependencyError(package="cattrs", install_package="cattrs")
+
+
+def ensure_fsspec() -> None:
+    """Ensure fsspec is available for filesystem operations.
+
+    Raises:
+        MissingDependencyError: If fsspec is not installed.
+    """
+    if not FSSPEC_INSTALLED:
+        raise MissingDependencyError(package="fsspec", install_package="fsspec")
+
+
+def ensure_litestar() -> None:
+    """Ensure Litestar is available.
+
+    Raises:
+        MissingDependencyError: If litestar is not installed.
+    """
+    if not LITESTAR_INSTALLED:
+        raise MissingDependencyError(package="litestar", install_package="litestar")
+
+
+def ensure_msgspec() -> None:
+    """Ensure msgspec is available for serialization.
+
+    Raises:
+        MissingDependencyError: If msgspec is not installed.
+    """
+    if not MSGSPEC_INSTALLED:
+        raise MissingDependencyError(package="msgspec", install_package="msgspec")
+
+
+def ensure_numpy() -> None:
+    """Ensure NumPy is available for array operations.
+
+    Raises:
+        MissingDependencyError: If numpy is not installed.
+    """
+    if not NUMPY_INSTALLED:
+        raise MissingDependencyError(package="numpy", install_package="numpy")
+
+
+def ensure_obstore() -> None:
+    """Ensure obstore is available for object storage operations.
+
+    Raises:
+        MissingDependencyError: If obstore is not installed.
+    """
+    if not OBSTORE_INSTALLED:
+        raise MissingDependencyError(package="obstore", install_package="obstore")
+
+
+def ensure_opentelemetry() -> None:
+    """Ensure OpenTelemetry is available for tracing.
+
+    Raises:
+        MissingDependencyError: If opentelemetry-api is not installed.
+    """
+    if not OPENTELEMETRY_INSTALLED:
+        raise MissingDependencyError(package="opentelemetry-api", install_package="opentelemetry")
+
+
+def ensure_orjson() -> None:
+    """Ensure orjson is available for fast JSON operations.
+
+    Raises:
+        MissingDependencyError: If orjson is not installed.
+    """
+    if not ORJSON_INSTALLED:
+        raise MissingDependencyError(package="orjson", install_package="orjson")
+
+
+def ensure_pandas() -> None:
+    """Ensure pandas is available for DataFrame operations.
+
+    Raises:
+        MissingDependencyError: If pandas is not installed.
+    """
+    if not PANDAS_INSTALLED:
+        raise MissingDependencyError(package="pandas", install_package="pandas")
+
+
+def ensure_pgvector() -> None:
+    """Ensure pgvector is available for vector operations.
+
+    Raises:
+        MissingDependencyError: If pgvector is not installed.
+    """
+    if not PGVECTOR_INSTALLED:
+        raise MissingDependencyError(package="pgvector", install_package="pgvector")
+
+
+def ensure_polars() -> None:
+    """Ensure Polars is available for DataFrame operations.
+
+    Raises:
+        MissingDependencyError: If polars is not installed.
+    """
+    if not POLARS_INSTALLED:
+        raise MissingDependencyError(package="polars", install_package="polars")
+
+
+def ensure_prometheus() -> None:
+    """Ensure Prometheus client is available for metrics.
+
+    Raises:
+        MissingDependencyError: If prometheus-client is not installed.
+    """
+    if not PROMETHEUS_INSTALLED:
+        raise MissingDependencyError(package="prometheus-client", install_package="prometheus")
+
+
+def ensure_pyarrow() -> None:
+    """Ensure PyArrow is available for Arrow operations.
+
+    Raises:
+        MissingDependencyError: If pyarrow is not installed.
+    """
+    if not PYARROW_INSTALLED:
+        raise MissingDependencyError(package="pyarrow", install_package="pyarrow")
+
+
+def ensure_pydantic() -> None:
+    """Ensure Pydantic is available for data validation.
+
+    Raises:
+        MissingDependencyError: If pydantic is not installed.
+    """
+    if not PYDANTIC_INSTALLED:
+        raise MissingDependencyError(package="pydantic", install_package="pydantic")

sqlspec/utils/portal.py

@@ -0,0 +1,311 @@
+"""Portal provider for calling async functions from synchronous contexts.
+
+Provides a background thread with an event loop to execute async database operations
+from sync frameworks like Flask. Based on the portal pattern from Advanced Alchemy.
+"""
+
+import asyncio
+import functools
+import queue
+import threading
+from typing import TYPE_CHECKING, Any, TypeVar
+
+from sqlspec.exceptions import ImproperConfigurationError
+from sqlspec.utils.logging import get_logger
+from sqlspec.utils.singleton import SingletonMeta
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Coroutine
+
+__all__ = ("Portal", "PortalManager", "PortalProvider", "get_global_portal")
+
+logger = get_logger("utils.portal")
+
+_R = TypeVar("_R")
+
+
+class PortalProvider:
+    """Manages a background thread with event loop for async operations.
+
+    Creates a daemon thread running an event loop to execute async functions
+    from synchronous contexts (Flask routes, etc.).
+    """
+
+    def __init__(self) -> None:
+        """Initialize the PortalProvider."""
+        self._request_queue: queue.Queue[
+            tuple[
+                Callable[..., Coroutine[Any, Any, Any]],
+                tuple[Any, ...],
+                dict[str, Any],
+                queue.Queue[tuple[Any | None, Exception | None]],
+            ]
+        ] = queue.Queue()
+        self._loop: asyncio.AbstractEventLoop | None = None
+        self._thread: threading.Thread | None = None
+        self._ready_event: threading.Event = threading.Event()
+
+    @property
+    def portal(self) -> "Portal":
+        """The portal instance for calling async functions.
+
+        Returns:
+            Portal instance.
+        """
+        return Portal(self)
+
+    @property
+    def is_running(self) -> bool:
+        """Check if portal provider is running.
+
+        Returns:
+            True if thread is alive, False otherwise.
+        """
+        return self._thread is not None and self._thread.is_alive()
+
+    @property
+    def is_ready(self) -> bool:
+        """Check if portal provider is ready.
+
+        Returns:
+            True if ready event is set, False otherwise.
+        """
+        return self._ready_event.is_set()
+
+    @property
+    def loop(self) -> "asyncio.AbstractEventLoop":
+        """Get the event loop.
+
+        Returns:
+            The event loop.
+
+        Raises:
+            ImproperConfigurationError: If portal provider not started.
+        """
+        if self._loop is None:
+            msg = "Portal provider not started. Call start() first."
+            raise ImproperConfigurationError(msg)
+        return self._loop
+
+    def start(self) -> None:
+        """Start the background thread and event loop.
+
+        Creates a daemon thread running an event loop for async operations.
+        """
+        if self._thread is not None:
+            logger.debug("Portal provider already started")
+            return
+
+        self._thread = threading.Thread(target=self._run_event_loop, daemon=True)
+        self._thread.start()
+        self._ready_event.wait()
+        logger.debug("Portal provider started")
+
+    def stop(self) -> None:
+        """Stop the background thread and event loop.
+
+        Gracefully shuts down the event loop and waits for thread to finish.
+        """
+        if self._loop is None or self._thread is None:
+            logger.debug("Portal provider not running")
+            return
+
+        self._loop.call_soon_threadsafe(self._loop.stop)
+        self._thread.join(timeout=5)
+
+        if self._thread.is_alive():
+            logger.warning("Portal thread did not stop within 5 seconds")
+
+        self._loop.close()
+        self._loop = None
+        self._thread = None
+        self._ready_event.clear()
+        logger.debug("Portal provider stopped")
+
+    def _run_event_loop(self) -> None:
+        """Main function of the background thread.
+
+        Creates event loop and runs forever until stopped.
+        """
+        if self._loop is None:
+            self._loop = asyncio.new_event_loop()
+
+        asyncio.set_event_loop(self._loop)
+        self._ready_event.set()
+        self._loop.run_forever()
+
+    @staticmethod
+    async def _async_caller(
+        func: "Callable[..., Coroutine[Any, Any, _R]]", args: "tuple[Any, ...]", kwargs: "dict[str, Any]"
+    ) -> _R:
+        """Wrapper to run async function.
+
+        Args:
+            func: The async function to call.
+            args: Positional arguments.
+            kwargs: Keyword arguments.
+
+        Returns:
+            Result of the async function.
+        """
+        result: _R = await func(*args, **kwargs)
+        return result
+
+    def call(self, func: "Callable[..., Coroutine[Any, Any, _R]]", *args: Any, **kwargs: Any) -> _R:
+        """Call an async function from synchronous context.
+
+        Executes the async function in the background event loop and blocks
+        until the result is available.
+
+        Args:
+            func: The async function to call.
+            *args: Positional arguments to the function.
+            **kwargs: Keyword arguments to the function.
+
+        Returns:
+            Result of the async function.
+
+        Raises:
+            ImproperConfigurationError: If portal provider not started.
+        """
+        if self._loop is None:
+            msg = "Portal provider not started. Call start() first."
+            raise ImproperConfigurationError(msg)
+
+        local_result_queue: queue.Queue[tuple[_R | None, Exception | None]] = queue.Queue()
+
+        self._request_queue.put((func, args, kwargs, local_result_queue))
+
+        self._loop.call_soon_threadsafe(self._process_request)
+
+        result, exception = local_result_queue.get()
+
+        if exception:
+            raise exception
+        return result  # type: ignore[return-value]
+
+    def _process_request(self) -> None:
+        """Process a request from the request queue in the event loop."""
+        if self._loop is None:
+            return
+
+        if not self._request_queue.empty():
+            func, args, kwargs, local_result_queue = self._request_queue.get()
+            future = asyncio.run_coroutine_threadsafe(self._async_caller(func, args, kwargs), self._loop)
+
+            future.add_done_callback(
+                functools.partial(self._handle_future_result, local_result_queue=local_result_queue)  # pyright: ignore[reportArgumentType]
+            )
+
+    @staticmethod
+    def _handle_future_result(
+        future: "asyncio.Future[Any]", local_result_queue: "queue.Queue[tuple[Any | None, Exception | None]]"
+    ) -> None:
+        """Handle result or exception from completed future.
+
+        Args:
+            future: The completed future.
+            local_result_queue: Queue to put result in.
+        """
+        try:
+            result = future.result()
+            local_result_queue.put((result, None))
+        except Exception as exc:
+            local_result_queue.put((None, exc))
+
+
+class Portal:
+    """Portal for calling async functions using PortalProvider."""
+
+    def __init__(self, provider: "PortalProvider") -> None:
+        """Initialize Portal with provider.
+
+        Args:
+            provider: The portal provider instance.
+        """
+        self._provider = provider
+
+    def call(self, func: "Callable[..., Coroutine[Any, Any, _R]]", *args: Any, **kwargs: Any) -> _R:
+        """Call an async function using the portal provider.
+
+        Args:
+            func: The async function to call.
+            *args: Positional arguments to the function.
+            **kwargs: Keyword arguments to the function.
+
+        Returns:
+            Result of the async function.
+        """
+        return self._provider.call(func, *args, **kwargs)
+
+
+class PortalManager(metaclass=SingletonMeta):
+    """Singleton manager for global portal instance.
+
+    Provides a global portal for use by sync_tools and other utilities
+    that need to call async functions from synchronous contexts without
+    an existing event loop.
+
+    Example:
+        manager = PortalManager()
+        portal = manager.get_or_create_portal()
+        result = portal.call(some_async_function, arg1, arg2)
+    """
+
+    def __init__(self) -> None:
+        """Initialize the PortalManager singleton."""
+        self._provider: PortalProvider | None = None
+        self._portal: Portal | None = None
+        self._lock = threading.Lock()
+
+    def get_or_create_portal(self) -> Portal:
+        """Get or create the global portal instance.
+
+        Lazily creates and starts the portal provider on first access.
+        Thread-safe via locking.
+
+        Returns:
+            Global portal instance.
+        """
+        if self._portal is None:
+            with self._lock:
+                if self._portal is None:
+                    self._provider = PortalProvider()
+                    self._provider.start()
+                    self._portal = Portal(self._provider)
+                    logger.debug("Global portal provider created and started")
+
+        return self._portal
+
+    @property
+    def is_running(self) -> bool:
+        """Check if global portal is running.
+
+        Returns:
+            True if portal provider exists and is running, False otherwise.
+        """
+        return self._provider is not None and self._provider.is_running
+
+    def stop(self) -> None:
+        """Stop the global portal provider.
+
+        Should typically only be called during application shutdown.
+        """
+        if self._provider is not None:
+            self._provider.stop()
+            self._provider = None
+            self._portal = None
+            logger.debug("Global portal provider stopped")
+
+
+def get_global_portal() -> Portal:
+    """Get the global portal instance for async-to-sync bridging.
+
+    Convenience function that creates and returns the singleton portal.
+    Used by sync_tools and other utilities.
+
+    Returns:
+        Global portal instance.
+    """
+    manager = PortalManager()
+    return manager.get_or_create_portal()