sqlspec 0.7.1__py3-none-any.whl → 0.9.0__py3-none-any.whl

sqlspec/adapters/duckdb/config.py

@@ -1,10 +1,11 @@
 from contextlib import contextmanager
-from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, Union, cast
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast
 
 from duckdb import DuckDBPyConnection
-from typing_extensions import NotRequired, TypedDict
+from typing_extensions import Literal, NotRequired, TypedDict
 
+from sqlspec.adapters.duckdb.driver import DuckDBDriver
 from sqlspec.base import NoPoolSyncConfig
 from sqlspec.exceptions import ImproperConfigurationError
 from sqlspec.typing import Empty, EmptyType, dataclass_to_dict
@@ -29,6 +30,8 @@ class ExtensionConfig(TypedDict):
     """The name of the extension to install"""
     config: "NotRequired[dict[str, Any]]"
     """Optional configuration settings to apply after installation"""
+    install_if_missing: "NotRequired[bool]"
+    """Whether to install if missing"""
     force_install: "NotRequired[bool]"
     """Whether to force reinstall if already present"""
     repository: "NotRequired[str]"
@@ -39,8 +42,34 @@ class ExtensionConfig(TypedDict):
     """Optional version of the extension to install"""
 
 
+class SecretConfig(TypedDict):
+    """Configuration for a secret to store in a connection.
+
+    This class provides configuration options for storing a secret in a connection for later retrieval.
+
+    For details see: https://duckdb.org/docs/stable/configuration/secrets_manager
+    """
+
+    secret_type: Union[
+        Literal[
+            "azure", "gcs", "s3", "r2", "huggingface", "http", "mysql", "postgres", "bigquery", "openai", "open_prompt"  # noqa: PYI051
+        ],
+        str,
+    ]
+    provider: NotRequired[str]
+    """The provider of the secret"""
+    name: str
+    """The name of the secret to store"""
+    value: dict[str, Any]
+    """The secret value to store"""
+    persist: NotRequired[bool]
+    """Whether to persist the secret"""
+    replace_if_exists: NotRequired[bool]
+    """Whether to replace the secret if it already exists"""
+
+
 @dataclass
-class DuckDBConfig(NoPoolSyncConfig[DuckDBPyConnection]):
+class DuckDBConfig(NoPoolSyncConfig["DuckDBPyConnection", "DuckDBDriver"]):
     """Configuration for DuckDB database connections.
 
     This class provides configuration options for DuckDB database connections, wrapping all parameters
@@ -49,7 +78,7 @@ class DuckDBConfig(NoPoolSyncConfig[DuckDBPyConnection]):
     For details see: https://duckdb.org/docs/api/python/overview#connection-options
     """
 
-    database: "Union[str, EmptyType]" = Empty
+    database: "Union[str, EmptyType]" = field(default=":memory:")
     """The path to the database file to be opened. Pass ":memory:" to open a connection to a database that resides in RAM instead of on disk. If not specified, an in-memory database will be created."""
 
     read_only: "Union[bool, EmptyType]" = Empty
@@ -63,6 +92,18 @@ class DuckDBConfig(NoPoolSyncConfig[DuckDBPyConnection]):
 
     extensions: "Union[Sequence[ExtensionConfig], ExtensionConfig, EmptyType]" = Empty
     """A sequence of extension configurations to install and configure upon connection creation."""
+    secrets: "Union[Sequence[SecretConfig], SecretConfig , EmptyType]" = Empty
+    """A dictionary of secrets to store in the connection for later retrieval."""
+    auto_update_extensions: "bool" = False
+    """Whether to automatically update on connection creation"""
+    on_connection_create: "Optional[Callable[[DuckDBPyConnection], Optional[DuckDBPyConnection]]]" = None
+    """A callable to be called after the connection is created."""
+    connection_type: "type[DuckDBPyConnection]" = field(init=False, default_factory=lambda: DuckDBPyConnection)
+    """The type of connection to create. Defaults to DuckDBPyConnection."""
+    driver_type: "type[DuckDBDriver]" = field(init=False, default_factory=lambda: DuckDBDriver)  # type: ignore[type-abstract,unused-ignore]
+    """The type of driver to use. Defaults to DuckDBDriver."""
+    pool_instance: "None" = field(init=False, default=None)
+    """The pool instance to use. Defaults to None."""
 
     def __post_init__(self) -> None:
         """Post-initialization validation and processing.
@@ -73,9 +114,10 @@ class DuckDBConfig(NoPoolSyncConfig[DuckDBPyConnection]):
         """
         if self.config is Empty:
             self.config = {}
-
         if self.extensions is Empty:
             self.extensions = []
+        if self.secrets is Empty:
+            self.secrets = []
         if isinstance(self.extensions, dict):
             self.extensions = [self.extensions]
         # this is purely for mypy
@@ -119,9 +161,104 @@ class DuckDBConfig(NoPoolSyncConfig[DuckDBPyConnection]):
 
         for extension in cast("list[ExtensionConfig]", self.extensions):
             self._configure_extension(connection, extension)
+        if self.auto_update_extensions:
+            connection.execute("update extensions")
 
     @staticmethod
-    def _configure_extension(connection: "DuckDBPyConnection", extension: ExtensionConfig) -> None:
+    def _secret_exists(connection: "DuckDBPyConnection", name: "str") -> bool:
+        """Check if a secret exists in the connection.
+
+        Args:
+            connection: The DuckDB connection to check for the secret.
+            name: The name of the secret to check for.
+
+        Returns:
+            bool: True if the secret exists, False otherwise.
+        """
+        results = connection.execute("select 1 from duckdb_secrets() where name=?", [name]).fetchone()  # pyright: ignore[reportUnknownMemberType,reportUnknownVariableType]
+        return results is not None
+
+    @classmethod
+    def _is_community_extension(cls, connection: "DuckDBPyConnection", name: "str") -> bool:
+        """Check if an extension is a community extension.
+
+        Args:
+            connection: The DuckDB connection to check for the extension.
+            name: The name of the extension to check.
+
+        Returns:
+            bool: True if the extension is a community extension, False otherwise.
+        """
+        results = connection.execute(  # pyright: ignore[reportUnknownMemberType,reportUnknownVariableType]
+            "select 1 from duckdb_extensions() where extension_name=?", [name]
+        ).fetchone()
+        return results is None
+
+    @classmethod
+    def _extension_installed(cls, connection: "DuckDBPyConnection", name: "str") -> bool:
+        """Check if a extension exists in the connection.
+
+        Args:
+            connection: The DuckDB connection to check for the secret.
+            name: The name of the secret to check for.
+
+        Returns:
+            bool: True if the extension is installed, False otherwise.
+        """
+        results = connection.execute(  # pyright: ignore[reportUnknownMemberType,reportUnknownVariableType]
+            "select 1 from duckdb_extensions() where extension_name=? and installed=true", [name]
+        ).fetchone()
+        return results is not None
+
+    @classmethod
+    def _extension_loaded(cls, connection: "DuckDBPyConnection", name: "str") -> bool:
+        """Check if a extension is loaded in the connection.
+
+        Args:
+            connection: The DuckDB connection to check for the extension.
+            name: The name of the extension to check for.
+
+        Returns:
+            bool: True if the extension is loaded, False otherwise.
+        """
+        results = connection.execute(  # pyright: ignore[reportUnknownMemberType,reportUnknownVariableType]
+            "select 1 from duckdb_extensions() where extension_name=? and loaded=true", [name]
+        ).fetchone()
+        return results is not None
+
+    @classmethod
+    def _configure_secrets(
+        cls,
+        connection: "DuckDBPyConnection",
+        secrets: "Sequence[SecretConfig]",
+    ) -> None:
+        """Configure persistent secrets for the connection.
+
+        Args:
+            connection: The DuckDB connection to configure secrets for.
+            secrets: The list of secrets to store in the connection.
+
+        Raises:
+            ImproperConfigurationError: If a secret could not be stored in the connection.
+        """
+        try:
+            for secret in secrets:
+                secret_exists = cls._secret_exists(connection, secret["name"])
+                if not secret_exists or secret.get("replace_if_exists", False):
+                    provider_type = "" if not secret.get("provider") else f"provider {secret.get('provider')},"
+                    connection.execute(
+                        f"""create or replace {"persistent" if secret.get("persist", False) else ""} secret {secret["name"]} (
+                        type {secret["secret_type"]},
+                        {provider_type}
+                        {" ,".join([f"{k} '{v}'" for k, v in secret["value"].items()])}
+                    ) """
+                    )
+        except Exception as e:
+            msg = f"Failed to store secret. Error: {e!s}"
+            raise ImproperConfigurationError(msg) from e
+
+    @classmethod
+    def _configure_extension(cls, connection: "DuckDBPyConnection", extension: "ExtensionConfig") -> None:
         """Configure a single extension for the connection.
 
         Args:
@@ -132,16 +269,32 @@ class DuckDBConfig(NoPoolSyncConfig[DuckDBPyConnection]):
             ImproperConfigurationError: If extension installation or configuration fails.
         """
         try:
-            if extension.get("force_install"):
+            # Install extension if needed
+            if (
+                not cls._extension_installed(connection, extension["name"])
+                and extension.get("install_if_missing", True)
+            ) or extension.get("force_install", False):
+                repository = extension.get("repository", None)
+                repository_url = (
+                    "https://community-extensions.duckdb.org"
+                    if repository is None
+                    and cls._is_community_extension(connection, extension["name"])
+                    and extension.get("repository_url") is None
+                    else extension.get("repository_url", None)
+                )
                 connection.install_extension(
                     extension=extension["name"],
                     force_install=extension.get("force_install", False),
-                    repository=extension.get("repository"),
-                    repository_url=extension.get("repository_url"),
+                    repository=repository,
+                    repository_url=repository_url,
                     version=extension.get("version"),
                 )
-            connection.load_extension(extension["name"])
 
+            # Load extension if not already loaded
+            if not cls._extension_loaded(connection, extension["name"]):
+                connection.load_extension(extension["name"])
+
+            # Apply any configuration settings
             if extension.get("config"):
                 for key, value in extension.get("config", {}).items():
                     connection.execute(f"SET {key}={value}")
@@ -156,7 +309,20 @@ class DuckDBConfig(NoPoolSyncConfig[DuckDBPyConnection]):
         Returns:
             A string keyed dict of config kwargs for the duckdb.connect() function.
         """
-        config = dataclass_to_dict(self, exclude_empty=True, exclude={"extensions"}, convert_nested=False)
+        config = dataclass_to_dict(
+            self,
+            exclude_empty=True,
+            exclude={
+                "extensions",
+                "pool_instance",
+                "secrets",
+                "on_connection_create",
+                "auto_update_extensions",
+                "driver_type",
+                "connection_type",
+            },
+            convert_nested=False,
+        )
         if not config.get("database"):
             config["database"] = ":memory:"
         return config
@@ -175,7 +341,11 @@ class DuckDBConfig(NoPoolSyncConfig[DuckDBPyConnection]):
         try:
             connection = duckdb.connect(**self.connection_config_dict)  # pyright: ignore[reportUnknownMemberType]
             self._configure_extensions(connection)
+            self._configure_secrets(connection, cast("list[SecretConfig]", self.secrets))
             self._configure_connection(connection)
+            if self.on_connection_create:
+                self.on_connection_create(connection)
+
         except Exception as e:
             msg = f"Could not configure the DuckDB connection. Error: {e!s}"
             raise ImproperConfigurationError(msg) from e
@@ -196,3 +366,15 @@ class DuckDBConfig(NoPoolSyncConfig[DuckDBPyConnection]):
             yield connection
         finally:
             connection.close()
+
+    @contextmanager
+    def provide_session(self, *args: Any, **kwargs: Any) -> "Generator[DuckDBDriver, None, None]":
+        """Create and provide a database connection.
+
+        Yields:
+            A DuckDB connection instance.
+
+
+        """
+        with self.provide_connection(*args, **kwargs) as connection:
+            yield self.driver_type(connection, use_cursor=True)

sqlspec/adapters/duckdb/driver.py

@@ -0,0 +1,225 @@
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Any, Optional, Union, cast
+
+from sqlspec.base import SyncArrowBulkOperationsMixin, SyncDriverAdapterProtocol, T
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
+    from duckdb import DuckDBPyConnection
+
+    from sqlspec.typing import ArrowTable, ModelDTOT, StatementParameterType
+
+__all__ = ("DuckDBDriver",)
+
+
+class DuckDBDriver(SyncArrowBulkOperationsMixin["DuckDBPyConnection"], SyncDriverAdapterProtocol["DuckDBPyConnection"]):
+    """DuckDB Sync Driver Adapter."""
+
+    connection: "DuckDBPyConnection"
+    use_cursor: bool = True
+    dialect: str = "duckdb"
+
+    def __init__(self, connection: "DuckDBPyConnection", use_cursor: bool = True) -> None:
+        self.connection = connection
+        self.use_cursor = use_cursor
+
+    # --- Helper Methods --- #
+    def _cursor(self, connection: "DuckDBPyConnection") -> "DuckDBPyConnection":
+        if self.use_cursor:
+            return connection.cursor()
+        return connection
+
+    @contextmanager
+    def _with_cursor(self, connection: "DuckDBPyConnection") -> "Generator[DuckDBPyConnection, None, None]":
+        if self.use_cursor:
+            cursor = self._cursor(connection)
+            try:
+                yield cursor
+            finally:
+                cursor.close()
+        else:
+            yield connection
+
+    # --- Public API Methods --- #
+
+    def select(
+        self,
+        sql: str,
+        parameters: Optional["StatementParameterType"] = None,
+        /,
+        *,
+        connection: Optional["DuckDBPyConnection"] = None,
+        schema_type: "Optional[type[ModelDTOT]]" = None,
+        **kwargs: Any,
+    ) -> "list[Union[ModelDTOT, dict[str, Any]]]":
+        connection = self._connection(connection)
+        sql, parameters = self._process_sql_params(sql, parameters, **kwargs)
+        with self._with_cursor(connection) as cursor:
+            cursor.execute(sql, parameters)  # pyright: ignore[reportUnknownMemberType]
+            results = cursor.fetchall()  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
+            if not results:
+                return []
+
+            column_names = [col[0] for col in cursor.description or []]  # pyright: ignore[reportUnknownMemberType,reportUnknownVariableType]
+
+            if schema_type is not None:
+                return [cast("ModelDTOT", schema_type(**dict(zip(column_names, row)))) for row in results]  # pyright: ignore[reportUnknownArgumentType]
+            return [dict(zip(column_names, row)) for row in results]  # pyright: ignore[reportUnknownArgumentType]
+
+    def select_one(
+        self,
+        sql: str,
+        parameters: Optional["StatementParameterType"] = None,
+        /,
+        *,
+        connection: Optional["DuckDBPyConnection"] = None,
+        schema_type: "Optional[type[ModelDTOT]]" = None,
+        **kwargs: Any,
+    ) -> "Union[ModelDTOT, dict[str, Any]]":
+        connection = self._connection(connection)
+        sql, parameters = self._process_sql_params(sql, parameters, **kwargs)
+        with self._with_cursor(connection) as cursor:
+            cursor.execute(sql, parameters)  # pyright: ignore[reportUnknownMemberType]
+            result = cursor.fetchone()  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
+            result = self.check_not_found(result)  # pyright: ignore
+
+            column_names = [col[0] for col in cursor.description or []]  # pyright: ignore[reportUnknownMemberType,reportUnknownVariableType]
+            if schema_type is not None:
+                return cast("ModelDTOT", schema_type(**dict(zip(column_names, result))))  # pyright: ignore[reportUnknownArgumentType]
+            # Always return dictionaries
+            return dict(zip(column_names, result))  # pyright: ignore[reportUnknownArgumentType,reportUnknownVariableType]
+
+    def select_one_or_none(
+        self,
+        sql: str,
+        parameters: Optional["StatementParameterType"] = None,
+        /,
+        *,
+        connection: Optional["DuckDBPyConnection"] = None,
+        schema_type: "Optional[type[ModelDTOT]]" = None,
+        **kwargs: Any,
+    ) -> "Optional[Union[ModelDTOT, dict[str, Any]]]":
+        connection = self._connection(connection)
+        sql, parameters = self._process_sql_params(sql, parameters, **kwargs)
+        with self._with_cursor(connection) as cursor:
+            cursor.execute(sql, parameters)  # pyright: ignore[reportUnknownMemberType]
+            result = cursor.fetchone()  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
+            if result is None:
+                return None
+
+            column_names = [col[0] for col in cursor.description or []]  # pyright: ignore[reportUnknownMemberType,reportUnknownVariableType]
+            if schema_type is not None:
+                return cast("ModelDTOT", schema_type(**dict(zip(column_names, result))))  # pyright: ignore[reportUnknownArgumentType]
+            return dict(zip(column_names, result))  # pyright: ignore[reportUnknownArgumentType,reportUnknownVariableType]
+
+    def select_value(
+        self,
+        sql: str,
+        parameters: "Optional[StatementParameterType]" = None,
+        /,
+        *,
+        connection: "Optional[DuckDBPyConnection]" = None,
+        schema_type: "Optional[type[T]]" = None,
+        **kwargs: Any,
+    ) -> "Union[T, Any]":
+        connection = self._connection(connection)
+        sql, parameters = self._process_sql_params(sql, parameters, **kwargs)
+        with self._with_cursor(connection) as cursor:
+            cursor.execute(sql, parameters)  # pyright: ignore[reportUnknownMemberType]
+            result = cursor.fetchone()  # pyright: ignore[reportUnknownMemberType,reportUnknownVariableType]
+            result = self.check_not_found(result)  # pyright: ignore
+            if schema_type is None:
+                return result[0]  # pyright: ignore
+            return schema_type(result[0])  # type: ignore[call-arg]
+
+    def select_value_or_none(
+        self,
+        sql: str,
+        parameters: "Optional[StatementParameterType]" = None,
+        /,
+        *,
+        connection: "Optional[DuckDBPyConnection]" = None,
+        schema_type: "Optional[type[T]]" = None,
+        **kwargs: Any,
+    ) -> "Optional[Union[T, Any]]":
+        connection = self._connection(connection)
+        sql, parameters = self._process_sql_params(sql, parameters, **kwargs)
+        with self._with_cursor(connection) as cursor:
+            cursor.execute(sql, parameters)  # pyright: ignore[reportUnknownMemberType]
+            result = cursor.fetchone()  # pyright: ignore[reportUnknownMemberType,reportUnknownVariableType]
+            if result is None:
+                return None
+            if schema_type is None:
+                return result[0]  # pyright: ignore
+            return schema_type(result[0])  # type: ignore[call-arg]
+
+    def insert_update_delete(
+        self,
+        sql: str,
+        parameters: Optional["StatementParameterType"] = None,
+        /,
+        *,
+        connection: Optional["DuckDBPyConnection"] = None,
+        **kwargs: Any,
+    ) -> int:
+        connection = self._connection(connection)
+        sql, parameters = self._process_sql_params(sql, parameters, **kwargs)
+        with self._with_cursor(connection) as cursor:
+            cursor.execute(sql, parameters)  # pyright: ignore[reportUnknownMemberType]
+            return getattr(cursor, "rowcount", -1)  # pyright: ignore[reportUnknownMemberType]
+
+    def insert_update_delete_returning(
+        self,
+        sql: str,
+        parameters: Optional["StatementParameterType"] = None,
+        /,
+        *,
+        connection: Optional["DuckDBPyConnection"] = None,
+        schema_type: "Optional[type[ModelDTOT]]" = None,
+        **kwargs: Any,
+    ) -> "Optional[Union[dict[str, Any], ModelDTOT]]":
+        connection = self._connection(connection)
+        sql, parameters = self._process_sql_params(sql, parameters, **kwargs)
+        with self._with_cursor(connection) as cursor:
+            cursor.execute(sql, parameters)  # pyright: ignore[reportUnknownMemberType]
+            result = cursor.fetchall()  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
+            if not result:
+                return None  # pyright: ignore[reportUnknownArgumentType]
+            column_names = [col[0] for col in cursor.description or []]  # pyright: ignore[reportUnknownMemberType,reportUnknownVariableType]
+            if schema_type is not None:
+                return cast("ModelDTOT", schema_type(**dict(zip(column_names, result[0]))))  # pyright: ignore[reportUnknownArgumentType]
+            # Always return dictionaries
+            return dict(zip(column_names, result[0]))  # pyright: ignore[reportUnknownArgumentType,reportUnknownVariableType]
+
+    def execute_script(
+        self,
+        sql: str,
+        parameters: Optional["StatementParameterType"] = None,
+        /,
+        *,
+        connection: Optional["DuckDBPyConnection"] = None,
+        **kwargs: Any,
+    ) -> str:
+        connection = self._connection(connection)
+        sql, parameters = self._process_sql_params(sql, parameters, **kwargs)
+        with self._with_cursor(connection) as cursor:
+            cursor.execute(sql, parameters)  # pyright: ignore[reportUnknownMemberType]
+            return cast("str", getattr(cursor, "statusmessage", "DONE"))  # pyright: ignore[reportUnknownMemberType]
+
+    # --- Arrow Bulk Operations ---
+
+    def select_arrow(  # pyright: ignore[reportUnknownParameterType]
+        self,
+        sql: str,
+        parameters: "Optional[StatementParameterType]" = None,
+        /,
+        *,
+        connection: "Optional[DuckDBPyConnection]" = None,
+        **kwargs: Any,
+    ) -> "ArrowTable":
+        connection = self._connection(connection)
+        sql, parameters = self._process_sql_params(sql, parameters, **kwargs)
+        with self._with_cursor(connection) as cursor:
+            cursor.execute(sql, parameters)  # pyright: ignore[reportUnknownMemberType]
+            return cast("ArrowTable", cursor.fetch_arrow_table())  # pyright: ignore[reportUnknownMemberType]

sqlspec/adapters/oracledb/__init__.py

@@ -1,13 +1,16 @@
 from sqlspec.adapters.oracledb.config import (
-    OracleAsyncDatabaseConfig,
+    OracleAsyncConfig,
     OracleAsyncPoolConfig,
-    OracleSyncDatabaseConfig,
+    OracleSyncConfig,
     OracleSyncPoolConfig,
 )
+from sqlspec.adapters.oracledb.driver import OracleAsyncDriver, OracleSyncDriver
 
 __all__ = (
-    "OracleAsyncDatabaseConfig",
+    "OracleAsyncConfig",
+    "OracleAsyncDriver",
     "OracleAsyncPoolConfig",
-    "OracleSyncDatabaseConfig",
+    "OracleSyncConfig",
+    "OracleSyncDriver",
     "OracleSyncPoolConfig",
 )

sqlspec/adapters/oracledb/config/__init__.py

@@ -1,9 +1,9 @@
-from sqlspec.adapters.oracledb.config._asyncio import OracleAsyncDatabaseConfig, OracleAsyncPoolConfig
-from sqlspec.adapters.oracledb.config._sync import OracleSyncDatabaseConfig, OracleSyncPoolConfig
+from sqlspec.adapters.oracledb.config._asyncio import OracleAsyncConfig, OracleAsyncPoolConfig
+from sqlspec.adapters.oracledb.config._sync import OracleSyncConfig, OracleSyncPoolConfig
 
 __all__ = (
-    "OracleAsyncDatabaseConfig",
+    "OracleAsyncConfig",
     "OracleAsyncPoolConfig",
-    "OracleSyncDatabaseConfig",
+    "OracleSyncConfig",
     "OracleSyncPoolConfig",
 )