detquantlib 5.3.1__tar.gz → 5.5.0__tar.gz

{detquantlib-5.3.1 → detquantlib-5.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: detquantlib
-Version: 5.3.1
+Version: 5.5.0
 Summary: An internal library containing functions and classes that can be used across Quant models.
 License-File: LICENSE.txt
 Author: Dynamic Energy Trading
@@ -13,6 +13,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: 3.15
 Requires-Dist: colorlog (>=6.10.1,<7.0.0)
+Requires-Dist: cryptography (>=50.0.2,<51.0.0)
 Requires-Dist: numpy (>=2.2.6,<3.0.0)
 Requires-Dist: pandas (>=2.3.3,<3.0.0)
 Requires-Dist: paramiko (>=4.0.0,<5.0.0)
@@ -22,6 +23,7 @@ Requires-Dist: pyodbc (>=5.2.0,<6.0.0)
 Requires-Dist: python-dotenv (>=1.1.0,<2.0.0)
 Requires-Dist: redis (>=7.4.0,<8.0.0)
 Requires-Dist: scipy (>=1.15.2,<2.0.0)
+Requires-Dist: snowflake-connector-python[pandas] (>=4.8.0,<5.0.0)
 Requires-Dist: sqlalchemy (>=2.0.43,<3.0.0)
 Description-Content-Type: text/markdown
 

detquantlib-5.5.0/detquantlib/data/__init__.py

@@ -0,0 +1,15 @@
+from .databases.detdatabase import DetDatabase
+from .databases.helpers import *
+from .databases.snowflake_client import SnowflakeClient, SnowflakeConfig
+from .entsoe.entsoe import Entsoe
+from .sftp.sftp import Sftp
+
+__all__ = [
+    "DetDatabase",
+    "Entsoe",
+    "Sftp",
+    "list_to_sql_set",
+    "list_to_sql_columns",
+    "SnowflakeClient",
+    "SnowflakeConfig",
+]

detquantlib-5.5.0/detquantlib/data/databases/snowflake_client.py

@@ -0,0 +1,515 @@
+"""
+Client for working with a Snowflake database from Python.
+
+This module provides a configuration dataclass and a client class that wrap
+``snowflake-connector-python`` with a small, safe API for interacting with Snowflake. Data is
+exchanged as pandas DataFrames.
+"""
+
+# Python built-in packages
+import logging
+import os
+import re
+from dataclasses import dataclass, field
+from types import TracebackType
+from typing import Any, Iterator, Optional, Sequence, Union
+
+# Third-party packages
+import pandas as pd
+import snowflake.connector
+from snowflake.connector import SnowflakeConnection
+from snowflake.connector.errors import Error as SnowflakeError
+from snowflake.connector.pandas_tools import write_pandas
+
+logger = logging.getLogger(__name__)
+
+_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_$]*$")
+
+
+@dataclass
+class SnowflakeConfig:
+    """
+    Connection settings for a Snowflake account.
+
+    Exactly one authentication method is used, chosen in this order of precedence:
+    - ``private_key_path`` (i.e. key-pair)
+    - ``authenticator`` (e.g. SSO)
+    - ``password``
+
+    Attributes:
+        account: Snowflake account identifier, e.g. ``"xy12345.eu-west-1"``.
+        user: Login name of the Snowflake user.
+        password: Password for password authentication.
+        warehouse: Default virtual warehouse for the session.
+        database: Default database for the session.
+        schema: Default schema for the session.
+        role: Role to assume for the session.
+        authenticator: Alternative authenticator, e.g. ``"externalbrowser"`` for browser-based SSO.
+        private_key_path: Path to a PEM-encoded private key for key-pair authentication.
+        private_key_passphrase: Passphrase for an encrypted private key.
+        extra: Additional keyword arguments passed verbatim to ``snowflake.connector.connect``.
+    """
+
+    account: Optional[str] = None
+    user: Optional[str] = None
+    password: Optional[str] = None
+    warehouse: Optional[str] = None
+    database: Optional[str] = None
+    schema: Optional[str] = None
+    role: Optional[str] = None
+    authenticator: Optional[str] = None
+    private_key_path: Optional[str] = None
+    private_key_passphrase: Optional[str] = None
+    extra: dict[str, Any] = field(default_factory=dict)
+
+    @classmethod
+    def from_env(cls, prefix: str = "SNOWFLAKE_") -> "SnowflakeConfig":
+        """
+        Builds a config from environment variables.
+
+        Reads variables named ``<prefix><FIELD>``, e.g. ``SNOWFLAKE_ACCOUNT``, ``SNOWFLAKE_USER``,
+        ``SNOWFLAKE_PASSWORD``, ``SNOWFLAKE_WAREHOUSE``, ``SNOWFLAKE_DATABASE``,
+        ``SNOWFLAKE_SCHEMA``, ``SNOWFLAKE_ROLE``, ``SNOWFLAKE_AUTHENTICATOR``,
+        ``SNOWFLAKE_PRIVATE_KEY_PATH`` and ``SNOWFLAKE_PRIVATE_KEY_PASSPHRASE``. Unset variables
+        become ``None``.
+
+        Args:
+            prefix: Prefix prepended to each variable name.
+
+        Returns:
+            A ``SnowflakeConfig`` populated from the environment.
+        """
+        g = lambda name: os.getenv(f"{prefix}{name}")
+        return cls(
+            account=g("ACCOUNT"),
+            user=g("USER"),
+            password=g("PASSWORD"),
+            warehouse=g("WAREHOUSE"),
+            database=g("DATABASE"),
+            schema=g("SCHEMA"),
+            role=g("ROLE"),
+            authenticator=g("AUTHENTICATOR"),
+            private_key_path=g("PRIVATE_KEY_PATH"),
+            private_key_passphrase=g("PRIVATE_KEY_PASSPHRASE"),
+        )
+
+
+class SnowflakeClient:
+    """
+    Client for interacting with a Snowflake database.
+
+    Wraps a single Snowflake connection and exposes methods for running SQL and exchanging data as
+    pandas DataFrames. The connection is opened lazily on first use and reused for subsequent
+    operations. The class can be used as a context manager so the connection is closed
+    automatically.
+
+    Attributes:
+        config: The connection settings used by this client.
+
+    Examples:
+        Example 1: Loading a table
+        >>> with SnowflakeClient() as client:
+        ...     customers = client.load_table("CUSTOMERS", columns=["ID", "NAME"], limit=100)
+
+        Example 2: Standard query method
+        >>> with SnowflakeClient() as client:
+        ...     df = client.query("SELECT * FROM orders WHERE status = %s", ("SHIPPED",))
+
+        Example 3: Batch query method to stream a large table in chunks
+        >>> with SnowflakeClient(SnowflakeConfig.from_env()) as client:
+        ...     for chunk in client.query_batches("SELECT * FROM BIG_TABLE"):
+        ...         process(chunk)
+
+        Example 4: Writing a DataFrame to a table, creating the table if needed
+        >>> with SnowflakeClient() as client:
+        ...     client.write_table(df, "DB.SCHEMA.ORDERS", auto_create_table=True)
+
+        Example 5: Running a statement that does not return data
+        >>> with SnowflakeClient() as client:
+        ...     client.execute("DELETE FROM ORDERS WHERE STATUS = %s", ("CANCELLED",))
+    """
+
+    def __init__(self, config: Optional[SnowflakeConfig] = None) -> None:
+        """
+        Initializes the client without opening a connection.
+
+        Args:
+            config: Connection settings. If ``None``, settings are read from environment variables
+                via ``SnowflakeConfig.from_env()``.
+        """
+        self.config = config or SnowflakeConfig.from_env()
+        self._conn: Optional[SnowflakeConnection] = None
+
+    # ---------------------------------------------------------------------------------------------
+    # Connection management
+    # ---------------------------------------------------------------------------------------------
+    def _connection_params(self) -> dict[str, Any]:
+        """
+        Builds the keyword arguments for ``snowflake.connector.connect``.
+
+        Selects a single authentication method (key-pair, authenticator or password, in that order)
+        and drops any parameters that are ``None``.
+
+        Returns:
+            A dict of connection parameters.
+
+        Raises:
+            ValueError: If ``account`` or ``user`` is missing, or if no credentials are configured.
+        """
+        c = self.config
+        if not c.account or not c.user:
+            raise ValueError("Snowflake 'account' and 'user' are required.")
+
+        params: dict[str, Any] = {
+            "account": c.account,
+            "user": c.user,
+            "warehouse": c.warehouse,
+            "database": c.database,
+            "schema": c.schema,
+            "role": c.role,
+            **c.extra,
+        }
+
+        if c.private_key_path:
+            params["private_key"] = self._load_private_key(
+                c.private_key_path, c.private_key_passphrase
+            )
+        elif c.authenticator:
+            params["authenticator"] = c.authenticator
+        elif c.password:
+            params["password"] = c.password
+        else:
+            raise ValueError(
+                "No credentials provided: set password, private_key_path, or authenticator."
+            )
+
+        return {k: v for k, v in params.items() if v is not None}
+
+    @staticmethod
+    def _load_private_key(path: str, passphrase: Optional[str]) -> bytes:
+        """
+        Loads a PEM private key and converts it to unencrypted DER bytes.
+
+        Args:
+            path: Path to the PEM-encoded private key file.
+            passphrase: Passphrase for an encrypted key, or ``None`` if the key is not encrypted.
+
+        Returns:
+            The private key as PKCS#8 DER bytes, as expected by the Snowflake connector.
+        """
+        from cryptography.hazmat.primitives import serialization
+
+        with open(path, "rb") as f:
+            key = serialization.load_pem_private_key(
+                f.read(),
+                password=passphrase.encode() if passphrase else None,
+            )
+        return key.private_bytes(
+            encoding=serialization.Encoding.DER,
+            format=serialization.PrivateFormat.PKCS8,
+            encryption_algorithm=serialization.NoEncryption(),
+        )
+
+    def connect(self) -> SnowflakeConnection:
+        """
+        Opens the connection if it is not already open.
+
+        Returns:
+            The active Snowflake connection.
+        """
+        if self._conn is None or self._conn.is_closed():
+            logger.info("Connecting to Snowflake account %s", self.config.account)
+            self._conn = snowflake.connector.connect(**self._connection_params())
+        return self._conn
+
+    def close(self) -> None:
+        """
+        Closes the connection if it is open.
+
+        Safe to call multiple times, or when no connection was ever opened.
+        """
+        if self._conn is not None and not self._conn.is_closed():
+            self._conn.close()
+            logger.info("Snowflake connection closed")
+        self._conn = None
+
+    def __enter__(self) -> "SnowflakeClient":
+        """
+        Opens the connection when entering a ``with`` block.
+
+        Returns:
+            This client instance.
+        """
+        self.connect()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[type[BaseException]],
+        exc: Optional[BaseException],
+        tb: Optional[TracebackType],
+    ) -> None:
+        """
+        Closes the connection when leaving a ``with`` block.
+
+        Exceptions raised inside the block are not suppressed.
+
+        Args:
+            exc_type: Type of the exception raised in the block, if any.
+            exc: The exception instance raised in the block, if any.
+            tb: The traceback of the exception, if any.
+        """
+        self.close()
+
+    # ---------------------------------------------------------------------------------------------
+    # Reading data
+    # ---------------------------------------------------------------------------------------------
+    def query(
+        self,
+        sql: str,
+        params: Optional[Union[Sequence[Any], dict[str, Any]]] = None,
+        lowercase_columns: bool = False,
+    ) -> pd.DataFrame:
+        """
+        Runs a SQL query and returns the full result as a DataFrame.
+
+        Always pass user-supplied values through ``params`` (``%s`` or ``%(name)s`` placeholders)
+        rather than formatting them into the SQL string, to avoid SQL injection.
+
+        Args:
+            sql: The SQL statement to execute.
+            params: Values bound to the placeholders in ``sql``: a sequence for positional ``%s``
+                placeholders, or a dict for named ``%(name)s`` placeholders.
+            lowercase_columns: If ``True``, column names in the result are converted to lowercase
+                (Snowflake returns them uppercase by default).
+
+        Returns:
+            A DataFrame containing every row returned by the query.
+
+        Raises:
+            SnowflakeError: If the query fails. The error is logged before being re-raised.
+        """
+        conn = self.connect()
+        try:
+            with conn.cursor() as cur:
+                cur.execute(sql, params)
+                df = cur.fetch_pandas_all()
+        except SnowflakeError as e:
+            logger.error("Query failed: %s", e)
+            raise
+        if lowercase_columns:
+            df.columns = [c.lower() for c in df.columns]
+        logger.info("Loaded %d rows", len(df))
+        return df
+
+    def query_batches(
+        self,
+        sql: str,
+        params: Optional[Union[Sequence[Any], dict[str, Any]]] = None,
+    ) -> Iterator[pd.DataFrame]:
+        """
+        Streams a large result set as a sequence of DataFrames.
+
+        Useful for results too large to hold in memory at once. Batch sizes are determined by the
+        Snowflake connector.
+
+        Args:
+            sql: The SQL statement to execute.
+            params: Values bound to the placeholders in ``sql``, as in ``query()``.
+
+        Yields:
+            DataFrames, each holding one batch of rows.
+        """
+        conn = self.connect()
+        with conn.cursor() as cur:
+            cur.execute(sql, params)
+            for batch in cur.fetch_pandas_batches():
+                yield batch
+
+    def load_table(
+        self,
+        table: str,
+        columns: Optional[Sequence[str]] = None,
+        where: Optional[str] = None,
+        params: Optional[Union[Sequence[Any], dict[str, Any]]] = None,
+        limit: Optional[int] = None,
+    ) -> pd.DataFrame:
+        """
+        Loads a table, optionally filtered, into a DataFrame.
+
+        Args:
+            table: Table name, optionally fully qualified, e.g. ``"DB.SCHEMA.TABLE"``. Each part is
+                validated as an identifier.
+            columns: Columns to select. If ``None``, all columns are selected.
+            where: Optional SQL condition, without the ``WHERE`` keyword. Use placeholders and pass
+                the values via ``params``; this string is inserted into the query as-is.
+            params: Values bound to the placeholders in ``where``.
+            limit: Maximum number of rows to return.
+
+        Returns:
+            A DataFrame containing the selected rows.
+        """
+        table_sql = ".".join(self._check_identifier(p) for p in table.split("."))
+        cols_sql = ", ".join(self._check_identifier(c) for c in columns) if columns else "*"
+        sql = f"SELECT {cols_sql} FROM {table_sql}"
+        if where:
+            sql += f" WHERE {where}"
+        if limit is not None:
+            sql += f" LIMIT {int(limit)}"
+        return self.query(sql, params)
+
+    def list_tables(self, schema: Optional[str] = None) -> pd.DataFrame:
+        """
+        Lists the tables visible in a schema.
+
+        Args:
+            schema: Schema to inspect, optionally qualified with a database, e.g. ``"DB.SCHEMA"``.
+                If ``None``, the session's current schema is used.
+
+        Returns:
+            A DataFrame with one row per table, using the columns returned by Snowflake's
+            ``SHOW TABLES`` command.
+        """
+        sql = "SHOW TABLES"
+        if schema:
+            sql += " IN SCHEMA " + ".".join(self._check_identifier(p) for p in schema.split("."))
+        conn = self.connect()
+        with conn.cursor() as cur:
+            cur.execute(sql)
+            cols = [d[0] for d in cur.description]
+            return pd.DataFrame(cur.fetchall(), columns=cols)
+
+    # ---------------------------------------------------------------------------------------------
+    # Writing data
+    # ---------------------------------------------------------------------------------------------
+    def write_table(
+        self,
+        df: pd.DataFrame,
+        table: str,
+        overwrite: bool = False,
+        auto_create_table: bool = False,
+        chunk_size: Optional[int] = None,
+    ) -> int:
+        """
+        Writes a DataFrame to a table.
+
+        Data is uploaded in bulk with the connector's ``write_pandas`` (staged Parquet files loaded
+        with ``COPY INTO``), which is much faster than row-by-row inserts. Table and column names
+        are treated as unquoted identifiers, so they are case-insensitive and stored in uppercase,
+        consistent with ``load_table()``.
+
+        Args:
+            df: The data to write. Column names must be valid unquoted identifiers and, unless the
+                table is created by this call, must match the table's columns.
+            table: Table name, optionally qualified, e.g. ``"SCHEMA.TABLE"`` or
+                ``"DB.SCHEMA.TABLE"``. Omitted parts default to the session's current database and
+                schema.
+            overwrite: If ``True``, the existing rows are replaced by ``df``. If ``False``, ``df``
+                is appended. Combined with ``auto_create_table``, the table itself is recreated
+                with column types inferred from ``df``.
+            auto_create_table: If ``True``, the table is created if it does not exist, with column
+                types inferred from ``df``.
+            chunk_size: Number of rows per uploaded file. If ``None``, the whole DataFrame is
+                uploaded at once.
+
+        Returns:
+            The number of rows written. If ``df`` is empty, nothing is written, the table is left
+            unchanged and ``0`` is returned.
+
+        Raises:
+            ValueError: If the table name has more than three parts, or if the table name or a
+                column name is not a valid identifier.
+            RuntimeError: If Snowflake reports that the upload did not complete.
+            SnowflakeError: If the write fails. The error is logged before being re-raised.
+        """
+        parts = [self._check_identifier(p) for p in table.split(".")]
+        if len(parts) > 3:
+            raise ValueError(f"Invalid table name: {table!r}. Expected at most DB.SCHEMA.TABLE.")
+        for col in df.columns:
+            self._check_identifier(str(col))
+
+        if df.empty:
+            logger.warning("DataFrame is empty; nothing written to %s", table)
+            return 0
+
+        # Pad the qualifier on the left, so "T" -> (None, None), "S.T" -> (None, S), etc.
+        *qualifier, table_name = parts
+        database, schema = ([None, None] + qualifier)[-2:]
+
+        conn = self.connect()
+        try:
+            success, n_chunks, n_rows, _ = write_pandas(
+                conn,
+                df,
+                table_name,
+                database=database,
+                schema=schema,
+                chunk_size=chunk_size,
+                quote_identifiers=False,
+                auto_create_table=auto_create_table,
+                overwrite=overwrite,
+                use_logical_type=True,
+            )
+        except SnowflakeError as e:
+            logger.error("Write to %s failed: %s", table, e)
+            raise
+        if not success:
+            raise RuntimeError(f"Write to {table} did not complete successfully.")
+        logger.info("Wrote %d rows to %s in %d chunk(s)", n_rows, table, n_chunks)
+        return n_rows
+
+    def execute(
+        self,
+        sql: str,
+        params: Optional[Union[Sequence[Any], dict[str, Any]]] = None,
+    ) -> Optional[int]:
+        """
+        Runs a SQL statement that does not return data.
+
+        Intended for statements such as ``INSERT``, ``UPDATE``, ``DELETE``, ``MERGE``,
+        ``CREATE TABLE`` or ``USE SCHEMA``. Always pass user-supplied values through ``params``, as
+        in ``query()``. Changes are committed automatically, as the connection runs in Snowflake's
+        default autocommit mode.
+
+        Args:
+            sql: The SQL statement to execute.
+            params: Values bound to the placeholders in ``sql``, as in ``query()``.
+
+        Returns:
+            The number of rows affected, as reported by Snowflake, or ``None`` if unavailable. For
+            statements that do not modify rows (e.g. DDL), the value is not meaningful.
+
+        Raises:
+            SnowflakeError: If the statement fails. The error is logged before being re-raised.
+        """
+        conn = self.connect()
+        try:
+            with conn.cursor() as cur:
+                cur.execute(sql, params)
+                rowcount = cur.rowcount
+        except SnowflakeError as e:
+            logger.error("Statement failed: %s", e)
+            raise
+        logger.info("Statement affected %s rows", rowcount)
+        return rowcount
+
+    @staticmethod
+    def _check_identifier(name: str) -> str:
+        """
+        Validates an unquoted Snowflake identifier.
+
+        Guards against SQL injection in table, schema and column names, which cannot be passed as
+        bind parameters.
+
+        Args:
+            name: The identifier to validate.
+
+        Returns:
+            The identifier, unchanged.
+
+        Raises:
+            ValueError: If ``name`` is not a valid unquoted identifier.
+        """
+        if not _IDENTIFIER_RE.match(name):
+            raise ValueError(f"Invalid Snowflake identifier: {name!r}")
+        return name

{detquantlib-5.3.1 → detquantlib-5.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "detquantlib"
-version = "5.3.1"
+version = "5.5.0"
 description = "An internal library containing functions and classes that can be used across Quant models."
 authors = ["Dynamic Energy Trading"]
 readme = "README.md"
@@ -21,6 +21,8 @@ pyarrow = "^21.0.0"
 scipy = "^1.15.2"
 colorlog = "^6.10.1"
 redis = "^7.4.0"
+snowflake-connector-python = {extras = ["pandas"], version = "^4.8.0"}
+cryptography = "^50.0.2"
 
 [tool.poetry.group.dev.dependencies]
 toml = "^0.10.2"

detquantlib-5.3.1/detquantlib/data/__init__.py

@@ -1,6 +0,0 @@
-from .databases.detdatabase import DetDatabase
-from .databases.helpers import *
-from .entsoe.entsoe import Entsoe
-from .sftp.sftp import Sftp
-
-__all__ = ["DetDatabase", "Entsoe", "Sftp", "list_to_sql_set", "list_to_sql_columns"]