detquantlib 5.10.1__tar.gz → 5.10.3__tar.gz

{detquantlib-5.10.1 → detquantlib-5.10.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: detquantlib
-Version: 5.10.1
+Version: 5.10.3
 Summary: An internal library containing functions and classes that can be used across Quant models.
 License-File: LICENSE.txt
 Author: Dynamic Energy Trading

{detquantlib-5.10.1 → detquantlib-5.10.3}/detquantlib/data/databases/azure_client.py

@@ -11,6 +11,7 @@ import logging
 import os
 import re
 from dataclasses import dataclass, field
+from datetime import datetime, timezone
 from types import TracebackType
 from typing import Any, Iterator, Optional, Sequence
 
@@ -18,6 +19,7 @@ from typing import Any, Iterator, Optional, Sequence
 import pandas as pd
 import pyodbc
 from sqlalchemy import Engine, create_engine, inspect, text
+from sqlalchemy.dialects.mssql import DATETIMEOFFSET
 from sqlalchemy.exc import SQLAlchemyError
 
 logger = logging.getLogger(__name__)
@@ -107,6 +109,16 @@ class AzureClient:
 
     Query parameters use SQLAlchemy's named placeholder style, e.g. ``:status``.
 
+    Timezone-aware datetimes (in parameters and in DataFrames written with ``write_table()``) are
+    converted to UTC before being sent, because the ODBC driver would otherwise silently drop
+    their timezone. As a result, ``DATETIMEOFFSET`` columns are compared with, and store, the
+    correct point in time.
+    WARNING: ``DATETIME``/``DATETIME2`` database columns are assumed to hold UTC times. If they
+    instead hold times in a local timezone, first set timezone-aware datetimes (in parameters and
+    in DataFrames) to the same local timezone and then convert them to timezone-naive datetimes,
+    before passing them to the AzureClient. This ensures that the AzureClient will not convert
+    them to UTC.
+
     Attributes:
         config: The connection settings used by this client.
 
@@ -254,7 +266,6 @@ class AzureClient:
         database = self._resolve_database(database)
         if database not in self._engines:
             conn_str = self._to_connection_string(self._connection_params(database))
-            logger.info("Connecting to Azure SQL database %s on %s", database, self.config.server)
             self._engines[database] = create_engine(
                 "mssql+pyodbc://",
                 creator=lambda: pyodbc.connect(conn_str),
@@ -268,9 +279,8 @@ class AzureClient:
 
         Safe to call multiple times, or when no connection was ever opened.
         """
-        for database, engine in self._engines.items():
+        for engine in self._engines.values():
             engine.dispose()
-            logger.info("Azure SQL connection to database %s closed", database)
         self._engines = {}
 
     def __enter__(self) -> "AzureClient":
@@ -320,7 +330,9 @@ class AzureClient:
 
         Args:
             sql: The SQL statement to execute.
-            params: Values bound to the named placeholders in ``sql``.
+            params: Values bound to the named placeholders in ``sql``. Timezone-aware datetimes
+                (including pandas Timestamps) are converted to UTC, and ``NaT`` to ``None``; see
+                ``_prepare_params()``.
             database: Database to run the query in. If ``None``, ``config.database`` is used.
 
         Returns:
@@ -333,7 +345,7 @@ class AzureClient:
         engine = self.connect(database)
         try:
             with engine.connect() as conn:
-                result = conn.execute(text(sql), params or {})
+                result = conn.execute(text(sql), self._prepare_params(params))
                 cols = list(result.keys())
                 df = pd.DataFrame.from_records(result.fetchall(), columns=cols, coerce_float=True)
         except SQLAlchemyError as e:
@@ -366,7 +378,7 @@ class AzureClient:
         """
         engine = self.connect(database)
         with engine.connect() as conn:
-            result = conn.execute(text(sql), params or {})
+            result = conn.execute(text(sql), self._prepare_params(params))
             cols = list(result.keys())
             while rows := result.fetchmany(batch_size):
                 yield pd.DataFrame.from_records(rows, columns=cols, coerce_float=True)
@@ -459,9 +471,14 @@ class AzureClient:
         is much faster than row-by-row inserts. The whole write runs in a single transaction: if
         any part fails, the table is left unchanged.
 
+        Timezone-aware datetime columns are written as UTC times. A ``DATETIMEOFFSET`` column
+        stores them with a ``+00:00`` offset (i.e. the same point in time), and a ``DATETIME2``
+        column stores the UTC time. Tables created by this method use ``DATETIMEOFFSET`` for
+        these columns.
+
         Args:
             df: The data to write. The index is not written. Unless the table is created by this
-                call, column names must match the table's columns.
+                call, column names must match the table's columns. ``df`` itself is not modified.
             table: Table name, optionally qualified, e.g. ``"SCHEMA.TABLE"`` or
                 ``"DB.SCHEMA.TABLE"``. A database part selects the database to write to. Parts may
                 be wrapped in square brackets. If no schema is given, the user's default schema
@@ -496,6 +513,14 @@ class AzureClient:
         schema, table_name = ([None] + parts)[-2:]
         table_sql = ".".join(self._quote_identifier(p) for p in parts)
 
+        # The ODBC driver drops timezones, so send UTC times. Without an explicit type, pandas
+        # would create these columns as TIMESTAMP, which is a row version in SQL Server.
+        tz_cols = [c for c in df.columns if isinstance(df[c].dtype, pd.DatetimeTZDtype)]
+        if tz_cols:
+            df = df.copy()
+            for c in tz_cols:
+                df[c] = df[c].dt.tz_convert("UTC").dt.tz_localize(None)
+
         engine = self.connect(database)
         # pandas' to_sql always creates missing tables, so the table must be checked beforehand
         if not auto_create_table and not inspect(engine).has_table(table_name, schema=schema):
@@ -513,6 +538,7 @@ class AzureClient:
                     if_exists="replace" if overwrite and auto_create_table else "append",
                     index=False,
                     chunksize=chunk_size,
+                    dtype={c: DATETIMEOFFSET() for c in tz_cols} or None,
                 )
         except SQLAlchemyError as e:
             logger.error("Write to %s failed: %s", table, e)
@@ -549,7 +575,7 @@ class AzureClient:
         engine = self.connect(database)
         try:
             with engine.begin() as conn:
-                rowcount = conn.execute(text(sql), params or {}).rowcount
+                rowcount = conn.execute(text(sql), self._prepare_params(params)).rowcount
         except SQLAlchemyError as e:
             logger.error("Statement failed: %s", e)
             raise
@@ -558,6 +584,40 @@ class AzureClient:
         logger.info("Statement affected %s rows", rowcount)
         return rowcount
 
+    # ---------------------------------------------------------------------------------------------
+    # Parameter handling
+    # ---------------------------------------------------------------------------------------------
+    @staticmethod
+    def _prepare_params(params: Optional[dict[str, Any]]) -> dict[str, Any]:
+        """
+        Converts parameter values so the ODBC driver binds them correctly.
+
+        Timezone-aware datetimes, including pandas Timestamps, are converted to naive UTC
+        datetimes, because the driver would otherwise drop their timezone and send the local
+        clock time. SQL Server interprets naive values as UTC (``+00:00``) when comparing them
+        with ``DATETIMEOFFSET`` columns, so the correct point in time is used. ``NaT`` becomes
+        ``None`` and pandas Timestamps become Python datetimes; Python datetimes only have
+        microsecond precision, so nanoseconds are dropped (pandas warns when this happens). Other
+        values are passed through unchanged.
+
+        Args:
+            params: Values bound to the placeholders of a query, as in ``query()``.
+
+        Returns:
+            The converted values, as a new dict.
+        """
+        converted = {}
+        for key, value in (params or {}).items():
+            if value is pd.NaT:
+                value = None
+            elif isinstance(value, datetime):
+                if value.tzinfo is not None:
+                    value = value.astimezone(timezone.utc).replace(tzinfo=None)
+                if isinstance(value, pd.Timestamp):
+                    value = value.to_pydatetime()
+            converted[key] = value
+        return converted
+
     # ---------------------------------------------------------------------------------------------
     # Identifier handling
     # ---------------------------------------------------------------------------------------------

{detquantlib-5.10.1 → detquantlib-5.10.3}/detquantlib/data/databases/snowflake_client.py

@@ -22,6 +22,7 @@ from snowflake.connector.errors import Error as SnowflakeError
 from snowflake.connector.pandas_tools import write_pandas
 
 logger = logging.getLogger(__name__)
+logging.getLogger("snowflake.connector").setLevel(logging.WARNING)
 
 _IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_$]*$")
 
@@ -225,7 +226,6 @@ class SnowflakeClient:
             The active Snowflake connection.
         """
         if self._conn is None or self._conn.is_closed():
-            logger.info("Connecting to Snowflake account %s", self.config.account)
             self._conn = snowflake.connector.connect(**self._connection_params())
         return self._conn
 
@@ -237,7 +237,6 @@ class SnowflakeClient:
         """
         if self._conn is not None and not self._conn.is_closed():
             self._conn.close()
-            logger.info("Snowflake connection closed")
         self._conn = None
 
     def __enter__(self) -> "SnowflakeClient":

{detquantlib-5.10.1 → detquantlib-5.10.3}/detquantlib/utils/logging.py

@@ -29,6 +29,7 @@ def setup_logger(log_dir: Path):
         fmt=(
             "%(log_color)s%(asctime)-8s%(reset)s "
             "%(log_color)s[%(levelname)s]%(reset)s "
+            "%(log_color)s[%(name)s]%(reset)s "
             "%(log_color)s%(message)s"
         ),
         datefmt="%Y-%m-%d %H:%M:%S",
@@ -40,7 +41,7 @@ def setup_logger(log_dir: Path):
 
     # File handler (plain, no colors)
     file_formatter = logging.Formatter(
-        fmt="%(asctime)s [%(levelname)s] %(message)s",
+        fmt="%(asctime)s [%(levelname)s] [%(name)s] %(message)s",
         datefmt="%Y-%m-%d %H:%M:%S",
     )
     file_handler = logging.FileHandler(log_dir, mode="a", encoding="utf-8")

{detquantlib-5.10.1 → detquantlib-5.10.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "detquantlib"
-version = "5.10.1"
+version = "5.10.3"
 description = "An internal library containing functions and classes that can be used across Quant models."
 authors = ["Dynamic Energy Trading"]
 readme = "README.md"