detquantlib 5.10.2__tar.gz → 5.10.3__tar.gz

{detquantlib-5.10.2 → detquantlib-5.10.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: detquantlib
-Version: 5.10.2
+Version: 5.10.3
 Summary: An internal library containing functions and classes that can be used across Quant models.
 License-File: LICENSE.txt
 Author: Dynamic Energy Trading

{detquantlib-5.10.2 → detquantlib-5.10.3}/detquantlib/data/databases/azure_client.py

@@ -11,6 +11,7 @@ import logging
 import os
 import re
 from dataclasses import dataclass, field
+from datetime import datetime, timezone
 from types import TracebackType
 from typing import Any, Iterator, Optional, Sequence
 
@@ -18,6 +19,7 @@ from typing import Any, Iterator, Optional, Sequence
 import pandas as pd
 import pyodbc
 from sqlalchemy import Engine, create_engine, inspect, text
+from sqlalchemy.dialects.mssql import DATETIMEOFFSET
 from sqlalchemy.exc import SQLAlchemyError
 
 logger = logging.getLogger(__name__)
@@ -107,6 +109,16 @@ class AzureClient:
 
     Query parameters use SQLAlchemy's named placeholder style, e.g. ``:status``.
 
+    Timezone-aware datetimes (in parameters and in DataFrames written with ``write_table()``) are
+    converted to UTC before being sent, because the ODBC driver would otherwise silently drop
+    their timezone. As a result, ``DATETIMEOFFSET`` columns are compared with, and store, the
+    correct point in time.
+    WARNING: ``DATETIME``/``DATETIME2`` database columns are assumed to hold UTC times. If they
+    instead hold times in a local timezone, first set timezone-aware datetimes (in parameters and
+    in DataFrames) to the same local timezone and then convert them to timezone-naive datetimes,
+    before passing them to the AzureClient. This ensures that the AzureClient will not convert
+    them to UTC.
+
     Attributes:
         config: The connection settings used by this client.
 
@@ -318,7 +330,9 @@ class AzureClient:
 
         Args:
             sql: The SQL statement to execute.
-            params: Values bound to the named placeholders in ``sql``.
+            params: Values bound to the named placeholders in ``sql``. Timezone-aware datetimes
+                (including pandas Timestamps) are converted to UTC, and ``NaT`` to ``None``; see
+                ``_prepare_params()``.
             database: Database to run the query in. If ``None``, ``config.database`` is used.
 
         Returns:
@@ -331,7 +345,7 @@ class AzureClient:
         engine = self.connect(database)
         try:
             with engine.connect() as conn:
-                result = conn.execute(text(sql), params or {})
+                result = conn.execute(text(sql), self._prepare_params(params))
                 cols = list(result.keys())
                 df = pd.DataFrame.from_records(result.fetchall(), columns=cols, coerce_float=True)
         except SQLAlchemyError as e:
@@ -364,7 +378,7 @@ class AzureClient:
         """
         engine = self.connect(database)
         with engine.connect() as conn:
-            result = conn.execute(text(sql), params or {})
+            result = conn.execute(text(sql), self._prepare_params(params))
             cols = list(result.keys())
             while rows := result.fetchmany(batch_size):
                 yield pd.DataFrame.from_records(rows, columns=cols, coerce_float=True)
@@ -457,9 +471,14 @@ class AzureClient:
         is much faster than row-by-row inserts. The whole write runs in a single transaction: if
         any part fails, the table is left unchanged.
 
+        Timezone-aware datetime columns are written as UTC times. A ``DATETIMEOFFSET`` column
+        stores them with a ``+00:00`` offset (i.e. the same point in time), and a ``DATETIME2``
+        column stores the UTC time. Tables created by this method use ``DATETIMEOFFSET`` for
+        these columns.
+
         Args:
             df: The data to write. The index is not written. Unless the table is created by this
-                call, column names must match the table's columns.
+                call, column names must match the table's columns. ``df`` itself is not modified.
             table: Table name, optionally qualified, e.g. ``"SCHEMA.TABLE"`` or
                 ``"DB.SCHEMA.TABLE"``. A database part selects the database to write to. Parts may
                 be wrapped in square brackets. If no schema is given, the user's default schema
@@ -494,6 +513,14 @@ class AzureClient:
         schema, table_name = ([None] + parts)[-2:]
         table_sql = ".".join(self._quote_identifier(p) for p in parts)
 
+        # The ODBC driver drops timezones, so send UTC times. Without an explicit type, pandas
+        # would create these columns as TIMESTAMP, which is a row version in SQL Server.
+        tz_cols = [c for c in df.columns if isinstance(df[c].dtype, pd.DatetimeTZDtype)]
+        if tz_cols:
+            df = df.copy()
+            for c in tz_cols:
+                df[c] = df[c].dt.tz_convert("UTC").dt.tz_localize(None)
+
         engine = self.connect(database)
         # pandas' to_sql always creates missing tables, so the table must be checked beforehand
         if not auto_create_table and not inspect(engine).has_table(table_name, schema=schema):
@@ -511,6 +538,7 @@ class AzureClient:
                     if_exists="replace" if overwrite and auto_create_table else "append",
                     index=False,
                     chunksize=chunk_size,
+                    dtype={c: DATETIMEOFFSET() for c in tz_cols} or None,
                 )
         except SQLAlchemyError as e:
             logger.error("Write to %s failed: %s", table, e)
@@ -547,7 +575,7 @@ class AzureClient:
         engine = self.connect(database)
         try:
             with engine.begin() as conn:
-                rowcount = conn.execute(text(sql), params or {}).rowcount
+                rowcount = conn.execute(text(sql), self._prepare_params(params)).rowcount
         except SQLAlchemyError as e:
             logger.error("Statement failed: %s", e)
             raise
@@ -556,6 +584,40 @@ class AzureClient:
         logger.info("Statement affected %s rows", rowcount)
         return rowcount
 
+    # ---------------------------------------------------------------------------------------------
+    # Parameter handling
+    # ---------------------------------------------------------------------------------------------
+    @staticmethod
+    def _prepare_params(params: Optional[dict[str, Any]]) -> dict[str, Any]:
+        """
+        Converts parameter values so the ODBC driver binds them correctly.
+
+        Timezone-aware datetimes, including pandas Timestamps, are converted to naive UTC
+        datetimes, because the driver would otherwise drop their timezone and send the local
+        clock time. SQL Server interprets naive values as UTC (``+00:00``) when comparing them
+        with ``DATETIMEOFFSET`` columns, so the correct point in time is used. ``NaT`` becomes
+        ``None`` and pandas Timestamps become Python datetimes; Python datetimes only have
+        microsecond precision, so nanoseconds are dropped (pandas warns when this happens). Other
+        values are passed through unchanged.
+
+        Args:
+            params: Values bound to the placeholders of a query, as in ``query()``.
+
+        Returns:
+            The converted values, as a new dict.
+        """
+        converted = {}
+        for key, value in (params or {}).items():
+            if value is pd.NaT:
+                value = None
+            elif isinstance(value, datetime):
+                if value.tzinfo is not None:
+                    value = value.astimezone(timezone.utc).replace(tzinfo=None)
+                if isinstance(value, pd.Timestamp):
+                    value = value.to_pydatetime()
+            converted[key] = value
+        return converted
+
     # ---------------------------------------------------------------------------------------------
     # Identifier handling
     # ---------------------------------------------------------------------------------------------

{detquantlib-5.10.2 → detquantlib-5.10.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "detquantlib"
-version = "5.10.2"
+version = "5.10.3"
 description = "An internal library containing functions and classes that can be used across Quant models."
 authors = ["Dynamic Energy Trading"]
 readme = "README.md"