detquantlib 5.10.2__tar.gz → 5.11.0__tar.gz

{detquantlib-5.10.2 → detquantlib-5.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: detquantlib
-Version: 5.10.2
+Version: 5.11.0
 Summary: An internal library containing functions and classes that can be used across Quant models.
 License-File: LICENSE.txt
 Author: Dynamic Energy Trading

{detquantlib-5.10.2 → detquantlib-5.11.0}/detquantlib/data/databases/azure_client.py

@@ -11,6 +11,7 @@ import logging
 import os
 import re
 from dataclasses import dataclass, field
+from datetime import datetime, timezone
 from types import TracebackType
 from typing import Any, Iterator, Optional, Sequence
 
@@ -18,6 +19,7 @@ from typing import Any, Iterator, Optional, Sequence
 import pandas as pd
 import pyodbc
 from sqlalchemy import Engine, create_engine, inspect, text
+from sqlalchemy.dialects.mssql import DATETIMEOFFSET
 from sqlalchemy.exc import SQLAlchemyError
 
 logger = logging.getLogger(__name__)
@@ -107,6 +109,16 @@ class AzureClient:
 
     Query parameters use SQLAlchemy's named placeholder style, e.g. ``:status``.
 
+    Timezone-aware datetimes (in parameters and in DataFrames written with ``write_table()``) are
+    converted to UTC before being sent, because the ODBC driver would otherwise silently drop
+    their timezone. As a result, ``DATETIMEOFFSET`` columns are compared with, and store, the
+    correct point in time.
+    WARNING: ``DATETIME``/``DATETIME2`` database columns are assumed to hold UTC times. If they
+    instead hold times in a local timezone, first set timezone-aware datetimes (in parameters and
+    in DataFrames) to the same local timezone and then convert them to timezone-naive datetimes,
+    before passing them to the AzureClient. This ensures that the AzureClient will not convert
+    them to UTC.
+
     Attributes:
         config: The connection settings used by this client.
 
@@ -318,7 +330,9 @@ class AzureClient:
 
         Args:
             sql: The SQL statement to execute.
-            params: Values bound to the named placeholders in ``sql``.
+            params: Values bound to the named placeholders in ``sql``. Timezone-aware datetimes
+                (including pandas Timestamps) are converted to UTC, and ``NaT`` to ``None``; see
+                ``_prepare_params()``.
             database: Database to run the query in. If ``None``, ``config.database`` is used.
 
         Returns:
@@ -331,7 +345,7 @@ class AzureClient:
         engine = self.connect(database)
         try:
             with engine.connect() as conn:
-                result = conn.execute(text(sql), params or {})
+                result = conn.execute(text(sql), self._prepare_params(params))
                 cols = list(result.keys())
                 df = pd.DataFrame.from_records(result.fetchall(), columns=cols, coerce_float=True)
         except SQLAlchemyError as e:
@@ -364,7 +378,7 @@ class AzureClient:
         """
         engine = self.connect(database)
         with engine.connect() as conn:
-            result = conn.execute(text(sql), params or {})
+            result = conn.execute(text(sql), self._prepare_params(params))
             cols = list(result.keys())
             while rows := result.fetchmany(batch_size):
                 yield pd.DataFrame.from_records(rows, columns=cols, coerce_float=True)
@@ -457,9 +471,14 @@ class AzureClient:
         is much faster than row-by-row inserts. The whole write runs in a single transaction: if
         any part fails, the table is left unchanged.
 
+        Timezone-aware datetime columns are written as UTC times. A ``DATETIMEOFFSET`` column
+        stores them with a ``+00:00`` offset (i.e. the same point in time), and a ``DATETIME2``
+        column stores the UTC time. Tables created by this method use ``DATETIMEOFFSET`` for
+        these columns.
+
         Args:
             df: The data to write. The index is not written. Unless the table is created by this
-                call, column names must match the table's columns.
+                call, column names must match the table's columns. ``df`` itself is not modified.
             table: Table name, optionally qualified, e.g. ``"SCHEMA.TABLE"`` or
                 ``"DB.SCHEMA.TABLE"``. A database part selects the database to write to. Parts may
                 be wrapped in square brackets. If no schema is given, the user's default schema
@@ -494,6 +513,14 @@ class AzureClient:
         schema, table_name = ([None] + parts)[-2:]
         table_sql = ".".join(self._quote_identifier(p) for p in parts)
 
+        # The ODBC driver drops timezones, so send UTC times. Without an explicit type, pandas
+        # would create these columns as TIMESTAMP, which is a row version in SQL Server.
+        tz_cols = [c for c in df.columns if isinstance(df[c].dtype, pd.DatetimeTZDtype)]
+        if tz_cols:
+            df = df.copy()
+            for c in tz_cols:
+                df[c] = df[c].dt.tz_convert("UTC").dt.tz_localize(None)
+
         engine = self.connect(database)
         # pandas' to_sql always creates missing tables, so the table must be checked beforehand
         if not auto_create_table and not inspect(engine).has_table(table_name, schema=schema):
@@ -511,6 +538,7 @@ class AzureClient:
                     if_exists="replace" if overwrite and auto_create_table else "append",
                     index=False,
                     chunksize=chunk_size,
+                    dtype={c: DATETIMEOFFSET() for c in tz_cols} or None,
                 )
         except SQLAlchemyError as e:
             logger.error("Write to %s failed: %s", table, e)
@@ -547,7 +575,7 @@ class AzureClient:
         engine = self.connect(database)
         try:
             with engine.begin() as conn:
-                rowcount = conn.execute(text(sql), params or {}).rowcount
+                rowcount = conn.execute(text(sql), self._prepare_params(params)).rowcount
         except SQLAlchemyError as e:
             logger.error("Statement failed: %s", e)
             raise
@@ -556,6 +584,40 @@ class AzureClient:
         logger.info("Statement affected %s rows", rowcount)
         return rowcount
 
+    # ---------------------------------------------------------------------------------------------
+    # Parameter handling
+    # ---------------------------------------------------------------------------------------------
+    @staticmethod
+    def _prepare_params(params: Optional[dict[str, Any]]) -> dict[str, Any]:
+        """
+        Converts parameter values so the ODBC driver binds them correctly.
+
+        Timezone-aware datetimes, including pandas Timestamps, are converted to naive UTC
+        datetimes, because the driver would otherwise drop their timezone and send the local
+        clock time. SQL Server interprets naive values as UTC (``+00:00``) when comparing them
+        with ``DATETIMEOFFSET`` columns, so the correct point in time is used. ``NaT`` becomes
+        ``None`` and pandas Timestamps become Python datetimes; Python datetimes only have
+        microsecond precision, so nanoseconds are dropped (pandas warns when this happens). Other
+        values are passed through unchanged.
+
+        Args:
+            params: Values bound to the placeholders of a query, as in ``query()``.
+
+        Returns:
+            The converted values, as a new dict.
+        """
+        converted = {}
+        for key, value in (params or {}).items():
+            if value is pd.NaT:
+                value = None
+            elif isinstance(value, datetime):
+                if value.tzinfo is not None:
+                    value = value.astimezone(timezone.utc).replace(tzinfo=None)
+                if isinstance(value, pd.Timestamp):
+                    value = value.to_pydatetime()
+            converted[key] = value
+        return converted
+
     # ---------------------------------------------------------------------------------------------
     # Identifier handling
     # ---------------------------------------------------------------------------------------------

{detquantlib-5.10.2 → detquantlib-5.11.0}/detquantlib/snapshot_testing/README.md

@@ -10,8 +10,10 @@ either be fixed or explicitly accepted, and is reviewed in the pull request diff
 
 Key properties:
 
-- **No database access needed in CI.** Database calls are recorded once, locally, with database
+- **No database access needed in CI.** Database reads are recorded once, locally, with database
   access, and replayed from disk in all other runs.
+- **Recording needs read access only.** The tests never execute database writes: the data the
+  model would write is saved and compared with the snapshot instead.
 - **Diffable recordings.** Recorded tables are stored as CSV + JSON files, so changes to recorded
   data or queries show up as line diffs in pull requests.
 - **Explicit failures.** A changed query, an unrecorded database call or an attempt to reach the
@@ -42,21 +44,24 @@ A snapshot test runs one test case in one of three modes:
 |---------|-------------------------------------------|--------------------|--------------------------------------------------------------|
 | replay  | `pytest tests/snapshot`                   | replayed from disk | Compares the outputs with the snapshot (what CI runs)        |
 | update  | `pytest tests/snapshot --snapshot-update` | replayed from disk | Overwrites the snapshot with the outputs                     |
-| record  | `pytest tests/snapshot --snapshot-record` | live (local only)  | Re-records the database calls, and overwrites the snapshot   |
+| record  | `pytest tests/snapshot --snapshot-record` | live reads (local) | Re-records the database reads, and overwrites the snapshot   |
 
 A run goes through the following steps (see `run_snapshot_case()` in `pytest_plugin.py`):
 
 1. The outputs folder `tests/snapshot/.actual/CaseXX/Outputs` is emptied. It is kept after the run
    for inspection, and uploaded as an artifact by CI when the test fails.
 2. The database functions are patched:
-   - record mode: they run normally, and their results are saved to the case's `db_cache` folder;
-   - replay and update modes: they never run; their recorded results are returned instead.
+   - reads, in record mode: they run normally, and their results are saved to the case's
+     `db_cache` folder;
+   - reads, in replay and update modes: they never run; their recorded results are returned
+     instead;
+   - writes, in every mode: they never run; each call is saved in the outputs folder
+     (`_db_writes`), and a simulated result is returned (see [Database Writes](#database-writes)).
 3. In replay and update modes, the network is blocked and the local `.env` file is ignored, so
    that local runs behave exactly like CI (which has no credentials).
 4. The model runs through its real entry point, with the time frozen if the test case says so,
-   and with its logging isolated from the rest of the pytest process. The test only prints the
-   name and description of the test case: everything the model prints or logs to the console is
-   written to `tests/snapshot/.actual/CaseXX/console.txt` instead.
+   and isolated from the rest of the pytest process (see [Isolation](#isolation)). The test only
+   prints the name and description of the test case.
 5. The outputs are compared with the snapshot (replay mode), or replace it (update and record
    modes).
 
@@ -69,8 +74,7 @@ name and of its meaningful arguments.
 - SQL text is whitespace-normalized: reformatting a query does not invalidate the recording.
   SQLAlchemy `text()` objects are keyed on their SQL.
 - Parameters, dates and other plain values are part of the key.
-- DataFrames (e.g. the data passed to `write_table()`) are keyed on their shape, column names,
-  data types and content.
+- DataFrame arguments are keyed on their shape, column names, data types and content.
 - Clients, connections and engines are replaced by a constant placeholder, so that a real client
   (record mode) and a client without credentials (replay mode) produce the same key.
 
@@ -86,6 +90,35 @@ Every module-level alias of a patched function is patched too (e.g. created by
 `from pkg.db import query`), so a model cannot bypass the recording by importing a function by
 name.
 
+### Database Writes
+
+Functions that write to a database (e.g. `AzureClient.write_table`) are _write targets_. They
+never run, so the tests never write to a database, and recording only needs read access. Each call
+is saved in the outputs folder instead:
+
+```
+ExpectedOutputs/
+├── ...                                   the model's own outputs
+└── _db_writes/
+    ├── calls.json                        function, arguments and simulated result of each call
+    ├── 001_AzureClient_write_table.csv   the DataFrame passed to the 1st call (rows written)
+    └── ...
+```
+
+- Files are numbered in call order, across all clients (e.g. `001_AzureClient_write_table.csv`,
+  `002_SnowflakeClient_write_table.csv`). A call with several DataFrame arguments gets one file per
+  DataFrame (`001_<function>.csv`, `001_<function>_2.csv`, ...).
+- `calls.json` lists every call: the function, its arguments (SQL text, table name, options, and
+  the CSV file names in place of DataFrames) and the value it returned.
+- DataFrames and parameters are saved as the model passes them, before any conversion by the
+  client (e.g. `AzureClient` sends timezone-aware datetimes as UTC). DataFrames are saved without
+  their index, as the clients write them, and timezone-aware values with their UTC offset.
+- These files are compared with the snapshot like any other output, so a change in the data the
+  model writes shows up as a readable difference in the test failure and in the pull request.
+- The replacement returns a _simulated_ result, computed from the call arguments: the number of
+  rows of the DataFrame for `write_table()`, and `None` for `execute()` (there is no database to
+  count the affected rows).
+
 ### Network Guard
 
 In replay and update modes, any connection to a non-local address fails immediately with a
@@ -116,9 +149,9 @@ created by that run, which are also detached afterwards. The exception hooks rep
 `detquantlib.utils.setup_logger` are restored as well. The working directory and the
 `INPUT_DIR` / `OUTPUT_DIR` environment variables are restored after each test.
 
-The model's console output (stdout, stderr, and console log handlers created by the model) is
-muted and written to `tests/snapshot/.actual/CaseXX/console.txt`, also when the model crashes.
-The run log written to the outputs folder is not affected.
+Everything the model prints or logs to the console (stdout, stderr, and console log handlers
+created by the model) is written to `tests/snapshot/.actual/CaseXX/console.txt` instead, also when
+the model crashes. The run log written to the outputs folder is not affected.
 
 ### Output Comparison
 
@@ -131,9 +164,9 @@ Each file of the outputs folder is compared with its snapshot according to its t
 | Text (`.txt`, `.sql`, ...)  | Line by line (line endings ignored), with a unified diff            |
 | Anything else               | Byte by byte                                                        |
 
-Missing and unexpected files are reported too. Run logs (`*.log`, timestamps) and HTML files
-(`*.html`, e.g. Plotly figures with random element identifiers and ~5 MB of embedded plotly.js) are
-not compared by default.
+This includes the captured database writes (`_db_writes`). Missing and unexpected files are
+reported too. Run logs (`*.log`, timestamps) and HTML files (`*.html`, e.g. Plotly figures with
+random element identifiers and ~5 MB of embedded plotly.js) are not compared by default.
 
 ## Supported Database Clients
 
@@ -145,8 +178,8 @@ The default configuration covers the database clients of `detquantlib.data`:
 | `load_table()`                  | through `query()`               | through `query()`               |
 | `list_tables()`                 | recorded                        | through `query()`               |
 | `query_batches()`               | recorded (batches)              | recorded (batches)              |
-| `write_table()`                 | recorded (row count)            | recorded (row count)            |
-| `execute()`                     | recorded (row count)            | recorded (row count)            |
+| `write_table()`                 | never run; returns rows written | never run; returns rows written |
+| `execute()`                     | never run; returns `None`       | never run; returns `None`       |
 | `connect()`                     | offline stub in replay mode     | offline stub in replay mode     |
 
 Notes:
@@ -155,10 +188,8 @@ Notes:
   results would be recorded twice.
 - `query_batches()` returns a generator. In record mode, it is consumed entirely during the call,
   and each batch is stored separately. In replay mode, the batches are returned by a generator.
-- Writes (`write_table()`, `execute()`) are _executed_ against the live databases in record mode,
-  as in a normal model run. In replay mode, they return the recorded row count. Since the written
-  data is part of the cache key, a change in the written data fails the test with a
-  `DbCacheMissError`.
+- Writes (`write_table()`, `execute()`) are saved as outputs (see
+  [Database Writes](#database-writes)).
 
 ## Setting Up a Model Repository
 
@@ -184,11 +215,8 @@ The framework expects the standard layout of Quant model repositories: `run.py`,
                 └── ExpectedOutputs/     created by --snapshot-record / --snapshot-update
 ```
 
-pytest requires two files: command line options can only be registered from a `conftest.py` file
-(or a plugin loaded at startup), and tests are only collected from `test_*.py` files.
-
 `tests/snapshot/conftest.py` activates the command line options and the `snapshot_mode` fixture
-for the snapshot tests only:
+for the snapshot tests only (pytest reads command line options from `conftest.py` files):
 
 ```python
 """pytest configuration of the snapshot tests (detquantlib.snapshot_testing)."""
@@ -226,12 +254,9 @@ def test_snapshot(case, snapshot_mode, monkeypatch):
     run_snapshot_case(case, snapshot_mode, monkeypatch, CONFIG, entry_point=model_entry_point)
 ```
 
-Code outside the tests that needs the test cases (e.g. Invoke tasks listing or creating them)
-should build the folder from the snapshot tests folder, e.g.
-`SnapshotConfig(snapshot_dir=...).cases_dir`, rather than import `test_snapshots.py`. Importing
-it would also import the model, and imports through the `tests` folder (e.g.
-`from tests.snapshot... import`) break when another package named `tests` is on the Python path
-(e.g. the one of an editable install of detquantlib).
+Code outside the tests that needs the test cases (e.g. Invoke tasks listing or creating them) can
+get their folder with `SnapshotConfig(snapshot_dir=...).cases_dir`, without importing
+`test_snapshots.py` (which imports the model).
 
 ### Dependencies and pytest Settings
 
@@ -277,7 +302,7 @@ jobs:
 
       - name: Upload outputs of snapshot tests on failure
         if: ${{ !cancelled() && steps.snapshot_tests.outcome == 'failure' }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: snapshot-actual-outputs
           path: tests/snapshot/.actual/
@@ -343,6 +368,7 @@ ignore = ["plots/*.png"]
 ### Creating a Test Case
 
 Step 1: Execute `create_case(inputs_dir, cases_root)`. The function handles the following actions:
+
 - Creates the folder of a new test case, with the lowest free case number.
 - Copies the given inputs folder (typically the project's default `Inputs` folder).
 - Writes a `case.toml` file to complete, which freezes the time at the moment of creation.
@@ -382,7 +408,7 @@ db_cache/
 ├── manifest.json                       cache key -> function, call arguments, file
 ├── 001_SnowflakeClient_query.csv       values of the 1st distinct table returned
 ├── 001_SnowflakeClient_query.dtypes.json   its column data types
-├── 002_AzureClient_execute.pkl         a result that is not a table (e.g. a row count)
+├── 002_<function>.pkl                  a result that is not a table (e.g. a number)
 ├── 003_AzureClient_query_batches_001.csv   the batches returned by query_batches()
 └── ...
 ```
@@ -410,8 +436,8 @@ data types). The round trip is exact, and verified when recording:
 
 Tables that cannot be stored exactly (categorical columns, columns mixing value types, columns
 with both empty strings and NULLs) are stored as pickle files instead, with a warning: replay
-works, but the recording is not diffable. Results that are not tables (e.g. row counts) are
-always stored as pickle files.
+works, but the recording is not diffable. Results that are not tables (e.g. numbers) are always
+stored as pickle files.
 
 Do not store recordings with Git LFS: GitHub does not show diffs of LFS files. Keep recordings
 small instead, by querying only the data that a test case needs.
@@ -424,14 +450,21 @@ small instead, by querying only the data that a test case needs.
 |-------------------------|---------------------------------|-----------------------------------------------|
 | `snapshot_dir`          | (required)                      | Folder of the snapshot tests                  |
 | `project_root`          | grandparent of `snapshot_dir`   | Folder in which the model runs                |
-| `db_query_targets`      | `DEFAULT_DB_QUERY_TARGETS`      | Functions recorded and replayed               |
+| `db_query_targets`      | `DEFAULT_DB_QUERY_TARGETS`      | Read functions, recorded and replayed         |
 | `db_connection_targets` | `DEFAULT_DB_CONNECTION_TARGETS` | Connection factories (stubbed in replay mode) |
+| `db_write_targets`      | `DEFAULT_DB_WRITE_TARGETS`      | Write functions, never run (saved as outputs) |
 | `ignore_patterns`       | `DEFAULT_IGNORE_PATTERNS`       | Output files not compared (glob patterns)     |
 | `rtol`                  | `1e-8`                          | Relative tolerance of numerical comparisons   |
 | `atol`                  | `1e-10`                         | Absolute tolerance of numerical comparisons   |
 
 `cases_dir` (`snapshot_dir / "cases"`) and `actual_dir` (`snapshot_dir / ".actual"`) are derived
-from `snapshot_dir`.
+from `snapshot_dir`. A function can be listed in only one kind of target: `SnapshotConfig` raises
+a `ValueError` otherwise.
+
+`db_write_targets` maps each write function to the function computing its simulated return value
+from the call arguments, or to `None` (the replacement then returns `None`). `rows_written`
+(from `detquantlib.snapshot_testing.recording`) returns the number of rows of the first
+DataFrame argument.
 
 ### Adding Database Functions
 
@@ -442,13 +475,16 @@ functions), add those functions to the targets of `CONFIG`, in `tests/snapshot/t
 from detquantlib.snapshot_testing import (
     DEFAULT_DB_CONNECTION_TARGETS,
     DEFAULT_DB_QUERY_TARGETS,
+    DEFAULT_DB_WRITE_TARGETS,
     SnapshotConfig,
 )
+from detquantlib.snapshot_testing.recording import rows_written
 
 CONFIG = SnapshotConfig(
     snapshot_dir=Path(__file__).resolve().parent,
     db_query_targets=[*DEFAULT_DB_QUERY_TARGETS, "src.data_access:load_prices"],
     db_connection_targets=[*DEFAULT_DB_CONNECTION_TARGETS, "src.data_access:connect"],
+    db_write_targets={**DEFAULT_DB_WRITE_TARGETS, "src.data_access:save_results": rows_written},
 )
 ```
 
@@ -456,14 +492,17 @@ CONFIG = SnapshotConfig(
 - Prefer the highest-level function that both connects and returns data.
 - Do not list a function that only calls another listed function: its results would be recorded
   twice, and the inner recordings would be reported as unused in replay mode.
-- List functions that only create connections as connection targets.
+- List functions that only create connections as connection targets, and functions that write to
+  a database as write targets.
 
 ## Troubleshooting
 
 | Error or warning                                    | Cause and solution                                                                                                                                |
 |-----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------|
 | `DbCacheMissError: No DB cache found`               | The test case was never recorded. Record it locally with `--snapshot-record`.                                                                    |
-| `DbCacheMissError: No recorded result for ...`      | A query, its parameters or the written data changed, or a new database call was added. Re-record the test case. If the call depends on the current date, freeze the time in `case.toml`. |
+| `DbCacheMissError: No recorded result for ...`      | A query or its parameters changed, or a new database read was added. Re-record the test case. If the call depends on the current date, freeze the time in `case.toml`. |
+| `[changed] _db_writes/...` in a snapshot mismatch   | The data the model writes to a database changed. Fix the model, or accept the change with `--snapshot-update`.                                    |
+| `... listed in ... and in another kind of target`   | A function is listed in two kinds of targets. List each function once: reads in `db_query_targets`, connection factories in `db_connection_targets`, writes in `db_write_targets`. |
 | `NetworkAccessBlockedError`                         | The model reaches the network or a database through a function that is not recorded. Add the function to the targets.                           |
 | `Snapshot mismatch for case ...`                    | The outputs changed. Fix the model, or accept the change with `--snapshot-update`.                                                               |
 | `No snapshot for case ...`                          | The test case has a DB cache but no snapshot. Create it with `--snapshot-update`.                                                                |
@@ -479,8 +518,11 @@ CONFIG = SnapshotConfig(
   which differs between record mode (credentials) and replay mode (none). Identical SQL sent to
   different default databases in one run shares one recording: qualify table names or pass
   `database=...` in that case.
-- **Writes in record mode.** Recording runs the model against the live databases, including its
-  writes. Use credentials of a development environment.
+- **Writes are simulated.** Since writes never run, a model that reads back data it wrote earlier
+  in the same run does not find it in the recording. Logic that depends on the number of rows
+  affected by `execute()` receives `None`. Checks and conversions made by the clients before
+  writing (e.g. the validation of table names) do not run either.
+- **Reserved folder name.** `_db_writes` is reserved inside the outputs folder.
 - **Multiprocessing.** Patches do not reach child processes: database calls made in child
   processes are neither recorded nor blocked.
 - **Module-level caches.** Query results cached at module level (e.g. with `functools.cache`)
@@ -500,7 +542,7 @@ CONFIG = SnapshotConfig(
 |--------------------|------------------------------------------------------------------------------|
 | `config.py`        | `SnapshotConfig` and the default targets, ignore patterns and tolerances     |
 | `cases.py`         | `SnapshotCase`, `discover_cases()`, `print_cases()`, `create_case()`         |
-| `recording.py`     | Cache keys, `DbRecorder` (record/replay), patching of the targets            |
+| `recording.py`     | Cache keys, `DbRecorder` (reads), `DbWriteCapture` (writes), patching        |
 | `tables.py`        | DataFrame <-> CSV + dtypes JSON, with an exact round trip                    |
 | `isolation.py`     | Network guard, frozen time, logging isolation                                |
 | `comparison.py`    | Comparison of the outputs with the snapshot                                  |

{detquantlib-5.10.2 → detquantlib-5.11.0}/detquantlib/snapshot_testing/__init__.py

@@ -5,6 +5,7 @@ from .cases import SnapshotCase, create_case, discover_cases, print_cases
 from .config import (
     DEFAULT_DB_CONNECTION_TARGETS,
     DEFAULT_DB_QUERY_TARGETS,
+    DEFAULT_DB_WRITE_TARGETS,
     DEFAULT_IGNORE_PATTERNS,
     SnapshotConfig,
 )
@@ -13,6 +14,7 @@ from .errors import DbCacheMissError, NetworkAccessBlockedError
 __all__ = [
     "DEFAULT_DB_CONNECTION_TARGETS",
     "DEFAULT_DB_QUERY_TARGETS",
+    "DEFAULT_DB_WRITE_TARGETS",
     "DEFAULT_IGNORE_PATTERNS",
     "DbCacheMissError",
     "NetworkAccessBlockedError",

{detquantlib-5.10.2 → detquantlib-5.11.0}/detquantlib/snapshot_testing/config.py

@@ -5,7 +5,10 @@ Configuration of the snapshot tests of a model repository.
 # Python built-in packages
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Optional
+from typing import Callable, Optional
+
+# Internal modules
+from detquantlib.snapshot_testing.recording import rows_written
 
 _SNOWFLAKE_CLIENT = "detquantlib.data.databases.snowflake_client:SnowflakeClient"
 _AZURE_CLIENT = "detquantlib.data.databases.azure_client:AzureClient"
@@ -19,14 +22,21 @@ DEFAULT_DB_QUERY_TARGETS: list[str] = [
     f"{_SNOWFLAKE_CLIENT}.query",
     f"{_SNOWFLAKE_CLIENT}.query_batches",
     f"{_SNOWFLAKE_CLIENT}.list_tables",
-    f"{_SNOWFLAKE_CLIENT}.write_table",
-    f"{_SNOWFLAKE_CLIENT}.execute",
     f"{_AZURE_CLIENT}.query",
     f"{_AZURE_CLIENT}.query_batches",
-    f"{_AZURE_CLIENT}.write_table",
-    f"{_AZURE_CLIENT}.execute",
 ]
 
+# Functions that write to a database, with the function computing their simulated return value
+# from the call arguments (None: the replacement returns None). They never run, in any mode, so
+# recording only needs read access. Each call is saved in the outputs folder instead
+# (_db_writes/), and compared with the snapshot like any other output.
+DEFAULT_DB_WRITE_TARGETS: dict[str, Optional[Callable]] = {
+    f"{_SNOWFLAKE_CLIENT}.write_table": rows_written,
+    f"{_SNOWFLAKE_CLIENT}.execute": None,  # no database to count the affected rows
+    f"{_AZURE_CLIENT}.write_table": rows_written,
+    f"{_AZURE_CLIENT}.execute": None,
+}
+
 # Functions that only create a connection/engine/client. In replay mode they return an offline
 # stub instead of connecting (they raise without credentials). In record mode they run normally.
 DEFAULT_DB_CONNECTION_TARGETS: list[str] = [
@@ -57,6 +67,9 @@ class SnapshotConfig:
             ``snapshot_dir`` (i.e. the repository root).
         db_query_targets: Functions whose results are recorded and replayed.
         db_connection_targets: Functions that only create connections (stubbed in replay mode).
+        db_write_targets: Functions that write to a database, never executed: their calls are
+            saved as outputs instead. Maps each function to the function computing its
+            simulated return value, or to None.
         ignore_patterns: Output files that are not compared.
         rtol: Default relative tolerance of numerical comparisons.
         atol: Default absolute tolerance of numerical comparisons.
@@ -68,17 +81,40 @@ class SnapshotConfig:
     db_connection_targets: list[str] = field(
         default_factory=lambda: list(DEFAULT_DB_CONNECTION_TARGETS)
     )
+    db_write_targets: dict[str, Optional[Callable]] = field(
+        default_factory=lambda: dict(DEFAULT_DB_WRITE_TARGETS)
+    )
     ignore_patterns: list[str] = field(default_factory=lambda: list(DEFAULT_IGNORE_PATTERNS))
     rtol: float = DEFAULT_RTOL
     atol: float = DEFAULT_ATOL
 
     def __post_init__(self):
-        """Converts the folders to absolute paths and sets the default project root."""
+        """
+        Converts the folders to absolute paths, sets the default project root, and checks that
+        no function is listed in more than one kind of target.
+
+        Raises:
+            ValueError: Raises an error if a function is listed in more than one kind of target.
+        """
         self.snapshot_dir = Path(self.snapshot_dir).resolve()
         if self.project_root is None:
             self.project_root = self.snapshot_dir.parents[1]
         self.project_root = Path(self.project_root).resolve()
 
+        kinds = {
+            "db_query_targets": set(self.db_query_targets),
+            "db_connection_targets": set(self.db_connection_targets),
+            "db_write_targets": set(self.db_write_targets),
+        }
+        for kind, targets in kinds.items():
+            others = set().union(*(t for k, t in kinds.items() if k != kind))
+            if targets & others:
+                raise ValueError(
+                    f"Function(s) {sorted(targets & others)} listed in {kind} and in another "
+                    "kind of target. Writes (e.g. write_table, execute) belong in "
+                    "db_write_targets only."
+                )
+
     @property
     def cases_dir(self) -> Path:
         """Folder containing the test cases (Case01, Case02, ...)."""

{detquantlib-5.10.2 → detquantlib-5.11.0}/detquantlib/snapshot_testing/pytest_plugin.py

@@ -37,7 +37,13 @@ from detquantlib.snapshot_testing.isolation import (
     isolated_logging,
     muted_console,
 )
-from detquantlib.snapshot_testing.recording import DbRecorder, install_db_patches
+from detquantlib.snapshot_testing.recording import (
+    WRITES_DIRNAME,
+    DbRecorder,
+    DbWriteCapture,
+    install_db_patches,
+    install_write_capture,
+)
 
 MODES = ("replay", "update", "record")
 OUTPUTS_DIRNAME = "Outputs"
@@ -123,7 +129,9 @@ def run_snapshot_case(
         mode="update": as "replay", but the snapshot is overwritten with the produced outputs.
         mode="record": the live databases are queried; the DB cache and the snapshot are
             overwritten.
-    All patches are undone by `monkeypatch` after the test.
+    In every mode, database writes are never executed: they are saved in the outputs folder
+    (_db_writes/) and compared with the snapshot. All patches are undone by `monkeypatch` after
+    the test.
 
     Args:
         case: Test case
@@ -156,6 +164,9 @@ def run_snapshot_case(
     install_db_patches(
         monkeypatch, recorder, config.db_query_targets, config.db_connection_targets
     )
+    # Database writes never run, in any mode: they are saved in the outputs folder instead
+    capture = DbWriteCapture(output_dir / WRITES_DIRNAME)
+    install_write_capture(monkeypatch, capture, config.db_write_targets)
     if not record:
         block_network(monkeypatch)
         # Ignore a developer's local .env so local runs behave exactly like CI

{detquantlib-5.10.2 → detquantlib-5.11.0}/detquantlib/snapshot_testing/recording.py

@@ -1,6 +1,7 @@
 """
-Database record/replay: the results of database calls are recorded once (locally, with database
-access) as CSV + JSON files, and replayed from disk afterwards (locally and in CI).
+Database record/replay: the results of database reads are recorded once (locally, with database
+access) as CSV + JSON files, and replayed from disk afterwards (locally and in CI). Database
+writes are never executed: they are captured as outputs instead.
 """
 
 from __future__ import annotations
@@ -33,6 +34,8 @@ from detquantlib.snapshot_testing.errors import DbCacheMissError, NetworkAccessB
 from detquantlib.snapshot_testing.tables import UnsupportedTableError, read_table, write_table
 
 MANIFEST_FILENAME = "manifest.json"
+WRITES_DIRNAME = "_db_writes"
+CALLS_FILENAME = "calls.json"
 RECORD_HINT = "pytest tests/snapshot --snapshot-record -k {case}"
 
 
@@ -134,10 +137,11 @@ class DbRecorder:
         manifest.json                   cache key -> function, call arguments (e.g. SQL), file
         001_<function>.csv              values of the 1st distinct table returned
         001_<function>.dtypes.json      its column data types
-        002_<function>.pkl              a result that is not a table (e.g. a row count)
+        002_<function>.pkl              a result that is not a table (e.g. a number)
         003_<function>_001.csv ...      the batches returned by a generator (e.g. query_batches)
-    Files are numbered in call order, so re-recording a changed query overwrites the same file
-    names and the change shows up as a normal line-by-line diff in the pull request.
+    Files are numbered in call order, and the manifest lists its entries in the same order, so
+    re-recording a changed query overwrites the same file names and the same manifest position,
+    and the change shows up as a normal line-by-line diff in the pull request.
 
     In record mode, live database calls run on the real clock (`real_time`, set by the caller),
     even when the case freezes time: database drivers use the clock, e.g. for authentication.
@@ -285,7 +289,7 @@ class DbRecorder:
     def save(self):
         """Atomically replaces the case's DB cache with the newly recorded one."""
         self._staging.mkdir(parents=True, exist_ok=True)
-        manifest = json.dumps(self._manifest, indent=2, sort_keys=True, default=str) + "\n"
+        manifest = json.dumps(self._manifest, indent=2, default=str) + "\n"
         (self._staging / MANIFEST_FILENAME).write_text(manifest, encoding="utf-8")
         shutil.rmtree(self.cache_dir, ignore_errors=True)
         self._staging.rename(self.cache_dir)
@@ -312,8 +316,8 @@ class DbRecorder:
             call = json.dumps(_normalize({"args": list(args), "kwargs": kwargs}), indent=2)
             raise DbCacheMissError(
                 f"No recorded result for '{target}' called with:\n{textwrap.shorten(call, 2000)}\n"
-                "The query text/parameters or the data written changed, or a new database call "
-                f"was added. Re-record locally with: {self._record_hint}"
+                "The query text or parameters changed, or a new database call was added. "
+                f"Re-record locally with: {self._record_hint}"
             )
         if entry["format"] == "batches":
             return (self._read_value(batch) for batch in entry["batches"])
@@ -344,6 +348,105 @@ class DbRecorder:
         return set(self._manifest) - self.used_keys
 
 
+# -------------------------------------------------------------------------------------------------
+# Database writes
+# -------------------------------------------------------------------------------------------------
+def rows_written(*args, **kwargs) -> int:
+    """
+    Simulated return value of a write_table() call: the number of rows of the DataFrame written.
+
+    Args:
+        *args: Positional arguments of the call
+        **kwargs: Keyword arguments of the call
+
+    Returns:
+        Number of rows of the first DataFrame argument (0 if there is none)
+    """
+    frames = [value for value in (*args, *kwargs.values()) if isinstance(value, pd.DataFrame)]
+    return len(frames[0]) if frames else 0
+
+
+class DbWriteCapture:
+    """
+    Replaces database write functions, in every mode: the real function never runs, so recording
+    needs read access only, and nothing is ever written to a database by the tests. Each call is
+    saved in the outputs folder instead, and is compared with the snapshot like any other output:
+        _db_writes/calls.json               function, arguments and simulated result of each call
+        _db_writes/001_<function>.csv       the DataFrame passed to the 1st call (rows written)
+        _db_writes/001_<function>_2.csv     a 2nd DataFrame argument of the same call, if any
+    Files are numbered in call order. DataFrames are saved without their index, as the database
+    clients write them.
+    """
+
+    def __init__(self, writes_dir: Path):
+        """
+        Constructor method.
+
+        Args:
+            writes_dir: Folder receiving the captured calls (inside the outputs folder)
+        """
+        self.writes_dir = writes_dir
+        self.calls: list[dict] = []
+
+    def wrap(self, target: str, func: Callable, simulate: Callable | None) -> Callable:
+        """
+        Builds the replacement of a database write function.
+
+        Args:
+            target: Name of the function, e.g. "pkg.module:Class.method"
+            func: Original function (never called)
+            simulate: Function computing the simulated return value from the call arguments, or
+                None to return None
+
+        Returns:
+            Replacement function
+        """
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            result = simulate(*args, **kwargs) if simulate is not None else None
+            self._save_call(target, args, kwargs, result)
+            return result
+
+        return wrapper
+
+    def _save_call(self, target: str, args: tuple, kwargs: dict, result: Any):
+        """
+        Saves a captured call: its DataFrame arguments as CSV files, and its description in
+        calls.json (rewritten after every call, so it is complete even if the model crashes).
+
+        Args:
+            target: Name of the function called
+            args: Positional arguments of the call
+            kwargs: Keyword arguments of the call
+            result: Simulated return value
+        """
+        self.writes_dir.mkdir(parents=True, exist_ok=True)
+        function_name = re.sub(r"[^\w-]", "_", target.split(":")[-1])  # valid on every OS
+        stem = f"{len(self.calls) + 1:03d}_{function_name}"
+        files: list[str] = []
+
+        def describe(value: Any) -> Any:
+            if not isinstance(value, (pd.DataFrame, pd.Series)):
+                return _normalize(value)
+            frame = value.to_frame() if isinstance(value, pd.Series) else value
+            name = f"{stem}.csv" if not files else f"{stem}_{len(files) + 1}.csv"
+            frame.to_csv(self.writes_dir / name, index=False, lineterminator="\n")
+            files.append(name)
+            return {"file": name}
+
+        self.calls.append(
+            {
+                "target": target,
+                "args": [describe(value) for value in args],
+                "kwargs": {name: describe(value) for name, value in sorted(kwargs.items())},
+                "result": _normalize(result),
+            }
+        )
+        calls = json.dumps(self.calls, indent=2, default=str) + "\n"
+        (self.writes_dir / CALLS_FILENAME).write_text(calls, encoding="utf-8")
+
+
 # -------------------------------------------------------------------------------------------------
 # Patching
 # -------------------------------------------------------------------------------------------------
@@ -490,3 +593,22 @@ def install_db_patches(
             replace_everywhere(monkeypatch, owner, attr, lambda f, t=target: _offline_stub(t))
         else:
             replace_everywhere(monkeypatch, owner, attr, recorder.wrap_connection)
+
+
+def install_write_capture(
+    monkeypatch, capture: DbWriteCapture, write_targets: dict[str, Callable | None]
+):
+    """
+    Replaces the database write functions with the capture, in every mode.
+
+    Args:
+        monkeypatch: pytest monkeypatch fixture (undoes every patch after the test)
+        capture: Write capture of the test case
+        write_targets: Functions that write to a database, mapped to the function computing
+            their simulated return value (or None)
+    """
+    for target, simulate in write_targets.items():
+        owner, attr = _resolve_target(target)
+        replace_everywhere(
+            monkeypatch, owner, attr, lambda f, t=target, s=simulate: capture.wrap(t, f, s)
+        )

{detquantlib-5.10.2 → detquantlib-5.11.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "detquantlib"
-version = "5.10.2"
+version = "5.11.0"
 description = "An internal library containing functions and classes that can be used across Quant models."
 authors = ["Dynamic Energy Trading"]
 readme = "README.md"