detquantlib 5.5.0__tar.gz → 5.7.0__tar.gz

{detquantlib-5.5.0 → detquantlib-5.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: detquantlib
-Version: 5.5.0
+Version: 5.7.0
 Summary: An internal library containing functions and classes that can be used across Quant models.
 License-File: LICENSE.txt
 Author: Dynamic Energy Trading
@@ -83,9 +83,9 @@ poetry run python docs.py
 
 Classes:
 
-- `Battery`:
-  - `from detquantlib.assets import Battery`
-  - `from detquantlib.assets.battery import Battery`
+- `BatteryState`:
+  - `from detquantlib.assets import BatteryState`
+  - `from detquantlib.assets.battery import BatteryState`
 - `DetDatabase`:
   - `from detquantlib.data import DetDatabase`
   - `from detquantlib.data.databases.detdatabase import DetDatabase`
@@ -104,6 +104,18 @@ Classes:
 - `Sftp`:
   - `from detquantlib.data import Sftp`
   - `from detquantlib.data.sftp.sftp import Sftp`
+- `SnowflakeClient`:
+  - `from detquantlib.data import SnowflakeClient`
+  - `from detquantlib.data.databases.snowflake_client import SnowflakeClient`
+- `SnowflakeConfig`:
+  - `from detquantlib.data import SnowflakeConfig`
+  - `from detquantlib.data.databases.snowflake_client import SnowflakeConfig`
+- `TennetAPIError`:
+  - `from detquantlib.data import TennetAPIError`
+  - `from detquantlib.data.tennet.tennet_client import TennetAPIError`
+- `TennetClient`:
+  - `from detquantlib.data import TennetClient`
+  - `from detquantlib.data.tennet.tennet_client import TennetClient`
 - `WindTurbine`:
   - `from detquantlib.assets import WindTurbine`
   - `from detquantlib.assets.wind_turbine import WindTurbine`

{detquantlib-5.5.0 → detquantlib-5.7.0}/README.md

@@ -54,9 +54,9 @@ poetry run python docs.py
 
 Classes:
 
-- `Battery`:
-  - `from detquantlib.assets import Battery`
-  - `from detquantlib.assets.battery import Battery`
+- `BatteryState`:
+  - `from detquantlib.assets import BatteryState`
+  - `from detquantlib.assets.battery import BatteryState`
 - `DetDatabase`:
   - `from detquantlib.data import DetDatabase`
   - `from detquantlib.data.databases.detdatabase import DetDatabase`
@@ -75,6 +75,18 @@ Classes:
 - `Sftp`:
   - `from detquantlib.data import Sftp`
   - `from detquantlib.data.sftp.sftp import Sftp`
+- `SnowflakeClient`:
+  - `from detquantlib.data import SnowflakeClient`
+  - `from detquantlib.data.databases.snowflake_client import SnowflakeClient`
+- `SnowflakeConfig`:
+  - `from detquantlib.data import SnowflakeConfig`
+  - `from detquantlib.data.databases.snowflake_client import SnowflakeConfig`
+- `TennetAPIError`:
+  - `from detquantlib.data import TennetAPIError`
+  - `from detquantlib.data.tennet.tennet_client import TennetAPIError`
+- `TennetClient`:
+  - `from detquantlib.data import TennetClient`
+  - `from detquantlib.data.tennet.tennet_client import TennetClient`
 - `WindTurbine`:
   - `from detquantlib.assets import WindTurbine`
   - `from detquantlib.assets.wind_turbine import WindTurbine`

detquantlib-5.7.0/detquantlib/assets/__init__.py

@@ -0,0 +1,4 @@
+from .battery import BatteryState
+from .wind_turbine import WindTurbine, WindTurbineModels
+
+__all__ = ["BatteryState", "WindTurbine", "WindTurbineModels"]

detquantlib-5.7.0/detquantlib/assets/battery.py

@@ -0,0 +1,397 @@
+# Python built-in packages
+import json
+import logging
+import os
+import uuid
+from dataclasses import dataclass
+from functools import cached_property
+from typing import Literal
+
+# Third-party packages
+import numpy as np
+import pandas as pd
+import redis
+
+# Internal modules
+from detquantlib.data import DetDatabase
+from detquantlib.utils import list_to_str
+
+
+@dataclass(frozen=True)
+class BatteryState:
+    """
+    A class containing the characteristics and current state of a battery.
+
+    Attributes:
+        capacity_mwh: Capacity (MWh)
+        power_mw: Power (MW)
+        soc_mwh: State of charge (MWh)
+        dod_mwh: Depth of discharge (MWh)
+        rte: Roundtrip efficiency (%)
+        site_node_ids: Site node IDs of the physical batteries aggregated into this group. An
+            entry is None when the source system doesn't report a node ID.
+    """
+
+    capacity_mwh: float
+    power_mw: float
+    soc_mwh: float
+    dod_mwh: float
+    rte: float
+    site_node_ids: list
+
+    @cached_property
+    def cp_ratio(self) -> float:
+        """Calculates the battery's capacity:power ratio."""
+        return round(self.capacity_mwh / self.power_mw)
+
+    def __post_init__(self) -> None:
+        self._validate_obj()
+
+    def _validate_obj(self) -> None:
+        """Validates the values of the object's attributes."""
+        errors: list[str] = []
+
+        # Type checks (bool is a subclass of int, so exclude it explicitly)
+        for name in ("capacity_mwh", "power_mw", "soc_mwh", "dod_mwh", "rte"):
+            value = getattr(self, name)
+            if isinstance(value, bool) or not isinstance(value, (int, float)):
+                errors.append(f"{name} must be a float, got {type(value).__name__}.")
+
+        # Skip range checks if types are wrong
+        if errors:
+            raise ValueError("\n".join(errors))
+
+        # Range checks
+        if not self.capacity_mwh >= 0:
+            errors.append(f"capacity_mwh must be >= 0, got {self.capacity_mwh}.")
+        if not self.power_mw >= 0:
+            errors.append(f"power_mw must be >= 0, got {self.power_mw}.")
+        if not self.soc_mwh >= 0:
+            errors.append(f"soc_mwh must be >= 0, got {self.soc_mwh}.")
+        if not self.dod_mwh >= 0:
+            errors.append(f"dod_mwh must be >= 0, got {self.dod_mwh}.")
+        if not 0 <= self.rte <= 1:
+            errors.append(f"rte must be in [0, 1], got {self.rte}.")
+        if self.soc_mwh > self.capacity_mwh:
+            raise ValueError(
+                f"State of charge ({self.soc_mwh} MWh) must be lower than battery capacity "
+                f"({self.capacity_mwh} MWh)."
+            )
+        if self.soc_mwh < self.dod_mwh:
+            logging.warning(
+                f"State of charge ({self.soc_mwh} MWh) is lower than depth of discharge "
+                f"({self.dod_mwh} MWh)."
+            )
+
+        if errors:
+            raise ValueError("\n".join(errors))
+
+    @staticmethod
+    def build_from_redis(
+        strategy_id: uuid.UUID,
+        ems: Literal["eneris", "smartbox"],
+        rte: float,
+        dod_ratio: float = 0.0,
+        cp_ratio: int | None = None,
+        steering_margin_power: float = 0.0,
+        steering_margin_capacity: float = 0.0,
+    ) -> "BatteryState | list[BatteryState]":
+        """
+        Loads live battery states from Redis and builds the corresponding battery object(s).
+
+        Args:
+            strategy_id: Strategy ID
+            ems: Energy management system (can take value 'eneris' or 'smartbox')
+            rte: Batteries roundtrip efficiency
+            dod_ratio: Batteries depth of discharge (ratio)
+            cp_ratio: Desired capacity-to-power ratio. If provided, only batteries matching that
+                ratio will be retained. If None, all batteries will be retained.
+            steering_margin_capacity: Steering margin (ratio) to be applied to the BESS capacity
+            steering_margin_power: Steering margin (ratio) to be applied to the BESS power
+
+        Returns:
+            Battery object (if input argument 'cp_ratio' is specified), or list of battery
+                objects otherwise
+
+        Raises:
+            ValueError: Raises an error if the input 'ems' is not valid.
+            ValueError: Raises an error if the Redis cache is empty.
+            ValueError: Raises an error if the capacity:power ratio is not found.
+        """
+        # Validate inputs
+        _ems_eneris = "eneris"
+        _ems_smartbox = "smartbox"
+        _ems_accepted_values = [_ems_eneris, _ems_smartbox]
+        if ems not in _ems_accepted_values:
+            raise ValueError(
+                f"Invalid ems value '{ems}'. Accepted values: {list_to_str(_ems_accepted_values)}."
+            )
+
+        # Get battery serial numbers
+        serial_numbers = BatteryState.load_serial_map(strategy_id)
+
+        # Connect to redis cache
+        r = redis.Redis(
+            host=os.environ["DET_REDIS_HOST"],
+            port=int(os.environ["DET_REDIS_PORT"]),
+            password=os.environ["DET_REDIS_PASSWORD"],
+            ssl=True,
+            db=int(os.environ["DET_REDIS_DB"]),
+        )
+
+        # Loop over cache data
+        battery_characteristics = []
+        missing_batteries = []
+        for key in r.scan_iter("*"):
+            # Parse data
+            raw = r.hget(key, "live")
+            data = json.loads(raw)
+
+            # Temporary code to distinguish between Eneris and SmartBox batteries
+            is_smartbox = "powermeter" in data.keys()
+
+            if (
+                ems == _ems_smartbox
+                and is_smartbox
+                and str(data.get("serial")).upper() in serial_numbers
+            ):
+                missing_keys = BatteryState.find_missing_battery_keys_smartbox(data)
+                if missing_keys:
+                    # Skip battery if info cannot be found
+                    missing_batteries.append(
+                        dict(
+                            site_node_id=None,
+                            error_msg=f"Serial {data['serial']}: missing or null keys: {missing_keys}",
+                        )
+                    )
+                    continue
+
+                capacity_mwh = (
+                    data["batteries"]["charge_capacity"] + data["batteries"]["discharge_capacity"]
+                ) / 1000
+                power_mw = data["batteries"]["rated_power"] / 1000
+                soc_mwh = data["batteries"]["discharge_capacity"] / 1000
+                timestamp = data["ts"]
+                site_node_id = None
+
+                # Calculate depth of discharge
+                dod_mwh = dod_ratio * capacity_mwh
+
+                # Store characteristics
+                battery_characteristics.append(
+                    dict(
+                        capacity_mwh=capacity_mwh,
+                        power_mw=power_mw,
+                        soc_mwh=soc_mwh,
+                        dod_mwh=dod_mwh,
+                        rte=rte,
+                        timestamp=timestamp,
+                        site_node_id=site_node_id,
+                    )
+                )
+
+            elif ems == _ems_eneris and not is_smartbox:
+                missing_keys = BatteryState.find_missing_battery_keys_eniris(data)
+                if missing_keys:
+                    # Skip battery if info cannot be found
+                    missing_batteries.append(
+                        dict(
+                            site_node_id=data.get("siteNodeId", None),
+                            error_msg=f"Missing keys: {missing_keys}",
+                        )
+                    )
+                    continue
+
+                storage = data["data"]["state"]["storage"]
+
+                # Convert settings
+                capacity_mwh = storage["energy_capacity_Wh"] / 1e6
+                soc_mwh = storage["energy_stored_Wh"] / 1e6
+                charge_rate_mw = storage["max_charge_power_W"] / 1e6
+                discharge_rate_mw = storage["max_discharge_power_W"] / 1e6
+
+                # Set (dis)charge rate
+                power_mw = max(charge_rate_mw, -discharge_rate_mw)
+
+                # Calculate depth of discharge
+                dod_mwh = dod_ratio * capacity_mwh
+
+                # Get BESS group data
+                timestamp = data["time"]
+                site_node_id = str(data["siteNodeId"])
+
+                # Store characteristics
+                battery_characteristics.append(
+                    dict(
+                        capacity_mwh=capacity_mwh,
+                        power_mw=power_mw,
+                        soc_mwh=soc_mwh,
+                        dod_mwh=dod_mwh,
+                        rte=rte,
+                        timestamp=timestamp,
+                        site_node_id=site_node_id,
+                    )
+                )
+
+        nr_miss = len(missing_batteries)
+        if nr_miss > 0:
+            nr_found = len(battery_characteristics)
+            msg = (
+                f"Failed to retrieve the battery storage data of {nr_miss}/{nr_found + nr_miss} "
+                f"batteries. Details:"
+            )
+            for mb in missing_batteries:
+                msg += (
+                    f"\n- Site node ID: {mb['site_node_id']}. Error message: '{mb['error_msg']}'."
+                )
+            logging.warning(msg)
+
+        # Convert to dataframe
+        df_bess = pd.DataFrame(battery_characteristics)
+        if df_bess.empty:
+            raise ValueError("Empty Redis cache. No batteries found.")
+
+        # Calculate capacity:power ratio
+        df_bess["cp_ratio"] = (df_bess["capacity_mwh"] / df_bess["power_mw"]).round(0)
+        df_bess.sort_values("cp_ratio", ignore_index=True, inplace=True)
+
+        # Calculate aggregated VPP's
+        df_vpp = (
+            df_bess[
+                [
+                    "capacity_mwh",
+                    "power_mw",
+                    "soc_mwh",
+                    "dod_mwh",
+                    "rte",
+                    "site_node_id",
+                    "cp_ratio",
+                ]
+            ]
+            .groupby("cp_ratio", as_index=False)
+            .agg(
+                {
+                    "capacity_mwh": "sum",
+                    "power_mw": "sum",
+                    "soc_mwh": "sum",
+                    "dod_mwh": "sum",
+                    "rte": "mean",
+                    "site_node_id": list,
+                }
+            )
+            .reset_index(drop=True)
+            # The aggregated column holds every node ID in the group, hence the plural field name
+            .rename(columns={"site_node_id": "site_node_ids"})
+        )
+
+        df_vpp.drop(columns="cp_ratio", inplace=True)
+
+        # Round to avoid machine precision issues caused by 'group by'
+        df_vpp = df_vpp.round(9)
+
+        # Add margin to available steerable batteries
+        df_vpp["power_mw"] *= 1 - steering_margin_power
+        df_vpp["capacity_mwh"] *= 1 - steering_margin_capacity
+        df_vpp["soc_mwh"] = np.clip(
+            a=df_vpp["soc_mwh"], a_min=df_vpp["dod_mwh"], a_max=df_vpp["capacity_mwh"]
+        )
+
+        # Create battery objects
+        batteries = [BatteryState(**df_vpp.loc[i, :].to_dict()) for i in df_vpp.index]
+
+        # Filter on desired capacity:power ratio
+        if cp_ratio is not None:
+            batteries = next((b for b in batteries if b.cp_ratio == cp_ratio), None)
+            if batteries is None:
+                raise ValueError(f"Capacity:power ratio of {cp_ratio} not found.")
+
+        return batteries
+
+    @staticmethod
+    def load_serial_map(strategy_id: uuid.UUID) -> list[str]:
+        """Loads the serial number of all active batteries linked to a given strategy."""
+        query = (
+            f"SELECT SerialNumber FROM [META].[StrategySerialMap] "
+            f"WHERE StrategyId = '{strategy_id}' "
+            f"AND Active = 1"
+        )
+        serial_numbers = DetDatabase("det-algo-trading").query_db(query)
+        serial_numbers = serial_numbers["SerialNumber"].str.upper().to_list()
+        return serial_numbers
+
+    @staticmethod
+    def find_missing_battery_keys_eniris(data: dict) -> list[str]:
+        """
+        Checks that a live Eniris payload contains all required keys, including
+        nested ones, and returns the list of keys that are missing.
+
+        Expected structure:
+            - Top level: "data", "siteNodeId", "time"
+            - data["data"]: "state"
+            - data["data"]["state"]: "storage"
+            - data["data"]["state"]["storage"]: "energy_capacity_Wh",
+              "energy_stored_Wh", "max_charge_power_W", "max_discharge_power_W"
+
+        Args:
+            data: Raw battery payload
+
+        Returns:
+            list: Names of the required keys that are missing (empty if all present)
+        """
+        # Each entry maps a path of parent keys to the keys expected there.
+        required_key_paths = [
+            ([], ["data", "siteNodeId", "time"]),
+            (["data"], ["state"]),
+            (["data", "state"], ["storage"]),
+            (
+                ["data", "state", "storage"],
+                [
+                    "energy_capacity_Wh",
+                    "energy_stored_Wh",
+                    "max_charge_power_W",
+                    "max_discharge_power_W",
+                ],
+            ),
+        ]
+
+        missing_keys = []
+        for path, keys in required_key_paths:
+            # Walk down to the sub-dict at `path`, stop if a parent is missing or is not a dict
+            node = data
+            for parent in path:
+                node = node.get(parent) if isinstance(node, dict) else None
+                if node is None:
+                    break
+            if not isinstance(node, dict):
+                continue
+            missing_keys.extend(key for key in keys if key not in node)
+
+        return missing_keys
+
+    @staticmethod
+    def find_missing_battery_keys_smartbox(data: dict) -> list[str]:
+        """
+        Checks that a live SmartBox payload contains all required keys with
+        non-null values, and returns the list of keys that are missing or null.
+
+        Expected structure:
+            - Top level: "serial", "ts", "batteries"
+            - data["batteries"]: "charge_capacity", "discharge_capacity", "rated_power"
+
+        Args:
+            data: Raw battery payload
+
+        Returns:
+            list: Names of the required keys that are missing or null (empty if all present)
+        """
+        missing_keys = [key for key in ["serial", "ts", "batteries"] if data.get(key) is None]
+
+        batteries = data.get("batteries")
+        if isinstance(batteries, dict):
+            missing_keys.extend(
+                key
+                for key in ["charge_capacity", "discharge_capacity", "rated_power"]
+                if batteries.get(key) is None
+            )
+
+        return missing_keys

{detquantlib-5.5.0 → detquantlib-5.7.0}/detquantlib/data/__init__.py

@@ -3,6 +3,7 @@ from .databases.helpers import *
 from .databases.snowflake_client import SnowflakeClient, SnowflakeConfig
 from .entsoe.entsoe import Entsoe
 from .sftp.sftp import Sftp
+from .tennet.tennet_client import TennetAPIError, TennetClient
 
 __all__ = [
     "DetDatabase",
@@ -12,4 +13,6 @@ __all__ = [
     "list_to_sql_columns",
     "SnowflakeClient",
     "SnowflakeConfig",
+    "TennetClient",
+    "TennetAPIError",
 ]

detquantlib-5.7.0/detquantlib/data/tennet/tennet_client.py

@@ -0,0 +1,417 @@
+# Python built-in packages
+import os
+import re
+import time
+from datetime import date, datetime
+from io import StringIO
+from math import ceil
+from typing import Optional, Union
+
+# Third-party packages
+import pandas as pd
+import requests
+
+# Custom data type
+DateLike = Union[str, date, datetime, pd.Timestamp]
+
+
+class TennetAPIError(Exception):
+    """Raised when the TenneT API returns an error response."""
+
+    def __init__(self, message: str, status_code: Optional[int] = None) -> None:
+        """
+        Initialise the error.
+
+        Args:
+            message: Human-readable error message.
+            status_code: HTTP status code of the failed response, if any.
+        """
+        super().__init__(message)
+        self.status_code = status_code
+
+
+class TennetClient:
+    """
+    Thin wrapper around the TenneT publications API (https://developer.tennet.eu).
+
+    Attributes:
+        base_url: Base URL used for all requests (production or acceptance).
+        timeout: Request timeout in seconds.
+        max_retries: Number of retries on HTTP 429 / 5xx responses and connection errors.
+        backoff: Base back-off in seconds used when no ``Retry-After`` header is returned.
+        session: The ``requests.Session`` used for all requests.
+
+    Example:
+        >>> client = TennetClient(api_key="...")  # or set TENNET_API_KEY
+        >>> prices = client.get_settlement_prices("2026-10-01", "2026-10-02")
+        >>> delta = client.get_balance_delta_high_res("2026-10-14 10:00", "2026-10-14 12:00")
+        >>> latest = client.get_balance_delta_high_res_latest()
+    """
+
+    BASE_URL = "https://api.tennet.eu/publications/v1/"
+    ACC_BASE_URL = "https://api.acc.tennet.eu/publications/v1/"
+    LOCAL_TZ = "Europe/Amsterdam"
+    DATE_FORMAT_UTC = "%Y-%m-%dT%H:%M:%SZ"
+
+    # Maximum period TenneT accepts in a single balance delta high res request.
+    BALANCE_DELTA_HIGH_RES_MAX_PERIOD = pd.Timedelta(hours=4)
+
+    _RETRY_STATUS = {429, 500, 502, 503, 504}
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        *,
+        acceptance: bool = False,
+        timeout: float = 30.0,
+        max_retries: int = 3,
+        backoff: float = 2.0,
+        session: Optional[requests.Session] = None,
+    ) -> None:
+        """
+        Initialise the client.
+
+        Args:
+            api_key: API key from the TenneT developer portal. Falls back to the
+                ``TENNET_API_KEY`` environment variable.
+            acceptance: Use the acceptance (test) environment instead of production.
+            timeout: Request timeout in seconds.
+            max_retries: How many times to retry on HTTP 429 / 5xx responses.
+            backoff: Base back-off in seconds (exponential) when no ``Retry-After``
+                header is returned.
+            session: Optional pre-configured ``requests.Session`` (handy for testing or
+                proxies).
+
+        Raises:
+            ValueError: If no API key is provided or found in the environment.
+        """
+        api_key = api_key or os.environ.get("TENNET_API_KEY")
+        if not api_key:
+            raise ValueError(
+                "No API key provided. Pass `api_key` or set the TENNET_API_KEY "
+                "environment variable."
+            )
+        self.base_url = self.ACC_BASE_URL if acceptance else self.BASE_URL
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.backoff = backoff
+        self.session = session or requests.Session()
+        self.session.headers.update(
+            {"apikey": api_key, "Accept": "text/csv", "User-Agent": "TennetClient/1.0"}
+        )
+
+    # ---------------------------------------------------------------------------------------------
+    # Public
+    # ---------------------------------------------------------------------------------------------
+
+    def get_settlement_prices(
+        self, start: DateLike, end: DateLike, *, chunk_size: Optional[pd.Timedelta] = None
+    ) -> pd.DataFrame:
+        """
+        Get imbalance settlement prices per 15-minute ISP.
+
+        Args:
+            start: Start of the period. Naive values are interpreted as Europe/Amsterdam.
+            end: End of the period. Naive values are interpreted as Europe/Amsterdam.
+            chunk_size: Optional maximum period per request; longer ranges are split.
+
+        Returns:
+            Settlement prices.
+        """
+        return self._query_range(
+            endpoint="settlement-prices",
+            start=start,
+            end=end,
+            resolution=pd.Timedelta(minutes=15),
+            chunk_size=chunk_size,
+        )
+
+    def get_balance_delta_high_res(
+        self,
+        start: DateLike,
+        end: DateLike,
+        *,
+        chunk_size: Optional[pd.Timedelta] = BALANCE_DELTA_HIGH_RES_MAX_PERIOD,
+    ) -> pd.DataFrame:
+        """
+        Get the balance delta at 12-second resolution.
+
+        TenneT accepts at most 4 hours per request, so longer ranges are split into
+        ``chunk_size`` windows (default 4 hours) and concatenated.
+
+        Args:
+            start: Start of the period. Naive values are interpreted as Europe/Amsterdam.
+            end: End of the period. Naive values are interpreted as Europe/Amsterdam.
+            chunk_size: Maximum period per request. Pass ``None`` to send a single request.
+
+        Returns:
+            Balance delta at 12-second resolution.
+        """
+        return self._query_range(
+            endpoint="balance-delta-high-res",
+            start=start,
+            end=end,
+            resolution=pd.Timedelta(seconds=12),
+            chunk_size=chunk_size,
+        )
+
+    def get_balance_delta_high_res_latest(self) -> pd.DataFrame:
+        """
+        Get the most recent balance delta high res publication (roughly the last 30 min).
+
+        Returns:
+            Most recent balance delta at 12-second resolution.
+        """
+        return self._parse_csv(
+            text=self._request(endpoint="balance-delta-high-res/latest"),
+            resolution=pd.Timedelta(seconds=12),
+        )
+
+    # ---------------------------------------------------------------------------------------------
+    # Internal
+    # ---------------------------------------------------------------------------------------------
+
+    def _query_range(
+        self,
+        endpoint: str,
+        start: DateLike,
+        end: DateLike,
+        resolution: pd.Timedelta,
+        chunk_size: Optional[pd.Timedelta],
+    ) -> pd.DataFrame:
+        """
+        Query a date-ranged endpoint, splitting the range into chunks if needed.
+
+        Args:
+            endpoint: Endpoint path relative to the base URL.
+            start: Start of the period.
+            end: End of the period.
+            resolution: Length of one interval, used to rebuild timestamps from ISPs.
+            chunk_size: Maximum period per request, or ``None`` for a single request.
+
+        Returns:
+            Concatenated DataFrame.
+
+        Raises:
+            ValueError: If ``start`` is not before ``end``, or ``chunk_size`` is not positive.
+        """
+        start_ts, end_ts = self._to_utc(start), self._to_utc(end)
+        if start_ts >= end_ts:
+            raise ValueError(f"`start` ({start_ts}) must be before `end` ({end_ts}).")
+        if chunk_size is not None and chunk_size <= pd.Timedelta(0):
+            raise ValueError(f"`chunk_size` ({chunk_size}) must be positive.")
+
+        # Set batch windows
+        chunk_size = end_ts - start_ts if chunk_size is None else chunk_size
+        windows = [
+            (start_ts + chunk_size * t, min(start_ts + chunk_size * (t + 1), end_ts))
+            for t in range(ceil((end_ts - start_ts) / chunk_size))
+        ]
+
+        # Query in batches
+        frames = []
+        for w_start, w_end in windows:
+            params = {
+                "date_from": w_start.strftime(self.DATE_FORMAT_UTC),
+                "date_to": w_end.strftime(self.DATE_FORMAT_UTC),
+            }
+            frames.append(self._parse_csv(self._request(endpoint, params), resolution))
+
+        frames = [f for f in frames if not f.empty]
+        if not frames:
+            return self._empty_frame()
+        df = pd.concat(frames)
+        df = df.sort_values(["timeinterval_start", "timeinterval_end"], ignore_index=True)
+        return df
+
+    @classmethod
+    def _to_utc(cls, value: DateLike) -> pd.Timestamp:
+        """
+        Convert a date-like value to a UTC timestamp.
+
+        Args:
+            value: Date-like value. Naive values are interpreted as Europe/Amsterdam,
+                timezone-aware values are converted.
+
+        Returns:
+            Timezone-aware UTC timestamp.
+        """
+        ts = pd.Timestamp(value)
+        if ts.tzinfo is None:
+            ts = ts.tz_localize(cls.LOCAL_TZ, ambiguous="raise", nonexistent="shift_forward")
+        ts = ts.tz_convert("UTC")
+        return ts
+
+    def _request(self, endpoint: str, params: Optional[dict[str, str]] = None) -> str:
+        """
+        Perform a GET request with retries.
+
+        Args:
+            endpoint: Endpoint path relative to the base URL.
+            params: Optional query parameters.
+
+        Returns:
+            The response body as text.
+
+        Raises:
+            TennetAPIError: If the request fails or the API returns an error status.
+        """
+        url = self.base_url + endpoint
+        for attempt in range(self.max_retries + 1):
+            try:
+                resp = self.session.get(url, params=params, timeout=self.timeout)
+            except requests.RequestException as exc:
+                if attempt >= self.max_retries:
+                    raise TennetAPIError(f"Request to {url} failed: {exc}") from exc
+                time.sleep(self.backoff * 2**attempt)
+                continue
+
+            if resp.status_code in self._RETRY_STATUS and attempt < self.max_retries:
+                time.sleep(self._retry_delay(resp, attempt))
+                continue
+            if resp.status_code >= 400:
+                raise TennetAPIError(
+                    f"TenneT API error {resp.status_code} for {url}: {resp.text[:500]}",
+                    status_code=resp.status_code,
+                )
+            return resp.text
+        raise TennetAPIError(f"Giving up on {url} after {self.max_retries} retries.")
+
+    def _retry_delay(self, resp: requests.Response, attempt: int) -> float:
+        """
+        Compute how long to wait before retrying.
+
+        Args:
+            resp: The response that triggered the retry.
+            attempt: Zero-based attempt number.
+
+        Returns:
+            Delay in seconds, taken from ``Retry-After`` or exponential back-off.
+        """
+        retry_after = (resp.headers or {}).get("Retry-After")
+        try:
+            return float(retry_after)
+        except (TypeError, ValueError):
+            return self.backoff * 2**attempt
+
+    @classmethod
+    def _parse_csv(cls, text: str, resolution: pd.Timedelta) -> pd.DataFrame:
+        """
+        Parse a TenneT CSV response into a DataFrame.
+
+        Interval columns (``timeinterval_start/end_utc`` or ``_loc``) are converted to
+        Europe/Amsterdam timestamps and renamed to ``timeinterval_start`` / ``timeinterval_end``.
+        UTC columns take precedence: if both are present, the local ones are dropped. Rows are
+        then sorted chronologically.
+
+        Args:
+            text: CSV response body.
+            resolution: Length of one interval, used to disambiguate local timestamps via ISPs.
+
+        Returns:
+            DataFrame with snake_case columns and Europe/Amsterdam interval timestamps.
+
+        Raises:
+            TennetAPIError: If only one column of a start/end interval pair is present.
+        """
+        if not text or not text.strip():
+            return cls._empty_frame()
+        df = pd.read_csv(StringIO(text))
+        if df.empty:
+            return cls._empty_frame()
+
+        # Convert column names to snake case
+        df.columns = [cls._snake(c) for c in df.columns]
+
+        # Find complete start/end interval column pairs per suffix
+        cols = ["timeinterval_start", "timeinterval_end"]
+        pairs = {}
+        for sfx in ["utc", "loc"]:
+            pair = {f"{c}_{sfx}": c for c in cols}
+            n_present = sum(c in df.columns for c in pair.keys())
+            if n_present == 1:
+                raise TennetAPIError(
+                    f"Incomplete interval columns in TenneT response: {list(pair.keys())}."
+                )
+            if n_present == 2:
+                pairs[sfx] = pair
+
+        # Convert interval columns to Europe/Amsterdam timestamps (UTC is unambiguous, so it is
+        # preferred over local)
+        if "utc" in pairs:
+            df = df.drop(columns=list(pairs.get("loc", {}).keys())).rename(columns=pairs["utc"])
+            for c in cols:
+                df[c] = pd.to_datetime(df[c], utc=True).dt.tz_convert(cls.LOCAL_TZ)
+        elif "loc" in pairs:
+            df = df.rename(columns=pairs["loc"])
+            df[cols[0]], df[cols[1]] = cls._localize_intervals(df, resolution)
+
+        # Sort chronologically
+        if pairs:
+            df = df.sort_values(cols, ignore_index=True)
+
+        return df
+
+    @classmethod
+    def _localize_intervals(
+        cls, df: pd.DataFrame, resolution: pd.Timedelta
+    ) -> tuple[pd.Series, pd.Series]:
+        """
+        Convert local (Europe/Amsterdam) interval columns to timezone-aware timestamps.
+
+        Local wall-clock times are ambiguous in the repeated hour when DST ends, so:
+        - Start: rebuilt from the delivery date + ISP number (adding a Timedelta to an aware
+          timestamp is absolute-time arithmetic, hence DST-safe), then checked against the
+          wall-clock start times in the response.
+        - End: converted from the response values. For an ambiguous wall-clock time, the
+          earliest reading that falls after the start is used.
+
+        Args:
+            df: DataFrame with naive ``timeinterval_start``, ``timeinterval_end`` and ``isp``
+                columns.
+            resolution: Length of one ISP.
+
+        Returns:
+            Timezone-aware start and end timestamps.
+
+        Raises:
+            TennetAPIError: If the ``isp`` column is missing, if the ISP-based start times don't
+                match the response (e.g. an inconsistent ``resolution``), or if an end time is not
+                after its start time.
+        """
+        if "isp" not in df.columns:
+            raise TennetAPIError(
+                "Cannot parse local date columns without column 'isp', because local "
+                "timestamps are ambiguous around DST."
+            )
+        start_naive = pd.to_datetime(df["timeinterval_start"])
+        end_naive = pd.to_datetime(df["timeinterval_end"])
+
+        # Start: delivery date + ISP number, cross-checked with the response
+        day = start_naive.dt.normalize().dt.tz_localize(cls.LOCAL_TZ)
+        start = day + (df["isp"].astype(int) - 1) * resolution
+        if not (start.dt.tz_localize(None) == start_naive).all():
+            raise TennetAPIError(
+                f"Input resolution '{resolution}' and ISP numbers are inconsistent with TenneT "
+                f"response start timestamps."
+            )
+
+        # End: localize both readings of ambiguous times (DST is the earlier one in absolute
+        # time), and keep the earliest that falls after the start
+        end_dst = end_naive.dt.tz_localize(cls.LOCAL_TZ, ambiguous=True)
+        end_std = end_naive.dt.tz_localize(cls.LOCAL_TZ, ambiguous=False)
+        end = end_dst.where(end_dst > start, end_std)
+        if not (end > start).all():
+            raise TennetAPIError("TenneT response contains intervals that end before they start.")
+
+        return start, end
+
+    @classmethod
+    def _empty_frame(cls) -> pd.DataFrame:
+        """Return an empty DataFrame, used when TenneT returns no data."""
+        return pd.DataFrame()
+
+    @staticmethod
+    def _snake(name: str) -> str:
+        """Convert a column name to snake_case."""
+        return re.sub(r"[^0-9a-z]+", "_", name.strip().lower()).strip("_")

{detquantlib-5.5.0 → detquantlib-5.7.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "detquantlib"
-version = "5.5.0"
+version = "5.7.0"
 description = "An internal library containing functions and classes that can be used across Quant models."
 authors = ["Dynamic Energy Trading"]
 readme = "README.md"

detquantlib-5.5.0/detquantlib/assets/__init__.py

@@ -1,4 +0,0 @@
-from .battery import Battery
-from .wind_turbine import WindTurbine, WindTurbineModels
-
-__all__ = ["Battery", "WindTurbine", "WindTurbineModels"]

detquantlib-5.5.0/detquantlib/assets/battery.py

@@ -1,174 +0,0 @@
-# Python built-in packages
-import json
-import os
-
-# Third-party packages
-import pandas as pd
-import redis
-
-# Internal modules
-from detquantlib.utils import list_to_str
-
-
-class Battery:
-    """A class containing the characteristics of a battery."""
-
-    def __init__(
-        self,
-        capacity_mwh: float = None,
-        power_mw: float = None,
-        state_of_charge_mwh: float = None,
-        roundtrip_eff: float = None,
-        group_id: int = None,
-    ):
-        """
-        Constructor method.
-
-        Args:
-            capacity_mwh: Battery capacity in MWh
-            power_mw: Battery power in MW
-            state_of_charge_mwh: Current state of charge in MWh
-            roundtrip_eff: Roundtrip efficiency
-            group_id: Group ID (used to differentiate batteries with different characteristics
-        """
-        # Assign inputs
-        self.capacity_mwh = capacity_mwh
-        self.power_mw = power_mw
-        self.state_of_charge_mwh = state_of_charge_mwh
-        self.roundtrip_eff = roundtrip_eff
-        self.group_id = group_id
-
-        # Calculate power-to-capacity ratio
-        if self.capacity_mwh and self.power_mw:
-            self.capacity_power_ratio = round(self.capacity_mwh / self.power_mw, 0)
-
-    @staticmethod
-    def build_from_redis() -> (list["Battery"], str):
-        """
-        Builds Battery objects from the individual battery units found in the redis cache.
-
-        Returns:
-            Tuple:
-                - List of Battery objects
-                - Message listing all the batteries whose information could not be retrieved
-                    from the redis cache
-        """
-        # Load individual battery units
-        battery_units, missing_units = Battery.load_battery_units_from_redis()
-
-        # Print missing battery units
-        msg_missing_units = ""
-        nr_found, nr_miss = len(battery_units), len(missing_units)
-        if nr_miss > 0:
-            msg_missing_units = (
-                f"Failed to retrieve the battery storage data of {nr_miss}/{nr_found + nr_miss} "
-                f"batteries. Details:"
-            )
-            for mb in missing_units:
-                msg_missing_units += (
-                    f"\n- Site node ID: {mb['site_node_id']}. Error message: '{mb['error_msg']}'."
-                )
-
-        if nr_found == 0:
-            return list(), msg_missing_units
-
-        # Set battery groups based on capacity:power ratio
-        battery_units = pd.DataFrame(battery_units)
-        battery_units.sort_values("cp_ratio", ignore_index=True, inplace=True)
-        battery_units["cp_group"] = battery_units["cp_ratio"].round(0)
-
-        # Calculate aggregated virtual batteries
-        agg_units = (
-            battery_units[["capacity_mwh", "power_mw", "state_of_charge_mwh", "cp_group"]]
-            .groupby("cp_group", as_index=False)
-            .agg({"capacity_mwh": "sum", "power_mw": "sum", "state_of_charge_mwh": "sum"})
-            .reset_index(drop=True)
-        )
-        agg_units["group_id"] = agg_units["cp_group"].rank(method="dense")
-        agg_units.drop(columns="cp_group", inplace=True)
-
-        # Round to avoid machine precision issues caused by 'group by'
-        agg_units = agg_units.round(9)
-
-        # Create battery objects
-        batteries = [Battery(**agg_units.loc[i, :].to_dict()) for i in agg_units.index]
-
-        return batteries, msg_missing_units
-
-    @staticmethod
-    def load_battery_units_from_redis() -> (list[dict], list[dict]):
-        """
-        Loads individual battery units from the redis cache.
-
-        Returns:
-            Tuple:
-                - Individual battery units whose information could be found
-                - Individual battery units whose information could not be found
-
-        Raises:
-            ConnectionError: Raises an error if environment variables are not defined.
-        """
-        # Check mandatory environment variables to connect to Redis cache
-        mandatory_vars = ["DET_REDIS_HOST", "DET_REDIS_PORT", "DET_REDIS_PASSWORD", "DET_REDIS_DB"]
-        for mv in mandatory_vars:
-            if mv not in os.environ:
-                raise ConnectionError(
-                    f"Environment variable '{mv}' not found. Connection to Redis cache requires "
-                    f"the following environment variables: {list_to_str(mandatory_vars)}."
-                )
-
-        # Connect to redis cache
-        r = redis.Redis(
-            host=os.environ["DET_REDIS_HOST"],
-            port=int(os.environ["DET_REDIS_PORT"]),
-            password=os.environ["DET_REDIS_PASSWORD"],
-            ssl=True,
-            db=int(os.environ["DET_REDIS_DB"]),
-        )
-
-        # Loop over cache data
-        battery_units = []
-        missing_units = []
-        for key in r.scan_iter("*"):
-            # Parse data
-            raw = r.get(key)
-            data = json.loads(raw)
-
-            try:
-                storage = data["data"]["state"]["storage"]
-            except KeyError:
-                # Skip battery if info cannot be found
-                missing_units.append(
-                    dict(site_node_id=data["siteNodeId"], error_msg=data["data"].get("errors", ""))
-                )
-                continue
-
-            # Convert settings
-            capacity_mwh = storage["energy_capacity_Wh"] / 1e6
-            state_of_charge_mwh = storage["energy_stored_Wh"] / 1e6
-            charge_rate_mw = storage["max_charge_power_W"] / 1e6
-            discharge_rate_mw = storage["max_discharge_power_W"] / 1e6
-
-            # Set (dis)charge rate
-            power_mw = max(charge_rate_mw, -discharge_rate_mw)
-
-            # Calculate capacity:power ratio
-            cp_ratio = capacity_mwh / power_mw
-
-            # Get BESS group data
-            timestamp = data["time"]
-            site_node_id = data["siteNodeId"]
-
-            # Store characteristics
-            battery_units.append(
-                dict(
-                    capacity_mwh=capacity_mwh,
-                    power_mw=power_mw,
-                    state_of_charge_mwh=state_of_charge_mwh,
-                    cp_ratio=cp_ratio,
-                    timestamp=timestamp,
-                    site_node_id=site_node_id,
-                )
-            )
-
-        return battery_units, missing_units