detquantlib 3.8.1__tar.gz → 3.10.0__tar.gz

{detquantlib-3.8.1 → detquantlib-3.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: detquantlib
-Version: 3.8.1
+Version: 3.10.0
 Summary: An internal library containing functions and classes that can be used across Quant models.
 License-File: LICENSE.txt
 Author: DET
@@ -15,9 +15,9 @@ Requires-Dist: numpy (>=2.2.6,<3.0.0)
 Requires-Dist: pandas (>=2.3.3,<3.0.0)
 Requires-Dist: paramiko (>=4.0.0,<5.0.0)
 Requires-Dist: plotly (>=6.3.1,<7.0.0)
-Requires-Dist: pyodbc (>=5.2.0,<6.0.0)
 Requires-Dist: python-dotenv (>=1.1.0,<2.0.0)
 Requires-Dist: scipy (>=1.15.2,<2.0.0)
+Requires-Dist: sqlalchemy (>=2.0.43,<3.0.0)
 Project-URL: Repository, https://github.com/Dynamic-Energy-Trading/detquantlib
 Description-Content-Type: text/markdown
 

{detquantlib-3.8.1 → detquantlib-3.10.0}/detquantlib/data/databases/detdatabase.py

@@ -1,13 +1,13 @@
 # Python built-in packages
 import os
-import warnings
 from datetime import datetime
+from urllib.parse import quote_plus
 from zoneinfo import ZoneInfo
 
 # Third-party packages
 import pandas as pd
-import pyodbc
 from dateutil.relativedelta import *
+from sqlalchemy import Engine, create_engine, text
 
 
 class DetDatabase:
@@ -15,36 +15,34 @@ class DetDatabase:
     A class to easily interact with the DET database, including fetching and processing data.
     """
 
-    def __init__(
-        self,
-        connection: pyodbc.Connection = None,
-        driver: str = "{ODBC Driver 18 for SQL Server}",
-    ):
+    def __init__(self, engine: Engine = None, driver: str = "ODBC Driver 18 for SQL Server"):
         """
         Constructor method.
 
         Args:
-            connection: Database connection object. This argument does not have to be passed
-                when creating the object. It can be set after the object has been created, using
-                the open_connection() method.
+            engine: Instance of SQLAlchemy engine. If None, the instance is automatically
+                initialized.
             driver: ODBC driver
+        """
+        # Only allow object creation if mandatory environment variables exist
+        DetDatabase.check_environment_variables()
+
+        self.driver = driver
+        self.engine = engine if engine is not None else self.initialize_engine()
+
+    @staticmethod
+    def check_environment_variables():
+        """
+        Checks if environment variables needed by the class are defined.
 
         Raises:
             EnvironmentError: Raises an error if environment variables are not defined
         """
-        self.connection = connection
-        self.driver = driver
-
-        # Check if environment variables needed by the class are defined
         required_env_vars = [
-            dict(name="DET_DB_NAME", value=None, description="DET database name"),
-            dict(name="DET_DB_SERVER", value=None, description="DET database server name"),
-            dict(
-                name="DET_DB_USERNAME", value=None, description="Username to connect to database"
-            ),
-            dict(
-                name="DET_DB_PASSWORD", value=None, description="Password to connect to database"
-            ),
+            dict(name="DET_DB_NAME", value=None, description="Database name"),
+            dict(name="DET_DB_SERVER", value=None, description="Server name"),
+            dict(name="DET_DB_USERNAME", value=None, description="Username"),
+            dict(name="DET_DB_PASSWORD", value=None, description="Password"),
         ]
         available_env_vars = os.environ
         for d in required_env_vars:
@@ -57,21 +55,28 @@ class DetDatabase:
                     f"(description: '{d['description']}') not found."
                 )
 
-    def open_connection(self):
-        """Opens a connection to the database."""
+    def initialize_engine(self) -> Engine:
+        """
+        Initializes the SQLAlchemy engine.
+
+        Returns:
+            Instance of SQLAlchemy engine
+        """
         # Create the connection string
         connection_str = (
             f"DRIVER={self.driver};"
             f"SERVER={os.getenv('DET_DB_SERVER')};"
             f"DATABASE={os.getenv('DET_DB_NAME')};"
             f"UID={os.getenv('DET_DB_USERNAME')};"
-            f"PWD={os.getenv('DET_DB_PASSWORD')}"
+            f"PWD={quote_plus(os.getenv('DET_DB_PASSWORD'))}"
         )
-        self.connection = pyodbc.connect(connection_str)
+        url = quote_plus(connection_str)
+        engine = create_engine(f"mssql+pyodbc:///?odbc_connect={url}")
+        return engine
 
-    def close_connection(self):
-        """Closes the connection to the database."""
-        self.connection.close()
+    def terminate_engine(self):
+        """Disconnects the SQLAlchemy engine and releases all underlying resources."""
+        self.engine.dispose()
 
     def query_db(self, query: str) -> pd.DataFrame:
         """
@@ -86,20 +91,34 @@ class DetDatabase:
         Raises:
             Exception: Raises an error if the SQL query fails
         """
-        with warnings.catch_warnings():
-            # Pandas UserWarning returned when using pandas with pyodbc. Disable warning
-            # temporarily for the SQL query.
-            warnings.simplefilter("ignore", category=UserWarning)
-
-            try:
-                df = pd.read_sql(query, self.connection)
-            except Exception as e:
-                # If query fails, close connection before raising the error
-                self.close_connection()
-                raise
+        try:
+            df = pd.read_sql_query(query, con=self.engine)
+        except Exception as e:
+            # If query fails, close connection before raising the error
+            self.terminate_engine()
+            raise
 
         return df
 
+    @staticmethod
+    def get_table_name(def_key: str) -> str:
+        """
+        Gets the name of a database table, according to the following logic:
+            1) First, try to get the table name from the corresponding environment variable.
+            2) If the environment variable is not defined, revert to the default table name.
+
+        Args:
+            def_key: Definitions dictionary key corresponding to the desired table
+
+        Returns:
+            Database table name
+        """
+        definition = DetDatabaseDefinitions.DEFINITIONS[def_key]
+        table = os.getenv(definition["env_variable"])
+        if table is None:
+            table = definition["default_table_name"]
+        return table
+
     def load_entsoe_day_ahead_spot_prices(
         self,
         commodity_name: str,
@@ -210,19 +229,15 @@ class DetDatabase:
         end_delivery_date = end_delivery_date.astimezone(ZoneInfo("UTC"))
         end_date_str = end_delivery_date.strftime("%Y-%m-%d %H:%M:%S")
 
-        # Create query
-        table = DetDatabaseDefinitions.DEFINITIONS["table_name_entsoe_day_ahead_spot_price"]
+        # Query db
+        table = DetDatabase.get_table_name("entsoe_day_ahead_spot_price")
         query = (
             f"SELECT {columns_str} FROM {table} "
             f"WHERE MapCode='{map_code}' "
             f"AND [DateTime(UTC)]>='{start_date_str}' "
             f"AND [DateTime(UTC)]<'{end_date_str}' "
         )
-
-        # Query db
-        self.open_connection()
         df = self.query_db(query)
-        self.close_connection()
 
         if df.empty:
             raise ValueError("No price data found for user-defined inputs.")
@@ -409,19 +424,15 @@ class DetDatabase:
         end_delivery_date = end_delivery_date.astimezone(ZoneInfo("UTC"))
         end_date_str = end_delivery_date.strftime("%Y-%m-%d %H:%M:%S")
 
-        # Create query
-        table = DetDatabaseDefinitions.DEFINITIONS["table_name_entsoe_imbalance_price"]
+        # Query db
+        table = DetDatabase.get_table_name("entsoe_imbalance_price")
         query = (
             f"SELECT {columns_str} FROM {table} "
             f"WHERE MapCode='{map_code}' "
             f"AND [DateTime(UTC)]>='{start_date_str}' "
             f"AND [DateTime(UTC)]<'{end_date_str}' "
         )
-
-        # Query db
-        self.open_connection()
         df = self.query_db(query)
-        self.close_connection()
 
         if df.empty:
             raise ValueError("No price data found for user-defined inputs.")
@@ -542,8 +553,8 @@ class DetDatabase:
         start_trading_date_str = start_trading_date.strftime("%Y-%m-%d")
         end_trading_date_str = end_trading_date.strftime("%Y-%m-%d")
 
-        # Create query
-        table = DetDatabaseDefinitions.DEFINITIONS["table_name_futures_eod_settlement_price"]
+        # Query db
+        table = DetDatabase.get_table_name("futures_tt")
         query = (
             f"SELECT {columns_str} FROM {table} "
             f"WHERE CommodityName='{commodity_name}' "
@@ -552,11 +563,7 @@ class DetDatabase:
             f"AND Tenor IN {tenors_str} "
             f"AND DeliveryType='{delivery_type}'"
         )
-
-        # Query db
-        self.open_connection()
         df = self.query_db(query)
-        self.close_connection()
 
         if df.empty:
             raise ValueError("No price data found for user-defined inputs.")
@@ -614,14 +621,10 @@ class DetDatabase:
         else:
             columns_str = f"[{'], ['.join(columns)}]"
 
-        # Create query
-        table = DetDatabaseDefinitions.DEFINITIONS["table_name_commodity"]
-        query = f"SELECT {columns_str} FROM {table} {conditions}"
-
         # Query db
-        self.open_connection()
+        table = DetDatabase.get_table_name("commodity")
+        query = f"SELECT {columns_str} FROM {table} {conditions}"
         df = self.query_db(query)
-        self.close_connection()
 
         return df
 
@@ -697,18 +700,14 @@ class DetDatabase:
         start_trading_date_str = start_trading_date.strftime("%Y-%m-%d")
         end_trading_date_str = end_trading_date.strftime("%Y-%m-%d")
 
-        # Create query
-        table = DetDatabaseDefinitions.DEFINITIONS["table_name_account_position"]
+        # Query db
+        table = DetDatabase.get_table_name("account_position")
         query = (
             f"SELECT {columns_str} FROM {table} "
             f"WHERE InsertionTimestamp>='{start_trading_date_str}' "
             f"AND InsertionTimestamp<'{end_trading_date_str}'"
         )
-
-        # Query db
-        self.open_connection()
         df = self.query_db(query)
-        self.close_connection()
 
         # Assert data
         if df.empty:
@@ -755,14 +754,10 @@ class DetDatabase:
         # Convert tenors from list to string
         identifiers_str = ", ".join(f"'{i}'" for i in identifiers)
 
-        # Create query
-        table = DetDatabaseDefinitions.DEFINITIONS["table_name_instruments"]
-        query = f"SELECT {columns_str} FROM {table} WHERE [id] IN ({identifiers_str})"
-
         # Query db
-        self.open_connection()
+        table = DetDatabase.get_table_name("instrument")
+        query = f"SELECT {columns_str} FROM {table} WHERE [id] IN ({identifiers_str})"
         df = self.query_db(query)
-        self.close_connection()
 
         # Assert data
         if df.empty:
@@ -816,19 +811,15 @@ class DetDatabase:
         start_trading_date_str = start_trading_date.strftime("%Y-%m-%d")
         end_trading_date_str = end_trading_date.strftime("%Y-%m-%d")
 
-        # Create query
-        table = DetDatabaseDefinitions.DEFINITIONS["table_name_eex_eod_price"]
+        # Query db
+        table = DetDatabase.get_table_name("futures_eex")
         query = (
             f"SELECT {columns_str} FROM {table} "
             f"WHERE Product LIKE '{product_code}' "
             f"AND TradingDate>='{start_trading_date_str}' "
             f"AND TradingDate<='{end_trading_date_str}'"
         )
-
-        # Query db
-        self.open_connection()
         df = self.query_db(query)
-        self.close_connection()
 
         # Assert data
         if df.empty:
@@ -925,8 +916,8 @@ class DetDatabase:
         # Convert dates from datetime to string
         forecast_date_str = forecast_date.strftime("%Y-%m-%d")
 
-        # Create query
-        table = DetDatabaseDefinitions.DEFINITIONS["table_name_forecast_customer_volume"]
+        # Query db
+        table = DetDatabase.get_table_name("client_volume_forecast")
         query = (
             f"SELECT {columns_str} FROM {table} "
             f"WHERE Profile='{profile}' "
@@ -934,11 +925,7 @@ class DetDatabase:
             f"AND Datetime>='{start_date_str}' "
             f"AND Datetime<'{end_date_str}'"
         )
-
-        # Query db
-        self.open_connection()
         df = self.query_db(query)
-        self.close_connection()
 
         # Assert data
         if df.empty:
@@ -1030,19 +1017,15 @@ class DetDatabase:
         else:
             columns_str = f"[{'], ['.join(columns)}]"
 
-        # Create query
-        table = DetDatabaseDefinitions.DEFINITIONS["table_name_customer_day_ahead_auction_bids"]
+        # Query db
+        table = DetDatabase.get_table_name("client_day_ahead_auction_bids")
         query = (
             f"SELECT {columns_str} FROM {table} "
             f"WHERE ClientId='{client_id}' "
             f"AND DeliveryStart>='{start_date_str}' "
             f"AND DeliveryStart<'{end_date_str}'"
         )
-
-        # Query db
-        self.open_connection()
         df = self.query_db(query)
-        self.close_connection()
 
         # Assert data
         if df.empty:
@@ -1101,14 +1084,10 @@ class DetDatabase:
         else:
             columns_str = f"[{'], ['.join(columns)}]"
 
-        # Create query
-        table = DetDatabaseDefinitions.DEFINITIONS["table_name_client"]
-        query = f"SELECT {columns_str} FROM {table} {conditions}"
-
         # Query db
-        self.open_connection()
+        table = DetDatabase.get_table_name("client")
+        query = f"SELECT {columns_str} FROM {table} {conditions}"
         df = self.query_db(query)
-        self.close_connection()
 
         return df
 
@@ -1149,19 +1128,72 @@ class DetDatabase:
 
         return client_info
 
+    def add_table(self, df: pd.DataFrame, table_name: str, schema: str):
+        """
+        Creates a new database table and populates it with the user-input data.
+
+        Args:
+            df: Data used to populate the database table
+            table_name: Table name
+            schema: Schema
+        """
+        df.to_sql(name=table_name, schema=schema, con=self.engine, if_exists="fail", index=False)
+
+    def remove_table(self, table_name: str, schema: str):
+        """
+        Deletes a database table.
+
+        Args:
+            table_name: Table name
+            schema: Schema
+        """
+        with self.engine.connect() as conn:
+            conn.execute(text(f"DROP TABLE IF EXISTS [{schema}].[{table_name}]"))
+            conn.commit()
+
 
 class DetDatabaseDefinitions:
     """A class containing some hard-coded definitions related to the DET database."""
 
     DEFINITIONS = dict(
-        table_name_commodity="[META].[Commodity]",
-        table_name_entsoe_day_ahead_spot_price="[ENTSOE].[DayAheadSpotPrice]",
-        table_name_entsoe_imbalance_price="[ENTSOE].[ImbalancePrice]",
-        table_name_futures_eod_settlement_price="[VW].[EODSettlementPrice]",
-        table_name_account_position="[TT].[AccountPosition]",
-        table_name_instruments="[TT].[Instrument]",
-        table_name_eex_eod_price="[EEX].[EODPrice]",
-        table_name_forecast_customer_volume="[DISP].[ForecastGold]",
-        table_name_customer_day_ahead_auction_bids="[TRADE].[ClientBid]",
-        table_name_client="[META].[Client]",
+        commodity=dict(
+            env_variable="DET_DB_TABLE_COMMODITY",
+            default_table_name="[META].[Commodity]",
+        ),
+        entsoe_day_ahead_spot_price=dict(
+            env_variable="DET_DB_TABLE_ENTSOE_DA",
+            default_table_name="[ENTSOE].[DayAheadSpotPrice]",
+        ),
+        entsoe_imbalance_price=dict(
+            env_variable="DET_DB_TABLE_ENTSOE_IB",
+            default_table_name="[ENTSOE].[ImbalancePrice]",
+        ),
+        futures_tt=dict(
+            env_variable="DET_DB_TABLE_FUTURES_TT",
+            default_table_name="[VW].[EODSettlementPrice]",
+        ),
+        futures_eex=dict(
+            env_variable="DET_DB_TABLE_FUTURES_EEX",
+            default_table_name="[EEX].[EODPrice]",
+        ),
+        account_position=dict(
+            env_variable="DET_DB_TABLE_ACCOUNT_POSITION",
+            default_table_name="[TT].[AccountPosition]",
+        ),
+        instrument=dict(
+            env_variable="DET_DB_TABLE_INSTRUMENT",
+            default_table_name="[TT].[Instrument]",
+        ),
+        client_volume_forecast=dict(
+            env_variable="DET_DB_TABLE_CLIENT_VOLUME_FORECAST",
+            default_table_name="[DISP].[ForecastGold]",
+        ),
+        client_day_ahead_auction_bids=dict(
+            env_variable="DET_DB_TABLE_CLIENT_DA_AUCTION_BIDS",
+            default_table_name="[TRADE].[ClientBid]",
+        ),
+        client=dict(
+            env_variable="DET_DB_TABLE_CLIENT",
+            default_table_name="[META].[Client]",
+        ),
     )

{detquantlib-3.8.1 → detquantlib-3.10.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "detquantlib"
-version = "3.8.1"
+version = "3.10.0"
 description = "An internal library containing functions and classes that can be used across Quant models."
 authors = ["DET"]
 readme = "README.md"
@@ -15,9 +15,9 @@ paramiko = "^4.0.0"
 numpy = "^2.2.6"
 pandas = "^2.3.3"
 plotly = "^6.3.1"
-pyodbc = "^5.2.0"
 python-dotenv = "^1.1.0"
 scipy = "^1.15.2"
+sqlalchemy = "^2.0.43"
 
 [tool.poetry.group.dev.dependencies]
 toml = "^0.10.2"