detquantlib 2.0.4__py3-none-any.whl

detquantlib/data/entsoe/entsoe.py

@@ -0,0 +1,174 @@
+# Python built-in packages
+import shutil
+from pathlib import Path
+
+# Third-party packages
+import pandas as pd
+
+# Internal modules
+from detquantlib.data.sftp.sftp import Sftp
+
+
+class Entsoe:
+    """
+    A class that handles ENTSOE data, including importing and processing data from their SFTP
+    server.
+    """
+
+    def __init__(self, sftp: Sftp = None):
+        self.sftp = sftp
+
+    @staticmethod
+    def get_sftp_folder_dir_day_ahead_spot_prices():
+        # Directory of SFTP folder containing day-ahead spot prices.
+        return "/TP_export/EnergyPrices_12.1.D_r3/"
+
+    @staticmethod
+    def get_sftp_filename_day_ahead_spot_prices(year: int, month: int) -> str:
+        """
+        Name of SFTP files containing day-ahead spot prices.
+
+        Args:
+            year: Delivery year
+            month: Delivery month
+
+        Returns:
+            Filename of day-ahead spot prices
+        """
+        year_str = str(year)
+        month_str = str(month) if month >= 10 else f"0{month}"
+        return f"{year_str}_{month_str}_EnergyPrices_12.1.D_r3.csv"
+
+    def import_day_ahead_spot_prices_file_from_sftp(
+        self, year: int, month: int, local_folder_dir: str
+    ):
+        """
+        Downloads the file containing the day-ahead spot prices of a given delivery month from
+        the ENTSOE SFTP server to a local directory.
+
+        Args:
+            year: Delivery year
+            month: Delivery month
+            local_folder_dir: Local directory where file will be copied
+        """
+        filename = Entsoe.get_sftp_filename_day_ahead_spot_prices(year=year, month=month)
+        remote_folder_dir = Entsoe.get_sftp_folder_dir_day_ahead_spot_prices()
+        remote_dir = f"/{remote_folder_dir}/{filename}"
+        local_dir = f"{local_folder_dir}/{filename}"
+
+        self.sftp.open_session()
+        self.sftp.get_file(remote_dir, local_dir)
+        self.sftp.close_session()
+
+    @staticmethod
+    def read_day_ahead_spot_prices_from_file(
+        country: str,
+        timezone: str,
+        year: int,
+        month: int,
+        local_folder_dir: str,
+    ) -> pd.DataFrame:
+        """
+        Reads and processes the content of the file containing day-ahead spot prices for a given
+        delivery month.
+
+        Args:
+            country: Country/area of requested day-ahead spot prices
+            timezone: Timezone of requested day-ahead spot prices
+            year: Delivery year
+            month: Delivery month
+            local_folder_dir: Local directory of folder containing the price data file
+
+        Returns:
+            Dataframe containing day-ahead spot prices
+        """
+        # Read data from csv
+        filename = Entsoe.get_sftp_filename_day_ahead_spot_prices(year=year, month=month)
+        file_dir = f"{local_folder_dir}/{filename}"
+        df = pd.read_csv(file_dir, sep="\t")
+
+        # Filter columns
+        columns = ["DateTime(UTC)", "ResolutionCode", "MapCode", "Price[Currency/MWh]", "Currency"]
+        df = df[columns]
+
+        # Filter rows for relevant country
+        idx = df["MapCode"] == country
+        df = df.loc[idx, :]
+        df.reset_index(drop=True, inplace=True)
+
+        # Convert dates from strings to datetime
+        df["DateTime(UTC)"] = pd.to_datetime(
+            df["DateTime(UTC)"], format="%Y-%m-%d %H:%M:%S", utc=True
+        )
+
+        # Convert from UTC timezone to local timezone
+        df["DateTime(UTC)"] = df["DateTime(UTC)"].dt.tz_convert(timezone)
+        df["DateTime(UTC)"] = df["DateTime(UTC)"].dt.tz_localize(None)
+
+        return df
+
+    def get_day_ahead_spot_prices_from_sftp(
+        self, country: str, timezone: str, year: int, month: int, keep_local_file: bool = False
+    ) -> pd.DataFrame:
+        """
+        Main method to fetch and process day-ahead spot price data from the ENSTOE SFTP server.
+
+        The function performs the following steps:
+        1) Download the file containing day-ahead spot prices from the SFTP to a temporary local
+            directory.
+        2) Read, process and store the price data into a dataframe.
+        3) Delete the temporary local directory containing the downloaded file, unless the user
+            indicates that the file should be kept.
+
+        Args:
+            country: Country/area of requested day-ahead spot prices
+            timezone: Timezone of requested day-ahead spot prices
+            year: Delivery year
+            month: Delivery month
+            keep_local_file: Indicates whether the downloaded data file should be kept or not
+
+        Returns:
+            Dataframe containing day-ahead spot prices
+        """
+        # Define local directory where price data file will be stored
+        input_folder_dir = Path.cwd().joinpath("Inputs")
+        entsoe_data_folder_dir = input_folder_dir.joinpath("EntsoeData")
+
+        # Check if folders already exist
+        input_folder_exists = input_folder_dir.exists()
+        entsoe_data_folder_exists = entsoe_data_folder_dir.exists()
+
+        # Create local directory if it doesn't exist
+        entsoe_data_folder_dir.mkdir(parents=True, exist_ok=True)
+        entsoe_data_folder_dir = str(entsoe_data_folder_dir)
+
+        # Import price data file from ENTSOE SFTP to local directory
+        self.import_day_ahead_spot_prices_file_from_sftp(
+            year=year, month=month, local_folder_dir=entsoe_data_folder_dir
+        )
+
+        # Read price data file
+        df = self.read_day_ahead_spot_prices_from_file(
+            country=country,
+            timezone=timezone,
+            year=year,
+            month=month,
+            local_folder_dir=entsoe_data_folder_dir,
+        )
+
+        # Delete price data file
+        if not keep_local_file:
+            if input_folder_exists and entsoe_data_folder_exists:
+                # Local directory already existed. Only delete the price data file.
+                filename = Entsoe.get_sftp_filename_day_ahead_spot_prices(year=year, month=month)
+                file_dir = Path(entsoe_data_folder_dir).joinpath(filename)
+                file_dir.unlink()
+            elif input_folder_exists:
+                # Only the Inputs folder already existed. Delete the EntsoeData folder and its
+                # content.
+                shutil.rmtree(entsoe_data_folder_dir)
+            else:
+                # The entire temporary local directory did not exist. Delete the entire directory.
+                shutil.rmtree(input_folder_dir)
+
+        return df

detquantlib/data/sftp/sftp.py

@@ -0,0 +1,89 @@
+import paramiko
+
+
+class Sftp:
+    """A class that handles imports and exports of data from and to an SFTP server."""
+
+    def __init__(
+        self,
+        hostname: str,
+        port: int,
+        username: str,
+        password: str = None,
+        private_key_dir: str = None,
+        authentication_type: str = "password",
+        sftp_session: paramiko.SFTPClient = None,
+        transport: paramiko.Transport = None,
+    ):
+        """
+        Constructor method.
+
+        Args:
+            hostname: SFTP server hostname
+            port: SFTP server port
+            username: Username to connect to the SFTP server
+            password: Password to connect to the SFTP server (only needed if
+                authentication_type="password")
+            private_key_dir: Directory of file containing the private key to connect to the SFTP
+                server (only needed if authentication_type="private_key")
+            authentication_type: Can take value "password" or "private_key"
+            sftp_session: SFTP session object
+            transport: Transport object
+
+        Raises:
+            ValueError: Raises an error in case of invalid authentication type
+        """
+        # Input validation
+        valid_authentication_types = ["password", "private_key"]
+        if authentication_type not in valid_authentication_types:
+            raise ValueError(
+                f"Invalid input 'authentication_type' value '{authentication_type}'. "
+                f"Supported values: {valid_authentication_types}."
+            )
+
+        self.hostname = hostname
+        self.port = port
+        self.username = username
+        self.password = password
+        self.private_key_dir = private_key_dir
+        self.authentication_type = authentication_type
+        self.sftp_session = sftp_session
+        self.transport = transport
+
+    def open_session(self):
+        """Opens an SFTP session."""
+        # Establish an SSH client and connect to the server
+        try:
+            self.transport = paramiko.Transport((self.hostname, self.port))
+            self.transport.connect(username=self.username, password=self.password)
+
+            # Open an SFTP session
+            self.sftp_session = paramiko.SFTPClient.from_transport(self.transport)
+
+        except Exception as e:
+            print(f"Error: {e}")
+
+    def close_session(self):
+        """Closes an SFTP session and transport."""
+        self.sftp_session.close()
+        self.transport.close()
+
+    def get_file(self, remote_dir: str, local_dir: str):
+        """
+        Imports a file from an SFTP server to a local directory.
+
+        Args:
+            remote_dir: SFTP server file directory
+            local_dir: Local file directory
+        """
+        self.sftp_session.get(remote_dir, local_dir)
+
+    def put_file(self, local_dir: str, remote_dir: str):
+        """
+        Exports a file from a local directory to an SFTP server.
+
+        Args:
+            local_dir: Local file directory
+            remote_dir: SFTP server file directory
+        """
+        self.sftp_session.put(local_dir, remote_dir)

detquantlib/dates/dates.py

@@ -0,0 +1,154 @@
+# Python built-in packages
+from datetime import datetime
+from typing import Literal
+
+# Third-party packages
+import pandas as pd
+from dateutil.relativedelta import *
+
+
+def count_delivery_periods(
+    start_date: datetime,
+    end_date: datetime,
+    delivery_frequency: str,
+    full_periods_only: bool = False,
+    timezone: str = None,
+) -> int:
+    """
+    Counts the number of delivery periods within a time interval.
+
+    Note: When the end date is exactly equal to the start of the next delivery period, that next
+    period is not included (because it has not yet started). For example, if the end date is
+    15-Jan-2025 00:00:00 and the delivery frequency is daily, then the period
+    15-Jan-2025 00:00:00 to 16-Jan-2025 00:00:00 is not included.
+
+    Args:
+        start_date: Delivery start date
+        end_date: Delivery end date
+        delivery_frequency: Delivery frequency, expressed as Pandas offset aliases
+        full_periods_only: Indicates whether to count only fully elapsed periods.
+            For example, suppose that start date is 15-Jan-2025 00:15:00, end date is
+            3-Feb-2025 03:45:00, and delivery frequency is hourly. Then:
+            - If full_periods_only=True, the number of periods is 2 (01:00:00-02:00:00 and
+                02:00:00-03:00:00), because the hours 00:00:00-01:00:00 and 00:03:00-04:00:00
+                are not full.
+            - If full_periods_only=False, the number of periods is 4, because incomplete hours
+                00:00:00-01:00:00 and 00:03:00-04:00:00 are also included. Note:
+                - If end date is 3-Feb-2025 04:00:00, hour 04:00:00-05:00:00 is not included.
+                - If end date is 3-Feb-2025 04:00:01, hour 04:00:00-05:00:00 is included.
+        timezone: Timezone (needed to account for DST switches)
+
+    Returns:
+        Number of delivery periods in the interval
+    """
+    # Convert to pandas timestamp
+    start_date = pd.Timestamp(start_date)
+    end_date = pd.Timestamp(end_date)
+
+    if full_periods_only:
+        start_date = start_date.ceil(delivery_frequency)
+        end_date = end_date.floor(delivery_frequency)
+    else:
+        start_date = start_date.floor(delivery_frequency)
+
+    periods = pd.date_range(
+        start=start_date,
+        end=end_date,
+        freq=delivery_frequency,
+        inclusive="left",
+        tz=timezone,
+    )
+    nr_periods = len(periods)
+    return nr_periods
+
+
+def calc_months_diff(
+    start_date: datetime,
+    end_date: datetime,
+    diff_method: Literal["month", "time", "full_months_only"] = "month",
+) -> int:
+    """
+    Calculate the month difference between 2 dates.
+
+    Args:
+        start_date: Start date
+        end_date: End date
+        diff_method: Method to count the month difference.
+            - diff_method="month": Counts the month difference based on the months of the start
+                date and end date, irrespective of the day. For example, suppose that start date
+                is 15-Jan-2025 and end date is 3-Mar-2025. Then, the month difference between
+                Jan-2025 and Mar-2025 is 2.
+            - diff_method="time": Counts the month difference, accounting the day and time of the
+                start and end dates. For example:
+                - Suppose that start date is 15-Jan-2025 and end date is 3-Mar-2025. Then:
+                    - start date + 1 months = 15-Feb-2025
+                    - start date + 2 months = 15-Mar-2025
+                    - The month difference is 1, because
+                        [start date + 1 months] <= end date < [start date + 2 months].
+                - Suppose that start date is 15-Jan-2025 and end date is 16-Mar-2025. Then:
+                    - start date + 2 months = 15-Mar-2025
+                    - start date + 3 months = 15-Apr-2025
+                    - The month difference is 2, because
+                        [start date + 2 months] <= end date < [start date + 3 months].
+            - diff_method="full_months_only": Counts the number of fully elapsed months between
+                the start date and the end date. For example, suppose that start date
+                is 15-Jan-2025 and end date is 3-Mar-2025. Then, the month difference is 1,
+                because only Feb-2025 was elapsed from start to finish.
+
+    Returns:
+        Month difference between 2 dates
+
+    Raises:
+        ValueError: Raises an error when end_date < start_date
+        ValueError: Raises an error when the input argument 'diff_method' is invalid
+    """
+    # Input validation
+    if end_date < start_date:
+        raise ValueError("End date cannot be smaller than start date.")
+
+    # Calculate month difference
+    start_mc = datetime_to_month_code(start_date)
+    end_mc = datetime_to_month_code(end_date)
+
+    if diff_method == "month":
+        diff = end_mc - start_mc
+    elif diff_method == "time":
+        # Check month difference, accounting for day and time. For example, suppose that start
+        # date is 10-Jan-2025 14:00:00. Then:
+        # - If end date is 10-Mar-2025 14:00:00, month difference is 2.
+        # - If end date is 10-Mar-2025 13:55:00, month difference is 1.
+        start_day = start_date + relativedelta(year=2000, month=1)
+        end_day = end_date + relativedelta(year=2000, month=1)
+        diff = end_mc - start_mc
+        if end_day < start_day:
+            diff -= 1
+    elif diff_method == "full_months_only":
+        # Calculate number of fully elapsed months
+        # Note: We need to use the max operator, otherwise diff = -1 when start month = end month.
+        if start_date > datetime(start_date.year, start_date.month, 1):
+            start_mc += 1
+        diff = max(end_mc - start_mc, 0)
+    else:
+        raise ValueError("Invalid value of input argument 'diff_method'.")
+
+    return diff
+
+
+def datetime_to_month_code(d: datetime) -> int:
+    """
+    Converts a datetime to its corresponding month code. A month code is calculated as the number
+    of months since 1 January 1900 (including the month of the input datetime).
+
+    Args:
+        d: Datetime
+
+    Returns:
+        Corresponding month code
+
+    Raises:
+        ValueError: Raises an error if the input datetime is before 1 January 1900
+    """
+    if d < datetime(1900, 1, 1):
+        raise ValueError("Input date cannot be before 1 January 1900.")
+    month_code = (d.year - 1900) * 12 + d.month
+    return month_code

detquantlib/figures/plotly_figures.py

@@ -0,0 +1,82 @@
+# Python built-in packages
+from pathlib import Path
+
+# Third-party packages
+import plotly.graph_objects as go
+import plotly.io as pio
+
+
+def get_default_plotly_folder_dir():
+    # Default path to output folder containing plotly figures.
+    return Path.cwd().joinpath("Outputs", "PlotlyFigures")
+
+
+def set_standard_layout(fig: go.Figure) -> go.Figure:
+    """
+    Short helper function to improve and standardize the layout of plotly figures.
+
+    Args:
+        fig: Plotly figure
+
+    Returns:
+        Plotly figure with updated layout options
+    """
+    fig.update_layout(
+        template="plotly_white",
+        title_x=0.5,
+        xaxis=dict(showline=True, linecolor="black", showgrid=False, ticks="outside"),
+        yaxis=dict(showline=True, linecolor="black", showgrid=True, ticks="outside"),
+        hovermode="x",
+    )
+    return fig
+
+
+def save_plotly_fig_to_json(
+    fig: go.Figure,
+    filename: str,
+    folder_dir: Path = get_default_plotly_folder_dir(),
+    standard_layout: bool = True,
+):
+    """
+    Saves a plotly figure object to a json file.
+
+    Args:
+        fig: Plotly figure
+        filename: Name of json file in which plotly figure will be saved
+        folder_dir: Folder directory where the json file will be saved
+        standard_layout: If true, enforces a standard plotly layout
+    """
+    # Create plotly folder if it doesn't exist yet
+    folder_dir.mkdir(parents=True, exist_ok=True)
+
+    # Define json file directory
+    file_dir = folder_dir.joinpath(f"{filename}.json")
+
+    # Adjust figure layout
+    if standard_layout:
+        fig = set_standard_layout(fig)
+
+    # Save figure to json
+    fig_json = fig.to_json()
+    with open(file_dir, "w") as f:
+        f.write(fig_json)
+
+
+def show_plotly_fig_json(filename: str, folder_dir: Path = get_default_plotly_folder_dir()):
+    """
+    Short helper function to display a plotly figure stored in a json file.
+
+    Args:
+        filename: Name of json file containing the plotly figure
+        folder_dir: Folder directory containing the json file
+    """
+    # Get json file directory
+    file_dir = folder_dir.joinpath(f"{filename}.json")
+
+    # Load figure from json
+    with open(file_dir, "r") as f:
+        fig_json = f.read()
+    fig = pio.from_json(fig_json)
+
+    # Display figure
+    fig.show()

detquantlib/stats/data_analysis.py

@@ -0,0 +1,143 @@
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+
+from detquantlib.figures.plotly_figures import set_standard_layout
+
+
+def filter_outliers(
+    df: pd.DataFrame,
+    column: str,
+    stats_radius: int,
+    std_dev_excl_factor: int = 3,
+    max_iter: int = 10,
+    plot_series_at_every_iter: bool = False,
+) -> pd.DataFrame:
+    """
+    Filters out outliers from a given time series.
+
+    Args:
+        df: Times series dataframe
+        column: Name of the column containing outliers to be filtered out
+        stats_radius: For an observation at a given time t, the mean and the standard deviation
+            will be computed over the interval [t - stats_radius, t + stats_radius]
+        std_dev_excl_factor: An observation at a given time t will be flagged as an outlier if it
+            is outside the interval
+                [mean(t) - std(t) * std_dev_excl_factor, mean(t) + std(t) * std_dev_excl_factor]
+        max_iter: Outliers affect the mean and standard deviation of neighbouring observations,
+            such that smaller neighbouring outliers may go undetected if we perform the filtering
+            process only once. Hence, the operation needs to be repeated multiple times. Parameter
+            'max_iter' determines the maximum number of iterations of the filtering processes.
+        plot_series_at_every_iter: If plot_series=True, the time series will be plotted at
+            every iteration. This parameter can be used to assess the effectiveness of the
+            other input parameters in the outliers filtering process.
+
+    Returns:
+        df: Time series dataframe with outliers removed
+    """
+    # Initialize number of outliers found in time series (filtering will stop when this reaches 0)
+    nr_outliers = 1
+
+    # Iteratively remove outliers
+    for count in range(max_iter):
+        if nr_outliers > 0:
+            # Count number of rows in df
+            nr_rows = df.shape[0]
+
+            # Calculate rolling mean and standard deviation per observation
+            for i, loc in enumerate(df.index):
+                # Determine time interval
+                min_i = max(i - stats_radius, 0)
+                max_i = min(i + stats_radius, nr_rows)
+
+                # Calculate mean and standard deviation around current observation
+                idx = df.index[min_i:max_i]
+                df.loc[loc, "mean"] = np.mean(df.loc[idx, column])
+                df.loc[loc, "std_dev"] = np.std(df.loc[idx, column], ddof=1)
+
+            # Identify lower and upper bounds
+            lower_bound = df["mean"] - df["std_dev"] * std_dev_excl_factor
+            upper_bound = df["mean"] + df["std_dev"] * std_dev_excl_factor
+
+            # Plot time series
+            if plot_series_at_every_iter:
+                create_outliers_iteration_plot(
+                    x_values=df.index,
+                    data=df[column],
+                    lower_bound=lower_bound,
+                    upper_bound=upper_bound,
+                    y_label=column,
+                    title=f"Outliers Filtering (Iteration {count+1}/{max_iter})",
+                )
+
+            # Exclude outliers, i.e. observations outside std dev boundaries
+            idx_keep = (df[column] >= lower_bound) & (df[column] <= upper_bound)
+            df = df.loc[idx_keep, :]
+
+            # Count number of outliers found in time series
+            nr_outliers = np.sum(np.invert(idx_keep))
+
+            if nr_outliers == 0:
+                print(f"No (more) outliers detected (iteration {count+1} out of max {max_iter}).")
+            else:
+                print(
+                    f"Outliers filtering excluded {nr_outliers}/{idx_keep.shape[0]}"
+                    f" observations (iteration {count+1} out of max {max_iter})."
+                )
+
+    # Plot time series
+    if plot_series_at_every_iter:
+        create_outliers_iteration_plot(
+            x_values=df.index,
+            data=df[column],
+            lower_bound=lower_bound,
+            upper_bound=upper_bound,
+            y_label=column,
+            title="Outliers Filtering (Final Data)",
+        )
+
+    # Dataframe post-processing
+    df.reset_index(drop=True, inplace=True)
+    df.drop(["mean", "std_dev"], axis=1, inplace=True)
+
+    return df
+
+
+def create_outliers_iteration_plot(
+    x_values: pd.Series,
+    data: pd.Series,
+    lower_bound: pd.Series,
+    upper_bound: pd.Series,
+    y_label: str,
+    title: str,
+):
+    """
+    Short utility function used in the 'filter_outliers()' function to plot outliers at every
+    iteration.
+
+    Args:
+        x_values: X-axis values
+        data: Time series data
+        lower_bound: Time series lower bound to identify outliers
+        upper_bound: Time series upper bound to identify outliers
+        y_label: Y-axis label
+        title: Plot title
+    """
+    # Define plot line colors
+    data_color = "blue"
+    data_line = dict(color=data_color)
+    bounds_color = "#ED595C"
+    bounds_line = dict(color=bounds_color)
+
+    # Create plot
+    fig = go.Figure()
+    fig.add_trace(
+        go.Scatter(x=x_values, y=upper_bound, mode="lines", name="Upper bound", line=bounds_line)
+    )
+    fig.add_trace(go.Scatter(x=x_values, y=data, mode="lines", name="Data", line=data_line))
+    fig.add_trace(
+        go.Scatter(x=x_values, y=lower_bound, mode="lines", name="Lower bound", line=bounds_line)
+    )
+    fig.update_layout(yaxis_title=y_label, title=title)
+    fig = set_standard_layout(fig)
+    fig.show()