openstef 3.4.29__py3-none-any.whl → 3.4.44__py3-none-any.whl

openstef/data/dutch_holidays.csv.license

@@ -0,0 +1,3 @@
+SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com>
+
+SPDX-License-Identifier: MPL-2.0

openstef/data_classes/prediction_job.py

@@ -9,7 +9,7 @@ from pydantic.v1 import BaseModel
 from openstef.data_classes.data_prep import DataPrepDataClass
 from openstef.data_classes.model_specifications import ModelSpecificationDataClass
 from openstef.data_classes.split_function import SplitFuncDataClass
-from openstef.enums import PipelineType
+from openstef.enums import PipelineType, BiddingZone
 
 
 class PredictionJobDataClass(BaseModel):
@@ -54,6 +54,8 @@ class PredictionJobDataClass(BaseModel):
     lon: Optional[float] = 5.291266
     """Longitude of the forecasted location in degrees. Used for fetching weather data in tasks, calculating derrived features and component splitting."""
     name: str
+    """Bidding zone is used to determine the electricity price. It is also used to determine the holidays that should be used. Currently only ENTSO-E bidding zones are supported."""
+    electricity_bidding_zone: Optional[BiddingZone] = BiddingZone.NL
     """Name of the forecast, e.g. the location name."""
     train_components: Optional[bool]
     """Whether splitting the forecasts in wind, solar, rest is desired."""

openstef/enums.py

@@ -4,8 +4,111 @@
 from enum import Enum
 
 
-# TODO replace this with ModelType (MLModelType == Machine Learning model type)
-class MLModelType(Enum):
+class BiddingZone(Enum):
+    DE_50HZ = "DE_50HZ"
+    AL = "AL"
+    DE_AMPRION = "DE_AMPRION"
+    AT = "AT"
+    BY = "BY"
+    BE = "BE"
+    BA = "BA"
+    BG = "BG"
+    CZ_DE_SK = "CZ_DE_SK"
+    HR = "HR"
+    CWE = "CWE"
+    CY = "CY"
+    CZ = "CZ"
+    DE_AT_LU = "DE_AT_LU"
+    DE_LU = "DE_LU"
+    DK = "DK"
+    DK_1 = "DK_1"
+    DK_1_NO_1 = "DK_1_NO_1"
+    DK_2 = "DK_2"
+    DK_CA = "DK_CA"
+    EE = "EE"
+    FI = "FI"
+    MK = "MK"
+    FR = "FR"
+    DE = "DE"
+    GR = "GR"
+    HU = "HU"
+    IS = "IS"
+    IE_SEM = "IE_SEM"
+    IE = "IE"
+    IT = "IT"
+    IT_SACO_AC = "IT_SACO_AC"
+    IT_CALA = "IT_CALA"
+    IT_SACO_DC = "IT_SACO_DC"
+    IT_BRNN = "IT_BRNN"
+    IT_CNOR = "IT_CNOR"
+    IT_CSUD = "IT_CSUD"
+    IT_FOGN = "IT_FOGN"
+    IT_GR = "IT_GR"
+    IT_MACRO_NORTH = "IT_MACRO_NORTH"
+    IT_MACRO_SOUTH = "IT_MACRO_SOUTH"
+    IT_MALTA = "IT_MALTA"
+    IT_NORD = "IT_NORD"
+    IT_NORD_AT = "IT_NORD_AT"
+    IT_NORD_CH = "IT_NORD_CH"
+    IT_NORD_FR = "IT_NORD_FR"
+    IT_NORD_SI = "IT_NORD_SI"
+    IT_PRGP = "IT_PRGP"
+    IT_ROSN = "IT_ROSN"
+    IT_SARD = "IT_SARD"
+    IT_SICI = "IT_SICI"
+    IT_SUD = "IT_SUD"
+    RU_KGD = "RU_KGD"
+    LV = "LV"
+    LT = "LT"
+    LU = "LU"
+    LU_BZN = "LU_BZN"
+    MT = "MT"
+    ME = "ME"
+    GB = "GB"
+    GE = "GE"
+    GB_IFA = "GB_IFA"
+    GB_IFA2 = "GB_IFA2"
+    GB_ELECLINK = "GB_ELECLINK"
+    UK = "UK"
+    NL = "NL"
+    NO_1 = "NO_1"
+    NO_1A = "NO_1A"
+    NO_2 = "NO_2"
+    NO_2_NSL = "NO_2_NSL"
+    NO_2A = "NO_2A"
+    NO_3 = "NO_3"
+    NO_4 = "NO_4"
+    NO_5 = "NO_5"
+    NO = "NO"
+    PL_CZ = "PL_CZ"
+    PL = "PL"
+    PT = "PT"
+    MD = "MD"
+    RO = "RO"
+    RU = "RU"
+    SE_1 = "SE_1"
+    SE_2 = "SE_2"
+    SE_3 = "SE_3"
+    SE_4 = "SE_4"
+    RS = "RS"
+    SK = "SK"
+    SI = "SI"
+    GB_NIR = "GB_NIR"
+    ES = "ES"
+    SE = "SE"
+    CH = "CH"
+    DE_TENNET = "DE_TENNET"
+    DE_TRANSNET = "DE_TRANSNET"
+    TR = "TR"
+    UA = "UA"
+    UA_DOBTPP = "UA_DOBTPP"
+    UA_BEI = "UA_BEI"
+    UA_IPS = "UA_IPS"
+    XK = "XK"
+    DE_AMP_LU = "DE_AMP_LU"
+
+
+class ModelType(Enum):
     XGB = "xgb"
     XGB_QUANTILE = "xgb_quantile"
     XGB_MULTIOUTPUT_QUANTILE = "xgb_multioutput_quantile"

openstef/feature_engineering/apply_features.py

@@ -14,16 +14,25 @@ Examples of features that are added:
 import pandas as pd
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
+from openstef.enums import BiddingZone
 from openstef.feature_engineering.holiday_features import (
     generate_holiday_feature_functions,
 )
 from openstef.feature_engineering.lag_features import generate_lag_feature_functions
+from openstef.feature_engineering.bidding_zone_to_country_mapping import (
+    BIDDING_ZONE_TO_COUNTRY_CODE_MAPPING,
+)
 from openstef.feature_engineering.weather_features import (
     add_additional_solar_features,
     add_additional_wind_features,
     add_humidity_features,
 )
 
+from openstef.feature_engineering.cyclic_features import (
+    add_seasonal_cyclic_features,
+    add_time_cyclic_features,
+)
+
 
 def apply_features(
     data: pd.DataFrame,
@@ -58,6 +67,7 @@ def apply_features(
 
         import pandas as pd
         import numpy as np
+        from geopy.geocoders import Nominatim
         index = pd.date_range(start = "2017-01-01 09:00:00",
         freq = '15T', periods = 200)
         data = pd.DataFrame(index = index,
@@ -66,6 +76,9 @@ def apply_features(
                             np.random.uniform(0.7,1.7, 200)))
 
     """
+    if pj is None:
+        pj = {"electricity_bidding_zone": BiddingZone.NL}
+
     # Get lag feature functions
     feature_functions = generate_lag_feature_functions(feature_names, horizon)
 
@@ -80,8 +93,14 @@ def apply_features(
         }
     )
 
+    # Get country code from bidding zone if available
+    electricity_bidding_zone = pj.get("electricity_bidding_zone", BiddingZone.NL)
+    country_code = BIDDING_ZONE_TO_COUNTRY_CODE_MAPPING[electricity_bidding_zone.name]
+
     # Get holiday feature functions
-    feature_functions.update(generate_holiday_feature_functions())
+    feature_functions.update(
+        generate_holiday_feature_functions(country_code=country_code)
+    )
 
     # Add the features to the dataframe using previously defined feature functions
     for key, featfunc in feature_functions.items():
@@ -99,5 +118,11 @@ def apply_features(
     # Add solar features; when pj is unavailable a default location is used.
     data = add_additional_solar_features(data, pj, feature_names)
 
+    # Adds cyclical features to capture seasonal and periodic patterns in time-based data.
+    data = add_seasonal_cyclic_features(data)
+
+    # Adds polar time features (sine and cosine) to capture periodic patterns based on the timestamp index.
+    data = add_time_cyclic_features(data)
+
     # Return dataframe including all requested features
     return data

openstef/feature_engineering/bidding_zone_to_country_mapping.py

@@ -0,0 +1,106 @@
+# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
+#
+# SPDX-License-Identifier: MPL-2.0
+BIDDING_ZONE_TO_COUNTRY_CODE_MAPPING = {
+    "DE_50HZ": "DE",
+    "AL": "AL",
+    "DE_AMPRION": "DE",
+    "AT": "AT",
+    "BY": "BY",
+    "BE": "BE",
+    "BA": "BA",
+    "BG": "BG",
+    "CZ_DE_SK": "CZ",
+    "HR": "HR",
+    "CWE": "CWE",
+    "CY": "CY",
+    "CZ": "CZ",
+    "DE_AT_LU": "DE",
+    "DE_LU": "DE",
+    "DK": "DK",
+    "DK_1": "DK",
+    "DK_1_NO_1": "DK",
+    "DK_2": "DK",
+    "DK_CA": "DK",
+    "EE": "EE",
+    "FI": "FI",
+    "MK": "MK",
+    "FR": "FR",
+    "DE": "DE",
+    "GR": "GR",
+    "HU": "HU",
+    "IS": "IS",
+    "IE_SEM": "IE",
+    "IE": "IE",
+    "IT": "IT",
+    "IT_SACO_AC": "IT",
+    "IT_CALA": "IT",
+    "IT_SACO_DC": "IT",
+    "IT_BRNN": "IT",
+    "IT_CNOR": "IT",
+    "IT_CSUD": "IT",
+    "IT_FOGN": "IT",
+    "IT_GR": "IT",
+    "IT_MACRO_NORTH": "IT",
+    "IT_MACRO_SOUTH": "IT",
+    "IT_MALTA": "IT",
+    "IT_NORD": "IT",
+    "IT_NORD_AT": "IT",
+    "IT_NORD_CH": "IT",
+    "IT_NORD_FR": "IT",
+    "IT_NORD_SI": "IT",
+    "IT_PRGP": "IT",
+    "IT_ROSN": "IT",
+    "IT_SARD": "IT",
+    "IT_SICI": "IT",
+    "IT_SUD": "IT",
+    "RU_KGD": "RU",
+    "LV": "LV",
+    "LT": "LT",
+    "LU": "LU",
+    "LU_BZN": "LU",
+    "MT": "MT",
+    "ME": "ME",
+    "GB": "GB",
+    "GE": "GE",
+    "GB_IFA": "GB",
+    "GB_IFA2": "GB",
+    "GB_ELECLINK": "GB",
+    "UK": "UK",
+    "NL": "NL",
+    "NO_1": "NO",
+    "NO_1A": "NO",
+    "NO_2": "NO",
+    "NO_2_NSL": "NO",
+    "NO_2A": "NO",
+    "NO_3": "NO",
+    "NO_4": "NO",
+    "NO_5": "NO",
+    "NO": "NO",
+    "PL_CZ": "PL",
+    "PL": "PL",
+    "PT": "PT",
+    "MD": "MD",
+    "RO": "RO",
+    "RU": "RU",
+    "SE_1": "SE",
+    "SE_2": "SE",
+    "SE_3": "SE",
+    "SE_4": "SE",
+    "RS": "RS",
+    "SK": "SK",
+    "SI": "SI",
+    "GB_NIR": "GB",
+    "ES": "ES",
+    "SE": "SE",
+    "CH": "CH",
+    "DE_TENNET": "DE",
+    "DE_TRANSNET": "DE",
+    "TR": "TR",
+    "UA": "UA",
+    "UA_DOBTPP": "UA",
+    "UA_BEI": "UA",
+    "UA_IPS": "UA",
+    "XK": "XK",
+    "DE_AMP_LU": "DE",
+}

openstef/feature_engineering/cyclic_features.py

@@ -0,0 +1,102 @@
+# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+# Module for adding temporal cyclic features to time-based data for capturing seasonality and periodic patterns.
+# Features include yearly, weekly, and monthly seasonality, as well as time-of-day periodicity.
+
+
+import numpy as np
+import pandas as pd
+
+import structlog
+import logging
+
+from openstef.settings import Settings
+
+structlog.configure(
+    wrapper_class=structlog.make_filtering_bound_logger(
+        logging.getLevelName(Settings.log_level)
+    )
+)
+logger = structlog.get_logger(__name__)
+
+
+NUM_SECONDS_IN_A_DAY = 24 * 60 * 60
+
+
+def add_time_cyclic_features(
+    data: pd.DataFrame,
+) -> pd.DataFrame:
+    """Adds time of the day features cyclically encoded using sine and cosine to the input data.
+
+    Args:
+        data: Dataframe indexed by datetime.
+
+    Returns:
+        DataFrame that is the same as input dataframe with extra columns for the added time of the day features.
+    """
+    # Ensure the index is a DatetimeIndex
+    if not isinstance(data.index, pd.DatetimeIndex):
+        raise ValueError("Index should be a pandas DatetimeIndex")
+
+    # Make a copy of the DataFrame to avoid modifying the original
+    data = data.copy()
+
+    second_of_the_day = (
+        data.index.second + data.index.minute * 60 + data.index.hour * 60 * 60
+    )
+    period_of_the_day = 2 * np.pi * second_of_the_day / NUM_SECONDS_IN_A_DAY
+
+    data["time0fday_sine"] = np.sin(period_of_the_day)
+    data["time0fday_cosine"] = np.cos(period_of_the_day)
+
+    return data
+
+
+def add_seasonal_cyclic_features(
+    data: pd.DataFrame, compute_features: list = None
+) -> pd.DataFrame:
+    """Adds cyclical features to capture seasonal and periodic patterns in time-based data.
+
+    Args:
+    - data (pd.DataFrame): DataFrame with a DatetimeIndex.
+    - compute_features (list): Optional. List of features to compute. Options are:
+      ['season', 'dayofweek', 'month']. Default is all features.
+
+    Returns:
+    - pd.DataFrame: DataFrame with added cyclical features.
+
+    Example:
+    >>> data = pd.DataFrame(index=pd.date_range(start='2023-01-01', periods=365, freq='D'))
+    >>> data_with_features = add_cyclical_features(data)
+    >>> print(data_with_features.head())
+    """
+    # Ensure the index is a DatetimeIndex
+    if not isinstance(data.index, pd.DatetimeIndex):
+        raise ValueError("The DataFrame index must be a DatetimeIndex.")
+
+    # Make a copy of the DataFrame to avoid modifying the original
+    data = data.copy()
+
+    # Default to all features if none specified
+    compute_features = compute_features or ["season", "dayofweek", "month"]
+
+    days_in_year = 365.25  # Account for leap years
+
+    # Add seasonality features (day of year)
+    if "season" in compute_features:
+        data["season_sine"] = np.sin(2 * np.pi * data.index.dayofyear / days_in_year)
+        data["season_cosine"] = np.cos(2 * np.pi * data.index.dayofyear / days_in_year)
+
+    # Add weekly features (day of the week)
+    if "dayofweek" in compute_features:
+        data["day0fweek_sine"] = np.sin(2 * np.pi * data.index.day_of_week / 7)
+        data["day0fweek_cosine"] = np.cos(2 * np.pi * data.index.day_of_week / 7)
+
+    # Add monthly features (month of the year)
+    if "month" in compute_features:
+        data["month_sine"] = np.sin(2 * np.pi * data.index.month / 12)
+        data["month_cosine"] = np.cos(2 * np.pi * data.index.month / 12)
+
+    return data

openstef/feature_engineering/holiday_features.py

@@ -10,13 +10,11 @@ import pandas as pd
 
 from openstef import PROJECT_ROOT
 
-HOLIDAY_CSV_PATH: str = (
-    PROJECT_ROOT / "openstef" / "data" / "dutch_holidays_2020-2022.csv"
-)
+HOLIDAY_CSV_PATH: str = PROJECT_ROOT / "openstef" / "data" / "dutch_holidays.csv"
 
 
 def generate_holiday_feature_functions(
-    country: str = "NL",
+    country_code: str = "NL",
     years: list = None,
     path_to_school_holidays_csv: str = HOLIDAY_CSV_PATH,
 ) -> dict:
@@ -46,12 +44,14 @@ def generate_holiday_feature_functions(
         - Pinksteren
         - Kerst
 
+
     The 'Brugdagen' are updated untill dec 2020. (Generated using agenda)
 
     Args:
         country: Country for which to create holiday features.
         years: years for which to create holiday features.
         path_to_school_holidays_csv: Filepath to csv with school holidays.
+        NOTE: Dutch holidays csv file is only until January 2026.
 
     Returns:
         Dictionary with functions that check if a given date is a holiday, keys
@@ -69,7 +69,7 @@ def generate_holiday_feature_functions(
             now.year + 1,
         ]
 
-    country_holidays = holidays.country_holidays(country, years=years)
+    country_holidays = holidays.country_holidays(country_code, years=years)
 
     # Make holiday function dict
     holiday_functions = {}
@@ -96,7 +96,7 @@ def generate_holiday_feature_functions(
 
         # Check for bridge day
         holiday_functions, bridge_days = check_for_bridge_day(
-            date, holiday_name, country, years, holiday_functions, bridge_days
+            date, holiday_name, country_code, years, holiday_functions, bridge_days
         )
 
     # Add feature function that includes all bridgedays
@@ -104,33 +104,42 @@ def generate_holiday_feature_functions(
         {"is_bridgeday": lambda x: np.isin(x.index.date, np.array(list(bridge_days)))}
     )
 
-    # Manully generated csv including all dutch schoolholidays for different regions
-    df_holidays = pd.read_csv(path_to_school_holidays_csv, index_col=None)
-    df_holidays["datum"] = pd.to_datetime(df_holidays.datum).apply(lambda x: x.date())
-
-    # Add check function that includes all holidays of the provided csv
-    holiday_functions.update(
-        {"is_schoolholiday": lambda x: np.isin(x.index.date, df_holidays.datum.values)}
-    )
-
-    # Loop over list of holidays names
-    for holiday_name in list(set(df_holidays.name)):
-        # Define function explicitely to mitigate 'late binding' problem
-        def make_holiday_func(holidayname=holiday_name):
-            return lambda x: np.isin(
-                x.index.date, df_holidays.datum[df_holidays.name == holidayname].values
-            )
+    # Add school holidays if country is NL
+    if country_code == "NL":
+        # Manully generated csv including all dutch schoolholidays for different regions
+        df_holidays = pd.read_csv(path_to_school_holidays_csv, index_col=None)
+        df_holidays["datum"] = pd.to_datetime(df_holidays.datum).apply(
+            lambda x: x.date()
+        )
 
-        # Create lag function for each holiday
+        # Add check function that includes all holidays of the provided csv
         holiday_functions.update(
             {
-                "is_"
-                + holiday_name.replace(" ", "_").lower(): make_holiday_func(
-                    holidayname=holiday_name
+                "is_schoolholiday": lambda x: np.isin(
+                    x.index.date, df_holidays.datum.values
                 )
             }
         )
 
+        # Loop over list of holidays names
+        for holiday_name in list(set(df_holidays.name)):
+            # Define function explicitely to mitigate 'late binding' problem
+            def make_holiday_func(holidayname=holiday_name):
+                return lambda x: np.isin(
+                    x.index.date,
+                    df_holidays.datum[df_holidays.name == holidayname].values,
+                )
+
+            # Create lag function for each holiday
+            holiday_functions.update(
+                {
+                    "is_"
+                    + holiday_name.replace(" ", "_").lower(): make_holiday_func(
+                        holidayname=holiday_name
+                    )
+                }
+            )
+
     return holiday_functions
 
 

openstef/feature_engineering/missing_values_transformer.py

@@ -7,7 +7,7 @@ import numpy as np
 import pandas as pd
 from sklearn.impute import SimpleImputer
 from sklearn.preprocessing import FunctionTransformer
-from sklearn.utils.validation import check_array
+from sklearn.utils.validation import check_array, check_is_fitted
 
 
 class MissingValuesTransformer:
@@ -27,6 +27,7 @@ class MissingValuesTransformer:
         missing_values: Union[int, float, str, None] = np.nan,
         imputation_strategy: str = None,
         fill_value: Union[str, int, float] = None,
+        no_fill_future_values_features: List[str] = None,
     ):
         """Initialize missing values handler.
 
@@ -37,11 +38,34 @@ class MissingValuesTransformer:
                 Can be one of "mean", "median", "most_frequent", "constant" or None.
             fill_value: When strategy == "constant", fill_value is used to replace all
                 occurrences of missing_values.
+            no_fill_future_values_features: The features for which it does not make sense
+                to fill future values. Rows that contain trailing null values for these
+                features will be removed from the data.
 
         """
         self.missing_values = missing_values
         self.imputation_strategy = imputation_strategy
         self.fill_value = fill_value
+        self.no_fill_future_values_features = no_fill_future_values_features or []
+        self.is_fitted_ = False
+
+        # Build the proper imputation transformer
+        # - Identity function if strategy is None
+        # - SimpleImputer with the dedicated strategy
+        if self.imputation_strategy is None:
+            self.imputer_ = FunctionTransformer(func=self._identity)
+        else:
+            self.imputer_ = SimpleImputer(
+                missing_values=self.missing_values,
+                strategy=self.imputation_strategy,
+                fill_value=self.fill_value,
+            ).set_output(transform="pandas")
+            self.imputer_._validate_params()
+
+    @staticmethod
+    def _determine_trailing_null_rows(x: pd.DataFrame) -> pd.Series:
+        """Determine rows with trailing null values in a DataFrame."""
+        return ~x.bfill().isnull().any(axis="columns")
 
     def fit(self, x, y=None):
         """Fit the imputer on the input data."""
@@ -55,33 +79,33 @@ class MissingValuesTransformer:
         # Remove always null columns
         is_column_null = x.isnull().all(axis="index")
         self.non_null_feature_names = list(x.columns[~is_column_null])
+        x = x[self.non_null_feature_names]
 
-        # Build the proper imputation transformer
-        # - Identity function if strategy is None
-        # - SimpleImputer with the dedicated strategy
-        if self.imputation_strategy is None:
-            self.imputer_ = FunctionTransformer(func=self._identity)
-        else:
-            self.imputer_ = SimpleImputer(
-                missing_values=self.missing_values,
-                strategy=self.imputation_strategy,
-                fill_value=self.fill_value,
-            ).set_output(transform="pandas")
+        # Remove trailing null rows for features that should
+        # not be imputed in the future
+        trailing_null_rows = self._determine_trailing_null_rows(
+            x[self.no_fill_future_values_features]
+        )
+        x = x.loc[trailing_null_rows]
 
         # Imputers do not support labels
         self.imputer_.fit(X=x, y=None)
+        self.is_fitted_ = True
 
     def transform(self, x) -> pd.DataFrame:
         """Transform the input data by imputing missing values."""
+        check_is_fitted(self)
         _ = check_array(x, force_all_finite="allow-nan")
         if not isinstance(x, pd.DataFrame):
             x = pd.DataFrame(np.asarray(x))
 
         x = x[self.non_null_feature_names]
 
-        return self.imputer_.transform(x)
+        transformed = self.imputer_.transform(x)
 
-    def fit_transform(self, x, y=None):
+        return transformed
+
+    def fit_transform(self, x, y=None) -> tuple[pd.DataFrame, Optional[pd.Series]]:
         """Fit the imputer on the input data and transform it.
 
         Returns:
@@ -89,7 +113,25 @@ class MissingValuesTransformer:
 
         """
         self.fit(x, y)
-        return self.transform(x)
+
+        if not isinstance(x, pd.DataFrame):
+            x = pd.DataFrame(np.asarray(x))
+
+        x = x[self.non_null_feature_names]
+
+        # Remove trailing null rows for features that should
+        # not be imputed in the future
+        non_trailing_null_rows = self._determine_trailing_null_rows(
+            x[self.no_fill_future_values_features]
+        )
+        x = x.loc[non_trailing_null_rows]
+
+        x = self.transform(x)
+
+        if y is not None:
+            y = y.loc[non_trailing_null_rows]
+
+        return x, y
 
     @classmethod
     def _identity(cls, x):