openstef 3.4.10__py3-none-any.whl → 3.4.44__py3-none-any.whl

openstef/data_classes/data_prep.py

@@ -5,7 +5,7 @@
 import inspect
 import json
 from importlib import import_module
-from typing import Any, Sequence, Union, TypeVar
+from typing import Any, Sequence, TypeVar, Union
 
 from pydantic.v1 import BaseModel
 

openstef/data_classes/prediction_job.py

@@ -6,10 +6,10 @@ from typing import Optional, Union
 
 from pydantic.v1 import BaseModel
 
+from openstef.data_classes.data_prep import DataPrepDataClass
 from openstef.data_classes.model_specifications import ModelSpecificationDataClass
 from openstef.data_classes.split_function import SplitFuncDataClass
-from openstef.data_classes.data_prep import DataPrepDataClass
-from openstef.enums import PipelineType
+from openstef.enums import PipelineType, BiddingZone
 
 
 class PredictionJobDataClass(BaseModel):
@@ -25,11 +25,15 @@ class PredictionJobDataClass(BaseModel):
         - ``"xgb_quantile"``
         - ``"lgb"``
         - ``"linear"``
-        - ``"proloaf"`` (extra dependencies requiered, see README)
+        - ``"linear_quantile"``
+        - ``"xgb_multioutput_quantile"``
+        - ``"flatliner"``
 
     If unsure what to pick, choose ``"xgb"``.
 
     """
+    model_kwargs: Optional[dict]
+    """The model parameters that should be used."""
     forecast_type: str
     """The type of forecasts that should be made.
 
@@ -41,15 +45,17 @@ class PredictionJobDataClass(BaseModel):
     If unsure what to pick, choose ``"demand"``.
 
     """
-    horizon_minutes: int = 2880
-    """The horizon of the desired forecast in minutes. Defaults to 2880 minutes (i.e. 2 days)."""
+    horizon_minutes: Optional[int] = 2880
+    """The horizon of the desired forecast in minutes used in tasks. Defaults to 2880 minutes (i.e. 2 days)."""
     resolution_minutes: int
     """The resolution of the desired forecast in minutes."""
-    lat: float
-    """Latitude of the forecasted location in degrees."""
-    lon: float
-    """Longitude of the forecasted location in degrees."""
+    lat: Optional[float] = 52.132633
+    """Latitude of the forecasted location in degrees. Used for fetching weather data in tasks, calculating derrived features and component splitting."""
+    lon: Optional[float] = 5.291266
+    """Longitude of the forecasted location in degrees. Used for fetching weather data in tasks, calculating derrived features and component splitting."""
     name: str
+    """Bidding zone is used to determine the electricity price. It is also used to determine the holidays that should be used. Currently only ENTSO-E bidding zones are supported."""
+    electricity_bidding_zone: Optional[BiddingZone] = BiddingZone.NL
     """Name of the forecast, e.g. the location name."""
     train_components: Optional[bool]
     """Whether splitting the forecasts in wind, solar, rest is desired."""

openstef/enums.py

@@ -4,14 +4,119 @@
 from enum import Enum
 
 
-# TODO replace this with ModelType (MLModelType == Machine Learning model type)
-class MLModelType(Enum):
+class BiddingZone(Enum):
+    DE_50HZ = "DE_50HZ"
+    AL = "AL"
+    DE_AMPRION = "DE_AMPRION"
+    AT = "AT"
+    BY = "BY"
+    BE = "BE"
+    BA = "BA"
+    BG = "BG"
+    CZ_DE_SK = "CZ_DE_SK"
+    HR = "HR"
+    CWE = "CWE"
+    CY = "CY"
+    CZ = "CZ"
+    DE_AT_LU = "DE_AT_LU"
+    DE_LU = "DE_LU"
+    DK = "DK"
+    DK_1 = "DK_1"
+    DK_1_NO_1 = "DK_1_NO_1"
+    DK_2 = "DK_2"
+    DK_CA = "DK_CA"
+    EE = "EE"
+    FI = "FI"
+    MK = "MK"
+    FR = "FR"
+    DE = "DE"
+    GR = "GR"
+    HU = "HU"
+    IS = "IS"
+    IE_SEM = "IE_SEM"
+    IE = "IE"
+    IT = "IT"
+    IT_SACO_AC = "IT_SACO_AC"
+    IT_CALA = "IT_CALA"
+    IT_SACO_DC = "IT_SACO_DC"
+    IT_BRNN = "IT_BRNN"
+    IT_CNOR = "IT_CNOR"
+    IT_CSUD = "IT_CSUD"
+    IT_FOGN = "IT_FOGN"
+    IT_GR = "IT_GR"
+    IT_MACRO_NORTH = "IT_MACRO_NORTH"
+    IT_MACRO_SOUTH = "IT_MACRO_SOUTH"
+    IT_MALTA = "IT_MALTA"
+    IT_NORD = "IT_NORD"
+    IT_NORD_AT = "IT_NORD_AT"
+    IT_NORD_CH = "IT_NORD_CH"
+    IT_NORD_FR = "IT_NORD_FR"
+    IT_NORD_SI = "IT_NORD_SI"
+    IT_PRGP = "IT_PRGP"
+    IT_ROSN = "IT_ROSN"
+    IT_SARD = "IT_SARD"
+    IT_SICI = "IT_SICI"
+    IT_SUD = "IT_SUD"
+    RU_KGD = "RU_KGD"
+    LV = "LV"
+    LT = "LT"
+    LU = "LU"
+    LU_BZN = "LU_BZN"
+    MT = "MT"
+    ME = "ME"
+    GB = "GB"
+    GE = "GE"
+    GB_IFA = "GB_IFA"
+    GB_IFA2 = "GB_IFA2"
+    GB_ELECLINK = "GB_ELECLINK"
+    UK = "UK"
+    NL = "NL"
+    NO_1 = "NO_1"
+    NO_1A = "NO_1A"
+    NO_2 = "NO_2"
+    NO_2_NSL = "NO_2_NSL"
+    NO_2A = "NO_2A"
+    NO_3 = "NO_3"
+    NO_4 = "NO_4"
+    NO_5 = "NO_5"
+    NO = "NO"
+    PL_CZ = "PL_CZ"
+    PL = "PL"
+    PT = "PT"
+    MD = "MD"
+    RO = "RO"
+    RU = "RU"
+    SE_1 = "SE_1"
+    SE_2 = "SE_2"
+    SE_3 = "SE_3"
+    SE_4 = "SE_4"
+    RS = "RS"
+    SK = "SK"
+    SI = "SI"
+    GB_NIR = "GB_NIR"
+    ES = "ES"
+    SE = "SE"
+    CH = "CH"
+    DE_TENNET = "DE_TENNET"
+    DE_TRANSNET = "DE_TRANSNET"
+    TR = "TR"
+    UA = "UA"
+    UA_DOBTPP = "UA_DOBTPP"
+    UA_BEI = "UA_BEI"
+    UA_IPS = "UA_IPS"
+    XK = "XK"
+    DE_AMP_LU = "DE_AMP_LU"
+
+
+class ModelType(Enum):
     XGB = "xgb"
     XGB_QUANTILE = "xgb_quantile"
+    XGB_MULTIOUTPUT_QUANTILE = "xgb_multioutput_quantile"
     LGB = "lgb"
     LINEAR = "linear"
-    ProLoaf = "proloaf"
+    LINEAR_QUANTILE = "linear_quantile"
     ARIMA = "arima"
+    FLATLINER = "flatliner"
 
 
 class ForecastType(Enum):
@@ -21,12 +126,6 @@ class ForecastType(Enum):
     BASECASE = "basecase"
 
 
-class TracyJobResult(Enum):
-    SUCCESS = "success"
-    FAILED = "failed"
-    UNKNOWN = "unknown"
-
-
 class PipelineType(Enum):
     FORECAST = "forecast"
     TRAIN = "train"

openstef/exceptions.py

@@ -5,7 +5,7 @@
 """Openstef custom exceptions."""
 
 
-# Define custom exception
+# Define custom exceptions
 class NoPredictedLoadError(Exception):
     """No predicted load for given datatime range."""
 

openstef/feature_engineering/apply_features.py

@@ -14,19 +14,25 @@ Examples of features that are added:
 import pandas as pd
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.feature_engineering.historic_features import (
-    add_historic_load_as_a_feature,
-)
+from openstef.enums import BiddingZone
 from openstef.feature_engineering.holiday_features import (
     generate_holiday_feature_functions,
 )
 from openstef.feature_engineering.lag_features import generate_lag_feature_functions
+from openstef.feature_engineering.bidding_zone_to_country_mapping import (
+    BIDDING_ZONE_TO_COUNTRY_CODE_MAPPING,
+)
 from openstef.feature_engineering.weather_features import (
     add_additional_solar_features,
     add_additional_wind_features,
     add_humidity_features,
 )
 
+from openstef.feature_engineering.cyclic_features import (
+    add_seasonal_cyclic_features,
+    add_time_cyclic_features,
+)
+
 
 def apply_features(
     data: pd.DataFrame,
@@ -61,6 +67,7 @@ def apply_features(
 
         import pandas as pd
         import numpy as np
+        from geopy.geocoders import Nominatim
         index = pd.date_range(start = "2017-01-01 09:00:00",
         freq = '15T', periods = 200)
         data = pd.DataFrame(index = index,
@@ -69,8 +76,8 @@ def apply_features(
                             np.random.uniform(0.7,1.7, 200)))
 
     """
-    # Add if needed the proloaf feature (historic_load)
-    data = add_historic_load_as_a_feature(data, pj)
+    if pj is None:
+        pj = {"electricity_bidding_zone": BiddingZone.NL}
 
     # Get lag feature functions
     feature_functions = generate_lag_feature_functions(feature_names, horizon)
@@ -86,8 +93,14 @@ def apply_features(
         }
     )
 
+    # Get country code from bidding zone if available
+    electricity_bidding_zone = pj.get("electricity_bidding_zone", BiddingZone.NL)
+    country_code = BIDDING_ZONE_TO_COUNTRY_CODE_MAPPING[electricity_bidding_zone.name]
+
     # Get holiday feature functions
-    feature_functions.update(generate_holiday_feature_functions())
+    feature_functions.update(
+        generate_holiday_feature_functions(country_code=country_code)
+    )
 
     # Add the features to the dataframe using previously defined feature functions
     for key, featfunc in feature_functions.items():
@@ -105,5 +118,11 @@ def apply_features(
     # Add solar features; when pj is unavailable a default location is used.
     data = add_additional_solar_features(data, pj, feature_names)
 
+    # Adds cyclical features to capture seasonal and periodic patterns in time-based data.
+    data = add_seasonal_cyclic_features(data)
+
+    # Adds polar time features (sine and cosine) to capture periodic patterns based on the timestamp index.
+    data = add_time_cyclic_features(data)
+
     # Return dataframe including all requested features
     return data

openstef/feature_engineering/bidding_zone_to_country_mapping.py

@@ -0,0 +1,106 @@
+# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
+#
+# SPDX-License-Identifier: MPL-2.0
+BIDDING_ZONE_TO_COUNTRY_CODE_MAPPING = {
+    "DE_50HZ": "DE",
+    "AL": "AL",
+    "DE_AMPRION": "DE",
+    "AT": "AT",
+    "BY": "BY",
+    "BE": "BE",
+    "BA": "BA",
+    "BG": "BG",
+    "CZ_DE_SK": "CZ",
+    "HR": "HR",
+    "CWE": "CWE",
+    "CY": "CY",
+    "CZ": "CZ",
+    "DE_AT_LU": "DE",
+    "DE_LU": "DE",
+    "DK": "DK",
+    "DK_1": "DK",
+    "DK_1_NO_1": "DK",
+    "DK_2": "DK",
+    "DK_CA": "DK",
+    "EE": "EE",
+    "FI": "FI",
+    "MK": "MK",
+    "FR": "FR",
+    "DE": "DE",
+    "GR": "GR",
+    "HU": "HU",
+    "IS": "IS",
+    "IE_SEM": "IE",
+    "IE": "IE",
+    "IT": "IT",
+    "IT_SACO_AC": "IT",
+    "IT_CALA": "IT",
+    "IT_SACO_DC": "IT",
+    "IT_BRNN": "IT",
+    "IT_CNOR": "IT",
+    "IT_CSUD": "IT",
+    "IT_FOGN": "IT",
+    "IT_GR": "IT",
+    "IT_MACRO_NORTH": "IT",
+    "IT_MACRO_SOUTH": "IT",
+    "IT_MALTA": "IT",
+    "IT_NORD": "IT",
+    "IT_NORD_AT": "IT",
+    "IT_NORD_CH": "IT",
+    "IT_NORD_FR": "IT",
+    "IT_NORD_SI": "IT",
+    "IT_PRGP": "IT",
+    "IT_ROSN": "IT",
+    "IT_SARD": "IT",
+    "IT_SICI": "IT",
+    "IT_SUD": "IT",
+    "RU_KGD": "RU",
+    "LV": "LV",
+    "LT": "LT",
+    "LU": "LU",
+    "LU_BZN": "LU",
+    "MT": "MT",
+    "ME": "ME",
+    "GB": "GB",
+    "GE": "GE",
+    "GB_IFA": "GB",
+    "GB_IFA2": "GB",
+    "GB_ELECLINK": "GB",
+    "UK": "UK",
+    "NL": "NL",
+    "NO_1": "NO",
+    "NO_1A": "NO",
+    "NO_2": "NO",
+    "NO_2_NSL": "NO",
+    "NO_2A": "NO",
+    "NO_3": "NO",
+    "NO_4": "NO",
+    "NO_5": "NO",
+    "NO": "NO",
+    "PL_CZ": "PL",
+    "PL": "PL",
+    "PT": "PT",
+    "MD": "MD",
+    "RO": "RO",
+    "RU": "RU",
+    "SE_1": "SE",
+    "SE_2": "SE",
+    "SE_3": "SE",
+    "SE_4": "SE",
+    "RS": "RS",
+    "SK": "SK",
+    "SI": "SI",
+    "GB_NIR": "GB",
+    "ES": "ES",
+    "SE": "SE",
+    "CH": "CH",
+    "DE_TENNET": "DE",
+    "DE_TRANSNET": "DE",
+    "TR": "TR",
+    "UA": "UA",
+    "UA_DOBTPP": "UA",
+    "UA_BEI": "UA",
+    "UA_IPS": "UA",
+    "XK": "XK",
+    "DE_AMP_LU": "DE",
+}

openstef/feature_engineering/cyclic_features.py

@@ -0,0 +1,102 @@
+# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+# Module for adding temporal cyclic features to time-based data for capturing seasonality and periodic patterns.
+# Features include yearly, weekly, and monthly seasonality, as well as time-of-day periodicity.
+
+
+import numpy as np
+import pandas as pd
+
+import structlog
+import logging
+
+from openstef.settings import Settings
+
+structlog.configure(
+    wrapper_class=structlog.make_filtering_bound_logger(
+        logging.getLevelName(Settings.log_level)
+    )
+)
+logger = structlog.get_logger(__name__)
+
+
+NUM_SECONDS_IN_A_DAY = 24 * 60 * 60
+
+
+def add_time_cyclic_features(
+    data: pd.DataFrame,
+) -> pd.DataFrame:
+    """Adds time of the day features cyclically encoded using sine and cosine to the input data.
+
+    Args:
+        data: Dataframe indexed by datetime.
+
+    Returns:
+        DataFrame that is the same as input dataframe with extra columns for the added time of the day features.
+    """
+    # Ensure the index is a DatetimeIndex
+    if not isinstance(data.index, pd.DatetimeIndex):
+        raise ValueError("Index should be a pandas DatetimeIndex")
+
+    # Make a copy of the DataFrame to avoid modifying the original
+    data = data.copy()
+
+    second_of_the_day = (
+        data.index.second + data.index.minute * 60 + data.index.hour * 60 * 60
+    )
+    period_of_the_day = 2 * np.pi * second_of_the_day / NUM_SECONDS_IN_A_DAY
+
+    data["time0fday_sine"] = np.sin(period_of_the_day)
+    data["time0fday_cosine"] = np.cos(period_of_the_day)
+
+    return data
+
+
+def add_seasonal_cyclic_features(
+    data: pd.DataFrame, compute_features: list = None
+) -> pd.DataFrame:
+    """Adds cyclical features to capture seasonal and periodic patterns in time-based data.
+
+    Args:
+    - data (pd.DataFrame): DataFrame with a DatetimeIndex.
+    - compute_features (list): Optional. List of features to compute. Options are:
+      ['season', 'dayofweek', 'month']. Default is all features.
+
+    Returns:
+    - pd.DataFrame: DataFrame with added cyclical features.
+
+    Example:
+    >>> data = pd.DataFrame(index=pd.date_range(start='2023-01-01', periods=365, freq='D'))
+    >>> data_with_features = add_cyclical_features(data)
+    >>> print(data_with_features.head())
+    """
+    # Ensure the index is a DatetimeIndex
+    if not isinstance(data.index, pd.DatetimeIndex):
+        raise ValueError("The DataFrame index must be a DatetimeIndex.")
+
+    # Make a copy of the DataFrame to avoid modifying the original
+    data = data.copy()
+
+    # Default to all features if none specified
+    compute_features = compute_features or ["season", "dayofweek", "month"]
+
+    days_in_year = 365.25  # Account for leap years
+
+    # Add seasonality features (day of year)
+    if "season" in compute_features:
+        data["season_sine"] = np.sin(2 * np.pi * data.index.dayofyear / days_in_year)
+        data["season_cosine"] = np.cos(2 * np.pi * data.index.dayofyear / days_in_year)
+
+    # Add weekly features (day of the week)
+    if "dayofweek" in compute_features:
+        data["day0fweek_sine"] = np.sin(2 * np.pi * data.index.day_of_week / 7)
+        data["day0fweek_cosine"] = np.cos(2 * np.pi * data.index.day_of_week / 7)
+
+    # Add monthly features (month of the year)
+    if "month" in compute_features:
+        data["month_sine"] = np.sin(2 * np.pi * data.index.month / 12)
+        data["month_cosine"] = np.cos(2 * np.pi * data.index.month / 12)
+
+    return data

openstef/feature_engineering/data_preparation.py

@@ -1,25 +1,27 @@
 # SPDX-FileCopyrightText: 2017-2023 Alliander N.V. <korte.termijn.prognoses@alliander.com> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
-import structlog
-
+import logging
 from abc import ABC, abstractmethod
+from datetime import timedelta
 from typing import Optional
 
 import pandas as pd
-from datetime import timedelta
+import structlog
+
 from openstef.data_classes.model_specifications import ModelSpecificationDataClass
 from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.model.regressors.regressor import OpenstfRegressor
 from openstef.feature_engineering.feature_applicator import (
-    TrainFeatureApplicator,
     OperationalPredictFeatureApplicator,
+    TrainFeatureApplicator,
 )
 from openstef.feature_engineering.general import (
     enforce_feature_order,
     remove_non_requested_feature_columns,
 )
+from openstef.model.regressors.regressor import OpenstfRegressor
 from openstef.pipeline.utils import generate_forecast_datetime_range
+from openstef.settings import Settings
 
 
 class AbstractDataPreparation(ABC):
@@ -120,6 +122,11 @@ class ARDataPreparation(AbstractDataPreparation):
     def prepare_forecast_data(
         self, data: pd.DataFrame
     ) -> tuple[pd.DataFrame, pd.DataFrame]:
+        structlog.configure(
+            wrapper_class=structlog.make_filtering_bound_logger(
+                logging.getLevelName(Settings.log_level)
+            )
+        )
         logger = structlog.get_logger(__name__)
         self.check_model()
         # Prep forecast input by selecting only the forecast datetime interval (this is much smaller than the input range)

openstef/feature_engineering/feature_applicator.py

@@ -149,11 +149,7 @@ class TrainFeatureApplicator(AbstractFeatureApplicator):
 
         # NOTE this is required since apply_features could add additional features
         if self.feature_names is not None:
-            # Add horizon to requested features else it is removed, and if needed the proloaf feature (historic_load)
-            if pj.get("model") == "proloaf":
-                features = self.feature_names + ["historic_load"] + ["horizon"]
-            else:
-                features = self.feature_names + ["horizon"]
+            features = self.feature_names + ["horizon"]
             result = remove_non_requested_feature_columns(result, features)
 
         # Sort all features except for the (first) load and (last) horizon columns

openstef/feature_engineering/general.py

@@ -3,10 +3,14 @@
 # SPDX-License-Identifier: MPL-2.0
 """This modelu contains various helper functions."""
 
+import logging
+
 import numpy as np
 import pandas as pd
 import structlog
 
+from openstef.settings import Settings
+
 
 def add_missing_feature_columns(
     input_data: pd.DataFrame, features: list[str]
@@ -30,6 +34,11 @@ def add_missing_feature_columns(
         Input dataframe with missing columns filled with ``np.N=nan``.
 
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
 
     if features is None:
@@ -61,6 +70,11 @@ def remove_non_requested_feature_columns(
         Model input data with features.
 
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
 
     if requested_features is None: