openstef 3.4.56__py3-none-any.whl → 4.0.0a3__py3-none-any.whl

openstef/feature_engineering/apply_features.py

@@ -1,138 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-"""This module provides functionality for applying features to the input data to improve forecast accuracy.
-
-Examples of features that are added:
-    - The load 1 day and 7 days ago at the same time.
-    - If a day is a weekday or a holiday.
-    - The extrapolated windspeed at 100m.
-    - The normalised wind power according to the turbine-specific power curve.
-
-"""
-
-import pandas as pd
-
-from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.enums import BiddingZone
-from openstef.feature_engineering.holiday_features import (
-    generate_holiday_feature_functions,
-)
-from openstef.feature_engineering.lag_features import generate_lag_feature_functions
-from openstef.feature_engineering.bidding_zone_to_country_mapping import (
-    BIDDING_ZONE_TO_COUNTRY_CODE_MAPPING,
-)
-from openstef.feature_engineering.rolling_features import add_rolling_aggregate_features
-from openstef.feature_engineering.weather_features import (
-    add_additional_solar_features,
-    add_additional_wind_features,
-    add_humidity_features,
-)
-
-from openstef.feature_engineering.cyclic_features import (
-    add_seasonal_cyclic_features,
-    add_time_cyclic_features,
-    add_daylight_terrestrial_feature,
-)
-
-
-def apply_features(
-    data: pd.DataFrame,
-    pj: PredictionJobDataClass = None,
-    feature_names: list[str] = None,
-    horizon: float = 24.0,
-    years: list[int] | None = None,
-) -> pd.DataFrame:
-    """Applies the feature functions defined in ``feature_functions.py`` and returns the complete dataframe.
-
-    Features requiring more recent label-data are omitted.
-
-    .. note::
-        For the time derived features only the ones in the features list will be added. But for the weather features all will be added at present.
-        These unrequested additional features have to be filtered out later.
-
-    Args:
-        data (pandas.DataFrame): a pandas dataframe with input data in the form:
-                                    pd.DataFrame(
-                                        index=datetime,
-                                        columns=[label, predictor_1,..., predictor_n]
-                                    )
-        pj (PredictionJobDataClass): Prediction job.
-        feature_names (list[str]): list of requested features
-        horizon (float): Forecast horizon limit in hours.
-        years (list[int] | None): years for which to create holiday features.
-
-    Returns:
-        pd.DataFrame(index = datetime, columns = [label, predictor_1,..., predictor_n, feature_1, ..., feature_m])
-
-    Example output:
-
-    .. code-block:: py
-
-        import pandas as pd
-        import numpy as np
-        from geopy.geocoders import Nominatim
-        index = pd.date_range(start = "2017-01-01 09:00:00",
-        freq = '15T', periods = 200)
-        data = pd.DataFrame(index = index,
-                            data = dict(load=
-                            np.sin(index.hour/24*np.pi)*
-                            np.random.uniform(0.7,1.7, 200)))
-
-    """
-    if pj is None:
-        pj = {"electricity_bidding_zone": BiddingZone.NL}
-
-    # Get lag feature functions
-    feature_functions = generate_lag_feature_functions(feature_names, horizon)
-
-    # Get timedrivenfeature functions
-    feature_functions.update(
-        {
-            "IsWeekendDay": lambda x: (x.index.weekday // 5) == 1,
-            "IsWeekDay": lambda x: x.index.weekday < 5,
-            "IsSunday": lambda x: x.index.weekday == 6,
-            "Month": lambda x: x.index.month,
-            "Quarter": lambda x: x.index.quarter,
-        }
-    )
-
-    # Get country code from bidding zone if available
-    electricity_bidding_zone = pj.get("electricity_bidding_zone", BiddingZone.NL)
-    country_code = BIDDING_ZONE_TO_COUNTRY_CODE_MAPPING[electricity_bidding_zone.name]
-
-    # Get holiday feature functions
-    feature_functions.update(
-        generate_holiday_feature_functions(country_code=country_code, years=years)
-    )
-
-    # Add the features to the dataframe using previously defined feature functions
-    for key, featfunc in feature_functions.items():
-        # Don't generate feature is not in features
-        if feature_names is not None and key not in feature_names:
-            continue
-        data.loc[:, key] = data.iloc[:, [0]].apply(featfunc)
-
-    # Add additional wind features
-    data = add_additional_wind_features(data, feature_names)
-
-    # Add humidity features
-    data = add_humidity_features(data, feature_names)
-
-    # Add solar features; when pj is unavailable a default location is used.
-    data = add_additional_solar_features(data, pj, feature_names)
-
-    # Adds cyclical features to capture seasonal and periodic patterns in time-based data.
-    data = add_seasonal_cyclic_features(data)
-
-    # Adds polar time features (sine and cosine) to capture periodic patterns based on the timestamp index.
-    data = add_time_cyclic_features(data)
-
-    # Adds daylight terrestrial feature
-    data = add_daylight_terrestrial_feature(data)
-
-    if pj.get("rolling_aggregate_features") is not None:
-        data = add_rolling_aggregate_features(data, pj=pj)
-
-    # Return dataframe including all requested features
-    return data

openstef/feature_engineering/bidding_zone_to_country_mapping.py

@@ -1,106 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-BIDDING_ZONE_TO_COUNTRY_CODE_MAPPING = {
-    "DE_50HZ": "DE",
-    "AL": "AL",
-    "DE_AMPRION": "DE",
-    "AT": "AT",
-    "BY": "BY",
-    "BE": "BE",
-    "BA": "BA",
-    "BG": "BG",
-    "CZ_DE_SK": "CZ",
-    "HR": "HR",
-    "CWE": "CWE",
-    "CY": "CY",
-    "CZ": "CZ",
-    "DE_AT_LU": "DE",
-    "DE_LU": "DE",
-    "DK": "DK",
-    "DK_1": "DK",
-    "DK_1_NO_1": "DK",
-    "DK_2": "DK",
-    "DK_CA": "DK",
-    "EE": "EE",
-    "FI": "FI",
-    "MK": "MK",
-    "FR": "FR",
-    "DE": "DE",
-    "GR": "GR",
-    "HU": "HU",
-    "IS": "IS",
-    "IE_SEM": "IE",
-    "IE": "IE",
-    "IT": "IT",
-    "IT_SACO_AC": "IT",
-    "IT_CALA": "IT",
-    "IT_SACO_DC": "IT",
-    "IT_BRNN": "IT",
-    "IT_CNOR": "IT",
-    "IT_CSUD": "IT",
-    "IT_FOGN": "IT",
-    "IT_GR": "IT",
-    "IT_MACRO_NORTH": "IT",
-    "IT_MACRO_SOUTH": "IT",
-    "IT_MALTA": "IT",
-    "IT_NORD": "IT",
-    "IT_NORD_AT": "IT",
-    "IT_NORD_CH": "IT",
-    "IT_NORD_FR": "IT",
-    "IT_NORD_SI": "IT",
-    "IT_PRGP": "IT",
-    "IT_ROSN": "IT",
-    "IT_SARD": "IT",
-    "IT_SICI": "IT",
-    "IT_SUD": "IT",
-    "RU_KGD": "RU",
-    "LV": "LV",
-    "LT": "LT",
-    "LU": "LU",
-    "LU_BZN": "LU",
-    "MT": "MT",
-    "ME": "ME",
-    "GB": "GB",
-    "GE": "GE",
-    "GB_IFA": "GB",
-    "GB_IFA2": "GB",
-    "GB_ELECLINK": "GB",
-    "UK": "UK",
-    "NL": "NL",
-    "NO_1": "NO",
-    "NO_1A": "NO",
-    "NO_2": "NO",
-    "NO_2_NSL": "NO",
-    "NO_2A": "NO",
-    "NO_3": "NO",
-    "NO_4": "NO",
-    "NO_5": "NO",
-    "NO": "NO",
-    "PL_CZ": "PL",
-    "PL": "PL",
-    "PT": "PT",
-    "MD": "MD",
-    "RO": "RO",
-    "RU": "RU",
-    "SE_1": "SE",
-    "SE_2": "SE",
-    "SE_3": "SE",
-    "SE_4": "SE",
-    "RS": "RS",
-    "SK": "SK",
-    "SI": "SI",
-    "GB_NIR": "GB",
-    "ES": "ES",
-    "SE": "SE",
-    "CH": "CH",
-    "DE_TENNET": "DE",
-    "DE_TRANSNET": "DE",
-    "TR": "TR",
-    "UA": "UA",
-    "UA_DOBTPP": "UA",
-    "UA_BEI": "UA",
-    "UA_IPS": "UA",
-    "XK": "XK",
-    "DE_AMP_LU": "DE",
-}

openstef/feature_engineering/cyclic_features.py

@@ -1,161 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-
-# Module for adding temporal cyclic features to time-based data for capturing seasonality and periodic patterns.
-# Features include yearly, weekly, and monthly seasonality, as well as time-of-day periodicity.
-
-
-import numpy as np
-import pandas as pd
-
-import structlog
-import logging
-
-from openstef.settings import Settings
-from openstef import PROJECT_ROOT
-
-
-structlog.configure(
-    wrapper_class=structlog.make_filtering_bound_logger(
-        logging.getLevelName(Settings.log_level)
-    )
-)
-logger = structlog.get_logger(__name__)
-
-TERRESTRIAL_RADIATION_CSV_PATH: str = (
-    PROJECT_ROOT / "openstef" / "data" / "NL_terrestrial_radiation.csv"
-)
-NUM_SECONDS_IN_A_DAY = 24 * 60 * 60
-
-
-def add_daylight_terrestrial_feature(
-    data: pd.DataFrame,
-    path_to_terrestrial_radiation_csv: str = TERRESTRIAL_RADIATION_CSV_PATH,
-) -> pd.DataFrame:
-    """Add daylight terrestrial radiation feature to the input dataset. This function processes terrestrial radiation
-    data and aligns it with the time indices of the input dataset. The terrestrial radiation data is normalized,
-    interpolated, and merged with the main dataset to provide a feature representing terrestrial radiation.
-
-    Args:
-        data (pd.DataFrame):
-            The input dataset containing a time-indexed DataFrame.
-        path_to_terrestrial_radiation_csv (str):
-            File path to the CSV file containing terrestrial radiation data. The CSV file
-            should have a time-based index.
-
-    Returns:
-        pd.DataFrame:
-            The input dataset with an added column for the terrestrial radiation feature.
-
-    Notes:
-        - The function assumes the input data and the terrestrial radiation data share
-          the same time zone and frequency alignment.
-        - The terrestrial radiation values are normalized using z-score normalization.
-
-    """
-    # Load the terrestrial radiation data
-    terrestrial_radiation = pd.read_csv(path_to_terrestrial_radiation_csv, index_col=0)
-    terrestrial_radiation.index = pd.to_datetime(terrestrial_radiation.index)
-
-    # Align the index with the input data's year
-    year_diff = data.index.min().year - terrestrial_radiation.index.min().year
-    terrestrial_radiation.index += pd.DateOffset(years=year_diff)
-
-    # Resample to 15-minute intervals, and interpolate missing values
-    terrestrial_radiation = terrestrial_radiation.resample("15min").mean().interpolate()
-
-    # Normalize the terrestrial radiation values using z-score normalization
-    terrestrial_radiation = (
-        terrestrial_radiation - terrestrial_radiation.mean(axis=0)
-    ) / terrestrial_radiation.std(axis=0)
-    terrestrial_radiation.columns = ["daylight_continuous"]
-
-    # Make a copy of the DataFrame to avoid modifying the original
-    data = data.copy()
-
-    # Merge the terrestrial radiation data with the input dataset
-    data = data.merge(
-        terrestrial_radiation, left_index=True, right_index=True, how="left"
-    )
-
-    return data
-
-
-def add_time_cyclic_features(
-    data: pd.DataFrame,
-) -> pd.DataFrame:
-    """Adds time of the day features cyclically encoded using sine and cosine to the input data.
-
-    Args:
-        data: Dataframe indexed by datetime.
-
-    Returns:
-        DataFrame that is the same as input dataframe with extra columns for the added time of the day features.
-
-    """
-    # Ensure the index is a DatetimeIndex
-    if not isinstance(data.index, pd.DatetimeIndex):
-        raise ValueError("Index should be a pandas DatetimeIndex")
-
-    # Make a copy of the DataFrame to avoid modifying the original
-    data = data.copy()
-
-    second_of_the_day = (
-        data.index.second + data.index.minute * 60 + data.index.hour * 60 * 60
-    )
-    period_of_the_day = 2 * np.pi * second_of_the_day / NUM_SECONDS_IN_A_DAY
-
-    data["time0fday_sine"] = np.sin(period_of_the_day)
-    data["time0fday_cosine"] = np.cos(period_of_the_day)
-
-    return data
-
-
-def add_seasonal_cyclic_features(
-    data: pd.DataFrame, compute_features: list = None
-) -> pd.DataFrame:
-    """Adds cyclical features to capture seasonal and periodic patterns in time-based data.
-
-    Args:
-    - data (pd.DataFrame): DataFrame with a DatetimeIndex.
-    - compute_features (list): Optional. List of features to compute. Options are:
-      ['season', 'dayofweek', 'month']. Default is all features.
-
-    Returns:
-    - pd.DataFrame: DataFrame with added cyclical features.
-
-    Example:
-    >>> data = pd.DataFrame(index=pd.date_range(start='2023-01-01', periods=365, freq='D'))
-    >>> data_with_features = add_cyclical_features(data)
-    >>> print(data_with_features.head())
-
-    """
-    # Ensure the index is a DatetimeIndex
-    if not isinstance(data.index, pd.DatetimeIndex):
-        raise ValueError("The DataFrame index must be a DatetimeIndex.")
-
-    # Make a copy of the DataFrame to avoid modifying the original
-    data = data.copy()
-
-    # Default to all features if none specified
-    compute_features = compute_features or ["season", "dayofweek", "month"]
-
-    days_in_year = 365.25  # Account for leap years
-
-    # Add seasonality features (day of year)
-    if "season" in compute_features:
-        data["season_sine"] = np.sin(2 * np.pi * data.index.dayofyear / days_in_year)
-        data["season_cosine"] = np.cos(2 * np.pi * data.index.dayofyear / days_in_year)
-
-    # Add weekly features (day of the week)
-    if "dayofweek" in compute_features:
-        data["day0fweek_sine"] = np.sin(2 * np.pi * data.index.day_of_week / 7)
-        data["day0fweek_cosine"] = np.cos(2 * np.pi * data.index.day_of_week / 7)
-
-    # Add monthly features (month of the year)
-    if "month" in compute_features:
-        data["month_sine"] = np.sin(2 * np.pi * data.index.month / 12)
-        data["month_cosine"] = np.cos(2 * np.pi * data.index.month / 12)
-
-    return data

openstef/feature_engineering/data_preparation.py

@@ -1,152 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Alliander N.V. <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-import logging
-from abc import ABC, abstractmethod
-from datetime import timedelta
-from typing import Optional
-
-import pandas as pd
-import structlog
-
-from openstef.data_classes.model_specifications import ModelSpecificationDataClass
-from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.feature_engineering.feature_applicator import (
-    OperationalPredictFeatureApplicator,
-    TrainFeatureApplicator,
-)
-from openstef.feature_engineering.general import (
-    enforce_feature_order,
-    remove_non_requested_feature_columns,
-)
-from openstef.model.regressors.regressor import OpenstfRegressor
-from openstef.pipeline.utils import generate_forecast_datetime_range
-from openstef.settings import Settings
-
-
-class AbstractDataPreparation(ABC):
-    def __init__(
-        self,
-        pj: PredictionJobDataClass,
-        model_specs: ModelSpecificationDataClass,
-        model: Optional[OpenstfRegressor] = None,
-        horizons: Optional[list[float]] = None,
-    ) -> None:
-        super().__init__()
-        self.pj = pj
-        self.model_specs = model_specs
-        self.model = model
-        self.horizons = horizons
-
-    @abstractmethod
-    def prepare_train_data(self, data: pd.DataFrame) -> pd.DataFrame:
-        pass
-
-    @abstractmethod
-    def prepare_forecast_data(
-        self, data: pd.DataFrame
-    ) -> tuple[pd.DataFrame, pd.DataFrame]:
-        pass
-
-    def check_model(self):
-        if self.model is None:
-            raise ValueError(
-                "If no model has been provided to the data prep class, it cannot perform preparation for forecast task!"
-            )
-
-
-class LegacyDataPreparation(AbstractDataPreparation):
-    def prepare_train_data(self, data: pd.DataFrame) -> pd.DataFrame:
-        if self.horizons:
-            horizons = self.horizons
-        else:
-            horizons = self.pj.resolution_minutes
-
-        features_applicator = TrainFeatureApplicator(
-            horizons=horizons,
-            feature_names=self.model_specs.feature_names,
-            feature_modules=self.model_specs.feature_modules,
-        )
-        return features_applicator.add_features(data, pj=self.pj)
-
-    def prepare_forecast_data(self, data: pd.DataFrame) -> pd.DataFrame:
-        self.check_model()
-
-        features_applicator = OperationalPredictFeatureApplicator(
-            horizons=[self.pj["resolution_minutes"] / 60.0],
-            feature_names=self.model.feature_names,
-            feature_modules=self.model_specs.feature_modules,
-        )
-        data_with_features = features_applicator.add_features(data)
-
-        # Prep forecast input by selecting only the forecast datetime interval (this is much smaller than the input range)
-        # Also drop the load column
-        forecast_start, forecast_end = generate_forecast_datetime_range(
-            data_with_features
-        )
-        forecast_input_data = data_with_features[forecast_start:forecast_end].drop(
-            columns="load"
-        )
-
-        return forecast_input_data, data_with_features
-
-
-class ARDataPreparation(AbstractDataPreparation):
-    def __init__(
-        self,
-        pj: PredictionJobDataClass,
-        model_specs: ModelSpecificationDataClass,
-        model: Optional[OpenstfRegressor] = None,
-        horizons: Optional[list[float]] = None,
-        historical_depth: Optional[int] = None,
-    ) -> None:
-        super().__init__(pj, model_specs, model, horizons)
-        self.historical_depth = historical_depth
-
-    def prepare_train_data(self, data: pd.DataFrame) -> pd.DataFrame:
-        # Add dummy horizon column
-        data["horizon"] = 0
-
-        # remove non requested feature
-        features = self.model_specs.feature_names + ["horizon"]
-        result = remove_non_requested_feature_columns(data, features)
-
-        # Sort all features except for the (first) load and (last) horizon columns
-        result = result[["load"] + [c for c in result.columns if c != "load"]]
-        result = result.sort_index()
-        result = enforce_feature_order(result)
-
-        result = result[result.iloc[:, 0].notna()]
-        return result
-
-    def prepare_forecast_data(
-        self, data: pd.DataFrame
-    ) -> tuple[pd.DataFrame, pd.DataFrame]:
-        structlog.configure(
-            wrapper_class=structlog.make_filtering_bound_logger(
-                logging.getLevelName(Settings.log_level)
-            )
-        )
-        logger = structlog.get_logger(__name__)
-        self.check_model()
-        # Prep forecast input by selecting only the forecast datetime interval (this is much smaller than the input range)
-        # Also drop the load column
-        data = data[["load"] + self.model.feature_names]
-        forecast_start, forecast_end = generate_forecast_datetime_range(data)
-        forecast_input_data = data[forecast_start:forecast_end].drop(columns="load")
-
-        historical_start = None
-        if self.historical_depth:
-            historical_start = forecast_start - self.historical_depth * timedelta(
-                minutes=self.pj.resolution_minutes
-            )
-        past_data = data[historical_start:forecast_start].iloc[:-1]
-        self.model.update_historic_data(
-            past_data.drop(columns="load"), past_data["load"]
-        )
-        logger.info(
-            "Watch-out side effect on the model performed in the feature builder to update the historical data."
-        )
-
-        data[self.model.feature_importance_dataframe.index.tolist()] = 0
-        return forecast_input_data, data