openstef 3.4.56__py3-none-any.whl → 4.0.0a3__py3-none-any.whl

openstef/feature_engineering/holiday_features.py

@@ -1,231 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-"""This module contains all holiday related features."""
-from datetime import datetime, timedelta
-
-import holidays
-import numpy as np
-import pandas as pd
-
-from openstef import PROJECT_ROOT
-
-HOLIDAY_CSV_PATH: str = PROJECT_ROOT / "openstef" / "data" / "dutch_holidays.csv"
-
-
-def generate_holiday_feature_functions(
-    country_code: str = "NL",
-    years: list[int] | None = None,
-    path_to_school_holidays_csv: str = HOLIDAY_CSV_PATH,
-) -> dict:
-    """Generates functions for creating holiday feature.
-
-    This improves forecast accuracy. Examples of features that are
-    added are: 2020-01-01 is 'Nieuwjaarsdag'.
-
-        2022-12-24 - 2023-01-08 is the 'Kerstvakantie'
-        2022-10-15 - 2022-10-23 is the 'HerfstvakantieNoord'
-
-
-    The holidays are based on a manually generated csv file.
-    The information is collected using:
-    https://www.schoolvakanties-nederland.nl/ and the python holiday function
-    The official following official ducth holidays are included untill 2023:
-        - Kerstvakantie
-        - Meivakantie
-        - Herstvakantie
-        - Bouwvak
-        - Zomervakantie
-        - Voorjaarsvakantie
-        - Nieuwjaarsdag
-        - Pasen
-        - Koningsdag
-        - Hemelvaart
-        - Pinksteren
-        - Kerst
-
-
-    The 'Brugdagen' are updated untill dec 2020. (Generated using agenda)
-
-    Args:
-        country_code: Country for which to create holiday features.
-        years: years for which to create holiday features. If None,
-            the last 4 years, the current and next year are used.
-        path_to_school_holidays_csv: Filepath to csv with school holidays.
-
-        NOTE: Dutch holidays csv file is only until January 2026.
-
-    Returns:
-        Dictionary with functions that check if a given date is a holiday, keys
-        consist of "Is" + the_name_of_the_holiday_to_be_checked
-
-    """
-    if years is None:
-        now = datetime.now()
-        years = [
-            now.year - 4,
-            now.year - 3,
-            now.year - 2,
-            now.year - 1,
-            now.year,
-            now.year + 1,
-        ]
-
-    country_holidays = holidays.country_holidays(country_code, years=years)
-
-    # Make holiday function dict
-    holiday_functions = {}
-    # Add check function that includes all holidays of the provided csv
-    holiday_functions.update(
-        {
-            "is_national_holiday": lambda x: np.isin(
-                x.index.date, np.array(list(country_holidays))
-            )
-        }
-    )
-    # Define empty list to keep track of bridgedays
-    bridge_days = []
-    # Loop over list of holidays names
-    for date, holiday_name in sorted(country_holidays.items()):
-        # Define function explicitely to mitigate 'late binding' problem
-        def make_holiday_func(requested_date):
-            return lambda x: np.isin(x.index.date, np.array([requested_date]))
-
-        # Create lag function for each holiday
-        holiday_functions.update(
-            {"is_" + holiday_name.replace(" ", "_").lower(): make_holiday_func(date)}
-        )
-
-        # Check for bridge day
-        holiday_functions, bridge_days = check_for_bridge_day(
-            date, holiday_name, country_code, years, holiday_functions, bridge_days
-        )
-
-    # Add feature function that includes all bridgedays
-    holiday_functions.update(
-        {"is_bridgeday": lambda x: np.isin(x.index.date, np.array(list(bridge_days)))}
-    )
-
-    # Add school holidays if country is NL
-    if country_code == "NL":
-        # Manully generated csv including all dutch schoolholidays for different regions
-        df_holidays = pd.read_csv(path_to_school_holidays_csv, index_col=None)
-        df_holidays["datum"] = pd.to_datetime(df_holidays.datum).apply(
-            lambda x: x.date()
-        )
-
-        # Add check function that includes all holidays of the provided csv
-        holiday_functions.update(
-            {
-                "is_schoolholiday": lambda x: np.isin(
-                    x.index.date, df_holidays.datum.values
-                )
-            }
-        )
-
-        # Loop over list of holidays names
-        for holiday_name in list(set(df_holidays.name)):
-            # Define function explicitely to mitigate 'late binding' problem
-            def make_holiday_func(holidayname=holiday_name):
-                return lambda x: np.isin(
-                    x.index.date,
-                    df_holidays.datum[df_holidays.name == holidayname].values,
-                )
-
-            # Create lag function for each holiday
-            holiday_functions.update(
-                {
-                    "is_"
-                    + holiday_name.replace(" ", "_").lower(): make_holiday_func(
-                        holidayname=holiday_name
-                    )
-                }
-            )
-
-    return holiday_functions
-
-
-# Check for bridgedays
-def check_for_bridge_day(
-    date: datetime,
-    holiday_name: str,
-    country: str,
-    years: list,
-    holiday_functions: dict,
-    bridge_days: list,
-) -> tuple[dict, list]:
-    """Checks for bridgedays associated to a specific holiday with date (date).
-
-    Any found bridgedays are appende dto the bridgedays list. Also a specific feature
-    function for the bridgeday is added to the general holidayfuncitons dictionary.
-
-    Args:
-        date: Date of holiday to check for associated bridgedays.
-        holiday_name: Name of the holiday.
-        country: Country for which to detect the bridgedays.
-        years: List of years for which to detect bridgedays.
-        holiday_functions: Dictionary to which the featurefunction has to be appended to in case of a bridgeday.
-        bridge_days: List of bridgedays to which any found bridgedays have to be appended.
-
-    Returns:
-        - Dict with holiday feature functions
-        - List of bridgedays
-
-    """
-    country_holidays = holidays.country_holidays(country, years=years)
-
-    # if the date is a holiday, it is not a bridgeday
-    if date in country_holidays:
-        return holiday_functions, bridge_days
-
-    # Define function explicitely to mitigate 'late binding' problem
-    def make_holiday_func(requested_date):
-        return lambda x: np.isin(x.index.date, np.array([requested_date]))
-
-    # Looking forward: If day after tomorow is a national holiday or
-    # a saturday check if tomorow is not a national holiday
-
-    is_saturday_in_two_days = (date + timedelta(days=2)).weekday() == 5
-    is_holiday_in_two_days = (date + timedelta(days=2)) in country_holidays
-
-    is_holiday_tommorow = (date + timedelta(days=1)) in country_holidays
-    is_weekend_tommorrow = (date + timedelta(days=1)).weekday() in [5, 6]
-
-    if (
-        (is_holiday_in_two_days or is_saturday_in_two_days)
-        and (not is_holiday_tommorow and not is_weekend_tommorrow)
-        and date not in country_holidays
-    ):
-        # Create feature function for each holiday
-        holiday_functions.update(
-            {
-                "is_bridgeday"
-                + holiday_name.replace(" ", "_").lower(): make_holiday_func(
-                    (date + timedelta(days=1))
-                )
-            }
-        )
-        bridge_days.append((date + timedelta(days=1)))
-
-    # Looking backward: If the day before is a national holiday
-    # or a sunday check if yesterday is a national holiday
-    is_saturday_two_days_ago = (date - timedelta(days=2)).weekday() == 6
-    is_holiday_two_days_ago = (date - timedelta(days=2)) in country_holidays
-    is_holiday_yesterday = (date - timedelta(days=1)) in country_holidays
-    is_weekend_yesterday = (date - timedelta(days=1)).weekday() in [5, 6]
-
-    if (is_saturday_two_days_ago or is_holiday_two_days_ago) and (
-        not is_holiday_yesterday and not is_weekend_yesterday
-    ):
-        # Create featurefunction for the bridge function
-        holiday_functions.update(
-            {
-                "is_bridgeday"
-                + holiday_name.replace(" ", "_").lower(): make_holiday_func(
-                    (date - timedelta(days=1))
-                )
-            }
-        )
-        bridge_days.append((date - timedelta(days=1)))
-
-    return holiday_functions, bridge_days

openstef/feature_engineering/lag_features.py

@@ -1,165 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-"""This module contains all lag features."""
-import re
-
-import numpy as np
-import pandas as pd
-import scipy.signal
-
-
-def generate_lag_feature_functions(
-    feature_names: list[str] = None, horizon: float = 24.0
-) -> dict:
-    """Creates functions to generate lag features in a dataset.
-
-    Args:
-        feature_names: minute lagtimes that where used during training
-            of the model. If empty a new set will be automatically generated.
-        horizon: Forecast horizon limit in hours.
-
-    Returns:
-        Lag functions.
-
-    Example:
-
-    .. code-block:: py
-
-        lag_functions = generate_lag_functions(data,minute_list,h_ahead)
-
-    """
-    # Use extracted lag features if provided.
-    if feature_names is not None:
-        lag_times_minutes, lag_time_days_list = extract_lag_features(
-            feature_names, horizon
-        )
-    else:
-        # Generate available lag_times if no features are provided
-        lag_times_minutes, lag_time_days_list = generate_trivial_lag_features(horizon)
-
-    # Empty dict to store all generated lag functions
-    lag_functions = {}
-    for minutes in lag_times_minutes:  # Add intraday-lag functions (lags in minutes)
-
-        def func(x, shift=minutes):
-            return x.shift(freq="min", periods=1 * shift)
-
-        new = {"T-" + str(int(minutes)) + "min": func}
-        lag_functions.update(new)
-
-    # Add day lag functions:
-    for day in lag_time_days_list:
-
-        def func(x, shift=day):
-            return x.shift(freq="1d", periods=1 * shift)
-
-        new = {"T-" + str(int(day)) + "d": func}
-        lag_functions.update(new)
-    return lag_functions
-
-
-def extract_lag_features(
-    feature_names: list[str], horizon: float = 24.0
-) -> tuple[list, list]:
-    """Creates a list of lag minutes and a list of lag days that were used during the training of the input model.
-
-    Args:
-        feature_names: All requested lag features
-        horizon: Forecast horizon limit in hours.
-
-    Returns:
-        - List of minute lags that were used as features during training.
-        - List of days lags that were used as features during training.
-
-    """
-    # Prepare empty lists to append on
-    minutes_list = []
-    days_list = []
-
-    for lag_feature in feature_names:
-        # Select the number of days or the number of minutes by matching with a regular expression
-        number_of_minutes = re.search(r"T-(\d+)min", lag_feature)
-        number_of_days = re.search(r"T-(\d+)d", lag_feature)
-
-        # Append to the appropriate list
-        if number_of_minutes is not None:
-            minutes_list.append(int(number_of_minutes[1]))
-        elif number_of_days is not None:
-            days_list.append(int(number_of_days[1]))
-
-    # Discard lag times that are not available for the specified horizon
-    minutes_list = list(set([i for i in minutes_list if i >= horizon * 60]))
-    days_list = list(set([i for i in days_list if i >= horizon / 24]))
-
-    return minutes_list, days_list
-
-
-def generate_trivial_lag_features(horizon: float) -> tuple[list, list]:
-    """Generates relevant lag times for lag feature function creation.
-
-    This function is mostly used during training of models and not during predicting.
-
-    Args:
-        horizon: Forecast horizon limit in hours.
-
-    Returns:
-        - List of minute lags that were used as features during training.
-        - List of days lags that were used as features during training.
-
-    """
-    mindays = min(int(np.ceil(horizon / 24)), 15)
-    lag_time_days_list = list(np.linspace(mindays, 14, 15 - mindays))
-
-    # Make list of trivial lag times
-    trivial_lag_minutes_list = np.linspace(60, 23 * 60, 23).tolist() + [15, 30, 45]
-
-    # Discard lag times that are not available for the specified horizon
-    trivial_lag_times_minutes = list(
-        set([i for i in trivial_lag_minutes_list if i >= horizon * 60])
-    )
-
-    return trivial_lag_times_minutes, lag_time_days_list
-
-
-def generate_non_trivial_lag_times(
-    data: pd.DataFrame, height_threshold: float = 0.1
-) -> list[int]:
-    """Calculate an autocorrelation curve of the load trace.
-
-    This curve is subsequently used to add additional lag times as features.
-
-    Args:
-        data: Dataframe with input data in the form pd.DataFrame(index = datetime,
-                             columns = [label, predictor_1,..., predictor_n])
-        height_threshold: Minimal autocorrelation value to be recognized as a peak.
-
-    Returns:
-        Aditional non-trivial minute lags
-
-    """
-
-    def autocorr(x: np.array, lags: range) -> np.array:
-        """Make an autocorrelation curve."""
-        mean = x.mean()
-        var = np.var(x)
-        xp = x - mean
-        corr = np.correlate(xp, xp, "full")[len(x) - 1 :] / var / len(x)
-
-        return corr[: len(lags)]
-
-    try:
-        # Get rid of nans as the autocorrelation handles these values badly
-        data = data[data.columns[0]].dropna()  # First column contains the load
-        # Get autocorrelation curve
-        y = autocorr(data, range(10000))
-        # Determine the peaks (positive and negative) larger than a specified threshold
-        peaks = scipy.signal.find_peaks(np.abs(y), height=height_threshold)
-        peaks = peaks[0]
-        # Convert peaks to lag times in minutes
-        peaks = peaks[peaks < (60 * 4)]
-        additional_minute_space = peaks * 15
-    except Exception:
-        return []
-    # Return list of additional minute lags to be procceses by apply features
-    return list(additional_minute_space)

openstef/feature_engineering/missing_values_transformer.py

@@ -1,141 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-from typing import Union, List, Optional
-
-import numpy as np
-import pandas as pd
-from sklearn.impute import SimpleImputer
-from sklearn.preprocessing import FunctionTransformer
-from sklearn.utils.validation import check_array, check_is_fitted
-
-
-class MissingValuesTransformer:
-    """MissingValuesTransformer handles missing values in data by imputing them with a given strategy.
-
-    It also removes columns that are always null from the data.
-
-    """
-
-    in_feature_names: Optional[List[str]] = None
-    _n_in_features: Optional[int] = None
-
-    non_null_feature_names: List[str] = None
-
-    def __init__(
-        self,
-        missing_values: Union[int, float, str, None] = np.nan,
-        imputation_strategy: str = None,
-        fill_value: Union[str, int, float] = None,
-        no_fill_future_values_features: List[str] = None,
-    ):
-        """Initialize missing values handler.
-
-        Args:
-            missing_values: The placeholder for the missing values. All occurrences of
-                `missing_values` will be imputed.
-            imputation_strategy: The imputation strategy to use
-                Can be one of "mean", "median", "most_frequent", "constant" or None.
-            fill_value: When strategy == "constant", fill_value is used to replace all
-                occurrences of missing_values.
-            no_fill_future_values_features: The features for which it does not make sense
-                to fill future values. Rows that contain trailing null values for these
-                features will be removed from the data.
-
-        """
-        self.missing_values = missing_values
-        self.imputation_strategy = imputation_strategy
-        self.fill_value = fill_value
-        self.no_fill_future_values_features = no_fill_future_values_features or []
-        self.is_fitted_ = False
-
-        # Build the proper imputation transformer
-        # - Identity function if strategy is None
-        # - SimpleImputer with the dedicated strategy
-        if self.imputation_strategy is None:
-            self.imputer_ = FunctionTransformer(func=self._identity)
-        else:
-            self.imputer_ = SimpleImputer(
-                missing_values=self.missing_values,
-                strategy=self.imputation_strategy,
-                fill_value=self.fill_value,
-            ).set_output(transform="pandas")
-            self.imputer_._validate_params()
-
-    @staticmethod
-    def _determine_trailing_null_rows(x: pd.DataFrame) -> pd.Series:
-        """Determine rows with trailing null values in a DataFrame."""
-        return ~x.bfill().isnull().any(axis="columns")
-
-    def fit(self, x, y=None):
-        """Fit the imputer on the input data."""
-        _ = check_array(x, force_all_finite="allow-nan")
-        if not isinstance(x, pd.DataFrame):
-            x = pd.DataFrame(np.asarray(x))
-
-        self.in_feature_names = list(x.columns)
-        self._n_in_features = x.shape[1]
-
-        # Remove always null columns
-        is_column_null = x.isnull().all(axis="index")
-        self.non_null_feature_names = list(x.columns[~is_column_null])
-        x = x[self.non_null_feature_names]
-
-        # Remove trailing null rows for features that should
-        # not be imputed in the future
-        trailing_null_rows = self._determine_trailing_null_rows(
-            x[self.no_fill_future_values_features]
-        )
-        x = x.loc[trailing_null_rows]
-
-        # Imputers do not support labels
-        self.imputer_.fit(X=x, y=None)
-        self.is_fitted_ = True
-
-    def transform(self, x) -> pd.DataFrame:
-        """Transform the input data by imputing missing values."""
-        check_is_fitted(self)
-        _ = check_array(x, force_all_finite="allow-nan")
-        if not isinstance(x, pd.DataFrame):
-            x = pd.DataFrame(np.asarray(x))
-
-        x = x[self.non_null_feature_names]
-
-        transformed = self.imputer_.transform(x)
-
-        return transformed
-
-    def fit_transform(self, x, y=None) -> tuple[pd.DataFrame, Optional[pd.Series]]:
-        """Fit the imputer on the input data and transform it.
-
-        Returns:
-            The data with missing values imputed.
-
-        """
-        self.fit(x, y)
-
-        if not isinstance(x, pd.DataFrame):
-            x = pd.DataFrame(np.asarray(x))
-
-        x = x[self.non_null_feature_names]
-
-        # Remove trailing null rows for features that should
-        # not be imputed in the future
-        non_trailing_null_rows = self._determine_trailing_null_rows(
-            x[self.no_fill_future_values_features]
-        )
-        x = x.loc[non_trailing_null_rows]
-
-        x = self.transform(x)
-
-        if y is not None:
-            y = y.loc[non_trailing_null_rows]
-
-        return x, y
-
-    @classmethod
-    def _identity(cls, x):
-        return x
-
-    def __sklearn_is_fitted__(self) -> bool:
-        return self.in_feature_names is not None

openstef/feature_engineering/rolling_features.py

@@ -1,58 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-from datetime import timedelta
-
-import pandas as pd
-
-from openstef.data_classes.prediction_job import PredictionJobDataClass
-from pydantic import TypeAdapter
-
-
-def convert_timedelta_to_isoformat(td: timedelta) -> str:
-    """
-    Converts a timedelta to an ISO 8601 formatted period string.
-
-    Args:
-        td: timedelta object to convert.
-
-    Returns:
-        ISO 8601 formatted period string.
-    """
-    timedelta_adapter = TypeAdapter(timedelta)
-    return timedelta_adapter.dump_python(td, mode="json")
-
-
-def add_rolling_aggregate_features(
-    data: pd.DataFrame,
-    pj: PredictionJobDataClass,
-    rolling_window: timedelta = timedelta(hours=24),
-) -> pd.DataFrame:
-    """
-    Adds rolling aggregate features to the input dataframe.
-
-    These features are calculated with an aggregation over a rolling window of the data.
-    A list of requested features is used to determine whether to add the rolling features
-    or not.
-
-    Args:
-        data: Input dataframe to which the rolling features will be added.
-        pj: Prediction job data.
-        rolling_window: Rolling window size for the aggregation.
-
-    Returns:
-        DataFrame with added rolling features.
-    """
-    # Ensure the index is a DatetimeIndex
-    if not isinstance(data.index, pd.DatetimeIndex):
-        raise ValueError("The DataFrame index must be a DatetimeIndex.")
-
-    if "load" not in data.columns:
-        raise ValueError("The DataFrame must contain a 'load' column.")
-    rolling_window_load = data["load"].rolling(window=rolling_window)
-
-    for aggregate_func in pj["rolling_aggregate_features"]:
-        data[
-            f"rolling_{aggregate_func.value}_load_{convert_timedelta_to_isoformat(rolling_window)}"
-        ] = rolling_window_load.aggregate(aggregate_func.value)
-    return data