openstef 3.4.56__py3-none-any.whl → 4.0.0a3__py3-none-any.whl

openstef/model/basecase.py

@@ -1,82 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-import numpy as np
-import pandas as pd
-from sklearn.base import BaseEstimator, RegressorMixin
-
-MINIMAL_RESOLUTION: int = 15  # Used for validating the forecast input
-
-
-class BaseCaseModel(BaseEstimator, RegressorMixin):
-    def predict(self, forecast_input_data: pd.DataFrame) -> pd.DataFrame:
-        """Predict using the basecase method. The basecase forecast is determined by the T-7d and T-14d load.
-
-        This means fitting the model is not required. However a fit method is still included to be fully comatible with sklearn.
-
-        Args:
-            forecast_input_data: Forecast input dataframe
-
-        Returns:
-            Basecase forecast
-
-        """
-        return self.make_basecase_forecast(forecast_input_data)
-
-    def fit(self):
-        return self
-
-    @staticmethod
-    def make_basecase_forecast(
-        forecast_input_data: pd.DataFrame, overwrite_delay_hours: int = 48
-    ) -> pd.DataFrame:
-        """Make a basecase forecast.
-
-        The idea of the basecase forecast is that if all else fails, this forecasts is
-        still available. Basecase example: the load of last week.
-
-        Args:
-            forecast_input_data: Forecast input dataframe
-            overwrite_delay_hours: times before this in the future are not
-                forecasted
-
-        Raises:
-            ValueError: if columns T-7d or T-14d is not present
-            ValueError: If the start of the forecast is before the horizon of the regular forecast
-        Returns:
-            Basecase forecast
-
-        """
-        # Check if required features are provided
-        if not all(
-            item in forecast_input_data.columns.to_list() for item in ["T-14d", "T-7d"]
-        ):
-            raise ValueError(
-                "Could not make basecase, features T-7d and T-14d are required! Tip:"
-                " Generate these features with a FeatureApplicator object."
-            )
-
-        # Make basecase forecast: Use load of last week
-        basecase_forecast = (
-            forecast_input_data[["T-7d"]].dropna().rename(columns={"T-7d": "forecast"})
-        )
-
-        # Maybe there is still missing data, for example if the cdb has been down for a
-        # while in this case, use the load of 2 weeks before
-        basecase_forecast = pd.concat(
-            [
-                basecase_forecast,
-                forecast_input_data[["T-14d"]]
-                .dropna()
-                .rename(columns={"T-14d": "forecast"}),
-            ]
-        )
-        basecase_forecast = basecase_forecast[
-            np.invert(basecase_forecast.index.duplicated())
-        ]
-
-        return basecase_forecast.sort_index()
-
-    @property
-    def can_predict_quantiles(self):
-        return False

openstef/model/confidence_interval_applicator.py

@@ -1,242 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-import logging
-from datetime import datetime
-
-import numpy as np
-import pandas as pd
-import structlog
-from scipy import stats
-from sklearn.base import RegressorMixin
-
-from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.exceptions import ModelWithoutStDev
-from openstef.settings import Settings
-
-
-class ConfidenceIntervalApplicator:
-    def __init__(self, model: RegressorMixin, forecast_input_data: pd.DataFrame):
-        self.model = model
-        self.forecast_input_data = forecast_input_data
-        structlog.configure(
-            wrapper_class=structlog.make_filtering_bound_logger(
-                logging.getLevelName(Settings.log_level)
-            )
-        )
-        self.logger = structlog.get_logger(self.__class__.__name__)
-
-    def add_confidence_interval(
-        self,
-        forecast: pd.DataFrame,
-        pj: PredictionJobDataClass,
-    ) -> pd.DataFrame:
-        """Add a confidence interval to a forecast.
-
-        Adds a confidence interval to a forecast in two ways:
-            1. "stdev" column, this is a column with a standard deviation that is
-                determined during training (ConfidenceGenerator)
-            2. Quantile columns, these columns give a more precise defenition of the
-                confidence interval. Quantile columns are determined with one of two
-                methods, depending on the model type group:
-
-                a. Default, using the "stdev" column and the assumption the error is
-                    normally distributed.
-                b. Quantile regression, this method is only available for quantile
-                    models and uses specifically trained models to estimate the
-                    quantiles of the confidence interval.
-
-                Depending on the model type (quantile or non quantile),
-                 a confidence interval is added to the forecast based on quantile
-                 regression or the default method.
-
-        Args:
-            forecast: Forecast DataFrame with columns: "forecast"
-            pj: Prediction job
-
-        Returns:
-            Forecast DataFrame with columns; "forecast", "stdev" and quantile columns.
-
-        """
-        temp_forecast = self._add_standard_deviation_to_forecast(forecast)
-
-        if self.model.can_predict_quantiles:
-            # Try to generate the quantiles that were requested
-            try:
-                result = self._add_quantiles_to_forecast_quantile_regression(
-                    temp_forecast, pj["quantiles"]
-                )
-                return result
-            except Exception:
-                # Fallback on quantiles of the model if the requested quantiles cant be generated by the model.
-                # Can happen when the model was trained on different quantiles than are requested
-                result = self._add_quantiles_to_forecast_quantile_regression(
-                    temp_forecast, self.model.quantiles
-                )
-                self.logger.warning(
-                    "Quantiles are requested the model was not trained on. Using the quantiles the model was trained on",
-                    requested_quantiles=pj["quantiles"],
-                    trained_quantiles=self.model.quantiles,
-                )
-                return result
-
-        return self._add_quantiles_to_forecast_default(temp_forecast, pj["quantiles"])
-
-    def _add_standard_deviation_to_forecast(
-        self, forecast: pd.DataFrame
-    ) -> pd.DataFrame:
-        """Add a standard deviation to a live forecast.
-
-        The stdev for intermediate forecast horizons is interpolated.
-
-        Args:
-            forecast: Forecast DataFrame with columns: "forecast"
-
-        Returns:
-            Forecast with added standard deviation. DataFrame with columns:
-                "forecast", "stdev"
-
-        Raises:
-            ModelWithoutStDev: If the model does not have a valid standard deviation.
-
-        """
-        minimal_resolution: int = 15  # Minimal time resolution in minutes
-        standard_deviation = self.model.standard_deviation
-
-        # raise an exception if no valid standard deviation is available
-        if standard_deviation is None:
-            raise ModelWithoutStDev("No stdev available")
-
-        if standard_deviation.empty:  # make separate statement to avoid None.empty
-            raise ModelWithoutStDev("No stdev available")
-
-        if standard_deviation.stdev.isnull().values.all():
-            raise ModelWithoutStDev("All stdev values are NA")
-
-        # Fill stdev nans with the mean of all stdev values
-        if standard_deviation.stdev.isnull().values.any():
-            self.logger.warning(
-                "Stdev for some hours is not known, filling in with mean."
-            )
-            standard_deviation["stdev"] = standard_deviation.stdev.fillna(
-                standard_deviation.stdev.mean()
-            )
-
-        # pivot to have a dataframe with columns [stdev, hour, horizon] for a
-        # 'near' and a 'far' horizon
-        stdev = standard_deviation.pivot_table(columns=["horizon"], index="hour")[
-            "stdev"
-        ]
-        # Prepare input dataframes for near and far horizon
-        near = stdev.columns.min()
-        far = stdev.columns.max()
-
-        forecast_copy = forecast.copy()
-        # add time ahead column if not already present
-        if "tAhead" not in forecast_copy.columns:
-            # Determine now, rounded on 15 minutes,
-            # Rounding helps to prevent fractional t_aheads
-            now = (
-                pd.Series(datetime.utcnow().replace(tzinfo=forecast_copy.index.tzinfo))
-                .min()
-                .round(f"{minimal_resolution}T")
-                .to_pydatetime()
-            )
-            # Determine t_aheads by subtracting with now
-            forecast_copy["tAhead"] = (
-                forecast_copy.index - now
-            ).total_seconds() / 3600.0
-
-        # add helper column hour
-        forecast_copy["hour"] = forecast_copy.index.hour
-
-        # Define functions which can be used to approximate the error for in-between
-        # time horizons
-        # Let's fit and exponential decay of accuracy
-        def calc_exp_dec(t, stdev_row, near, far):
-            # We use the formula sigma(t) = (1 - A * exp(-t/tau)) + b
-            # Strictly speaking, tau is specific for each time series.
-            # However, for simplicity, we use tau = Far/4.
-            # This represents a situation where the stdev at 25% of the Far horizon,
-            # has increased by two.
-            tau = far / 4.0
-            # Filling in the known sigma(Near) and sigma(Far) gives:
-            sf, sn = stdev_row[far], stdev_row[near]
-            A = (sf - sn) / ((1 - np.exp(-far / tau)) - (1 - np.exp(-near / tau)))
-            b = sn - A * (1 - np.exp(-near / tau))
-            value = A * (1 - np.exp(-t / tau)) + b
-            # cap the value to keep it between near and far
-            if value < sn:
-                return sn
-            return sf if value > sf else value
-
-        # If only one horizon is available use that one
-        if len(stdev.columns) == 1:
-            forecast_copy["stdev"] = forecast_copy.apply(
-                lambda x: stdev.loc[x.hour], axis=1
-            )
-        # If more are available do something fancy with interpolation
-        else:
-            # Add stdev to forecast_copy dataframe
-            forecast_copy["stdev"] = forecast_copy.apply(
-                lambda x: calc_exp_dec(x.tAhead, stdev.loc[x.hour], near, far), axis=1
-            )
-        return forecast_copy.drop(columns=["hour"])
-
-    @staticmethod
-    def _add_quantiles_to_forecast_default(
-        forecast: pd.DataFrame, quantiles: list[float]
-    ) -> pd.DataFrame:
-        """Add quantiles to forecast.
-
-            Use the standard deviation to calculate the quantiles.
-
-        Args:
-            forecast: Forecast (should contain a 'forecast' + 'stdev' column)
-            quantiles: List with desired quantiles,
-                for example: [0.01, 0.1, 0.9, 0.99]
-
-        Returns:
-            Forecast DataFrame with quantile (e.g. 'quantile_PXX')
-                columns added.
-
-        """
-        # Check if stdev and forecast are in the dataframe
-        if not all(elem in forecast.columns for elem in ["forecast", "stdev"]):
-            raise ValueError("Forecast should contain a 'forecast' and 'stdev' column")
-
-        for quantile in quantiles:
-            quantile_key = f"quantile_P{quantile * 100:02.0f}"
-            forecast[quantile_key] = (
-                forecast["forecast"] + stats.norm.ppf(quantile) * forecast["stdev"]
-            )
-
-        return forecast
-
-    def _add_quantiles_to_forecast_quantile_regression(
-        self, forecast: pd.DataFrame, quantiles: list[float]
-    ) -> pd.DataFrame:
-        """Add quantiles to forecast.
-
-            Use trained quantile regression model to calculate the quantiles.
-
-        Args:
-            forecast: Forecast
-            quantiles: List with desired quantiles
-
-        Returns:
-            Forecast DataFrame with quantile (e.g. 'quantile_PXX')
-                columns added.
-
-        """
-        # Only determine quantiles for datetimes in forecast
-        quantile_df = pd.DataFrame(index=self.forecast_input_data.index)
-        for quantile in quantiles:
-            quantile_key = f"quantile_P{quantile * 100:02.0f}"
-            quantile_df[quantile_key] = self.model.predict(
-                self.forecast_input_data, quantile=quantile
-            )
-
-        return forecast.merge(
-            quantile_df, left_index=True, right_index=True, how="left"
-        )

openstef/model/fallback.py

@@ -1,77 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-from datetime import datetime
-
-import pandas as pd
-
-
-def generate_fallback(
-    forecast_input: pd.DataFrame,
-    load: pd.DataFrame,
-    fallback_strategy: str = "extreme_day",
-) -> pd.DataFrame:
-    """Make a fall back forecast, Set the value of the forecast 'quality' column to 'substituted'.
-
-    Currently only fallback_strategy=extreme day is implemented which return historic profile of most extreme day.
-
-    Args:
-        forecast_input : dataframe desired for the forecast
-        load: index=datetime, columns=['load']
-        fallback_strategy: strategy to determine fallback. options:
-            - extreme_day: use daily profile of most extreme day
-    Returns:
-        Fallback forecast DataFrame with columns; 'forecast', 'quality'
-
-    Raises:
-        ValueError if len(load) == 0
-        NotImplementedError if fallback_strategy != 'extreme_day'
-
-    """
-    # Check if load is completely empty
-    if len(load.dropna()) == 0:
-        raise ValueError("No historic load data available")
-
-    if fallback_strategy != "extreme_day":
-        raise NotImplementedError(
-            f'fallback_strategy should be "extreme_day", received:{fallback_strategy}'
-        )
-
-    if fallback_strategy == "extreme_day":
-        # Execute this fallback strategy
-        # Find most extreme historic day and merge it by time-of-day to the requested moments
-
-        # Find most extreme historic day (do not count today as it is incomplete)
-        day_with_highest_load_date = (
-            load[load.index.tz_localize(None).date != datetime.utcnow().date()]
-            .idxmax()
-            .load.date()
-        )
-        # generate datetime range of the day with the highest load
-        from_datetime = pd.Timestamp(day_with_highest_load_date, tz=load.index.tz)
-        till_datetime = from_datetime + pd.Timedelta("1 days")
-
-        # slice load dataframe, only rows for the day with the highest load
-        highest_daily_loadprofile = load.loc[
-            (load.index >= from_datetime) & (load.index < till_datetime)
-        ]
-
-        # Match moments by time-of-day
-        highest_daily_loadprofile.loc[:, "time"] = highest_daily_loadprofile.index.time
-        forecast = pd.DataFrame(index=forecast_input.index)
-        forecast["time"] = forecast.index.time
-        forecast = (
-            forecast.reset_index()
-            .merge(
-                highest_daily_loadprofile, left_on="time", right_on="time", how="outer"
-            )
-            .set_index("index")
-        )
-
-    # Rename so column is called forecast
-    forecast = forecast[["load"]].rename(columns=dict(load="forecast"))
-
-    # Add a column quality.
-    forecast["quality"] = "substituted"
-
-    return forecast

openstef/model/metamodels/__init__.py

@@ -1,3 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0

openstef/model/metamodels/feature_clipper.py

@@ -1,90 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-from sklearn.base import BaseEstimator, TransformerMixin
-import pandas as pd
-from typing import List, Dict, Tuple, Optional
-
-
-class FeatureClipper(BaseEstimator, TransformerMixin):
-    """
-    A transformer that clips the values of specified columns to the minimum and
-    maximum values observed during training. This prevents the model from
-    extrapolating beyond these values during prediction.
-    """
-
-    def __init__(self, columns: List[str]):
-        """
-        Initialize the FeatureClipper.
-
-        Parameters:
-        ----------
-        columns : List[str]
-            List of column names to be clipped.
-        """
-        self.columns: List[str] = columns
-        self.feature_ranges: Dict[str, Tuple[float, float]] = {}
-
-    def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None) -> "FeatureClipper":
-        """
-        Fits the transformer on the training data by calculating the min and max
-        values for the specified columns.
-
-        Parameters:
-        ----------
-        X : pd.DataFrame
-            The input DataFrame containing training data.
-
-        y : Optional[pd.Series]
-            Ignored. This parameter exists for compatibility with scikit-learn's pipeline.
-
-        Returns:
-        -------
-        self : FeatureClipper
-            Fitted transformer.
-
-        Raises:
-        ------
-        ValueError:
-            If the input is not a pandas DataFrame.
-        """
-        if not isinstance(X, pd.DataFrame):
-            raise ValueError("Input must be a pandas DataFrame")
-
-        for col in self.columns:
-            if col in X.columns:
-                self.feature_ranges[col] = (X[col].min(), X[col].max())
-
-        return self
-
-    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
-        """
-        Transforms new data by clipping the specified columns' values to be within
-        the min and max range observed during fitting.
-
-        Parameters:
-        ----------
-        X : pd.DataFrame
-            The input DataFrame containing new data to be transformed.
-
-        Returns:
-        -------
-        X_ : pd.DataFrame
-            A copy of the input DataFrame with clipped values in the specified columns.
-
-        Raises:
-        ------
-        ValueError:
-            If the input is not a pandas DataFrame.
-        """
-        if not isinstance(X, pd.DataFrame):
-            raise ValueError("Input must be a pandas DataFrame")
-
-        X_copy = X.copy()
-
-        for col in self.columns:
-            if col in X_copy.columns and col in self.feature_ranges:
-                min_val, max_val = self.feature_ranges[col]
-                X_copy[col] = X_copy[col].clip(lower=min_val, upper=max_val)
-
-        return X_copy