openstef 3.4.10__py3-none-any.whl → 3.4.44__py3-none-any.whl

openstef/feature_engineering/holiday_features.py

@@ -10,13 +10,11 @@ import pandas as pd
 
 from openstef import PROJECT_ROOT
 
-HOLIDAY_CSV_PATH: str = (
-    PROJECT_ROOT / "openstef" / "data" / "dutch_holidays_2020-2022.csv"
-)
+HOLIDAY_CSV_PATH: str = PROJECT_ROOT / "openstef" / "data" / "dutch_holidays.csv"
 
 
 def generate_holiday_feature_functions(
-    country: str = "NL",
+    country_code: str = "NL",
     years: list = None,
     path_to_school_holidays_csv: str = HOLIDAY_CSV_PATH,
 ) -> dict:
@@ -46,12 +44,14 @@ def generate_holiday_feature_functions(
         - Pinksteren
         - Kerst
 
+
     The 'Brugdagen' are updated untill dec 2020. (Generated using agenda)
 
     Args:
         country: Country for which to create holiday features.
         years: years for which to create holiday features.
         path_to_school_holidays_csv: Filepath to csv with school holidays.
+        NOTE: Dutch holidays csv file is only until January 2026.
 
     Returns:
         Dictionary with functions that check if a given date is a holiday, keys
@@ -69,7 +69,7 @@ def generate_holiday_feature_functions(
             now.year + 1,
         ]
 
-    country_holidays = holidays.country_holidays(country, years=years)
+    country_holidays = holidays.country_holidays(country_code, years=years)
 
     # Make holiday function dict
     holiday_functions = {}
@@ -96,7 +96,7 @@ def generate_holiday_feature_functions(
 
         # Check for bridge day
         holiday_functions, bridge_days = check_for_bridge_day(
-            date, holiday_name, country, years, holiday_functions, bridge_days
+            date, holiday_name, country_code, years, holiday_functions, bridge_days
         )
 
     # Add feature function that includes all bridgedays
@@ -104,33 +104,42 @@ def generate_holiday_feature_functions(
         {"is_bridgeday": lambda x: np.isin(x.index.date, np.array(list(bridge_days)))}
     )
 
-    # Manully generated csv including all dutch schoolholidays for different regions
-    df_holidays = pd.read_csv(path_to_school_holidays_csv, index_col=None)
-    df_holidays["datum"] = pd.to_datetime(df_holidays.datum).apply(lambda x: x.date())
-
-    # Add check function that includes all holidays of the provided csv
-    holiday_functions.update(
-        {"is_schoolholiday": lambda x: np.isin(x.index.date, df_holidays.datum.values)}
-    )
-
-    # Loop over list of holidays names
-    for holiday_name in list(set(df_holidays.name)):
-        # Define function explicitely to mitigate 'late binding' problem
-        def make_holiday_func(holidayname=holiday_name):
-            return lambda x: np.isin(
-                x.index.date, df_holidays.datum[df_holidays.name == holidayname].values
-            )
+    # Add school holidays if country is NL
+    if country_code == "NL":
+        # Manully generated csv including all dutch schoolholidays for different regions
+        df_holidays = pd.read_csv(path_to_school_holidays_csv, index_col=None)
+        df_holidays["datum"] = pd.to_datetime(df_holidays.datum).apply(
+            lambda x: x.date()
+        )
 
-        # Create lag function for each holiday
+        # Add check function that includes all holidays of the provided csv
         holiday_functions.update(
             {
-                "is_"
-                + holiday_name.replace(" ", "_").lower(): make_holiday_func(
-                    holidayname=holiday_name
+                "is_schoolholiday": lambda x: np.isin(
+                    x.index.date, df_holidays.datum.values
                 )
             }
         )
 
+        # Loop over list of holidays names
+        for holiday_name in list(set(df_holidays.name)):
+            # Define function explicitely to mitigate 'late binding' problem
+            def make_holiday_func(holidayname=holiday_name):
+                return lambda x: np.isin(
+                    x.index.date,
+                    df_holidays.datum[df_holidays.name == holidayname].values,
+                )
+
+            # Create lag function for each holiday
+            holiday_functions.update(
+                {
+                    "is_"
+                    + holiday_name.replace(" ", "_").lower(): make_holiday_func(
+                        holidayname=holiday_name
+                    )
+                }
+            )
+
     return holiday_functions
 
 

openstef/feature_engineering/missing_values_transformer.py

@@ -0,0 +1,141 @@
+# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
+#
+# SPDX-License-Identifier: MPL-2.0
+from typing import Union, List, Optional
+
+import numpy as np
+import pandas as pd
+from sklearn.impute import SimpleImputer
+from sklearn.preprocessing import FunctionTransformer
+from sklearn.utils.validation import check_array, check_is_fitted
+
+
+class MissingValuesTransformer:
+    """MissingValuesTransformer handles missing values in data by imputing them with a given strategy.
+
+    It also removes columns that are always null from the data.
+
+    """
+
+    in_feature_names: Optional[List[str]] = None
+    _n_in_features: Optional[int] = None
+
+    non_null_feature_names: List[str] = None
+
+    def __init__(
+        self,
+        missing_values: Union[int, float, str, None] = np.nan,
+        imputation_strategy: str = None,
+        fill_value: Union[str, int, float] = None,
+        no_fill_future_values_features: List[str] = None,
+    ):
+        """Initialize missing values handler.
+
+        Args:
+            missing_values: The placeholder for the missing values. All occurrences of
+                `missing_values` will be imputed.
+            imputation_strategy: The imputation strategy to use
+                Can be one of "mean", "median", "most_frequent", "constant" or None.
+            fill_value: When strategy == "constant", fill_value is used to replace all
+                occurrences of missing_values.
+            no_fill_future_values_features: The features for which it does not make sense
+                to fill future values. Rows that contain trailing null values for these
+                features will be removed from the data.
+
+        """
+        self.missing_values = missing_values
+        self.imputation_strategy = imputation_strategy
+        self.fill_value = fill_value
+        self.no_fill_future_values_features = no_fill_future_values_features or []
+        self.is_fitted_ = False
+
+        # Build the proper imputation transformer
+        # - Identity function if strategy is None
+        # - SimpleImputer with the dedicated strategy
+        if self.imputation_strategy is None:
+            self.imputer_ = FunctionTransformer(func=self._identity)
+        else:
+            self.imputer_ = SimpleImputer(
+                missing_values=self.missing_values,
+                strategy=self.imputation_strategy,
+                fill_value=self.fill_value,
+            ).set_output(transform="pandas")
+            self.imputer_._validate_params()
+
+    @staticmethod
+    def _determine_trailing_null_rows(x: pd.DataFrame) -> pd.Series:
+        """Determine rows with trailing null values in a DataFrame."""
+        return ~x.bfill().isnull().any(axis="columns")
+
+    def fit(self, x, y=None):
+        """Fit the imputer on the input data."""
+        _ = check_array(x, force_all_finite="allow-nan")
+        if not isinstance(x, pd.DataFrame):
+            x = pd.DataFrame(np.asarray(x))
+
+        self.in_feature_names = list(x.columns)
+        self._n_in_features = x.shape[1]
+
+        # Remove always null columns
+        is_column_null = x.isnull().all(axis="index")
+        self.non_null_feature_names = list(x.columns[~is_column_null])
+        x = x[self.non_null_feature_names]
+
+        # Remove trailing null rows for features that should
+        # not be imputed in the future
+        trailing_null_rows = self._determine_trailing_null_rows(
+            x[self.no_fill_future_values_features]
+        )
+        x = x.loc[trailing_null_rows]
+
+        # Imputers do not support labels
+        self.imputer_.fit(X=x, y=None)
+        self.is_fitted_ = True
+
+    def transform(self, x) -> pd.DataFrame:
+        """Transform the input data by imputing missing values."""
+        check_is_fitted(self)
+        _ = check_array(x, force_all_finite="allow-nan")
+        if not isinstance(x, pd.DataFrame):
+            x = pd.DataFrame(np.asarray(x))
+
+        x = x[self.non_null_feature_names]
+
+        transformed = self.imputer_.transform(x)
+
+        return transformed
+
+    def fit_transform(self, x, y=None) -> tuple[pd.DataFrame, Optional[pd.Series]]:
+        """Fit the imputer on the input data and transform it.
+
+        Returns:
+            The data with missing values imputed.
+
+        """
+        self.fit(x, y)
+
+        if not isinstance(x, pd.DataFrame):
+            x = pd.DataFrame(np.asarray(x))
+
+        x = x[self.non_null_feature_names]
+
+        # Remove trailing null rows for features that should
+        # not be imputed in the future
+        non_trailing_null_rows = self._determine_trailing_null_rows(
+            x[self.no_fill_future_values_features]
+        )
+        x = x.loc[non_trailing_null_rows]
+
+        x = self.transform(x)
+
+        if y is not None:
+            y = y.loc[non_trailing_null_rows]
+
+        return x, y
+
+    @classmethod
+    def _identity(cls, x):
+        return x
+
+    def __sklearn_is_fitted__(self) -> bool:
+        return self.in_feature_names is not None

openstef/feature_engineering/weather_features.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: MPL-2.0
 
 """This module contains all wheather related functions used for feature engineering."""
+import logging
 from typing import Union
 
 import numpy as np
@@ -12,7 +13,13 @@ import structlog
 from pvlib.location import Location
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
+from openstef.settings import Settings
 
+structlog.configure(
+    wrapper_class=structlog.make_filtering_bound_logger(
+        logging.getLevelName(Settings.log_level)
+    )
+)
 logger = structlog.get_logger(__name__)
 
 

openstef/metrics/figure.py

@@ -71,6 +71,9 @@ def plot_data_series(
     Returns:
         A line plot of each passed data series.
 
+    Raises:
+        ValueError: If names is None and the number of series is greater than 3.
+
     """
     series_names = {
         1: ("series",),

openstef/metrics/metrics.py

@@ -25,6 +25,9 @@ def get_eval_metric_function(metric_name: str) -> Callable:
     Returns:
         Function to calculate the metric.
 
+    Raises:
+        KeyError: If the metric is not available.
+
     """
     evaluation_function = {
         "rmse": rmse,
@@ -130,6 +133,9 @@ def r_mae_highest(
 
     The range is based on the load range of the previous two weeks.
 
+    Raises:
+        ValueError: If the length of the realised and forecast arrays are not equal.
+
     """
     # Check if length of both arrays is equal
     if len(np.array(realised)) != len(np.array(forecast)):
@@ -395,7 +401,7 @@ def xgb_quantile_obj(
     Args:
         preds: numpy.ndarray
         dmatrix: xgboost.DMatrix
-        quantile: float
+        quantile: float between 0 and 1
 
     Returns:
         Gradient and Hessian
@@ -425,3 +431,54 @@ def xgb_quantile_obj(
     hess = np.ones_like(preds)
 
     return grad, hess
+
+
+def arctan_loss(y_true, y_pred, taus, s=0.1):
+    """Compute the arctan pinball loss.
+
+    Note that XGBoost outputs the predictions in a slightly peculiar manner.
+    Suppose we have 100 data points and we predict 10 quantiles. The predictions
+    will be an array of size (1000 x 1). We first resize this to a (100x10) array
+    where each row corresponds to the 10 predicted quantile for a single data
+    point. We then use a for-loop (over the 10 columns) to calculate the gradients
+    and second derivatives. Legibility was chosen over efficiency. This part
+    can be made more efficient.
+
+    Args:
+        y_true: An array containing the true observations.
+        y_pred: An array containing the predicted quantiles.
+        taus: A list containing the true desired coverage of the quantiles.
+        s: A smoothing parameter.
+
+    Returns:
+        grad: An array containing the (negative) gradients with respect to y_pred.
+        hess: An array containing the second derivative with respect to y_pred.
+
+    """
+    size = len(y_true)
+    n_dim = len(taus)  # The number of columns
+    n_rows = size // n_dim
+
+    # Resize the predictions and targets.
+    # Each column corresponds to a quantile, each row to a data point.
+    y_pred = np.reshape(y_pred, (n_rows, n_dim))
+    y_true = np.reshape(y_true, (n_rows, n_dim))
+
+    # Calculate the differences
+    u = y_true - y_pred
+
+    # Calculate the gradient and second derivatives
+    grad = np.zeros_like(y_pred)
+    hess = np.zeros_like(y_pred)
+    z = u / s
+    for i, tau in enumerate(taus):
+        x = 1 + z[:, i] ** 2
+        grad[:, i] = (
+            tau - 0.5 + 1 / np.pi * np.arctan(z[:, i]) + z[:, i] / (np.pi) * x**-1
+        )
+        hess[:, i] = 2 / (np.pi * s) * x ** (-2)
+
+    # Reshape back to the original shape.
+    grad = grad.reshape(size)
+    hess = hess.reshape(size)
+    return -grad / n_dim, hess / n_dim

openstef/metrics/reporter.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: MPL-2.0
 """Defines reporter class."""
+import logging
 import os
 import warnings
 from dataclasses import dataclass
@@ -16,6 +17,7 @@ from plotly.graph_objects import Figure
 from openstef.metrics import figure
 from openstef.metrics.metrics import bias, mae, nsme, r_mae, rmse
 from openstef.model.regressors.regressor import OpenstfRegressor
+from openstef.settings import Settings
 
 
 @dataclass
@@ -167,6 +169,11 @@ class Reporter:
     def write_report_to_disk(report: Report, report_folder: str):
         """Write report to disk; e.g. for viewing report of latest models using grafana."""
         # Initialize logger and serializer
+        structlog.configure(
+            wrapper_class=structlog.make_filtering_bound_logger(
+                logging.getLevelName(Settings.log_level)
+            )
+        )
         logger = structlog.get_logger(__name__)
         if report_folder:
             # create path if does not exist

openstef/model/confidence_interval_applicator.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
 from datetime import datetime
 
 import numpy as np
@@ -11,12 +12,18 @@ from sklearn.base import RegressorMixin
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
 from openstef.exceptions import ModelWithoutStDev
+from openstef.settings import Settings
 
 
 class ConfidenceIntervalApplicator:
     def __init__(self, model: RegressorMixin, forecast_input_data: pd.DataFrame):
         self.model = model
         self.forecast_input_data = forecast_input_data
+        structlog.configure(
+            wrapper_class=structlog.make_filtering_bound_logger(
+                logging.getLevelName(Settings.log_level)
+            )
+        )
         self.logger = structlog.get_logger(self.__class__.__name__)
 
     def add_confidence_interval(
@@ -54,9 +61,24 @@ class ConfidenceIntervalApplicator:
         temp_forecast = self._add_standard_deviation_to_forecast(forecast)
 
         if self.model.can_predict_quantiles:
-            return self._add_quantiles_to_forecast_quantile_regression(
-                temp_forecast, pj["quantiles"]
-            )
+            # Try to generate the quantiles that were requested
+            try:
+                result = self._add_quantiles_to_forecast_quantile_regression(
+                    temp_forecast, pj["quantiles"]
+                )
+                return result
+            except Exception:
+                # Fallback on quantiles of the model if the requested quantiles cant be generated by the model.
+                # Can happen when the model was trained on different quantiles than are requested
+                result = self._add_quantiles_to_forecast_quantile_regression(
+                    temp_forecast, self.model.quantiles
+                )
+                self.logger.warning(
+                    "Quantiles are requested the model was not trained on. Using the quantiles the model was trained on",
+                    requested_quantiles=pj["quantiles"],
+                    trained_quantiles=self.model.quantiles,
+                )
+                return result
 
         return self._add_quantiles_to_forecast_default(temp_forecast, pj["quantiles"])
 
@@ -74,6 +96,9 @@ class ConfidenceIntervalApplicator:
             Forecast with added standard deviation. DataFrame with columns:
                 "forecast", "stdev"
 
+        Raises:
+            ModelWithoutStDev: If the model does not have a valid standard deviation.
+
         """
         minimal_resolution: int = 15  # Minimal time resolution in minutes
         standard_deviation = self.model.standard_deviation

openstef/model/model_creator.py

@@ -1,28 +1,35 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
 from typing import Union
 
 import structlog
 
-from openstef.enums import MLModelType
+from openstef.enums import ModelType
+from openstef.model.regressors.arima import ARIMAOpenstfRegressor
 from openstef.model.regressors.custom_regressor import is_custom_type, load_custom_model
 from openstef.model.regressors.lgbm import LGBMOpenstfRegressor
 from openstef.model.regressors.linear import LinearOpenstfRegressor
+from openstef.model.regressors.linear_quantile import LinearQuantileOpenstfRegressor
 from openstef.model.regressors.regressor import OpenstfRegressor
+from openstef.model.regressors.flatliner import FlatlinerRegressor
 from openstef.model.regressors.xgb import XGBOpenstfRegressor
 from openstef.model.regressors.xgb_quantile import XGBQuantileOpenstfRegressor
-from openstef.model.regressors.arima import ARIMAOpenstfRegressor
+from openstef.model.regressors.xgb_multioutput_quantile import (
+    XGBMultiOutputQuantileOpenstfRegressor,
+)
+from openstef.settings import Settings
 
+structlog.configure(
+    wrapper_class=structlog.make_filtering_bound_logger(
+        logging.getLevelName(Settings.log_level)
+    )
+)
 logger = structlog.get_logger(__name__)
-try:
-    from openstef.model.regressors.proloaf import OpenstfProloafRegressor
-except ImportError:
-    logger.info("Proloaf not available, setting constructor to None")
-    OpenstfProloafRegressor = None
 
 valid_model_kwargs = {
-    MLModelType.XGB: [
+    ModelType.XGB: [
         "n_estimators",
         "objective",
         "max_depth",
@@ -53,7 +60,7 @@ valid_model_kwargs = {
         "validate_parameters",
         "early_stopping_rounds",
     ],
-    MLModelType.LGB: [
+    ModelType.LGB: [
         "boosting_type",
         "objective",
         "num_leaves",
@@ -75,7 +82,7 @@ valid_model_kwargs = {
         "importance_type",
         "early_stopping_rounds",
     ],
-    MLModelType.XGB_QUANTILE: [
+    ModelType.XGB_QUANTILE: [
         "quantiles",
         "gamma",
         "colsample_bytree",
@@ -84,33 +91,37 @@ valid_model_kwargs = {
         "max_depth",
         "early_stopping_rounds",
     ],
-    MLModelType.ProLoaf: [
-        "relu_leak",
-        "encoder_features",
-        "decoder_features",
-        "core_layers",
-        "rel_linear_hidden_size",
-        "rel_core_hidden_size",
-        "dropout_fc",
-        "dropout_core",
-        "training_metric",
-        "metric_options",
-        "optimizer_name",
-        "early_stopping_patience",
-        "early_stopping_margin",
-        "learning_rate",
-        "max_epochs",
-        "device",
-        "batch_size",
-        "history_horizon",
-        "horizon_minutes",
+    ModelType.XGB_MULTIOUTPUT_QUANTILE: [
+        "quantiles",
+        "gamma",
+        "colsample_bytree",
+        "subsample",
+        "min_child_weight",
+        "max_depth",
+        "early_stopping_rounds",
+        "arctan_smoothing",
+    ],
+    ModelType.LINEAR: [
+        "missing_values",
+        "imputation_strategy",
+        "fill_value",
+    ],
+    ModelType.FLATLINER: [
+        "quantiles",
     ],
-    MLModelType.LINEAR: [
+    ModelType.LINEAR_QUANTILE: [
+        "alpha",
+        "quantiles",
+        "solver",
         "missing_values",
         "imputation_strategy",
         "fill_value",
+        "weight_scale_percentile",
+        "weight_exponent",
+        "weight_floor",
+        "no_fill_future_values_features",
     ],
-    MLModelType.ARIMA: [
+    ModelType.ARIMA: [
         "backtest_max_horizon",
         "order",
         "seasonal_order",
@@ -124,16 +135,18 @@ class ModelCreator:
 
     # Set object mapping
     MODEL_CONSTRUCTORS = {
-        MLModelType.XGB: XGBOpenstfRegressor,
-        MLModelType.LGB: LGBMOpenstfRegressor,
-        MLModelType.XGB_QUANTILE: XGBQuantileOpenstfRegressor,
-        MLModelType.ProLoaf: OpenstfProloafRegressor,
-        MLModelType.LINEAR: LinearOpenstfRegressor,
-        MLModelType.ARIMA: ARIMAOpenstfRegressor,
+        ModelType.XGB: XGBOpenstfRegressor,
+        ModelType.LGB: LGBMOpenstfRegressor,
+        ModelType.XGB_QUANTILE: XGBQuantileOpenstfRegressor,
+        ModelType.XGB_MULTIOUTPUT_QUANTILE: XGBMultiOutputQuantileOpenstfRegressor,
+        ModelType.LINEAR: LinearOpenstfRegressor,
+        ModelType.LINEAR_QUANTILE: LinearQuantileOpenstfRegressor,
+        ModelType.ARIMA: ARIMAOpenstfRegressor,
+        ModelType.FLATLINER: FlatlinerRegressor,
     }
 
     @staticmethod
-    def create_model(model_type: Union[MLModelType, str], **kwargs) -> OpenstfRegressor:
+    def create_model(model_type: Union[ModelType, str], **kwargs) -> OpenstfRegressor:
         """Create a machine learning model based on model type.
 
         Args:
@@ -154,7 +167,7 @@ class ModelCreator:
                 model_class = load_custom_model(model_type)
                 valid_kwargs = model_class.valid_kwargs()
             else:
-                model_type = MLModelType(model_type)
+                model_type = ModelType(model_type)
                 model_class = ModelCreator.MODEL_CONSTRUCTORS[model_type]
                 valid_kwargs = valid_model_kwargs[model_type]
                 # Check if model as imported
@@ -165,7 +178,7 @@ class ModelCreator:
                         "Please refer to the ReadMe for instructions"
                     )
         except ValueError as e:
-            valid_types = [t.value for t in MLModelType]
+            valid_types = [t.value for t in ModelType]
             raise NotImplementedError(
                 f"No constructor for '{model_type}', "
                 f"valid model_types are: {valid_types} "