openstef 3.4.10__py3-none-any.whl → 3.4.44__py3-none-any.whl

openstef/model/regressors/linear_quantile.py

@@ -0,0 +1,296 @@
+# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
+#
+# SPDX-License-Identifier: MPL-2.0
+import re
+from typing import Dict, Union, Set, Optional, List
+
+import numpy as np
+import pandas as pd
+from sklearn.base import RegressorMixin
+from sklearn.linear_model import QuantileRegressor
+from sklearn.preprocessing import StandardScaler
+from sklearn.utils.validation import check_is_fitted
+
+from openstef.feature_engineering.missing_values_transformer import (
+    MissingValuesTransformer,
+)
+from openstef.model.regressors.regressor import OpenstfRegressor
+
+DEFAULT_QUANTILES: tuple[float, ...] = (0.9, 0.5, 0.1)
+
+
+class LinearQuantileOpenstfRegressor(OpenstfRegressor, RegressorMixin):
+    quantiles: tuple[float, ...]
+    alpha: float
+    solver: str
+
+    imputer_: MissingValuesTransformer
+    x_scaler_: StandardScaler
+    y_scaler_: StandardScaler
+    models_: Dict[float, QuantileRegressor]
+
+    is_fitted_: bool = False
+
+    FEATURE_IGNORE_LIST: Set[str] = {
+        "IsWeekendDay",
+        "IsWeekDay",
+        "IsSunday",
+        "Month",
+        "Quarter",
+    }
+
+    def __init__(
+        self,
+        quantiles: tuple[float, ...] = DEFAULT_QUANTILES,
+        alpha: float = 0.0,
+        solver: str = "highs",
+        missing_values: Union[int, float, str, None] = np.nan,
+        imputation_strategy: Optional[str] = "mean",
+        fill_value: Union[str, int, float] = None,
+        weight_scale_percentile: int = 95,
+        weight_exponent: float = 1,
+        weight_floor: float = 0.1,
+        no_fill_future_values_features: List[str] = None,
+    ):
+        """Initialize LinearQuantileOpenstfRegressor.
+
+        Model that provides quantile regression with SKLearn QuantileRegressor.
+        For each desired quantile an QuantileRegressor model is trained,
+        these can later be used to predict quantiles.
+
+        This model is sensitive to feature quality and therefore has logic to remove
+        some custom features produced by OpenSTEF. The features that are removed are:
+        - Holiday features (is_christmas, is_*)
+        - Lagged features (T-1d, T-*)
+        - Point in time features (IsWeekendDay, IsWeekDay, IsSunday, Month, Quarter)
+        - Infeed MFFBAS profiles (E*_I)
+
+        Args:
+            quantiles: Tuple with desired quantiles, quantile 0.5 is required.
+                For example: (0.1, 0.5, 0.9)
+            alpha: Regularization constant for L1 regularization
+            solver: Solver to use for optimization
+            missing_values: Value to be considered as missing value
+            imputation_strategy: Imputation strategy
+            fill_value: Fill value
+            weight_scale_percentile: Percentile used in scaling of the samples
+            weight_exponent: Exponent used in sample weighing
+            weight_floor: Minimum weight for samples
+            no_fill_future_values_features: The features for which it does not make sense
+                to fill future values. Rows that contain trailing null values for these
+                features will be removed from the data.
+
+        """
+        super().__init__()
+
+        # Check if quantile 0.5 is present. This is required.
+        if 0.5 not in quantiles:
+            raise ValueError(
+                "Cannot train quantile model as 0.5 is not in requested quantiles!"
+            )
+
+        self.quantiles = quantiles
+        self.alpha = alpha
+        self.solver = solver
+        self.weight_scale_percentile = weight_scale_percentile
+        self.weight_exponent = weight_exponent
+        self.weight_floor = weight_floor
+        self.imputer_ = MissingValuesTransformer(
+            missing_values=missing_values,
+            imputation_strategy=imputation_strategy,
+            fill_value=fill_value,
+            no_fill_future_values_features=no_fill_future_values_features,
+        )
+        self.x_scaler_ = StandardScaler()
+        self.y_scaler_ = StandardScaler()
+        self.models_ = {
+            quantile: QuantileRegressor(alpha=alpha, quantile=quantile, solver=solver)
+            for quantile in quantiles
+        }
+
+    @property
+    def feature_names(self) -> list:
+        """The names of the features used to train the model."""
+        check_is_fitted(self)
+        return self.imputer_.non_null_feature_names
+
+    @staticmethod
+    def _get_importance_names():
+        return {
+            "gain_importance_name": "total_gain",
+            "weight_importance_name": "weight",
+        }
+
+    @property
+    def can_predict_quantiles(self) -> bool:
+        """Attribute that indicates if the model predict particular quantiles."""
+        return True
+
+    def _is_feature_ignored(self, feature_name: str) -> bool:
+        """Check if a feature is ignored by the model.
+
+        Args:
+            feature_name: Feature name
+
+        Returns:
+            True if the feature is ignored, False otherwise
+
+        """
+        return (
+            # Ignore named features
+            feature_name in self.FEATURE_IGNORE_LIST
+            or
+            # Ignore holiday features
+            re.match(r"is_", feature_name) is not None
+            or
+            # Ignore lag features
+            re.match(r"T-", feature_name) is not None
+            or
+            # Ignore infeed MFFBAS profiles
+            re.match(r"E\d.*_I", feature_name) is not None
+        )
+
+    def _remove_ignored_features(self, x: pd.DataFrame) -> pd.DataFrame:
+        """Remove ignored features from the input data.
+
+        Args:
+            x: Input data
+
+        Returns:
+            Data without ignored features
+
+        """
+        return x.drop(columns=[c for c in x.columns if self._is_feature_ignored(c)])
+
+    def fit(self, x: pd.DataFrame, y: pd.Series, **kwargs) -> RegressorMixin:
+        """Fits linear quantile model.
+
+        Args:
+            x: Feature matrix
+            y: Labels
+
+        Returns:
+            Fitted LinearQuantile model
+
+        """
+        if not isinstance(y, pd.Series):
+            y = pd.Series(np.asarray(y), name="load")
+
+        x = self._remove_ignored_features(x)
+
+        # Fix nan columns
+        x, y = self.imputer_.fit_transform(x, y)
+        if x.isna().any().any():
+            raise ValueError(
+                "There are nan values in the input data. Set "
+                "imputation_strategy to solve them."
+            )
+
+        # Apply feature scaling
+        x_scaled = self.x_scaler_.fit_transform(x)
+        y_scaled = self.y_scaler_.fit_transform(y.to_frame())[:, 0]
+
+        # Add more focus on extreme / peak values
+        sample_weight = self._calculate_sample_weights(y.values.squeeze())
+
+        # Fit quantile regressors
+        for quantile in self.quantiles:
+            self.models_[quantile].fit(
+                X=x_scaled, y=y_scaled, sample_weight=sample_weight
+            )
+
+        self.is_fitted_ = True
+
+        self.feature_importances_ = self._get_feature_importance_from_linear()
+
+        return self
+
+    def _calculate_sample_weights(self, y: np.array):
+        """Calculate sample weights based on the y values of arbitrary scale.
+
+        The resulting weights are in the range [0,1] and are used to put more emphasis
+        on certain samples. The sample weighting function does:
+
+        * Rescale data to a [-1, 1] range using quantile scaling. 90% of the data will
+          be within this range. Rest is outside.
+        * Calculate the weight by taking the exponent of scaled data.
+          * exponent=0: Results in uniform weights for all samples.
+          * exponent=1: Results in linearly increasing weights for samples that are
+            closer to the extremes.
+          * exponent>1: Results in exponentially increasing weights for samples that are
+            closer to the extremes.
+        * Clip the data to [0, 1] range with weight_floor as the minimum weight.
+          * Weight floor is used to make sure that all the samples are considered.
+
+        """
+        return np.clip(
+            _weight_exp(
+                _scale_percentile(y, percentile=self.weight_scale_percentile),
+                exponent=self.weight_exponent,
+            ),
+            a_min=self.weight_floor,
+            a_max=1,
+        )
+
+    def predict(self, x: pd.DataFrame, quantile: float = 0.5, **kwargs) -> np.array:
+        """Makes a prediction for a desired quantile.
+
+        Args:
+            x: Feature matrix
+            quantile: Quantile for which a prediciton is desired,
+                note that only quantile are available for which a model is trained,
+                and that this is a quantile-model specific keyword
+
+        Returns:
+            Prediction
+
+        Raises:
+            ValueError in case no model is trained for the requested quantile
+
+        """
+        check_is_fitted(self)
+
+        # Preprocess input data
+        x = self._remove_ignored_features(x)
+        x = self.imputer_.transform(x)
+        x_scaled = self.x_scaler_.transform(x)
+
+        # Make prediction
+        y_pred = self.models_[quantile].predict(X=x_scaled)
+
+        # Inverse scaling
+        y_pred = self.y_scaler_.inverse_transform(y_pred.reshape(-1, 1))[:, 0]
+
+        return y_pred
+
+    def _get_feature_importance_from_linear(self, quantile: float = 0.5) -> np.array:
+        check_is_fitted(self)
+        feature_importance_linear = np.abs(self.models_[quantile].coef_)
+        reg_feature_importances_dict = dict(
+            zip(self.imputer_.non_null_feature_names, feature_importance_linear)
+        )
+        return np.array(
+            [
+                reg_feature_importances_dict.get(c, 0)
+                for c in self.imputer_.non_null_feature_names
+            ]
+        )
+
+    @classmethod
+    def _get_param_names(cls):
+        return [
+            "quantiles",
+            "alpha",
+            "solver",
+        ]
+
+    def __sklearn_is_fitted__(self) -> bool:
+        return self.is_fitted_
+
+
+def _scale_percentile(x: np.ndarray, percentile: int = 95):
+    return np.abs(x / np.percentile(np.abs(x), percentile))
+
+
+def _weight_exp(x: np.ndarray, exponent: float = 1):
+    return np.abs(x) ** exponent

openstef/model/regressors/xgb.py

@@ -1,6 +1,10 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+from typing import Optional
+
+import numpy as np
+from sklearn.base import RegressorMixin
 
 from xgboost import XGBRegressor
 
@@ -27,3 +31,22 @@ class XGBOpenstfRegressor(XGBRegressor, OpenstfRegressor):
             "gain_importance_name": "total_gain",
             "weight_importance_name": "weight",
         }
+
+    def fit(
+        self,
+        x: np.array,
+        y: np.array,
+        *,
+        early_stopping_rounds: Optional[int] = None,
+        callbacks: Optional[list] = None,
+        eval_metric: Optional[str] = None,
+        **kwargs
+    ):
+        if early_stopping_rounds is not None:
+            self.set_params(early_stopping_rounds=early_stopping_rounds)
+        if callbacks is not None:
+            self.set_params(callbacks=callbacks)
+        if eval_metric is not None:
+            self.set_params(eval_metric=eval_metric)
+
+        super().fit(x, y, **kwargs)

openstef/model/regressors/xgb_multioutput_quantile.py

@@ -0,0 +1,261 @@
+# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
+#
+# SPDX-License-Identifier: MPL-2.0
+from functools import partial
+from typing import Dict, Optional, Sequence, Tuple, Union
+
+import numpy as np
+import pandas as pd
+import sklearn.base
+import xgboost as xgb
+from sklearn.compose import TransformedTargetRegressor
+from sklearn.preprocessing import StandardScaler
+from sklearn.utils.validation import check_array, check_is_fitted, check_X_y
+from xgboost import Booster
+
+import openstef.metrics.metrics as metrics
+from openstef.model.regressors.regressor import OpenstfRegressor
+
+DEFAULT_QUANTILES: tuple[float, ...] = (0.9, 0.5, 0.1)
+
+
+class XGBMultiOutputQuantileOpenstfRegressor(OpenstfRegressor):
+    r"""Model that provides multioutput quantile regression with XGBoost by default using the arctan loss function.
+
+    Arctan loss:
+        Refence: https://github.com/LaurensSluyterman/XGBoost_quantile_regression/tree/master
+        The key idea is to use a smooth approximation of the pinball loss, the arctan
+        pinball loss, that has a relatively large second derivative.
+
+        The approximation is given by:
+        $$L^{(\text{arctan})}_{\tau, s}(u) = (\tau - 0.5 + \frac{\arctan (u/s)}{\pi})u   + \frac{s}{\pi}$$.  # noqa E501
+
+        Some important settings:
+
+        * The parameter in the loss function determines the amount of smoothing. A
+            smaller values gives a closer approximation but also a much smaller second
+            derivative. A larger value gives more conservative quantiles when
+            is larger than 0.5, the quantile becomes larger and vice versa.
+            Values between 0.05 and 0.1 appear to work well. It may be a good idea to
+            optimize this parameter.
+        * Set min-child-weight to zero. The second derivatives can be a lot smaller
+            than 1 and this parameter may prevent any splits.
+        * Use a relatively small max-delta-step. We used a default of 0.5.
+            This prevents excessive steps that could happen due to the relatively
+            small second derivative.
+        * For the same reason, use a slightly lower learning rate of 0.05.
+
+    """
+
+    estimator_: TransformedTargetRegressor
+    quantile_indices_: Dict[float, int]
+
+    @staticmethod
+    def _get_importance_names():
+        return {
+            "gain_importance_name": "total_gain",
+            "weight_importance_name": "weight",
+        }
+
+    def __init__(
+        self,
+        quantiles: tuple[float, ...] = DEFAULT_QUANTILES,
+        gamma: float = 0.0,
+        colsample_bytree: float = 1.0,
+        subsample: float = 1.0,
+        min_child_weight: int = 0,
+        max_depth: int = 6,
+        learning_rate: float = 0.22,
+        alpha: float = 0.0,
+        max_delta_step: int = 0.5,
+        arctan_smoothing: float = 0.055,
+        early_stopping_rounds: Optional[int] = None,
+    ):
+        """Initialize XGBMultiQuantileRegressor.
+
+        Model that provides quantile regression with XGBoost.
+        For each desired quantile an XGBoost model is trained,
+        these can later be used to predict quantiles.
+
+        Args:
+            quantiles: Tuple with desired quantiles, quantile 0.5 is required.
+                For example: (0.1, 0.5, 0.9)
+            gamma: Gamma.
+            colsample_bytree: Colsample by tree.
+            subsample: Subsample.
+            min_child_weight: Minimum child weight.
+            max_depth: Maximum depth.
+            learning_rate: Learning rate.
+            alpha: Alpha.
+            max_delta_step: Maximum delta step.
+            arctan_smoothing: smoothing parameter of the arctan loss function.
+            early_stopping_rounds: Number of rounds to stop training if no improvement
+                is made.
+
+        Raises:
+            ValueError in case quantile 0.5 is not in the requested quantiles.
+
+        """
+        super().__init__()
+        if 0.5 not in quantiles:
+            raise ValueError(
+                "Cannot train quantile model as 0.5 is not in requested quantiles!"
+            )
+
+        self.quantiles = quantiles
+
+        # Set attributes for hyper parameters
+        self.subsample = subsample
+        self.min_child_weight = min_child_weight
+        self.max_depth = max_depth
+        self.gamma = gamma
+        self.alpha = alpha
+        self.max_delta_step = max_delta_step
+        self.colsample_bytree = colsample_bytree
+        self.learning_rate = learning_rate
+        self.early_stopping_rounds = early_stopping_rounds
+        self.arctan_smoothing = arctan_smoothing
+
+        # Get fitting parameters - only those required for xgbooster's
+        xgb_regressor_params = {
+            key: value
+            for key, value in self.get_params().items()
+            if key in xgb.XGBRegressor().get_params().keys()
+        }
+
+        # Define the model
+        objective = partial(
+            metrics.arctan_loss, taus=self.quantiles, s=arctan_smoothing
+        )
+        xgb_model: xgb.XGBRegressor = xgb.XGBRegressor(
+            objective=objective,
+            base_score=0,
+            multi_strategy="one_output_per_tree",
+            **xgb_regressor_params,
+        )
+        self.estimator_ = TransformedTargetRegressor(
+            regressor=xgb_model, transformer=StandardScaler()
+        )
+
+        # Set quantile indices to remap multioutput predictions
+        self.quantile_indices_ = {
+            quantile: i for i, quantile in enumerate(self.quantiles)
+        }
+
+    def fit(
+        self,
+        x: np.array,
+        y: np.array,
+        eval_set: Optional[Sequence[Tuple[np.array, np.array]]] = None,
+        verbose: Optional[Union[bool, int]] = 0,
+        **kwargs
+    ) -> OpenstfRegressor:
+        """Fits xgb quantile model.
+
+        Args:
+            x: Feature matrix.
+            y: Labels.
+            eval_set: Evaluation set to monitor training performance.
+            verbose: Verbosity level (disabled by default).
+
+        Returns:
+            Fitted XGBQuantile model.
+
+        """
+        if isinstance(y, pd.Series):
+            y = y.to_numpy()
+
+        if not isinstance(x, pd.DataFrame):
+            x = pd.DataFrame(np.asarray(x))
+
+        # Check/validate input
+        check_X_y(x, y, force_all_finite="allow-nan")
+
+        # Prepare inputs
+        y_multioutput = replicate_for_multioutput(y, len(self.quantiles))
+
+        # Define watchlist if eval_set is defined
+        eval_set_multioutput = []
+        if eval_set:
+            for x_eval, y_eval in eval_set:
+                if isinstance(y_eval, pd.Series):
+                    y_eval = y_eval.to_numpy()
+
+                y_eval_multioutput = replicate_for_multioutput(
+                    y=y_eval, num_quantiles=len(self.quantiles)
+                )
+                eval_set_multioutput.append((x_eval, y_eval_multioutput))
+
+            eval_set_multioutput.append((x, y_multioutput))
+
+        self.estimator_.fit(
+            X=x.copy(deep=True),
+            y=y_multioutput,
+            eval_set=eval_set_multioutput,
+            verbose=verbose,
+        )
+
+        # Update state of the estimator
+        self.feature_importances_ = self.estimator_.regressor_.feature_importances_
+        self.is_fitted_ = True
+
+        return self
+
+    def predict(self, x: np.array, quantile: float = 0.5) -> np.array:
+        """Makes a prediction for a desired quantile.
+
+        Args:
+            x: Feature matrix.
+            quantile: Quantile for which a prediciton is desired,
+                note that only quantile are available for which a model is trained,
+                and that this is a quantile-model specific keyword.
+
+        Returns:
+            Prediction
+
+        Raises:
+            ValueError in case no model is trained for the requested quantile.
+
+        """
+        # Check if model is trained for this quantile
+        if quantile not in self.quantiles:
+            raise ValueError("No model trained for requested quantile!")
+
+        # Check/validate input
+        check_array(x, force_all_finite="allow-nan")
+        check_is_fitted(self)
+
+        # best_iteration is only available if early stopping was used during training
+        prediction: np.array
+        if hasattr(self.estimator_, "best_iteration"):
+            prediction = self.estimator_.predict(
+                X=x,
+                iteration_range=(0, self.estimator_.best_iteration + 1),
+            )
+        else:
+            prediction = self.estimator_.predict(X=x)
+
+        quantile_index = self.quantile_indices_[quantile]
+        return prediction[:, quantile_index]
+
+    @property
+    def feature_names(self):
+        return self.estimator_.feature_names_in_
+
+    @property
+    def can_predict_quantiles(self):
+        return True
+
+
+def replicate_for_multioutput(y: np.array, num_quantiles: int) -> np.array:
+    """Replicates a 1D array to a 2D array for multioutput regression.
+
+    Args:
+        y: 1D array.
+        num_quantiles: Number of columns in the output array.
+
+    Returns:
+        2D array with shape (len(y), num_quantiles)
+
+    """
+    return np.repeat(y[:, None], num_quantiles, axis=1)

openstef/model/regressors/xgb_quantile.py

@@ -52,6 +52,9 @@ class XGBQuantileOpenstfRegressor(OpenstfRegressor):
             alpha: Alpha
             max_delta_step: Maximum delta step
 
+        Raises:
+            ValueError in case quantile 0.5 is not in the requested quantiles
+
         """
         super().__init__()
         # Check if quantile 0.5 is pressent this is required

openstef/model/serializer.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: MPL-2.0
 import json
+import logging
 import os
 import shutil
 from datetime import datetime
@@ -20,10 +21,16 @@ from xgboost import XGBModel  # Temporary for backward compatibility
 from openstef.data_classes.model_specifications import ModelSpecificationDataClass
 from openstef.metrics.reporter import Report
 from openstef.model.regressors.regressor import OpenstfRegressor
+from openstef.settings import Settings
 
 
 class MLflowSerializer:
     def __init__(self, mlflow_tracking_uri: str):
+        structlog.configure(
+            wrapper_class=structlog.make_filtering_bound_logger(
+                logging.getLevelName(Settings.log_level)
+            )
+        )
         self.logger = structlog.get_logger(self.__class__.__name__)
         mlflow.set_tracking_uri(mlflow_tracking_uri)
         self.logger.debug(f"MLflow tracking uri at init= {mlflow_tracking_uri}")
@@ -147,6 +154,9 @@ class MLflowSerializer:
         Args:
             experiment_name: Name of the experiment, often the id of the predition job.
 
+        Raises:
+            LookupError: If model is not found in MLflow.
+
         """
         try:
             models_df = self._find_models(

openstef/model_selection/model_selection.py

@@ -106,7 +106,7 @@ def split_data_train_validation_test(
     validation_fraction: float = 0.15,
     back_test: bool = False,
     stratification_min_max: bool = True,
-) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
+) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]:
     """Split input data into train, test and validation set.
 
     Function for splitting data with features in a train, test and
@@ -140,6 +140,9 @@ def split_data_train_validation_test(
         - Validation data.
         - Test data.
 
+    Raises:
+        ValueError: When the test and validation fractions are too high.
+
     """
     test_fraction = test_fraction if back_test else 0
     train_fraction = 1 - (test_fraction + validation_fraction)

openstef/monitoring/performance_meter.py

@@ -20,8 +20,7 @@ class PerformanceMeter:
 
         Args:
             level_label: The label of the new level. This could i.e. be 'task'
-            level_name: The name of the specified level. This could i.e. be
-                'tracy_todo'
+            level_name: The name of the specified level.
             **kwargs: Any other kwargs are appended to the logging.
 
         Returns:

openstef/monitoring/teams.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
 from typing import Union
 
 import pandas as pd
@@ -8,6 +9,8 @@ import pymsteams
 import structlog
 from pymsteams import cardsection
 
+from openstef.settings import Settings
+
 
 def post_teams(
     msg: Union[str, dict],
@@ -38,6 +41,14 @@ def post_teams(
     Note:
         This function is namespace-specific.
     """
+    if not Settings.post_teams_messages:
+        return
+
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
     # If no url is passed, give warning and don't send teams message
     if url is None: