openstef 3.4.10__py3-none-any.whl → 3.4.29__py3-none-any.whl

openstef/model/regressors/xgb_multioutput_quantile.py

@@ -0,0 +1,261 @@
+# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
+#
+# SPDX-License-Identifier: MPL-2.0
+from functools import partial
+from typing import Dict, Optional, Sequence, Tuple, Union
+
+import numpy as np
+import pandas as pd
+import sklearn.base
+import xgboost as xgb
+from sklearn.compose import TransformedTargetRegressor
+from sklearn.preprocessing import StandardScaler
+from sklearn.utils.validation import check_array, check_is_fitted, check_X_y
+from xgboost import Booster
+
+import openstef.metrics.metrics as metrics
+from openstef.model.regressors.regressor import OpenstfRegressor
+
+DEFAULT_QUANTILES: tuple[float, ...] = (0.9, 0.5, 0.1)
+
+
+class XGBMultiOutputQuantileOpenstfRegressor(OpenstfRegressor):
+    r"""Model that provides multioutput quantile regression with XGBoost by default using the arctan loss function.
+
+    Arctan loss:
+        Refence: https://github.com/LaurensSluyterman/XGBoost_quantile_regression/tree/master
+        The key idea is to use a smooth approximation of the pinball loss, the arctan
+        pinball loss, that has a relatively large second derivative.
+
+        The approximation is given by:
+        $$L^{(\text{arctan})}_{\tau, s}(u) = (\tau - 0.5 + \frac{\arctan (u/s)}{\pi})u   + \frac{s}{\pi}$$.  # noqa E501
+
+        Some important settings:
+
+        * The parameter in the loss function determines the amount of smoothing. A
+            smaller values gives a closer approximation but also a much smaller second
+            derivative. A larger value gives more conservative quantiles when
+            is larger than 0.5, the quantile becomes larger and vice versa.
+            Values between 0.05 and 0.1 appear to work well. It may be a good idea to
+            optimize this parameter.
+        * Set min-child-weight to zero. The second derivatives can be a lot smaller
+            than 1 and this parameter may prevent any splits.
+        * Use a relatively small max-delta-step. We used a default of 0.5.
+            This prevents excessive steps that could happen due to the relatively
+            small second derivative.
+        * For the same reason, use a slightly lower learning rate of 0.05.
+
+    """
+
+    estimator_: TransformedTargetRegressor
+    quantile_indices_: Dict[float, int]
+
+    @staticmethod
+    def _get_importance_names():
+        return {
+            "gain_importance_name": "total_gain",
+            "weight_importance_name": "weight",
+        }
+
+    def __init__(
+        self,
+        quantiles: tuple[float, ...] = DEFAULT_QUANTILES,
+        gamma: float = 0.0,
+        colsample_bytree: float = 1.0,
+        subsample: float = 1.0,
+        min_child_weight: int = 0,
+        max_depth: int = 6,
+        learning_rate: float = 0.22,
+        alpha: float = 0.0,
+        max_delta_step: int = 0.5,
+        arctan_smoothing: float = 0.055,
+        early_stopping_rounds: Optional[int] = None,
+    ):
+        """Initialize XGBMultiQuantileRegressor.
+
+        Model that provides quantile regression with XGBoost.
+        For each desired quantile an XGBoost model is trained,
+        these can later be used to predict quantiles.
+
+        Args:
+            quantiles: Tuple with desired quantiles, quantile 0.5 is required.
+                For example: (0.1, 0.5, 0.9)
+            gamma: Gamma.
+            colsample_bytree: Colsample by tree.
+            subsample: Subsample.
+            min_child_weight: Minimum child weight.
+            max_depth: Maximum depth.
+            learning_rate: Learning rate.
+            alpha: Alpha.
+            max_delta_step: Maximum delta step.
+            arctan_smoothing: smoothing parameter of the arctan loss function.
+            early_stopping_rounds: Number of rounds to stop training if no improvement
+                is made.
+
+        Raises:
+            ValueError in case quantile 0.5 is not in the requested quantiles.
+
+        """
+        super().__init__()
+        if 0.5 not in quantiles:
+            raise ValueError(
+                "Cannot train quantile model as 0.5 is not in requested quantiles!"
+            )
+
+        self.quantiles = quantiles
+
+        # Set attributes for hyper parameters
+        self.subsample = subsample
+        self.min_child_weight = min_child_weight
+        self.max_depth = max_depth
+        self.gamma = gamma
+        self.alpha = alpha
+        self.max_delta_step = max_delta_step
+        self.colsample_bytree = colsample_bytree
+        self.learning_rate = learning_rate
+        self.early_stopping_rounds = early_stopping_rounds
+        self.arctan_smoothing = arctan_smoothing
+
+        # Get fitting parameters - only those required for xgbooster's
+        xgb_regressor_params = {
+            key: value
+            for key, value in self.get_params().items()
+            if key in xgb.XGBRegressor().get_params().keys()
+        }
+
+        # Define the model
+        objective = partial(
+            metrics.arctan_loss, taus=self.quantiles, s=arctan_smoothing
+        )
+        xgb_model: xgb.XGBRegressor = xgb.XGBRegressor(
+            objective=objective,
+            base_score=0,
+            multi_strategy="one_output_per_tree",
+            **xgb_regressor_params,
+        )
+        self.estimator_ = TransformedTargetRegressor(
+            regressor=xgb_model, transformer=StandardScaler()
+        )
+
+        # Set quantile indices to remap multioutput predictions
+        self.quantile_indices_ = {
+            quantile: i for i, quantile in enumerate(self.quantiles)
+        }
+
+    def fit(
+        self,
+        x: np.array,
+        y: np.array,
+        eval_set: Optional[Sequence[Tuple[np.array, np.array]]] = None,
+        verbose: Optional[Union[bool, int]] = 0,
+        **kwargs
+    ) -> OpenstfRegressor:
+        """Fits xgb quantile model.
+
+        Args:
+            x: Feature matrix.
+            y: Labels.
+            eval_set: Evaluation set to monitor training performance.
+            verbose: Verbosity level (disabled by default).
+
+        Returns:
+            Fitted XGBQuantile model.
+
+        """
+        if isinstance(y, pd.Series):
+            y = y.to_numpy()
+
+        if not isinstance(x, pd.DataFrame):
+            x = pd.DataFrame(np.asarray(x))
+
+        # Check/validate input
+        check_X_y(x, y, force_all_finite="allow-nan")
+
+        # Prepare inputs
+        y_multioutput = replicate_for_multioutput(y, len(self.quantiles))
+
+        # Define watchlist if eval_set is defined
+        eval_set_multioutput = []
+        if eval_set:
+            for x_eval, y_eval in eval_set:
+                if isinstance(y_eval, pd.Series):
+                    y_eval = y_eval.to_numpy()
+
+                y_eval_multioutput = replicate_for_multioutput(
+                    y=y_eval, num_quantiles=len(self.quantiles)
+                )
+                eval_set_multioutput.append((x_eval, y_eval_multioutput))
+
+            eval_set_multioutput.append((x, y_multioutput))
+
+        self.estimator_.fit(
+            X=x.copy(deep=True),
+            y=y_multioutput,
+            eval_set=eval_set_multioutput,
+            verbose=verbose,
+        )
+
+        # Update state of the estimator
+        self.feature_importances_ = self.estimator_.regressor_.feature_importances_
+        self.is_fitted_ = True
+
+        return self
+
+    def predict(self, x: np.array, quantile: float = 0.5) -> np.array:
+        """Makes a prediction for a desired quantile.
+
+        Args:
+            x: Feature matrix.
+            quantile: Quantile for which a prediciton is desired,
+                note that only quantile are available for which a model is trained,
+                and that this is a quantile-model specific keyword.
+
+        Returns:
+            Prediction
+
+        Raises:
+            ValueError in case no model is trained for the requested quantile.
+
+        """
+        # Check if model is trained for this quantile
+        if quantile not in self.quantiles:
+            raise ValueError("No model trained for requested quantile!")
+
+        # Check/validate input
+        check_array(x, force_all_finite="allow-nan")
+        check_is_fitted(self)
+
+        # best_iteration is only available if early stopping was used during training
+        prediction: np.array
+        if hasattr(self.estimator_, "best_iteration"):
+            prediction = self.estimator_.predict(
+                X=x,
+                iteration_range=(0, self.estimator_.best_iteration + 1),
+            )
+        else:
+            prediction = self.estimator_.predict(X=x)
+
+        quantile_index = self.quantile_indices_[quantile]
+        return prediction[:, quantile_index]
+
+    @property
+    def feature_names(self):
+        return self.estimator_.feature_names_in_
+
+    @property
+    def can_predict_quantiles(self):
+        return True
+
+
+def replicate_for_multioutput(y: np.array, num_quantiles: int) -> np.array:
+    """Replicates a 1D array to a 2D array for multioutput regression.
+
+    Args:
+        y: 1D array.
+        num_quantiles: Number of columns in the output array.
+
+    Returns:
+        2D array with shape (len(y), num_quantiles)
+
+    """
+    return np.repeat(y[:, None], num_quantiles, axis=1)

openstef/model/regressors/xgb_quantile.py

@@ -52,6 +52,9 @@ class XGBQuantileOpenstfRegressor(OpenstfRegressor):
             alpha: Alpha
             max_delta_step: Maximum delta step
 
+        Raises:
+            ValueError in case quantile 0.5 is not in the requested quantiles
+
         """
         super().__init__()
         # Check if quantile 0.5 is pressent this is required

openstef/model/serializer.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: MPL-2.0
 import json
+import logging
 import os
 import shutil
 from datetime import datetime
@@ -20,10 +21,16 @@ from xgboost import XGBModel  # Temporary for backward compatibility
 from openstef.data_classes.model_specifications import ModelSpecificationDataClass
 from openstef.metrics.reporter import Report
 from openstef.model.regressors.regressor import OpenstfRegressor
+from openstef.settings import Settings
 
 
 class MLflowSerializer:
     def __init__(self, mlflow_tracking_uri: str):
+        structlog.configure(
+            wrapper_class=structlog.make_filtering_bound_logger(
+                logging.getLevelName(Settings.log_level)
+            )
+        )
         self.logger = structlog.get_logger(self.__class__.__name__)
         mlflow.set_tracking_uri(mlflow_tracking_uri)
         self.logger.debug(f"MLflow tracking uri at init= {mlflow_tracking_uri}")
@@ -147,6 +154,9 @@ class MLflowSerializer:
         Args:
             experiment_name: Name of the experiment, often the id of the predition job.
 
+        Raises:
+            LookupError: If model is not found in MLflow.
+
         """
         try:
             models_df = self._find_models(

openstef/model_selection/model_selection.py

@@ -140,6 +140,9 @@ def split_data_train_validation_test(
         - Validation data.
         - Test data.
 
+    Raises:
+        ValueError: When the test and validation fractions are too high.
+
     """
     test_fraction = test_fraction if back_test else 0
     train_fraction = 1 - (test_fraction + validation_fraction)

openstef/monitoring/performance_meter.py

@@ -20,8 +20,7 @@ class PerformanceMeter:
 
         Args:
             level_label: The label of the new level. This could i.e. be 'task'
-            level_name: The name of the specified level. This could i.e. be
-                'tracy_todo'
+            level_name: The name of the specified level.
             **kwargs: Any other kwargs are appended to the logging.
 
         Returns:

openstef/monitoring/teams.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
 from typing import Union
 
 import pandas as pd
@@ -8,6 +9,8 @@ import pymsteams
 import structlog
 from pymsteams import cardsection
 
+from openstef.settings import Settings
+
 
 def post_teams(
     msg: Union[str, dict],
@@ -38,6 +41,14 @@ def post_teams(
     Note:
         This function is namespace-specific.
     """
+    if not Settings.post_teams_messages:
+        return
+
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
     # If no url is passed, give warning and don't send teams message
     if url is None:

openstef/pipeline/create_basecase_forecast.py

@@ -1,13 +1,14 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
 from pathlib import Path
 
 import pandas as pd
 import structlog
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.exceptions import NoRealisedLoadError, InputDataOngoingZeroFlatlinerError
+from openstef.exceptions import InputDataOngoingZeroFlatlinerError, NoRealisedLoadError
 from openstef.feature_engineering.feature_applicator import (
     OperationalPredictFeatureApplicator,
 )
@@ -18,6 +19,7 @@ from openstef.postprocessing.postprocessing import (
     add_components_base_case_forecast,
     add_prediction_job_properties_to_forecast,
 )
+from openstef.settings import Settings
 from openstef.validation import validation
 
 MODEL_LOCATION = Path(".")
@@ -38,7 +40,15 @@ def create_basecase_forecast_pipeline(
     Returns:
         Base case forecast
 
+    Raises:
+        NoRealisedLoadError: When no realised load for given datetime range.
+
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
 
     logger.info("Preprocessing data for basecase forecast")

openstef/pipeline/create_component_forecast.py

@@ -2,7 +2,10 @@
 #
 # SPDX-License-Identifier: MPL-2.0
 
+import logging
+
 import joblib
+import numpy as np
 import pandas as pd
 import structlog
 
@@ -11,12 +14,11 @@ from openstef import PROJECT_ROOT
 from openstef.data_classes.prediction_job import PredictionJobDataClass
 from openstef.enums import ForecastType
 from openstef.model.regressors.dazls import Dazls
-
-import numpy as np
+from openstef.settings import Settings
 
 # Set the path for the Dazls stored model
 DAZLS_STORED = str(
-    PROJECT_ROOT / "openstef" / "data" / "dazls_model_3.4.0" / "dazls_stored_3.4.0_"
+    PROJECT_ROOT / "openstef" / "data" / "dazls_model_3.4.24" / "dazls_stored_3.4.24_"
 )
 
 
@@ -96,6 +98,11 @@ def create_components_forecast_pipeline(
         "algtype"
 
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
     logger.info("Make components prediction", pid=pj["id"])
 
@@ -106,26 +113,8 @@ def create_components_forecast_pipeline(
         # Save and load the model as .sav file (or as .z file)
         # For the code contact: korte.termijn.prognoses@alliander.com
         dazls_model = Dazls()
-        dazls_model.domain_model = joblib.load(DAZLS_STORED + "domain_model.z")
-        dazls_model.domain_model_scaler = joblib.load(
-            DAZLS_STORED + "domain_model_scaler.z"
-        )
-        dazls_model.domain_model_input_columns = joblib.load(
-            DAZLS_STORED + "domain_model_features.z"
-        )
-
-        dazls_model.adaptation_model = joblib.load(DAZLS_STORED + "adaptation_model.z")
-        dazls_model.adaptation_model_scaler = joblib.load(
-            DAZLS_STORED + "adaptation_model_scaler.z"
-        )
-        dazls_model.adaptation_model_input_columns = joblib.load(
-            DAZLS_STORED + "adaptation_model_features.z"
-        )
-
-        dazls_model.target_columns = joblib.load(DAZLS_STORED + "target.z")
-        dazls_model.target_scaler = joblib.load(DAZLS_STORED + "target_scaler.z")
+        dazls_model.model_ = joblib.load(DAZLS_STORED + "baseline_model.z")
 
-        logger = structlog.get_logger(__name__)
         logger.info("DAZLS model loaded", dazls_model=str(dazls_model))
 
         # Use the predict function of Dazls model

openstef/pipeline/create_forecast.py

@@ -1,6 +1,8 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
+
 import pandas as pd
 import structlog
 
@@ -16,7 +18,9 @@ from openstef.model.serializer import MLflowSerializer
 from openstef.pipeline.utils import generate_forecast_datetime_range
 from openstef.postprocessing.postprocessing import (
     add_prediction_job_properties_to_forecast,
+    sort_quantiles,
 )
+from openstef.settings import Settings
 from openstef.validation import validation
 
 
@@ -40,6 +44,10 @@ def create_forecast_pipeline(
     Returns:
         DataFrame with the forecast
 
+    Raises:
+        InputDataOngoingZeroFlatlinerError: When all recent load measurements are zero.
+        LookupError: When no model is found for the given prediction job in MLflow.
+
     """
     prediction_model_pid = pj["id"]
     # Use the alternative forecast model if it's specify in the pj
@@ -64,7 +72,7 @@ def create_forecast_pipeline_core(
     Computes the forecasts and confidence intervals given a prediction job and input data.
     This pipeline has no database or persisitent storage dependencies.
 
-    Expected prediction job keys: "resolution_minutes", "horizon_minutes", "id", "type",
+    Expected prediction job keys: "resolution_minutes", "id", "type",
         "name", "quantiles"
 
     Args:
@@ -76,7 +84,15 @@ def create_forecast_pipeline_core(
     Returns:
         Forecast
 
+    Raises:
+        InputDataOngoingZeroFlatlinerError: When all recent load measurements are zero.
+
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
 
     fallback_strategy = "extreme_day"  # this can later be expanded
@@ -142,6 +158,9 @@ def create_forecast_pipeline_core(
         model, forecast_input_data
     ).add_confidence_interval(forecast, pj)
 
+    # Sort quantiles - prevents crossing and is statistically sound
+    forecast = sort_quantiles(forecast)
+
     # Prepare for output
     forecast = add_prediction_job_properties_to_forecast(
         pj,

openstef/pipeline/optimize_hyperparameters.py

@@ -1,8 +1,9 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
 import os
-from typing import Any, Union
+from typing import Any
 
 import optuna
 import pandas as pd
@@ -21,16 +22,22 @@ from openstef.model.objective import RegressorObjective
 from openstef.model.objective_creator import ObjectiveCreator
 from openstef.model.regressors.regressor import OpenstfRegressor
 from openstef.model.serializer import MLflowSerializer
+from openstef.model_selection.model_selection import split_data_train_validation_test
 from openstef.pipeline.train_model import (
     DEFAULT_TRAIN_HORIZONS_HOURS,
     train_model_pipeline_core,
 )
+from openstef.settings import Settings
 from openstef.validation import validation
-from openstef.model_selection.model_selection import split_data_train_validation_test
 
 optuna.logging.enable_propagation()  # Propagate logs to the root logger.
 optuna.logging.disable_default_handler()  # Stop showing logs in sys.stderr.
 
+structlog.configure(
+    wrapper_class=structlog.make_filtering_bound_logger(
+        logging.getLevelName(Settings.log_level)
+    )
+)
 logger = structlog.get_logger(__name__)
 
 # See https://optuna.readthedocs.io/en/stable/reference/generated/optuna.study.Study.html#optuna.study.Study.optimize
@@ -59,6 +66,9 @@ def optimize_hyperparameters_pipeline(
 
     Raises:
         ValueError: If the input_date is insufficient.
+        InputDataInsufficientError: If the input dataframe is empty.
+        InputDataWrongColumnOrderError: If the load column is missing in the input dataframe.
+        OldModelHigherScoreError: When old model is better than new model.
 
     Returns:
         Optimized hyperparameters.
@@ -119,6 +129,10 @@ def optimize_hyperparameters_pipeline_core(
 
     Raises:
         ValueError: If the input_date is insufficient.
+        InputDataInsufficientError: If the input dataframe is empty.
+        InputDataWrongColumnOrderError: If the load column is missing in the input dataframe.
+        OldModelHigherScoreError: When old model is better than new model.
+        InputDataOngoingZeroFlatlinerError: When all recent load measurements are zero.
 
     Returns:
         - Best model,
@@ -175,18 +189,6 @@ def optimize_hyperparameters_pipeline_core(
         horizons=horizons, feature_names=feature_names, feature_modules=feature_modules
     ).add_features(validated_data, pj=pj)
 
-    # Adds additional proloaf features to the input data, historic_load (equal to the load, first column)
-    if pj["model"] == "proloaf" and "historic_load" not in list(
-        validated_data_with_features.columns
-    ):
-        validated_data_with_features[
-            "historic_load"
-        ] = validated_data_with_features.iloc[:, 0]
-        # Make sure horizons is last column
-        temp_cols = validated_data_with_features.columns.tolist()
-        new_cols = temp_cols[:-2] + [temp_cols[-1]] + [temp_cols[-2]]
-        validated_data_with_features = validated_data_with_features[new_cols]
-
     # Create objective (NOTE: this is a callable class)
     objective = ObjectiveCreator.create_objective(model_type=pj["model"])
 
@@ -245,7 +247,7 @@ def optuna_optimization(
         - The objective object used by optuna
 
     """
-    model = ModelCreator.create_model(pj["model"])
+    model = ModelCreator.create_model(pj["model"], **(pj.model_kwargs or {}))
     # Apply set to default hyperparameters if they are specified in the pj
     if pj.default_modelspecs:
         valid_hyper_parameters = {
@@ -268,7 +270,7 @@ def optuna_optimization(
     if pj.train_split_func is None:
         split_func = split_data_train_validation_test
         split_args = {
-            "stratification_min_max": pj["model"] != "proloaf",
+            "stratification_min_max": True,
             "back_test": True,
         }
     else:

openstef/pipeline/train_create_forecast_backtest.py

@@ -56,10 +56,16 @@ def train_model_and_forecast_back_test(
         - Validation data sets (list[pd.DataFrame])
         - Test data sets (list[pd.DataFrame])
 
+    Raises:
+        InputDataInsufficientError: when input data is insufficient.
+        InputDataWrongColumnOrderError: when input data has a invalid column order.
+        ValueError: when the horizon is a string and the corresponding column in not in the input data
+        InputDataOngoingZeroFlatlinerError: when all recent load measurements are zero.
+
     """
     if pj.backtest_split_func is None:
         backtest_split_func = backtest_split_default
-        backtest_split_args = {"stratification_min_max": pj["model"] != "proloaf"}
+        backtest_split_args = {"stratification_min_max": True}
     else:
         backtest_split_func, backtest_split_args = pj.backtest_split_func.load(
             required_arguments=["data", "n_folds"]
@@ -124,6 +130,10 @@ def train_model_and_forecast_test_core(
         - The trained model
         - The forecast on the test set.
 
+    Raises:
+        NotImplementedError: When using invalid model type in the prediction job.
+        InputDataWrongColumnOrderError: When 'load' column is not first and 'horizon' column is not last.
+
     """
     model = train_model.train_pipeline_step_train_model(
         pj, modelspecs, train_data, validation_data