openstef 3.4.10__py3-none-any.whl → 3.4.29__py3-none-any.whl

openstef/app_settings.py

@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class AppSettings(BaseSettings):
+    """Global app settings."""
+
+    model_config = SettingsConfigDict(
+        env_prefix="openstef_", env_file=".env", extra="ignore"
+    )
+
+    post_teams_messages: bool = True
+
+    # Logging settings.
+    log_level: str = Field("INFO", description="Log level used for logging statements.")

openstef/data_classes/data_prep.py

@@ -5,7 +5,7 @@
 import inspect
 import json
 from importlib import import_module
-from typing import Any, Sequence, Union, TypeVar
+from typing import Any, Sequence, TypeVar, Union
 
 from pydantic.v1 import BaseModel
 

openstef/data_classes/prediction_job.py

@@ -6,9 +6,9 @@ from typing import Optional, Union
 
 from pydantic.v1 import BaseModel
 
+from openstef.data_classes.data_prep import DataPrepDataClass
 from openstef.data_classes.model_specifications import ModelSpecificationDataClass
 from openstef.data_classes.split_function import SplitFuncDataClass
-from openstef.data_classes.data_prep import DataPrepDataClass
 from openstef.enums import PipelineType
 
 
@@ -25,11 +25,15 @@ class PredictionJobDataClass(BaseModel):
         - ``"xgb_quantile"``
         - ``"lgb"``
         - ``"linear"``
-        - ``"proloaf"`` (extra dependencies requiered, see README)
+        - ``"linear_quantile"``
+        - ``"xgb_multioutput_quantile"``
+        - ``"flatliner"``
 
     If unsure what to pick, choose ``"xgb"``.
 
     """
+    model_kwargs: Optional[dict]
+    """The model parameters that should be used."""
     forecast_type: str
     """The type of forecasts that should be made.
 
@@ -41,14 +45,14 @@ class PredictionJobDataClass(BaseModel):
     If unsure what to pick, choose ``"demand"``.
 
     """
-    horizon_minutes: int = 2880
-    """The horizon of the desired forecast in minutes. Defaults to 2880 minutes (i.e. 2 days)."""
+    horizon_minutes: Optional[int] = 2880
+    """The horizon of the desired forecast in minutes used in tasks. Defaults to 2880 minutes (i.e. 2 days)."""
     resolution_minutes: int
     """The resolution of the desired forecast in minutes."""
-    lat: float
-    """Latitude of the forecasted location in degrees."""
-    lon: float
-    """Longitude of the forecasted location in degrees."""
+    lat: Optional[float] = 52.132633
+    """Latitude of the forecasted location in degrees. Used for fetching weather data in tasks, calculating derrived features and component splitting."""
+    lon: Optional[float] = 5.291266
+    """Longitude of the forecasted location in degrees. Used for fetching weather data in tasks, calculating derrived features and component splitting."""
     name: str
     """Name of the forecast, e.g. the location name."""
     train_components: Optional[bool]

openstef/enums.py

@@ -8,10 +8,12 @@ from enum import Enum
 class MLModelType(Enum):
     XGB = "xgb"
     XGB_QUANTILE = "xgb_quantile"
+    XGB_MULTIOUTPUT_QUANTILE = "xgb_multioutput_quantile"
     LGB = "lgb"
     LINEAR = "linear"
-    ProLoaf = "proloaf"
+    LINEAR_QUANTILE = "linear_quantile"
     ARIMA = "arima"
+    FLATLINER = "flatliner"
 
 
 class ForecastType(Enum):
@@ -21,12 +23,6 @@ class ForecastType(Enum):
     BASECASE = "basecase"
 
 
-class TracyJobResult(Enum):
-    SUCCESS = "success"
-    FAILED = "failed"
-    UNKNOWN = "unknown"
-
-
 class PipelineType(Enum):
     FORECAST = "forecast"
     TRAIN = "train"

openstef/exceptions.py

@@ -5,7 +5,7 @@
 """Openstef custom exceptions."""
 
 
-# Define custom exception
+# Define custom exceptions
 class NoPredictedLoadError(Exception):
     """No predicted load for given datatime range."""
 

openstef/feature_engineering/apply_features.py

@@ -14,9 +14,6 @@ Examples of features that are added:
 import pandas as pd
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.feature_engineering.historic_features import (
-    add_historic_load_as_a_feature,
-)
 from openstef.feature_engineering.holiday_features import (
     generate_holiday_feature_functions,
 )
@@ -69,9 +66,6 @@ def apply_features(
                             np.random.uniform(0.7,1.7, 200)))
 
     """
-    # Add if needed the proloaf feature (historic_load)
-    data = add_historic_load_as_a_feature(data, pj)
-
     # Get lag feature functions
     feature_functions = generate_lag_feature_functions(feature_names, horizon)
 

openstef/feature_engineering/data_preparation.py

@@ -1,25 +1,27 @@
 # SPDX-FileCopyrightText: 2017-2023 Alliander N.V. <korte.termijn.prognoses@alliander.com> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
-import structlog
-
+import logging
 from abc import ABC, abstractmethod
+from datetime import timedelta
 from typing import Optional
 
 import pandas as pd
-from datetime import timedelta
+import structlog
+
 from openstef.data_classes.model_specifications import ModelSpecificationDataClass
 from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.model.regressors.regressor import OpenstfRegressor
 from openstef.feature_engineering.feature_applicator import (
-    TrainFeatureApplicator,
     OperationalPredictFeatureApplicator,
+    TrainFeatureApplicator,
 )
 from openstef.feature_engineering.general import (
     enforce_feature_order,
     remove_non_requested_feature_columns,
 )
+from openstef.model.regressors.regressor import OpenstfRegressor
 from openstef.pipeline.utils import generate_forecast_datetime_range
+from openstef.settings import Settings
 
 
 class AbstractDataPreparation(ABC):
@@ -120,6 +122,11 @@ class ARDataPreparation(AbstractDataPreparation):
     def prepare_forecast_data(
         self, data: pd.DataFrame
     ) -> tuple[pd.DataFrame, pd.DataFrame]:
+        structlog.configure(
+            wrapper_class=structlog.make_filtering_bound_logger(
+                logging.getLevelName(Settings.log_level)
+            )
+        )
         logger = structlog.get_logger(__name__)
         self.check_model()
         # Prep forecast input by selecting only the forecast datetime interval (this is much smaller than the input range)

openstef/feature_engineering/feature_applicator.py

@@ -149,11 +149,7 @@ class TrainFeatureApplicator(AbstractFeatureApplicator):
 
         # NOTE this is required since apply_features could add additional features
         if self.feature_names is not None:
-            # Add horizon to requested features else it is removed, and if needed the proloaf feature (historic_load)
-            if pj.get("model") == "proloaf":
-                features = self.feature_names + ["historic_load"] + ["horizon"]
-            else:
-                features = self.feature_names + ["horizon"]
+            features = self.feature_names + ["horizon"]
             result = remove_non_requested_feature_columns(result, features)
 
         # Sort all features except for the (first) load and (last) horizon columns

openstef/feature_engineering/general.py

@@ -3,10 +3,14 @@
 # SPDX-License-Identifier: MPL-2.0
 """This modelu contains various helper functions."""
 
+import logging
+
 import numpy as np
 import pandas as pd
 import structlog
 
+from openstef.settings import Settings
+
 
 def add_missing_feature_columns(
     input_data: pd.DataFrame, features: list[str]
@@ -30,6 +34,11 @@ def add_missing_feature_columns(
         Input dataframe with missing columns filled with ``np.N=nan``.
 
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
 
     if features is None:
@@ -61,6 +70,11 @@ def remove_non_requested_feature_columns(
         Model input data with features.
 
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
 
     if requested_features is None:

openstef/feature_engineering/missing_values_transformer.py

@@ -0,0 +1,99 @@
+# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
+#
+# SPDX-License-Identifier: MPL-2.0
+from typing import Union, List, Optional
+
+import numpy as np
+import pandas as pd
+from sklearn.impute import SimpleImputer
+from sklearn.preprocessing import FunctionTransformer
+from sklearn.utils.validation import check_array
+
+
+class MissingValuesTransformer:
+    """MissingValuesTransformer handles missing values in data by imputing them with a given strategy.
+
+    It also removes columns that are always null from the data.
+
+    """
+
+    in_feature_names: Optional[List[str]] = None
+    _n_in_features: Optional[int] = None
+
+    non_null_feature_names: List[str] = None
+
+    def __init__(
+        self,
+        missing_values: Union[int, float, str, None] = np.nan,
+        imputation_strategy: str = None,
+        fill_value: Union[str, int, float] = None,
+    ):
+        """Initialize missing values handler.
+
+        Args:
+            missing_values: The placeholder for the missing values. All occurrences of
+                `missing_values` will be imputed.
+            imputation_strategy: The imputation strategy to use
+                Can be one of "mean", "median", "most_frequent", "constant" or None.
+            fill_value: When strategy == "constant", fill_value is used to replace all
+                occurrences of missing_values.
+
+        """
+        self.missing_values = missing_values
+        self.imputation_strategy = imputation_strategy
+        self.fill_value = fill_value
+
+    def fit(self, x, y=None):
+        """Fit the imputer on the input data."""
+        _ = check_array(x, force_all_finite="allow-nan")
+        if not isinstance(x, pd.DataFrame):
+            x = pd.DataFrame(np.asarray(x))
+
+        self.in_feature_names = list(x.columns)
+        self._n_in_features = x.shape[1]
+
+        # Remove always null columns
+        is_column_null = x.isnull().all(axis="index")
+        self.non_null_feature_names = list(x.columns[~is_column_null])
+
+        # Build the proper imputation transformer
+        # - Identity function if strategy is None
+        # - SimpleImputer with the dedicated strategy
+        if self.imputation_strategy is None:
+            self.imputer_ = FunctionTransformer(func=self._identity)
+        else:
+            self.imputer_ = SimpleImputer(
+                missing_values=self.missing_values,
+                strategy=self.imputation_strategy,
+                fill_value=self.fill_value,
+            ).set_output(transform="pandas")
+
+        # Imputers do not support labels
+        self.imputer_.fit(X=x, y=None)
+
+    def transform(self, x) -> pd.DataFrame:
+        """Transform the input data by imputing missing values."""
+        _ = check_array(x, force_all_finite="allow-nan")
+        if not isinstance(x, pd.DataFrame):
+            x = pd.DataFrame(np.asarray(x))
+
+        x = x[self.non_null_feature_names]
+
+        return self.imputer_.transform(x)
+
+    def fit_transform(self, x, y=None):
+        """Fit the imputer on the input data and transform it.
+
+        Returns:
+            The data with missing values imputed.
+
+        """
+        self.fit(x, y)
+        return self.transform(x)
+
+    @classmethod
+    def _identity(cls, x):
+        return x
+
+    def __sklearn_is_fitted__(self) -> bool:
+        return self.in_feature_names is not None

openstef/feature_engineering/weather_features.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: MPL-2.0
 
 """This module contains all wheather related functions used for feature engineering."""
+import logging
 from typing import Union
 
 import numpy as np
@@ -12,7 +13,13 @@ import structlog
 from pvlib.location import Location
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
+from openstef.settings import Settings
 
+structlog.configure(
+    wrapper_class=structlog.make_filtering_bound_logger(
+        logging.getLevelName(Settings.log_level)
+    )
+)
 logger = structlog.get_logger(__name__)
 
 

openstef/metrics/figure.py

@@ -71,6 +71,9 @@ def plot_data_series(
     Returns:
         A line plot of each passed data series.
 
+    Raises:
+        ValueError: If names is None and the number of series is greater than 3.
+
     """
     series_names = {
         1: ("series",),

openstef/metrics/metrics.py

@@ -25,6 +25,9 @@ def get_eval_metric_function(metric_name: str) -> Callable:
     Returns:
         Function to calculate the metric.
 
+    Raises:
+        KeyError: If the metric is not available.
+
     """
     evaluation_function = {
         "rmse": rmse,
@@ -130,6 +133,9 @@ def r_mae_highest(
 
     The range is based on the load range of the previous two weeks.
 
+    Raises:
+        ValueError: If the length of the realised and forecast arrays are not equal.
+
     """
     # Check if length of both arrays is equal
     if len(np.array(realised)) != len(np.array(forecast)):
@@ -395,7 +401,7 @@ def xgb_quantile_obj(
     Args:
         preds: numpy.ndarray
         dmatrix: xgboost.DMatrix
-        quantile: float
+        quantile: float between 0 and 1
 
     Returns:
         Gradient and Hessian
@@ -425,3 +431,54 @@ def xgb_quantile_obj(
     hess = np.ones_like(preds)
 
     return grad, hess
+
+
+def arctan_loss(y_true, y_pred, taus, s=0.1):
+    """Compute the arctan pinball loss.
+
+    Note that XGBoost outputs the predictions in a slightly peculiar manner.
+    Suppose we have 100 data points and we predict 10 quantiles. The predictions
+    will be an array of size (1000 x 1). We first resize this to a (100x10) array
+    where each row corresponds to the 10 predicted quantile for a single data
+    point. We then use a for-loop (over the 10 columns) to calculate the gradients
+    and second derivatives. Legibility was chosen over efficiency. This part
+    can be made more efficient.
+
+    Args:
+        y_true: An array containing the true observations.
+        y_pred: An array containing the predicted quantiles.
+        taus: A list containing the true desired coverage of the quantiles.
+        s: A smoothing parameter.
+
+    Returns:
+        grad: An array containing the (negative) gradients with respect to y_pred.
+        hess: An array containing the second derivative with respect to y_pred.
+
+    """
+    size = len(y_true)
+    n_dim = len(taus)  # The number of columns
+    n_rows = size // n_dim
+
+    # Resize the predictions and targets.
+    # Each column corresponds to a quantile, each row to a data point.
+    y_pred = np.reshape(y_pred, (n_rows, n_dim))
+    y_true = np.reshape(y_true, (n_rows, n_dim))
+
+    # Calculate the differences
+    u = y_true - y_pred
+
+    # Calculate the gradient and second derivatives
+    grad = np.zeros_like(y_pred)
+    hess = np.zeros_like(y_pred)
+    z = u / s
+    for i, tau in enumerate(taus):
+        x = 1 + z[:, i] ** 2
+        grad[:, i] = (
+            tau - 0.5 + 1 / np.pi * np.arctan(z[:, i]) + z[:, i] / (np.pi) * x**-1
+        )
+        hess[:, i] = 2 / (np.pi * s) * x ** (-2)
+
+    # Reshape back to the original shape.
+    grad = grad.reshape(size)
+    hess = hess.reshape(size)
+    return -grad / n_dim, hess / n_dim

openstef/metrics/reporter.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: MPL-2.0
 """Defines reporter class."""
+import logging
 import os
 import warnings
 from dataclasses import dataclass
@@ -16,6 +17,7 @@ from plotly.graph_objects import Figure
 from openstef.metrics import figure
 from openstef.metrics.metrics import bias, mae, nsme, r_mae, rmse
 from openstef.model.regressors.regressor import OpenstfRegressor
+from openstef.settings import Settings
 
 
 @dataclass
@@ -167,6 +169,11 @@ class Reporter:
     def write_report_to_disk(report: Report, report_folder: str):
         """Write report to disk; e.g. for viewing report of latest models using grafana."""
         # Initialize logger and serializer
+        structlog.configure(
+            wrapper_class=structlog.make_filtering_bound_logger(
+                logging.getLevelName(Settings.log_level)
+            )
+        )
         logger = structlog.get_logger(__name__)
         if report_folder:
             # create path if does not exist

openstef/model/confidence_interval_applicator.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
 from datetime import datetime
 
 import numpy as np
@@ -11,12 +12,18 @@ from sklearn.base import RegressorMixin
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
 from openstef.exceptions import ModelWithoutStDev
+from openstef.settings import Settings
 
 
 class ConfidenceIntervalApplicator:
     def __init__(self, model: RegressorMixin, forecast_input_data: pd.DataFrame):
         self.model = model
         self.forecast_input_data = forecast_input_data
+        structlog.configure(
+            wrapper_class=structlog.make_filtering_bound_logger(
+                logging.getLevelName(Settings.log_level)
+            )
+        )
         self.logger = structlog.get_logger(self.__class__.__name__)
 
     def add_confidence_interval(
@@ -54,9 +61,24 @@ class ConfidenceIntervalApplicator:
         temp_forecast = self._add_standard_deviation_to_forecast(forecast)
 
         if self.model.can_predict_quantiles:
-            return self._add_quantiles_to_forecast_quantile_regression(
-                temp_forecast, pj["quantiles"]
-            )
+            # Try to generate the quantiles that were requested
+            try:
+                result = self._add_quantiles_to_forecast_quantile_regression(
+                    temp_forecast, pj["quantiles"]
+                )
+                return result
+            except Exception:
+                # Fallback on quantiles of the model if the requested quantiles cant be generated by the model.
+                # Can happen when the model was trained on different quantiles than are requested
+                result = self._add_quantiles_to_forecast_quantile_regression(
+                    temp_forecast, self.model.quantiles
+                )
+                self.logger.warning(
+                    "Quantiles are requested the model was not trained on. Using the quantiles the model was trained on",
+                    requested_quantiles=pj["quantiles"],
+                    trained_quantiles=self.model.quantiles,
+                )
+                return result
 
         return self._add_quantiles_to_forecast_default(temp_forecast, pj["quantiles"])
 
@@ -74,6 +96,9 @@ class ConfidenceIntervalApplicator:
             Forecast with added standard deviation. DataFrame with columns:
                 "forecast", "stdev"
 
+        Raises:
+            ModelWithoutStDev: If the model does not have a valid standard deviation.
+
         """
         minimal_resolution: int = 15  # Minimal time resolution in minutes
         standard_deviation = self.model.standard_deviation

openstef/model/model_creator.py

@@ -1,25 +1,32 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
 from typing import Union
 
 import structlog
 
 from openstef.enums import MLModelType
+from openstef.model.regressors.arima import ARIMAOpenstfRegressor
 from openstef.model.regressors.custom_regressor import is_custom_type, load_custom_model
 from openstef.model.regressors.lgbm import LGBMOpenstfRegressor
 from openstef.model.regressors.linear import LinearOpenstfRegressor
+from openstef.model.regressors.linear_quantile import LinearQuantileOpenstfRegressor
 from openstef.model.regressors.regressor import OpenstfRegressor
+from openstef.model.regressors.flatliner import FlatlinerRegressor
 from openstef.model.regressors.xgb import XGBOpenstfRegressor
 from openstef.model.regressors.xgb_quantile import XGBQuantileOpenstfRegressor
-from openstef.model.regressors.arima import ARIMAOpenstfRegressor
+from openstef.model.regressors.xgb_multioutput_quantile import (
+    XGBMultiOutputQuantileOpenstfRegressor,
+)
+from openstef.settings import Settings
 
+structlog.configure(
+    wrapper_class=structlog.make_filtering_bound_logger(
+        logging.getLevelName(Settings.log_level)
+    )
+)
 logger = structlog.get_logger(__name__)
-try:
-    from openstef.model.regressors.proloaf import OpenstfProloafRegressor
-except ImportError:
-    logger.info("Proloaf not available, setting constructor to None")
-    OpenstfProloafRegressor = None
 
 valid_model_kwargs = {
     MLModelType.XGB: [
@@ -84,32 +91,32 @@ valid_model_kwargs = {
         "max_depth",
         "early_stopping_rounds",
     ],
-    MLModelType.ProLoaf: [
-        "relu_leak",
-        "encoder_features",
-        "decoder_features",
-        "core_layers",
-        "rel_linear_hidden_size",
-        "rel_core_hidden_size",
-        "dropout_fc",
-        "dropout_core",
-        "training_metric",
-        "metric_options",
-        "optimizer_name",
-        "early_stopping_patience",
-        "early_stopping_margin",
-        "learning_rate",
-        "max_epochs",
-        "device",
-        "batch_size",
-        "history_horizon",
-        "horizon_minutes",
+    MLModelType.XGB_MULTIOUTPUT_QUANTILE: [
+        "quantiles",
+        "gamma",
+        "colsample_bytree",
+        "subsample",
+        "min_child_weight",
+        "max_depth",
+        "early_stopping_rounds",
+        "arctan_smoothing",
     ],
     MLModelType.LINEAR: [
         "missing_values",
         "imputation_strategy",
         "fill_value",
     ],
+    MLModelType.FLATLINER: [
+        "quantiles",
+    ],
+    MLModelType.LINEAR_QUANTILE: [
+        "alpha",
+        "quantiles",
+        "solver",
+        "missing_values",
+        "imputation_strategy",
+        "fill_value",
+    ],
     MLModelType.ARIMA: [
         "backtest_max_horizon",
         "order",
@@ -127,9 +134,11 @@ class ModelCreator:
         MLModelType.XGB: XGBOpenstfRegressor,
         MLModelType.LGB: LGBMOpenstfRegressor,
         MLModelType.XGB_QUANTILE: XGBQuantileOpenstfRegressor,
-        MLModelType.ProLoaf: OpenstfProloafRegressor,
+        MLModelType.XGB_MULTIOUTPUT_QUANTILE: XGBMultiOutputQuantileOpenstfRegressor,
         MLModelType.LINEAR: LinearOpenstfRegressor,
+        MLModelType.LINEAR_QUANTILE: LinearQuantileOpenstfRegressor,
         MLModelType.ARIMA: ARIMAOpenstfRegressor,
+        MLModelType.FLATLINER: FlatlinerRegressor,
     }
 
     @staticmethod