openstef 3.4.70__py3-none-any.whl → 3.4.73__py3-none-any.whl

openstef/data_classes/prediction_job.py

@@ -21,7 +21,7 @@ class PredictionJobDataClass(BaseModel):
     )
     model: str = Field(
         ...,
-        description="The model type that should be used. Options are: 'xgb', 'xgb_quantile', 'lgb', 'linear', 'linear_quantile', 'gblinear_quantile', 'xgb_multioutput_quantile', 'flatliner'.",
+        description="The model type that should be used. Options are: 'xgb', 'xgb_quantile', 'lgb', 'linear', 'linear_quantile', 'gblinear_quantile', 'xgb_multioutput_quantile', 'flatliner', 'median'.",
     )
 
     model_kwargs: Optional[dict] = Field(

openstef/enums.py

@@ -125,6 +125,7 @@ class ModelType(Enum):
     GBLINEAR_QUANTILE = "gblinear_quantile"
     ARIMA = "arima"
     FLATLINER = "flatliner"
+    MEDIAN = "median"
 
 
 class ForecastType(Enum):

openstef/model/model_creator.py

@@ -17,6 +17,7 @@ from openstef.model.regressors.xgb import XGBOpenstfRegressor
 from openstef.model.regressors.xgb_multioutput_quantile import (
     XGBMultiOutputQuantileOpenstfRegressor,
 )
+from openstef.model.regressors.median import MedianRegressor
 from openstef.model.regressors.xgb_quantile import XGBQuantileOpenstfRegressor
 
 logger = get_logger(__name__)
@@ -73,7 +74,6 @@ valid_model_kwargs = {
         "n_jobs",
         "silent",
         "importance_type",
-        "early_stopping_rounds",
     ],
     ModelType.XGB_QUANTILE: [
         "quantiles",
@@ -139,6 +139,9 @@ valid_model_kwargs = {
         "seasonal_order",
         "trend",
     ],
+    ModelType.MEDIAN: [
+        # This model does not have any parameters
+    ],
 }
 
 
@@ -156,6 +159,7 @@ class ModelCreator:
         ModelType.GBLINEAR_QUANTILE: GBLinearQuantileOpenstfRegressor,
         ModelType.ARIMA: ARIMAOpenstfRegressor,
         ModelType.FLATLINER: FlatlinerRegressor,
+        ModelType.MEDIAN: MedianRegressor,
     }
 
     @staticmethod

openstef/model/objective.py

@@ -7,6 +7,8 @@ from typing import Any, Callable, Optional
 
 import optuna
 import pandas as pd
+from lightgbm import early_stopping
+from xgboost.callback import EarlyStopping
 
 from openstef.enums import ModelType
 from openstef.metrics import metrics
@@ -130,31 +132,42 @@ class RegressorObjective:
         # Configure evals for early stopping
         eval_set = [(train_x, train_y), (valid_x, valid_y)]
 
-        # get the parameters used in this trial
+        # Get the parameters used in this trial
         hyper_params = self.get_params(trial)
 
-        # insert parameters into model
+        # Insert parameters into model
         self.model.set_params(**hyper_params)
 
-        # create the specific pruning callback
+        callbacks = []
+
+        # Create the early stopping callback
+        early_stopping_callback = self.get_early_stopping_callback()
+        if early_stopping_callback is not None:
+            callbacks.append(early_stopping_callback)
+
+        # Create the specific pruning callback
         pruning_callback = self.get_pruning_callback(trial)
-        if pruning_callback is None:
-            callbacks = None
-        else:
-            callbacks = [pruning_callback]
+        if pruning_callback is not None:
+            callbacks.append(pruning_callback)
+
+        # Pass verbose argument to fit call if model is not LGB
+        fit_kwargs = {}
+        if self.model_type not in [ModelType.XGB, ModelType.LGB]:
+            fit_kwargs["early_stopping_rounds"] = EARLY_STOPPING_ROUNDS
+        elif self.model_type != ModelType.LGB:
+            fit_kwargs["verbose"] = self.verbose
 
         # validation_0 and validation_1 are available
         self.model.fit(
             train_x,
             train_y,
             eval_set=eval_set,
-            early_stopping_rounds=EARLY_STOPPING_ROUNDS,
-            verbose=self.verbose,
             eval_metric=self.eval_metric,
             callbacks=callbacks,
+            **fit_kwargs,
         )
 
-        self.model.feature_importance_dataframe = self.model.set_feature_importance()
+        self.model.feature_importance_dataframe = self.model.get_feature_importance()
 
         # Do confidence interval determination
         self.model = StandardDeviationGenerator(
@@ -203,6 +216,9 @@ class RegressorObjective:
     def get_pruning_callback(self, trial: optuna.trial.FrozenTrial):
         return None
 
+    def get_early_stopping_callback(self):
+        return None
+
     def get_trial_track(self) -> dict:
         """Get a dictionary of al trials.
 
@@ -272,6 +288,15 @@ class XGBRegressorObjective(RegressorObjective):
             trial, observation_key=f"validation_1-{self.eval_metric}"
         )
 
+    def get_early_stopping_callback(self):
+        return EarlyStopping(
+            rounds=EARLY_STOPPING_ROUNDS,
+            metric_name=self.eval_metric,
+            data_name=f"validation_1",
+            maximize=False,
+            save_best=True,
+        )
+
     @classmethod
     def get_default_values(cls) -> dict:
         default_parameter_values = super().get_default_values()
@@ -319,6 +344,11 @@ class LGBRegressorObjective(RegressorObjective):
             trial, metric=metric, valid_name="valid_1"
         )
 
+    def get_early_stopping_callback(self):
+        return early_stopping(
+            stopping_rounds=EARLY_STOPPING_ROUNDS, verbose=self.verbose
+        )
+
 
 class XGBQuantileRegressorObjective(RegressorObjective):
     def __init__(self, *args, **kwargs):

openstef/model/regressors/arima.py

@@ -126,7 +126,7 @@ class ARIMAOpenstfRegressor(OpenstfRegressor):
             predictions = self.predict_quantile(start, end, exog=x, quantile=quantile)
         return predictions
 
-    def set_feature_importance(self):
+    def get_feature_importance(self):
         """Because report needs 'weight' and 'gain' as importance metrics, we set the values to these names.
 
         - 'weight' is corresponding to the coefficients values

openstef/model/regressors/median.py

@@ -0,0 +1,78 @@
+"""This module contains the median regressor."""
+
+import numpy as np
+import pandas as pd
+
+from sklearn.base import RegressorMixin
+from sklearn.utils.validation import check_is_fitted
+
+from openstef.model.regressors.regressor import OpenstfRegressor
+
+
+class MedianRegressor(OpenstfRegressor, RegressorMixin):
+    """
+    Median regressor implementing the OpenSTEF regressor API.
+
+    This regressor is good for predicting two types of signals:
+    - Signals with very slow dynamics compared to the sampling rate, possibly
+      with a lot of noise.
+    - Signals that switch between two or more states, which random in nature or
+    depend on unknown features, but tend to be stable in each state. An example of
+    this may be waste heat delivered from an industrial process. Using a median
+    over the last few timesteps adds some hysterisis to avoid triggering on noise.
+    """
+
+    def __init__(self):
+        """Initialize MedianRegressor."""
+        super().__init__()
+
+    @property
+    def feature_names(self) -> list:
+        """Retrieve the model input feature names.
+
+        Returns:
+            The list of feature names
+
+        """
+        check_is_fitted(self, "feature_names_")
+        return self.feature_names_
+
+    @property
+    def can_predict_quantiles(self) -> bool:
+        return False
+
+    def predict(self, x: pd.DataFrame, **kwargs) -> np.array:
+        """
+        Predict the median of the lag features for each time step in the context window.
+
+        Args:
+            x (pd.DataFrame): The input data for prediction. This should be a pandas dataframe with lag features.
+
+        Returns:
+            np.array: The predicted median for each time step in the context window.
+            If any lag feature is NaN, this will be ignored.
+            If all lag features are NaN, the regressor will return NaN.
+        """
+
+        lag_df = x.loc[:, self.feature_names]
+        median = lag_df.median(axis=1, skipna=True)
+
+        return median
+
+    def fit(self, x: pd.DataFrame, y: pd.DataFrame, **kwargs) -> RegressorMixin:
+        """This model does not have any hyperparameters to fit,
+        but it does need to know the feature names of the lag features.
+
+        Which lag features are used is determined by the feature engineering step.
+        """
+        self.feature_names_ = list(x.columns[x.columns.str.startswith("T-")])
+        if len(self.feature_names_) == 0:
+            raise ValueError("No lag features found in the input data.")
+
+        self.feature_importances_ = np.ones(len(self.feature_names_)) / (
+            len(self.feature_names_) or 1.0
+        )
+        return self
+
+    def __sklearn_is_fitted__(self) -> bool:
+        return True

openstef/model/regressors/regressor.py

@@ -59,7 +59,7 @@ class OpenstfRegressor(BaseEstimator):
         """
 
     @abstractmethod
-    def fit(self, x: np.array, y: np.array, **kwargs) -> RegressorMixin:
+    def fit(self, x: pd.DataFrame, y: pd.DataFrame, **kwargs) -> RegressorMixin:
         """Fits the regressor.
 
         Args:
@@ -72,7 +72,7 @@ class OpenstfRegressor(BaseEstimator):
 
         """
 
-    def set_feature_importance(self) -> Union[pd.DataFrame, None]:
+    def get_feature_importance(self) -> Union[pd.DataFrame, None]:
         """Get feature importance.
 
         Returns:

openstef/model/regressors/xgb.py

@@ -36,13 +36,10 @@ class XGBOpenstfRegressor(XGBRegressor, OpenstfRegressor):
         x: np.array,
         y: np.array,
         *,
-        early_stopping_rounds: Optional[int] = None,
         callbacks: Optional[list] = None,
         eval_metric: Optional[str] = None,
         **kwargs
     ):
-        if early_stopping_rounds is not None:
-            self.set_params(early_stopping_rounds=early_stopping_rounds)
         if callbacks is not None:
             self.set_params(callbacks=callbacks)
         if eval_metric is not None:

openstef/pipeline/train_model.py

@@ -495,10 +495,9 @@ def train_pipeline_step_train_model(
         train_x,
         train_y,
         eval_set=eval_set,
-        verbose=False,
     )
     # Gets the feature importance df or None if we don't have feature importance
-    model.feature_importance_dataframe = model.set_feature_importance()
+    model.feature_importance_dataframe = model.get_feature_importance()
 
     logger.info("Fitted a new model, not yet stored")
 

{openstef-3.4.70.dist-info → openstef-3.4.73.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openstef
-Version: 3.4.70
+Version: 3.4.73
 Summary: Open short term energy forecaster
 Home-page: https://github.com/OpenSTEF/openstef
 Author: Alliander N.V
@@ -16,7 +16,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: holidays==0.21
 Requires-Dist: joblib==1.3.2
-Requires-Dist: lightgbm~=3.3
+Requires-Dist: lightgbm>=4.0
 Requires-Dist: matplotlib~=3.7
 Requires-Dist: mlflow~=2.3
 Requires-Dist: networkx~=3.1

{openstef-3.4.70.dist-info → openstef-3.4.73.dist-info}/RECORD

@@ -1,7 +1,7 @@
 openstef/__init__.py,sha256=93UM6m0LLQhO69-mSqLuUy73jgs4W7Iuxfo3Lm8c98g,419
 openstef/__main__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/app_settings.py,sha256=4SiEoPfPmM_Vc9gwWleT_IRxHIk36bZaYRCdNf3ewoo,762
-openstef/enums.py,sha256=FrP0m_Tk0kV7gSZ2hTY_8iD45KIKnexHrjNufhpKXpE,2829
+openstef/enums.py,sha256=kwAbEoMI_a--Ikk6RxeJmvR2JKYKDNHynhMUbbiczZs,2851
 openstef/exceptions.py,sha256=dgnvZe5WWuJWCZm_GES6suEATbusPlwbiEUfNQKeExY,1993
 openstef/settings.py,sha256=nSgkBqFxuqB3w7Rwo60i8j37c5ngDbt6vpjHS6QtJXQ,354
 openstef/data/NL_terrestrial_radiation.csv,sha256=A4kbW56GDzWi4tWUwY2C-4PiOvcKJCwkWQQtdg4ekPE,820246
@@ -17,7 +17,7 @@ openstef/data/dazls_model_3.4.24/dazls_stored_3.4.24_model_card.md.license,sha25
 openstef/data_classes/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/data_classes/data_prep.py,sha256=sANgFjfwmSWhLCfmLjfqXQnczuvVZfk2765jZd7LwuE,3691
 openstef/data_classes/model_specifications.py,sha256=PZeBLfH_MrP9-QorL1r0Hklp0befE8Nw05vNhTX9Y20,1338
-openstef/data_classes/prediction_job.py,sha256=e6_PFAovNd31tjzoTQJvqRNQyVM-M0XHffclAG9Ez8A,6721
+openstef/data_classes/prediction_job.py,sha256=omVzowGYUp5YD_7k3i3pKlDh1I7i_jU3-za7FJ3fK1s,6731
 openstef/data_classes/split_function.py,sha256=K8y1dsQC5exeIDh37f7UwJ11tV71_uVSNbnKmwXpnOM,3435
 openstef/feature_engineering/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/feature_engineering/apply_features.py,sha256=pro4eUmOFexX_9g9kJtDcbrQ1hWKzXjVpiJBmmBi89o,5326
@@ -46,8 +46,8 @@ openstef/model/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,16
 openstef/model/basecase.py,sha256=caI6Q-8y0ymlxGK9Js_H3Vh0q6ruNHlGD5RG0_kE5M0,2878
 openstef/model/confidence_interval_applicator.py,sha256=VOdHsDJhfeyaq_cnk9QMUaZ2IumbiBBoW1zo8AuqDg0,9559
 openstef/model/fallback.py,sha256=g6TEuEhV4w07SCGkR_AvPf2up9f0ixGKQojYC-Ewl6Y,2812
-openstef/model/model_creator.py,sha256=lbWtarr9_PlMfG9u9Qbnzyw5GNcYIExt4-V3PevTRn8,6506
-openstef/model/objective.py,sha256=qJdI6GAzv8l5Mxd8G7BIqQnfdJNM7aOlg9DMzMGjWqA,14558
+openstef/model/model_creator.py,sha256=fnhcVGUHskbuAys5kjlJ4GXKxbi9Eq5eAA19ex11Vv0,6658
+openstef/model/objective.py,sha256=0PZUbPzuyaYlpWEH_qPavO6ll7zwqTTUTfIrUzzFMbs,15585
 openstef/model/objective_creator.py,sha256=3jJgcmY1sm-Yoe3SfjKrJukrsqtYyloUFaPbBWqswhQ,2208
 openstef/model/serializer.py,sha256=k5GY8eRJdlii8mEY7Qheu4yb5USyIyxw77EYkSQJGYk,17034
 openstef/model/standard_deviation_generator.py,sha256=OorRvX2wRScU7f4SIBoiT24yJeeM50sETP3xC6m5IG4,2865
@@ -56,7 +56,7 @@ openstef/model/metamodels/feature_clipper.py,sha256=DNsyYdjUT7ZNimJJIyTvv1nmwTwD
 openstef/model/metamodels/grouped_regressor.py,sha256=yMN_a6TnQSyFaqlB_6Nifq-ydpb5hs6w_b97IaBbHj4,8337
 openstef/model/metamodels/missing_values_handler.py,sha256=glgAlkeubLZFWbD8trTYBik7_qOJi4GCPGl1sSybSkQ,5257
 openstef/model/regressors/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
-openstef/model/regressors/arima.py,sha256=wt7FVykjSvljpl7vjtliq61SiyjQ7KKtw8PF9x0xf04,7587
+openstef/model/regressors/arima.py,sha256=2MIcIvfxOAxYC5QtC00K9oQl2-qaOewm7P3gzoHgxgE,7587
 openstef/model/regressors/custom_regressor.py,sha256=T4JdJ-oTTt1PHQV0DdIEIhALvEEh07WCFlWxl8EFih0,1765
 openstef/model/regressors/dazls.py,sha256=Xt89yFHjkwpIUTkkhPmPZ74F8_tht_XV88INuP5GU2E,3994
 openstef/model/regressors/flatliner.py,sha256=T9u-ukhqFcatQmlgUtBL_G-1b_wQzgdVRq0ac64GnjQ,2789
@@ -64,8 +64,9 @@ openstef/model/regressors/gblinear_quantile.py,sha256=PKQL_TAXa3Kw9oZrKC6Uvo_n2N
 openstef/model/regressors/lgbm.py,sha256=zCdn1euEdSFxYJzH8XqQFFnb6R4JVUnmineKjX_Gy-g,800
 openstef/model/regressors/linear.py,sha256=uOvZMLGZH_9nXfmS5honCMfyVeyGXP1Cza9A_BdXlVw,3665
 openstef/model/regressors/linear_quantile.py,sha256=zIpGo9deMeTZdwFWoZ3FstX74mYdlAhfg-YOsPRFl0k,10534
-openstef/model/regressors/regressor.py,sha256=uJcx59AyCPE9f_yPcAQ59h2ZS7eNsDpIHJrladKvHIw,3461
-openstef/model/regressors/xgb.py,sha256=mYQpfVVQQbmNA-esDr_PGbYXVqdEzHdkHIJ0CoUyA2c,1466
+openstef/model/regressors/median.py,sha256=ITM5QhqfvjMjfk8fuHbyVWyWgld1NTiWajaaWAugbis,2697
+openstef/model/regressors/regressor.py,sha256=0um575rTEkzYb1E5IAOuTlsZDhmb7eI5byu5e062NRs,3469
+openstef/model/regressors/xgb.py,sha256=uhV9Wm90aOkjByTm-O2xpt2kpANRxAqQvv5mA0H1uBc,1294
 openstef/model/regressors/xgb_multioutput_quantile.py,sha256=xWzA7tymC_o-F1OS3I7vUKf9zP6RR1ZglEeY4NAgjU0,9146
 openstef/model/regressors/xgb_quantile.py,sha256=PzKIxqN_CnEPFmzXACNuzLSmZSHbooTuiJ5ckJ9vh_E,7805
 openstef/model_selection/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
@@ -79,7 +80,7 @@ openstef/pipeline/create_component_forecast.py,sha256=40fYKajdj4F9K7fzmL3euyvwTr
 openstef/pipeline/create_forecast.py,sha256=mVbbu7jM31NEwfaDeQPqF3Okps9H1oLfjhPPiJRL4zg,5582
 openstef/pipeline/optimize_hyperparameters.py,sha256=w5LpZhW3KVklCJzaogNzyHfpMJfNqeRAnvyV4vi35wg,10953
 openstef/pipeline/train_create_forecast_backtest.py,sha256=hBJPxfDkbrmFSSGZrRH1vTiIVqJP-SWe0ibVpHT_8Qg,6048
-openstef/pipeline/train_model.py,sha256=4ooyPve2uBvL6sWzdLX47kiMpYmaXH7agc4qdTEXtSg,19971
+openstef/pipeline/train_model.py,sha256=O1pyATMQUkNZQ01FlOwG8r3gtKwRcx7YD73f-91umuo,19948
 openstef/pipeline/utils.py,sha256=23mB31p19FoGWelLJzxNmqlzGwEr3fCDBEA37V2kpYY,2167
 openstef/plotting/__init__.py,sha256=KQjXzyafCt1bE7XDrSeV4TDUIO7MkwN_Br4ASOcNI2g,163
 openstef/plotting/load_forecast_plotter.py,sha256=GWHVmUB2YosNj7TnSrMnxYAfM2Z1mNg5oRV9A_lJmQY,8129
@@ -103,8 +104,8 @@ openstef/tasks/utils/predictionjobloop.py,sha256=Ysy3zF5lzPMz_asYDKeF5m0qgVT3tCt
 openstef/tasks/utils/taskcontext.py,sha256=O-LZ_wHEl5vbT8oB7EYtOeMkvk6EqCnI1-KiyER7Eu4,5407
 openstef/validation/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/validation/validation.py,sha256=r6UqkdH5TMjsGfn8Ta07K1jkqmrVmwcPGfyQvMmZyO4,11459
-openstef-3.4.70.dist-info/licenses/LICENSE,sha256=7Pm2fWFFHHUG5lDHed1vl5CjzxObIXQglnYsEdtjo_k,14907
-openstef-3.4.70.dist-info/METADATA,sha256=EQbXHDVITkJqoejoOe7mTzUl0kzKc-geXrjF65m0qX4,8834
-openstef-3.4.70.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-openstef-3.4.70.dist-info/top_level.txt,sha256=kD0H4PqrQoncZ957FvqwfBxa89kTrun4Z_RAPs_HhLs,9
-openstef-3.4.70.dist-info/RECORD,,
+openstef-3.4.73.dist-info/licenses/LICENSE,sha256=7Pm2fWFFHHUG5lDHed1vl5CjzxObIXQglnYsEdtjo_k,14907
+openstef-3.4.73.dist-info/METADATA,sha256=7Nxg4a2GePqCFf19ANnEIze222JfNmypS88wt05cdfg,8834
+openstef-3.4.73.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+openstef-3.4.73.dist-info/top_level.txt,sha256=kD0H4PqrQoncZ957FvqwfBxa89kTrun4Z_RAPs_HhLs,9
+openstef-3.4.73.dist-info/RECORD,,

{openstef-3.4.70.dist-info → openstef-3.4.73.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (79.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any