openstef 3.4.52__py3-none-any.whl → 3.4.53__py3-none-any.whl

openstef/data_classes/prediction_job.py

@@ -26,6 +26,7 @@ class PredictionJobDataClass(BaseModel):
         - ``"lgb"``
         - ``"linear"``
         - ``"linear_quantile"``
+        - ``"gblinear_quantile"``
         - ``"xgb_multioutput_quantile"``
         - ``"flatliner"``
 

openstef/enums.py

@@ -115,6 +115,7 @@ class ModelType(Enum):
     LGB = "lgb"
     LINEAR = "linear"
     LINEAR_QUANTILE = "linear_quantile"
+    GBLINEAR_QUANTILE = "gblinear_quantile"
     ARIMA = "arima"
     FLATLINER = "flatliner"
 

openstef/model/model_creator.py

@@ -9,6 +9,7 @@ import structlog
 from openstef.enums import ModelType
 from openstef.model.regressors.arima import ARIMAOpenstfRegressor
 from openstef.model.regressors.custom_regressor import is_custom_type, load_custom_model
+from openstef.model.regressors.gblinear_quantile import GBLinearQuantileOpenstfRegressor
 from openstef.model.regressors.lgbm import LGBMOpenstfRegressor
 from openstef.model.regressors.linear import LinearOpenstfRegressor
 from openstef.model.regressors.linear_quantile import LinearQuantileOpenstfRegressor
@@ -121,6 +122,25 @@ valid_model_kwargs = {
         "weight_floor",
         "no_fill_future_values_features",
     ],
+    ModelType.GBLINEAR_QUANTILE: [
+        "quantiles",
+        "missing_values",
+        "imputation_strategy",
+        "fill_value",
+        "weight_scale_percentile",
+        "weight_exponent",
+        "weight_floor",
+        "no_fill_future_values_features",
+        "clipped_features",
+        "learning_rate",
+        "num_boost_round",
+        "early_stopping_rounds",
+        "reg_alpha",
+        "reg_lambda",
+        "updater",
+        "feature_selector",
+        "top_k",
+    ],
     ModelType.ARIMA: [
         "backtest_max_horizon",
         "order",
@@ -141,6 +161,7 @@ class ModelCreator:
         ModelType.XGB_MULTIOUTPUT_QUANTILE: XGBMultiOutputQuantileOpenstfRegressor,
         ModelType.LINEAR: LinearOpenstfRegressor,
         ModelType.LINEAR_QUANTILE: LinearQuantileOpenstfRegressor,
+        ModelType.GBLINEAR_QUANTILE: GBLinearQuantileOpenstfRegressor,
         ModelType.ARIMA: ARIMAOpenstfRegressor,
         ModelType.FLATLINER: FlatlinerRegressor,
     }

openstef/model/objective_creator.py

@@ -28,6 +28,7 @@ class ObjectiveCreator:
         ModelType.XGB_MULTIOUTPUT_QUANTILE: XGBMultioutputQuantileRegressorObjective,
         ModelType.LINEAR: LinearRegressorObjective,
         ModelType.LINEAR_QUANTILE: LinearRegressorObjective,
+        ModelType.GBLINEAR_QUANTILE: LinearRegressorObjective,
         ModelType.ARIMA: ARIMARegressorObjective,
     }
 

openstef/model/regressors/gblinear_quantile.py

@@ -0,0 +1,334 @@
+# SPDX-FileCopyrightText: 2017-2025 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
+#
+# SPDX-License-Identifier: MPL-2.0
+import math
+import re
+from typing import Union, Optional, List
+
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import train_test_split
+import xgboost as xgb
+from sklearn.preprocessing import StandardScaler
+from sklearn.utils.validation import check_is_fitted
+
+from openstef.feature_engineering.missing_values_transformer import (
+    MissingValuesTransformer,
+)
+from openstef.model.metamodels.feature_clipper import FeatureClipper
+from openstef.model.regressors.regressor import OpenstfRegressor
+
+DEFAULT_QUANTILES: tuple[float, ...] = (0.9, 0.5, 0.1)
+
+
+class GBLinearQuantileOpenstfRegressor(OpenstfRegressor):
+    is_fitted_: bool = False
+
+    TO_KEEP_FEATURES: List[str] = [
+        "T-7d",
+        "T-1d",
+    ]
+    TO_IGNORE_FEATURES: List[str] = [
+        "Month",
+        "Quarter",
+    ]
+
+    def __init__(
+        self,
+        quantiles: tuple[float, ...] = DEFAULT_QUANTILES,
+        missing_values: Union[int, float, str, None] = np.nan,
+        imputation_strategy: Optional[str] = "mean",
+        fill_value: Union[str, int, float] = None,
+        weight_scale_percentile: int = 95,
+        weight_exponent: float = 1,
+        weight_floor: float = 0.1,
+        validation_fraction: float = 0.2,
+        no_fill_future_values_features: List[str] = None,
+        clipped_features: List[str] = None,
+        learning_rate: float = 0.15,
+        num_boost_round: int = 500,
+        early_stopping_rounds: int = 10,
+        reg_alpha: float = 0.0001,
+        reg_lambda: float = 0.1,
+        updater: str = "shotgun",
+        feature_selector: str = "shuffle",
+        top_k: int = 0,
+    ):
+        super().__init__()
+
+        # Check if quantile 0.5 is present. This is required.
+        if 0.5 not in quantiles:
+            raise ValueError(
+                "Cannot train quantile model as 0.5 is not in requested quantiles!"
+            )
+
+        if clipped_features is None:
+            clipped_features = ["APX"]
+
+        self.quantiles = quantiles
+        self.weight_scale_percentile = weight_scale_percentile
+        self.weight_exponent = weight_exponent
+        self.weight_floor = weight_floor
+        self.imputer_ = MissingValuesTransformer(
+            missing_values=missing_values,
+            imputation_strategy=imputation_strategy,
+            fill_value=fill_value,
+            no_fill_future_values_features=no_fill_future_values_features,
+        )
+        self.x_scaler_ = StandardScaler()
+        self.y_scaler_ = StandardScaler()
+        self.validation_fraction = validation_fraction
+        self.model_: xgb.Booster = None
+        self.feature_clipper_ = FeatureClipper(columns=clipped_features)
+
+        self.learning_rate = learning_rate
+        self.num_boost_round = num_boost_round
+        self.early_stopping_rounds = early_stopping_rounds
+        self.reg_alpha = reg_alpha
+        self.reg_labmda = reg_lambda
+        self.updater = updater
+        self.feature_selector = feature_selector
+        self.top_k = top_k
+
+    @property
+    def feature_names(self) -> list:
+        """The names of the features used to train the model."""
+        check_is_fitted(self)
+        return self.imputer_.non_null_feature_names
+
+    @staticmethod
+    def _get_importance_names():
+        return {
+            "gain_importance_name": "total_gain",
+            "weight_importance_name": "weight",
+        }
+
+    @property
+    def can_predict_quantiles(self) -> bool:
+        """Attribute that indicates if the model predict particular quantiles."""
+        return True
+
+    def _is_feature_ignored(self, feature_name: str) -> bool:
+        """Check if a feature is ignored by the model.
+
+        Args:
+            feature_name: Feature name
+
+        Returns:
+            True if the feature is ignored, False otherwise
+
+        """
+
+        if feature_name in self.TO_KEEP_FEATURES:
+            return False
+
+        return (
+            # Ignore named features
+            feature_name in self.TO_IGNORE_FEATURES
+            or
+            # Ignore holiday features
+            re.match(r"is_", feature_name) is not None
+            or
+            # Ignore lag features
+            re.match(r"T-", feature_name) is not None
+            or
+            # Ignore infeed MFFBAS profiles
+            re.match(r"E\d.*_I", feature_name) is not None
+        )
+
+    def _remove_ignored_features(self, x: pd.DataFrame) -> pd.DataFrame:
+        """Remove ignored features from the input data.
+
+        Args:
+            x: Input data
+
+        Returns:
+            Data without ignored features
+
+        """
+        return x.drop(columns=[c for c in x.columns if self._is_feature_ignored(c)])
+
+    def fit(self, x: pd.DataFrame, y: pd.Series, **kwargs) -> OpenstfRegressor:
+        if not isinstance(y, pd.Series):
+            y = pd.Series(np.asarray(y), name="load")
+
+        x = self._remove_ignored_features(x)
+        self.feature_clipper_.fit(x)
+
+        # Fix nan columns
+        x, y = self.imputer_.fit_transform(x, y)
+        if x.isna().any().any():
+            raise ValueError(
+                "There are nan values in the input data. Set "
+                "imputation_strategy to solve them."
+            )
+
+        # Apply feature scaling
+        x_scaled = self.x_scaler_.fit_transform(x)
+        y_scaled = self.y_scaler_.fit_transform(y.to_frame())[:, 0]
+
+        # Add more focus on extreme / peak values
+        sample_weight = self._calculate_sample_weights(y.values.squeeze())
+
+        # Split the data into training and validation sets
+        x_train, x_val, y_train, y_val, weight_train, weight_val = train_test_split(
+            x_scaled,
+            y_scaled,
+            sample_weight,
+            test_size=self.validation_fraction,
+            random_state=42,
+        )
+
+        # Preserve feature names
+        x_train = pd.DataFrame(x_train, columns=x.columns)
+        x_val = pd.DataFrame(x_val, columns=x.columns)
+
+        dtrain = xgb.DMatrix(x_train, label=y_train, weight=weight_train)
+        dval = xgb.DMatrix(x_val, label=y_val, weight=weight_val)
+
+        xgb_params = {
+            # Use the quantile objective function.
+            "objective": "reg:quantileerror",  # This is pinball loss
+            "booster": "gblinear",
+            "updater": self.updater,
+            "alpha": self.reg_alpha,
+            "lambda": self.reg_labmda,
+            "feature_selector": self.feature_selector,
+            "quantile_alpha": np.array(self.quantiles),
+            "learning_rate": self.learning_rate,
+        }
+
+        if self.top_k > 0:
+            xgb_params["top_k"] = self.top_k
+
+        self.model_ = xgb.train(
+            params=xgb_params,
+            dtrain=dtrain,
+            num_boost_round=self.num_boost_round,
+            early_stopping_rounds=self.early_stopping_rounds,
+            evals=[(dtrain, "train"), (dval, "val")],
+        )
+
+        self._Booster = self.model_
+
+        self.is_fitted_ = True
+
+        self.feature_importances_ = self._get_feature_importances_from_booster(
+            self.model_
+        )
+
+        return self
+
+    def _calculate_sample_weights(self, y: np.array):
+        """Calculate sample weights based on the y values of arbitrary scale.
+
+        The resulting weights are in the range [0,1] and are used to put more emphasis
+        on certain samples. The sample weighting function does:
+
+        * Rescale data to a [-1, 1] range using quantile scaling. 90% of the data will
+          be within this range. Rest is outside.
+        * Calculate the weight by taking the exponent of scaled data.
+          * exponent=0: Results in uniform weights for all samples.
+          * exponent=1: Results in linearly increasing weights for samples that are
+            closer to the extremes.
+          * exponent>1: Results in exponentially increasing weights for samples that are
+            closer to the extremes.
+        * Clip the data to [0, 1] range with weight_floor as the minimum weight.
+          * Weight floor is used to make sure that all the samples are considered.
+
+        """
+        return np.clip(
+            _weight_exp(
+                _scale_percentile(y, percentile=self.weight_scale_percentile),
+                exponent=self.weight_exponent,
+            ),
+            a_min=self.weight_floor,
+            a_max=1,
+        )
+
+    def predict(self, x: pd.DataFrame, quantile: float = 0.5, **kwargs) -> np.array:
+        check_is_fitted(self)
+
+        # Preprocess input data
+        x = self._remove_ignored_features(x)
+        x = self.feature_clipper_.transform(x)
+        x = self.imputer_.transform(x)
+        x_scaled = self.x_scaler_.transform(x)
+
+        # Preserve feature names
+        x_scaled = pd.DataFrame(x_scaled, columns=x.columns)
+
+        d_x_scaled = xgb.DMatrix(x_scaled)
+
+        # Make prediction
+        y_pred = self.model_.predict(d_x_scaled)
+
+        # When multiple quantiles are trained,
+        # we need to select the requested quantile
+        if len(self.quantiles) > 1:
+            # Get index of the quantile value in the quantiles list
+            quantile_index = self.quantiles.index(quantile)
+
+            # Get the quantile prediction
+            y_pred = y_pred[:, quantile_index]
+
+        # Inverse scaling
+        y_pred = self.y_scaler_.inverse_transform(y_pred.reshape(-1, 1))[:, 0]
+
+        return y_pred
+
+    @classmethod
+    def _get_feature_importances_from_booster(cls, booster: xgb.Booster) -> np.ndarray:
+        """Gets feature importances from a XGB booster.
+
+        This is based on the feature_importance_ property defined in:
+        https://github.com/dmlc/xgboost/blob/master/python-package/xgboost/sklearn.py.
+
+        Args:
+            booster: Booster object,
+                most of the times the median model (quantile=0.5) is preferred
+
+        Returns:
+            Ndarray with normalized feature importances.
+
+        """
+        # Get score
+        score = booster.get_score(importance_type="weight")
+
+        if type(next(iter(score.values()))) is list:
+            num_quantiles = len(next(iter(score.values())))
+
+            # Select middle quantile, assuming odd number of quantiles
+            quantile_index = num_quantiles // 2
+
+            score = {f: score[f][quantile_index] for f in score}
+
+        # Get feature names from booster
+        feature_names = booster.feature_names
+
+        # Get importance
+        feature_importance = [score.get(f, 0.0) for f in feature_names]
+        # Convert to array
+        features_importance_array = np.array(feature_importance, dtype=np.float32)
+
+        total = features_importance_array.sum()  # For normalizing
+        if total == 0:
+            return features_importance_array
+        return features_importance_array / total  # Normalize
+
+    @classmethod
+    def _get_param_names(cls):
+        return [
+            "quantiles",
+        ]
+
+    def __sklearn_is_fitted__(self) -> bool:
+        return self.is_fitted_
+
+
+def _scale_percentile(x: np.ndarray, percentile: int = 95):
+    return np.abs(x / np.percentile(np.abs(x), percentile))
+
+
+def _weight_exp(x: np.ndarray, exponent: float = 1):
+    return np.abs(x) ** exponent

{openstef-3.4.52.dist-info → openstef-3.4.53.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: openstef
-Version: 3.4.52
+Version: 3.4.53
 Summary: Open short term energy forecaster
 Home-page: https://github.com/OpenSTEF/openstef
 Author: Alliander N.V

{openstef-3.4.52.dist-info → openstef-3.4.53.dist-info}/RECORD

@@ -1,7 +1,7 @@
 openstef/__init__.py,sha256=93UM6m0LLQhO69-mSqLuUy73jgs4W7Iuxfo3Lm8c98g,419
 openstef/__main__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/app_settings.py,sha256=EJTDtimctFQQ-3f7ZcOQaRYohpZk3JD6aZBWPFYM2_A,582
-openstef/enums.py,sha256=Wmoag2p7G2cvENA1qt8FcVbAgo-MswXKxmq7vkxHaxs,2680
+openstef/enums.py,sha256=wToJV56CAqgl-UY2DU1Zf12Ft9WsSiUNMrHYLnULOhQ,2724
 openstef/exceptions.py,sha256=U4u2LTcdT6cmzpipT2Jh7kq9nCjT_-6gntn8yjuhGU0,1993
 openstef/settings.py,sha256=nSgkBqFxuqB3w7Rwo60i8j37c5ngDbt6vpjHS6QtJXQ,354
 openstef/data/NL_terrestrial_radiation.csv,sha256=A4kbW56GDzWi4tWUwY2C-4PiOvcKJCwkWQQtdg4ekPE,820246
@@ -17,7 +17,7 @@ openstef/data/dazls_model_3.4.24/dazls_stored_3.4.24_model_card.md.license,sha25
 openstef/data_classes/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/data_classes/data_prep.py,sha256=gRSL7UiHvZis8m8z7VoTCZc0Ccffhef5_hmSyApnqK0,3417
 openstef/data_classes/model_specifications.py,sha256=Uod1W3QzhRqVLb6zvXwxh9wRL3EHCzSvX0oDNd28cFk,1197
-openstef/data_classes/prediction_job.py,sha256=_o5_9HYv6ERTIWlcMpUE-mWwe7dRpaiP83dgNpqpa5Y,5657
+openstef/data_classes/prediction_job.py,sha256=hJNwLkCIIVsG5nzOmu_yY6rqTYzjuMt8FiLi7Aei3n4,5691
 openstef/data_classes/split_function.py,sha256=ljQIQQu1t1Y_CVWGAy25jrM6wG9odIVVQVimrT1n-1s,3358
 openstef/feature_engineering/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/feature_engineering/apply_features.py,sha256=EIxP9fvmnAjFRehQpVGBhZHt35GACL-HhDm65_ktHc0,5121
@@ -39,9 +39,9 @@ openstef/model/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,16
 openstef/model/basecase.py,sha256=caI6Q-8y0ymlxGK9Js_H3Vh0q6ruNHlGD5RG0_kE5M0,2878
 openstef/model/confidence_interval_applicator.py,sha256=Bx0mm4zGKlqopMZ589cVyDN_k6jfuyqtV1FoViXxc2Y,9775
 openstef/model/fallback.py,sha256=VV9ehgnoMZtWzqKk9H1t8wnERFh5CyC4TvDIuRP_ZDI,2861
-openstef/model/model_creator.py,sha256=EHWB5bIq9yYfWJpMWg62K5fns1p3aHFFB7_Izt2VWnA,6018
+openstef/model/model_creator.py,sha256=L84A4_HVYL7bRZY0D77j4c4kHRleVGCRXqZZkT9BZY0,6668
 openstef/model/objective.py,sha256=qJdI6GAzv8l5Mxd8G7BIqQnfdJNM7aOlg9DMzMGjWqA,14558
-openstef/model/objective_creator.py,sha256=cIO-uiCEYHjqYrgZizeFEjjgLHLLwab8le9O8DJOF8I,2145
+openstef/model/objective_creator.py,sha256=jqMvdXiVRc9GmOvSijQY0zuyxS07-ezkVXclvoW98g4,2208
 openstef/model/serializer.py,sha256=IUiiAWvoGVoWzmS-akI6LC7jHRY5Ln_vOCBZy1LnESY,17238
 openstef/model/standard_deviation_generator.py,sha256=Od9bzXi2TLb1v8Nz-VhBMZHSopWH6ssaDe8gYLlqO1I,2911
 openstef/model/metamodels/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
@@ -53,6 +53,7 @@ openstef/model/regressors/arima.py,sha256=wt7FVykjSvljpl7vjtliq61SiyjQ7KKtw8PF9x
 openstef/model/regressors/custom_regressor.py,sha256=Hsmxahc9nfSWD0aEZ6cm4pxW2noQ8B1SujS17_fmxcU,1768
 openstef/model/regressors/dazls.py,sha256=Xt89yFHjkwpIUTkkhPmPZ74F8_tht_XV88INuP5GU2E,3994
 openstef/model/regressors/flatliner.py,sha256=T9u-ukhqFcatQmlgUtBL_G-1b_wQzgdVRq0ac64GnjQ,2789
+openstef/model/regressors/gblinear_quantile.py,sha256=DSRjL_kadBfDKrDEgrOfU1N60grTiAovtcBszBa41TI,11271
 openstef/model/regressors/lgbm.py,sha256=zCdn1euEdSFxYJzH8XqQFFnb6R4JVUnmineKjX_Gy-g,800
 openstef/model/regressors/linear.py,sha256=uOvZMLGZH_9nXfmS5honCMfyVeyGXP1Cza9A_BdXlVw,3665
 openstef/model/regressors/linear_quantile.py,sha256=VAyIhp7GPayqbk8Vj_ONqPLNYuaOvxkFKDrRxx6yGY0,10510
@@ -93,8 +94,8 @@ openstef/tasks/utils/predictionjobloop.py,sha256=Ysy3zF5lzPMz_asYDKeF5m0qgVT3tCt
 openstef/tasks/utils/taskcontext.py,sha256=L9K14ycwgVxbIVUjH2DIn_QWbnu-OfxcGtQ1K9T6sus,5630
 openstef/validation/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/validation/validation.py,sha256=HVgreHvcZvPazfwC3NNE8_3lsMsZEd_42osCAg1_6W4,11128
-openstef-3.4.52.dist-info/LICENSE,sha256=7Pm2fWFFHHUG5lDHed1vl5CjzxObIXQglnYsEdtjo_k,14907
-openstef-3.4.52.dist-info/METADATA,sha256=r8E2LPnPIDJ-AhzrbVmYP3MPYIfx2OOpwIuYC9obtcE,8305
-openstef-3.4.52.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-openstef-3.4.52.dist-info/top_level.txt,sha256=kD0H4PqrQoncZ957FvqwfBxa89kTrun4Z_RAPs_HhLs,9
-openstef-3.4.52.dist-info/RECORD,,
+openstef-3.4.53.dist-info/LICENSE,sha256=7Pm2fWFFHHUG5lDHed1vl5CjzxObIXQglnYsEdtjo_k,14907
+openstef-3.4.53.dist-info/METADATA,sha256=MCAC7JnwfKToob21A5_a-eFE4CkG7PJ3Wov0BIDJ6Uc,8305
+openstef-3.4.53.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+openstef-3.4.53.dist-info/top_level.txt,sha256=kD0H4PqrQoncZ957FvqwfBxa89kTrun4Z_RAPs_HhLs,9
+openstef-3.4.53.dist-info/RECORD,,