openstef 3.4.45__py3-none-any.whl → 3.4.48__py3-none-any.whl

openstef/data/NL_terrestrial_radiation.csv.license

@@ -0,0 +1,3 @@
+SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com>
+
+SPDX-License-Identifier: MPL-2.0

openstef/feature_engineering/apply_features.py

@@ -31,6 +31,7 @@ from openstef.feature_engineering.weather_features import (
 from openstef.feature_engineering.cyclic_features import (
     add_seasonal_cyclic_features,
     add_time_cyclic_features,
+    add_daylight_terrestrial_feature,
 )
 
 
@@ -124,5 +125,8 @@ def apply_features(
     # Adds polar time features (sine and cosine) to capture periodic patterns based on the timestamp index.
     data = add_time_cyclic_features(data)
 
+    # Adds daylight terrestrial feature
+    data = add_daylight_terrestrial_feature(data)
+
     # Return dataframe including all requested features
     return data

openstef/feature_engineering/cyclic_features.py

@@ -13,6 +13,8 @@ import structlog
 import logging
 
 from openstef.settings import Settings
+from openstef import PROJECT_ROOT
+
 
 structlog.configure(
     wrapper_class=structlog.make_filtering_bound_logger(
@@ -21,10 +23,65 @@ structlog.configure(
 )
 logger = structlog.get_logger(__name__)
 
-
+TERRESTRIAL_RADIATION_CSV_PATH: str = (
+    PROJECT_ROOT / "openstef" / "data" / "NL_terrestrial_radiation.csv"
+)
 NUM_SECONDS_IN_A_DAY = 24 * 60 * 60
 
 
+def add_daylight_terrestrial_feature(
+    data: pd.DataFrame,
+    path_to_terrestrial_radiation_csv: str = TERRESTRIAL_RADIATION_CSV_PATH,
+) -> pd.DataFrame:
+    """Add daylight terrestrial radiation feature to the input dataset. This function processes terrestrial radiation
+    data and aligns it with the time indices of the input dataset. The terrestrial radiation data is normalized,
+    interpolated, and merged with the main dataset to provide a feature representing terrestrial radiation.
+
+    Args:
+        data (pd.DataFrame):
+            The input dataset containing a time-indexed DataFrame.
+        path_to_terrestrial_radiation_csv (str):
+            File path to the CSV file containing terrestrial radiation data. The CSV file
+            should have a time-based index.
+
+    Returns:
+        pd.DataFrame:
+            The input dataset with an added column for the terrestrial radiation feature.
+
+    Notes:
+        - The function assumes the input data and the terrestrial radiation data share
+          the same time zone and frequency alignment.
+        - The terrestrial radiation values are normalized using z-score normalization.
+
+    """
+    # Load the terrestrial radiation data
+    terrestrial_radiation = pd.read_csv(path_to_terrestrial_radiation_csv, index_col=0)
+    terrestrial_radiation.index = pd.to_datetime(terrestrial_radiation.index)
+
+    # Align the index with the input data's year
+    year_diff = data.index.min().year - terrestrial_radiation.index.min().year
+    terrestrial_radiation.index += pd.DateOffset(years=year_diff)
+
+    # Resample to 15-minute intervals, and interpolate missing values
+    terrestrial_radiation = terrestrial_radiation.resample("15min").mean().interpolate()
+
+    # Normalize the terrestrial radiation values using z-score normalization
+    terrestrial_radiation = (
+        terrestrial_radiation - terrestrial_radiation.mean(axis=0)
+    ) / terrestrial_radiation.std(axis=0)
+    terrestrial_radiation.columns = ["daylight_continuous"]
+
+    # Make a copy of the DataFrame to avoid modifying the original
+    data = data.copy()
+
+    # Merge the terrestrial radiation data with the input dataset
+    data = data.merge(
+        terrestrial_radiation, left_index=True, right_index=True, how="left"
+    )
+
+    return data
+
+
 def add_time_cyclic_features(
     data: pd.DataFrame,
 ) -> pd.DataFrame:
@@ -35,6 +92,7 @@ def add_time_cyclic_features(
 
     Returns:
         DataFrame that is the same as input dataframe with extra columns for the added time of the day features.
+
     """
     # Ensure the index is a DatetimeIndex
     if not isinstance(data.index, pd.DatetimeIndex):
@@ -71,6 +129,7 @@ def add_seasonal_cyclic_features(
     >>> data = pd.DataFrame(index=pd.date_range(start='2023-01-01', periods=365, freq='D'))
     >>> data_with_features = add_cyclical_features(data)
     >>> print(data_with_features.head())
+
     """
     # Ensure the index is a DatetimeIndex
     if not isinstance(data.index, pd.DatetimeIndex):

openstef/model/metamodels/feature_clipper.py

@@ -0,0 +1,90 @@
+# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
+#
+# SPDX-License-Identifier: MPL-2.0
+from sklearn.base import BaseEstimator, TransformerMixin
+import pandas as pd
+from typing import List, Dict, Tuple, Optional
+
+
+class FeatureClipper(BaseEstimator, TransformerMixin):
+    """
+    A transformer that clips the values of specified columns to the minimum and
+    maximum values observed during training. This prevents the model from
+    extrapolating beyond these values during prediction.
+    """
+
+    def __init__(self, columns: List[str]):
+        """
+        Initialize the FeatureClipper.
+
+        Parameters:
+        ----------
+        columns : List[str]
+            List of column names to be clipped.
+        """
+        self.columns: List[str] = columns
+        self.feature_ranges: Dict[str, Tuple[float, float]] = {}
+
+    def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None) -> "FeatureClipper":
+        """
+        Fits the transformer on the training data by calculating the min and max
+        values for the specified columns.
+
+        Parameters:
+        ----------
+        X : pd.DataFrame
+            The input DataFrame containing training data.
+
+        y : Optional[pd.Series]
+            Ignored. This parameter exists for compatibility with scikit-learn's pipeline.
+
+        Returns:
+        -------
+        self : FeatureClipper
+            Fitted transformer.
+
+        Raises:
+        ------
+        ValueError:
+            If the input is not a pandas DataFrame.
+        """
+        if not isinstance(X, pd.DataFrame):
+            raise ValueError("Input must be a pandas DataFrame")
+
+        for col in self.columns:
+            if col in X.columns:
+                self.feature_ranges[col] = (X[col].min(), X[col].max())
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        """
+        Transforms new data by clipping the specified columns' values to be within
+        the min and max range observed during fitting.
+
+        Parameters:
+        ----------
+        X : pd.DataFrame
+            The input DataFrame containing new data to be transformed.
+
+        Returns:
+        -------
+        X_ : pd.DataFrame
+            A copy of the input DataFrame with clipped values in the specified columns.
+
+        Raises:
+        ------
+        ValueError:
+            If the input is not a pandas DataFrame.
+        """
+        if not isinstance(X, pd.DataFrame):
+            raise ValueError("Input must be a pandas DataFrame")
+
+        X_copy = X.copy()
+
+        for col in self.columns:
+            if col in X_copy.columns and col in self.feature_ranges:
+                min_val, max_val = self.feature_ranges[col]
+                X_copy[col] = X_copy[col].clip(lower=min_val, upper=max_val)
+
+        return X_copy

openstef/model/regressors/linear_quantile.py

@@ -14,6 +14,7 @@ from sklearn.utils.validation import check_is_fitted
 from openstef.feature_engineering.missing_values_transformer import (
     MissingValuesTransformer,
 )
+from openstef.model.metamodels.feature_clipper import FeatureClipper
 from openstef.model.regressors.regressor import OpenstfRegressor
 
 DEFAULT_QUANTILES: tuple[float, ...] = (0.9, 0.5, 0.1)
@@ -28,6 +29,7 @@ class LinearQuantileOpenstfRegressor(OpenstfRegressor, RegressorMixin):
     x_scaler_: StandardScaler
     y_scaler_: StandardScaler
     models_: Dict[float, QuantileRegressor]
+    feature_clipper_: FeatureClipper
 
     is_fitted_: bool = False
 
@@ -51,6 +53,7 @@ class LinearQuantileOpenstfRegressor(OpenstfRegressor, RegressorMixin):
         weight_exponent: float = 1,
         weight_floor: float = 0.1,
         no_fill_future_values_features: List[str] = None,
+        clipped_features: List[str] = None,
     ):
         """Initialize LinearQuantileOpenstfRegressor.
 
@@ -89,6 +92,9 @@ class LinearQuantileOpenstfRegressor(OpenstfRegressor, RegressorMixin):
                 "Cannot train quantile model as 0.5 is not in requested quantiles!"
             )
 
+        if clipped_features is None:
+            clipped_features = ["APX"]
+
         self.quantiles = quantiles
         self.alpha = alpha
         self.solver = solver
@@ -103,6 +109,7 @@ class LinearQuantileOpenstfRegressor(OpenstfRegressor, RegressorMixin):
         )
         self.x_scaler_ = StandardScaler()
         self.y_scaler_ = StandardScaler()
+        self.feature_clipper_ = FeatureClipper(columns=clipped_features)
         self.models_ = {
             quantile: QuantileRegressor(alpha=alpha, quantile=quantile, solver=solver)
             for quantile in quantiles
@@ -177,6 +184,7 @@ class LinearQuantileOpenstfRegressor(OpenstfRegressor, RegressorMixin):
             y = pd.Series(np.asarray(y), name="load")
 
         x = self._remove_ignored_features(x)
+        self.feature_clipper_.fit(x)
 
         # Fix nan columns
         x, y = self.imputer_.fit_transform(x, y)
@@ -252,6 +260,7 @@ class LinearQuantileOpenstfRegressor(OpenstfRegressor, RegressorMixin):
 
         # Preprocess input data
         x = self._remove_ignored_features(x)
+        x = self.feature_clipper_.transform(x)
         x = self.imputer_.transform(x)
         x_scaled = self.x_scaler_.transform(x)
 

{openstef-3.4.45.dist-info → openstef-3.4.48.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: openstef
-Version: 3.4.45
+Version: 3.4.48
 Summary: Open short term energy forecaster
 Home-page: https://github.com/OpenSTEF/openstef
 Author: Alliander N.V
@@ -29,7 +29,7 @@ Requires-Dist: pvlib==0.10.5
 Requires-Dist: pydantic~=2.4
 Requires-Dist: pydantic-settings~=2.3
 Requires-Dist: pymsteams~=0.2.2
-Requires-Dist: scikit-learn~=1.3
+Requires-Dist: scikit-learn<1.6,>=1.3
 Requires-Dist: scipy~=1.10
 Requires-Dist: statsmodels<1.0.0,>=0.13.5
 Requires-Dist: structlog<25,>=23.1

{openstef-3.4.45.dist-info → openstef-3.4.48.dist-info}/RECORD

@@ -4,6 +4,8 @@ openstef/app_settings.py,sha256=EJTDtimctFQQ-3f7ZcOQaRYohpZk3JD6aZBWPFYM2_A,582
 openstef/enums.py,sha256=Wmoag2p7G2cvENA1qt8FcVbAgo-MswXKxmq7vkxHaxs,2680
 openstef/exceptions.py,sha256=U4u2LTcdT6cmzpipT2Jh7kq9nCjT_-6gntn8yjuhGU0,1993
 openstef/settings.py,sha256=nSgkBqFxuqB3w7Rwo60i8j37c5ngDbt6vpjHS6QtJXQ,354
+openstef/data/NL_terrestrial_radiation.csv,sha256=A4kbW56GDzWi4tWUwY2C-4PiOvcKJCwkWQQtdg4ekPE,820246
+openstef/data/NL_terrestrial_radiation.csv.license,sha256=AxxHusqwIXU5RHl5ZMU65LyXmgtbj6QlcnFaOEN4kEE,145
 openstef/data/dutch_holidays.csv,sha256=Cg8EYjXp1O0lcFOkIOmrS5HaOArrxZwOXsZ9pVkIcKI,49847
 openstef/data/dutch_holidays.csv.license,sha256=AxxHusqwIXU5RHl5ZMU65LyXmgtbj6QlcnFaOEN4kEE,145
 openstef/data/pv_single_coefs.csv,sha256=jadIEYdHvl1lnV_06X_FASkJZ6C3Hecs5xZnH1gPMvI,24779
@@ -18,9 +20,9 @@ openstef/data_classes/model_specifications.py,sha256=Uod1W3QzhRqVLb6zvXwxh9wRL3E
 openstef/data_classes/prediction_job.py,sha256=_o5_9HYv6ERTIWlcMpUE-mWwe7dRpaiP83dgNpqpa5Y,5657
 openstef/data_classes/split_function.py,sha256=ljQIQQu1t1Y_CVWGAy25jrM6wG9odIVVQVimrT1n-1s,3358
 openstef/feature_engineering/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
-openstef/feature_engineering/apply_features.py,sha256=9scyEpUZcSWQrhMXV4c7iT1KvmHDk1J_KSZ_qI63lfY,4866
+openstef/feature_engineering/apply_features.py,sha256=S9HSsooDPcSSyEw9kixEhjjk3MAkAfyK1VhZKg5R0gA,4995
 openstef/feature_engineering/bidding_zone_to_country_mapping.py,sha256=u9aabjFDImydkO6_cXiaQxBT4gb5zy0gGTg2EoIUO_Y,2106
-openstef/feature_engineering/cyclic_features.py,sha256=gmU49D40yR9-Fh9ajiv3SyIWVLQcnibvQ4fFnpvAOj4,3527
+openstef/feature_engineering/cyclic_features.py,sha256=0Z3wZeF_qrkmEcOq91gtdSMZucAq99kUoBuFDV0SHqk,5962
 openstef/feature_engineering/data_preparation.py,sha256=htca9LBO3ZN1D-iX4vXf0UN1fw_rRO7y6N3AuYVMpfk,5628
 openstef/feature_engineering/feature_adder.py,sha256=aSqDl_gUrB3H2TD3cNvU5JniY_KOb4u4a2A6J7zB2BQ,6835
 openstef/feature_engineering/feature_applicator.py,sha256=DR7jayrEMlra4BFL1Ps5WV2fxbkQ6VaOTa5RIKM-YNk,7447
@@ -43,6 +45,7 @@ openstef/model/objective_creator.py,sha256=cIO-uiCEYHjqYrgZizeFEjjgLHLLwab8le9O8
 openstef/model/serializer.py,sha256=IUiiAWvoGVoWzmS-akI6LC7jHRY5Ln_vOCBZy1LnESY,17238
 openstef/model/standard_deviation_generator.py,sha256=Od9bzXi2TLb1v8Nz-VhBMZHSopWH6ssaDe8gYLlqO1I,2911
 openstef/model/metamodels/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
+openstef/model/metamodels/feature_clipper.py,sha256=wDsf4k_2YuC6rrFlwE12LpgOdOwHHWuET2ZrJ_gr4yo,2861
 openstef/model/metamodels/grouped_regressor.py,sha256=yMN_a6TnQSyFaqlB_6Nifq-ydpb5hs6w_b97IaBbHj4,8337
 openstef/model/metamodels/missing_values_handler.py,sha256=veyvYZHhKvlYZxaUpxRQ7XoE033_3Lcg9LrbuKchlOk,5241
 openstef/model/regressors/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
@@ -52,7 +55,7 @@ openstef/model/regressors/dazls.py,sha256=Xt89yFHjkwpIUTkkhPmPZ74F8_tht_XV88INuP
 openstef/model/regressors/flatliner.py,sha256=T9u-ukhqFcatQmlgUtBL_G-1b_wQzgdVRq0ac64GnjQ,2789
 openstef/model/regressors/lgbm.py,sha256=zCdn1euEdSFxYJzH8XqQFFnb6R4JVUnmineKjX_Gy-g,800
 openstef/model/regressors/linear.py,sha256=uOvZMLGZH_9nXfmS5honCMfyVeyGXP1Cza9A_BdXlVw,3665
-openstef/model/regressors/linear_quantile.py,sha256=uj8Hd-Po14YymC_dUNSzxDayGqK9MjZ2dp_aLnpiv0s,10126
+openstef/model/regressors/linear_quantile.py,sha256=VAyIhp7GPayqbk8Vj_ONqPLNYuaOvxkFKDrRxx6yGY0,10510
 openstef/model/regressors/regressor.py,sha256=uJcx59AyCPE9f_yPcAQ59h2ZS7eNsDpIHJrladKvHIw,3461
 openstef/model/regressors/xgb.py,sha256=SH-UiYJtMbfmRBK6738dU0ZRfYfzNynnikwbxINCE7Q,1467
 openstef/model/regressors/xgb_multioutput_quantile.py,sha256=xWzA7tymC_o-F1OS3I7vUKf9zP6RR1ZglEeY4NAgjU0,9146
@@ -90,8 +93,8 @@ openstef/tasks/utils/predictionjobloop.py,sha256=Ysy3zF5lzPMz_asYDKeF5m0qgVT3tCt
 openstef/tasks/utils/taskcontext.py,sha256=L9K14ycwgVxbIVUjH2DIn_QWbnu-OfxcGtQ1K9T6sus,5630
 openstef/validation/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/validation/validation.py,sha256=HVgreHvcZvPazfwC3NNE8_3lsMsZEd_42osCAg1_6W4,11128
-openstef-3.4.45.dist-info/LICENSE,sha256=7Pm2fWFFHHUG5lDHed1vl5CjzxObIXQglnYsEdtjo_k,14907
-openstef-3.4.45.dist-info/METADATA,sha256=nszNfTz9kFGVZyiXyRL18H4_6WtSIlZDvLuAAleM5wM,8068
-openstef-3.4.45.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-openstef-3.4.45.dist-info/top_level.txt,sha256=kD0H4PqrQoncZ957FvqwfBxa89kTrun4Z_RAPs_HhLs,9
-openstef-3.4.45.dist-info/RECORD,,
+openstef-3.4.48.dist-info/LICENSE,sha256=7Pm2fWFFHHUG5lDHed1vl5CjzxObIXQglnYsEdtjo_k,14907
+openstef-3.4.48.dist-info/METADATA,sha256=w22ao36-l9Y4HXpsHvVZ8G9k3_3Iqt2EJ8-dGoArWIs,8073
+openstef-3.4.48.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+openstef-3.4.48.dist-info/top_level.txt,sha256=kD0H4PqrQoncZ957FvqwfBxa89kTrun4Z_RAPs_HhLs,9
+openstef-3.4.48.dist-info/RECORD,,