openstef 3.4.53__py3-none-any.whl → 3.4.55__py3-none-any.whl

openstef/data_classes/prediction_job.py

@@ -9,7 +9,7 @@ from pydantic.v1 import BaseModel
 from openstef.data_classes.data_prep import DataPrepDataClass
 from openstef.data_classes.model_specifications import ModelSpecificationDataClass
 from openstef.data_classes.split_function import SplitFuncDataClass
-from openstef.enums import PipelineType, BiddingZone
+from openstef.enums import PipelineType, BiddingZone, AggregateFunction
 
 
 class PredictionJobDataClass(BaseModel):
@@ -83,6 +83,8 @@ class PredictionJobDataClass(BaseModel):
     data_balancing_ratio: Optional[float] = None
     """If data balancing is enabled, the data will be balanced with data from 1 year 
     ago in the future."""
+    rolling_aggregate_features: Optional[list[AggregateFunction]] = None
+    """If not None, rolling aggregate(s) of load will be used as features in the model."""
     depends_on: Optional[list[Union[int, str]]]
     """Link to another prediction job on which this prediction job might depend."""
     sid: Optional[str]

openstef/enums.py

@@ -108,6 +108,13 @@ class BiddingZone(Enum):
     DE_AMP_LU = "DE_AMP_LU"
 
 
+class AggregateFunction(Enum):
+    MEAN = "mean"
+    MEDIAN = "median"
+    MAX = "max"
+    MIN = "min"
+
+
 class ModelType(Enum):
     XGB = "xgb"
     XGB_QUANTILE = "xgb_quantile"

openstef/feature_engineering/apply_features.py

@@ -22,6 +22,7 @@ from openstef.feature_engineering.lag_features import generate_lag_feature_funct
 from openstef.feature_engineering.bidding_zone_to_country_mapping import (
     BIDDING_ZONE_TO_COUNTRY_CODE_MAPPING,
 )
+from openstef.feature_engineering.rolling_features import add_rolling_aggregate_features
 from openstef.feature_engineering.weather_features import (
     add_additional_solar_features,
     add_additional_wind_features,
@@ -130,5 +131,8 @@ def apply_features(
     # Adds daylight terrestrial feature
     data = add_daylight_terrestrial_feature(data)
 
+    if pj.get("rolling_aggregate_features") is not None:
+        data = add_rolling_aggregate_features(data, pj=pj)
+
     # Return dataframe including all requested features
     return data

openstef/feature_engineering/feature_applicator.py

@@ -24,7 +24,9 @@ from openstef.feature_engineering.general import (
     remove_non_requested_feature_columns,
 )
 
-LATENCY_CONFIG = {"APX": 24}  # A specific latency is part of a specific feature.
+LATENCY_CONFIG = {
+    "day_ahead_electricity_price": 24
+}  # A specific latency is part of a specific feature.
 
 
 class AbstractFeatureApplicator(ABC):
@@ -94,7 +96,7 @@ class TrainFeatureApplicator(AbstractFeatureApplicator):
                 if not specified a default location is used
             latency_config: (Optional) Invalidate certain features that are not
                 available for a specific horizon due to data latency. Defaults to
-                ``{"APX": 24}``.
+                ``{"day_ahead_electricity_price": 24}``.
 
         Returns:
             Input DataFrame with an extra column for every added feature and sorted on the datetime index.

openstef/feature_engineering/rolling_features.py

@@ -0,0 +1,43 @@
+# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
+#
+# SPDX-License-Identifier: MPL-2.0
+from datetime import timedelta
+
+import pandas as pd
+
+from openstef.data_classes.prediction_job import PredictionJobDataClass
+
+
+def add_rolling_aggregate_features(
+    data: pd.DataFrame,
+    pj: PredictionJobDataClass,
+    rolling_window: timedelta = timedelta(hours=24),
+) -> pd.DataFrame:
+    """
+    Adds rolling aggregate features to the input dataframe.
+
+    These features are calculated with an aggregation over a rolling window of the data.
+    A list of requested features is used to determine whether to add the rolling features
+    or not.
+
+    Args:
+        data: Input dataframe to which the rolling features will be added.
+        pj: Prediction job data.
+        rolling_window: Rolling window size for the aggregation.
+
+    Returns:
+        DataFrame with added rolling features.
+    """
+    # Ensure the index is a DatetimeIndex
+    if not isinstance(data.index, pd.DatetimeIndex):
+        raise ValueError("The DataFrame index must be a DatetimeIndex.")
+
+    if "load" not in data.columns:
+        raise ValueError("The DataFrame must contain a 'load' column.")
+    rolling_window_load = data["load"].rolling(window=rolling_window)
+
+    for aggregate_func in pj["rolling_aggregate_features"]:
+        data[
+            f"rolling_{aggregate_func.value}_load_{rolling_window}"
+        ] = rolling_window_load.aggregate(aggregate_func.value)
+    return data

openstef/model/regressors/linear_quantile.py

@@ -93,7 +93,7 @@ class LinearQuantileOpenstfRegressor(OpenstfRegressor, RegressorMixin):
             )
 
         if clipped_features is None:
-            clipped_features = ["APX"]
+            clipped_features = ["day_ahead_electricity_price"]
 
         self.quantiles = quantiles
         self.alpha = alpha

openstef/tasks/create_components_forecast.py

@@ -6,7 +6,7 @@
 This code assumes trained models are available from the persistent storage.
 If these are not available run model_train.py to train all models.
 To provide the prognoses the following steps are carried out:
-  1. Get historic training data (TDCV, Load, Weather and APX price data)
+  1. Get historic training data (TDCV, Load, Weather and day_ahead_electricity_price price data)
   2. Apply features
   3. Load model
   4. Make component prediction

openstef/tasks/create_forecast.py

@@ -6,7 +6,7 @@
 This code assumes trained models are available from the persistent storage. If these
 are not available run model_train.py to train all models.
 To provide the prognoses the folowing steps are carried out:
-  1. Get historic training data (TDCV, Load, Weather and APX price data)
+  1. Get historic training data (TDCV, Load, Weather and day_ahead_electricity_price price data)
   2. Apply features
   3. Load model
   4. Make prediction
@@ -24,7 +24,7 @@ from datetime import datetime, timedelta
 from pathlib import Path
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.enums import ModelType, PipelineType
+from openstef.enums import BiddingZone, ModelType, PipelineType
 from openstef.exceptions import InputDataOngoingZeroFlatlinerError
 from openstef.pipeline.create_forecast import create_forecast_pipeline
 from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
@@ -82,8 +82,13 @@ def create_forecast_task(
         location=[pj["lat"], pj["lon"]],
         datetime_start=datetime_start,
         datetime_end=datetime_end,
+        market_price=pj.electricity_bidding_zone.value,
     )
 
+    # Add APX price to the input data for backward compatibility,remove this line when all models are retrained
+    if pj.electricity_bidding_zone == BiddingZone.NL:
+        input_data["APX"] = input_data["day_ahead_electricity_price"]
+
     try:
         # Make forecast with the forecast pipeline
         forecast = create_forecast_pipeline(

openstef/tasks/train_model.py

@@ -4,7 +4,7 @@
 """This module contains the CRON job that is periodically executed to retrain the prognosis models.
 
 For this the folowing steps are caried out:
-  1. Get historic training data (TDCV, Load, Weather and APX price data)
+  1. Get historic training data (TDCV, Load, Weather and day_ahead_electricity_price price data)
   2. Apply features
   3. Train and Test the new model
   4. Check if new model performs better than the old model

openstef/validation/validation.py

@@ -313,9 +313,9 @@ def calc_completeness_dataframe(
                     expected_numbers_timedelayed=value,
                 )
 
-        # Correct for APX being only expected to be available up to 24h
-        if "APX" in non_na_count.index:
-            non_na_count["APX"] += max([len(df) - 96, 0])
+        # Correct for day_ahead_electricity_price being only expected to be available up to 24h
+        if "day_ahead_electricity_price" in non_na_count.index:
+            non_na_count["day_ahead_electricity_price"] += max([len(df) - 96, 0])
 
         completeness_per_column_dataframe = non_na_count / (len(df))
 

{openstef-3.4.53.dist-info → openstef-3.4.55.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: openstef
-Version: 3.4.53
+Version: 3.4.55
 Summary: Open short term energy forecaster
 Home-page: https://github.com/OpenSTEF/openstef
 Author: Alliander N.V

{openstef-3.4.53.dist-info → openstef-3.4.55.dist-info}/RECORD

@@ -1,7 +1,7 @@
 openstef/__init__.py,sha256=93UM6m0LLQhO69-mSqLuUy73jgs4W7Iuxfo3Lm8c98g,419
 openstef/__main__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/app_settings.py,sha256=EJTDtimctFQQ-3f7ZcOQaRYohpZk3JD6aZBWPFYM2_A,582
-openstef/enums.py,sha256=wToJV56CAqgl-UY2DU1Zf12Ft9WsSiUNMrHYLnULOhQ,2724
+openstef/enums.py,sha256=FrP0m_Tk0kV7gSZ2hTY_8iD45KIKnexHrjNufhpKXpE,2829
 openstef/exceptions.py,sha256=U4u2LTcdT6cmzpipT2Jh7kq9nCjT_-6gntn8yjuhGU0,1993
 openstef/settings.py,sha256=nSgkBqFxuqB3w7Rwo60i8j37c5ngDbt6vpjHS6QtJXQ,354
 openstef/data/NL_terrestrial_radiation.csv,sha256=A4kbW56GDzWi4tWUwY2C-4PiOvcKJCwkWQQtdg4ekPE,820246
@@ -17,19 +17,20 @@ openstef/data/dazls_model_3.4.24/dazls_stored_3.4.24_model_card.md.license,sha25
 openstef/data_classes/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/data_classes/data_prep.py,sha256=gRSL7UiHvZis8m8z7VoTCZc0Ccffhef5_hmSyApnqK0,3417
 openstef/data_classes/model_specifications.py,sha256=Uod1W3QzhRqVLb6zvXwxh9wRL3EHCzSvX0oDNd28cFk,1197
-openstef/data_classes/prediction_job.py,sha256=hJNwLkCIIVsG5nzOmu_yY6rqTYzjuMt8FiLi7Aei3n4,5691
+openstef/data_classes/prediction_job.py,sha256=OFGg6h0XQZOIkJEYr1EoT3LE2oV6YULaCCxhkaES4wA,5874
 openstef/data_classes/split_function.py,sha256=ljQIQQu1t1Y_CVWGAy25jrM6wG9odIVVQVimrT1n-1s,3358
 openstef/feature_engineering/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
-openstef/feature_engineering/apply_features.py,sha256=EIxP9fvmnAjFRehQpVGBhZHt35GACL-HhDm65_ktHc0,5121
+openstef/feature_engineering/apply_features.py,sha256=9Yzg61Whd4n0osQBfrcW8cI0gaUiv7u8KnQIQPR40fY,5327
 openstef/feature_engineering/bidding_zone_to_country_mapping.py,sha256=u9aabjFDImydkO6_cXiaQxBT4gb5zy0gGTg2EoIUO_Y,2106
 openstef/feature_engineering/cyclic_features.py,sha256=0Z3wZeF_qrkmEcOq91gtdSMZucAq99kUoBuFDV0SHqk,5962
 openstef/feature_engineering/data_preparation.py,sha256=htca9LBO3ZN1D-iX4vXf0UN1fw_rRO7y6N3AuYVMpfk,5628
 openstef/feature_engineering/feature_adder.py,sha256=aSqDl_gUrB3H2TD3cNvU5JniY_KOb4u4a2A6J7zB2BQ,6835
-openstef/feature_engineering/feature_applicator.py,sha256=DR7jayrEMlra4BFL1Ps5WV2fxbkQ6VaOTa5RIKM-YNk,7447
+openstef/feature_engineering/feature_applicator.py,sha256=bU1Pu5V1fxMCQCwh6HG66nmctBjrNa7gHUYqOqPmLTU,7501
 openstef/feature_engineering/general.py,sha256=tgU4_1stag9jJmaQAfWCMhfBscznVuQvW5hPK_z9_9g,4438
 openstef/feature_engineering/holiday_features.py,sha256=CbolIP5bfiQkqDct-9TbD828-lhC48bfeNQ2-VFnsJA,8274
 openstef/feature_engineering/lag_features.py,sha256=Dr6qS8UhdgEHPZZSe-w6ibtjl_lcbcQohhqdZN9fqEU,5652
 openstef/feature_engineering/missing_values_transformer.py,sha256=o_zCVEOCPn2tWzvlY44XZuDysV0TuxqeVYhilYU54YY,5010
+openstef/feature_engineering/rolling_features.py,sha256=596UWr0VIEgD2t5UhgPRjvOvcUI9v477M9yRaoYsnSk,1536
 openstef/feature_engineering/weather_features.py,sha256=Lr9DItyHvJ2CpWQ1r6A83tJKtR2k_Wwn32FdFTGblO0,15750
 openstef/metrics/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/metrics/figure.py,sha256=KDoezYem9wdS13kUx7M7FOy-4u88Sg3OX1DuhNT6kgQ,9751
@@ -56,7 +57,7 @@ openstef/model/regressors/flatliner.py,sha256=T9u-ukhqFcatQmlgUtBL_G-1b_wQzgdVRq
 openstef/model/regressors/gblinear_quantile.py,sha256=DSRjL_kadBfDKrDEgrOfU1N60grTiAovtcBszBa41TI,11271
 openstef/model/regressors/lgbm.py,sha256=zCdn1euEdSFxYJzH8XqQFFnb6R4JVUnmineKjX_Gy-g,800
 openstef/model/regressors/linear.py,sha256=uOvZMLGZH_9nXfmS5honCMfyVeyGXP1Cza9A_BdXlVw,3665
-openstef/model/regressors/linear_quantile.py,sha256=VAyIhp7GPayqbk8Vj_ONqPLNYuaOvxkFKDrRxx6yGY0,10510
+openstef/model/regressors/linear_quantile.py,sha256=sI5cl6_W-hh13mg4Gf09LQ1caZmBy7COc8_5BBJxySQ,10534
 openstef/model/regressors/regressor.py,sha256=uJcx59AyCPE9f_yPcAQ59h2ZS7eNsDpIHJrladKvHIw,3461
 openstef/model/regressors/xgb.py,sha256=SH-UiYJtMbfmRBK6738dU0ZRfYfzNynnikwbxINCE7Q,1467
 openstef/model/regressors/xgb_multioutput_quantile.py,sha256=xWzA7tymC_o-F1OS3I7vUKf9zP6RR1ZglEeY4NAgjU0,9146
@@ -81,21 +82,21 @@ openstef/preprocessing/preprocessing.py,sha256=bM_cSSSb2vGTD79RGzUrI6KoELbzlCyJw
 openstef/tasks/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/tasks/calculate_kpi.py,sha256=O8m-K7lmkU3tY2H4E3gLqRAhhALZ75Y18x8Y_-D8Htw,11876
 openstef/tasks/create_basecase_forecast.py,sha256=rexAt6jGbW3YTXvDo606rzJvYETCoLVYCsBRihZas9U,4213
-openstef/tasks/create_components_forecast.py,sha256=Gyhrb_l3HGLBD5loUZt_3bYoPqMlmcVGJbEd5I8_A38,6155
-openstef/tasks/create_forecast.py,sha256=gLH3Z-t3fkFjYUyxiLXlywZ2fd9Dnhqq84C0UEO31g4,5766
+openstef/tasks/create_components_forecast.py,sha256=bhBVN1v8mhBQhYP1fBfbhag58DmQlxSKDTSz5YfTYGc,6179
+openstef/tasks/create_forecast.py,sha256=OOXfM3wTPEMa5CnlFPXNPe9kNN4LQG-3fHbeAUsiOKo,6096
 openstef/tasks/create_solar_forecast.py,sha256=cZiIoCVHlLlDrsWeH3ZX4zfcMMrgGgqkG2CmbCp8lqM,15074
 openstef/tasks/create_wind_forecast.py,sha256=RhshkmNSyFWx4Y6yQn02GzHjWTREbN5A5GAeWv0JpcE,2907
 openstef/tasks/optimize_hyperparameters.py,sha256=meiOn5S4yBrk5ANCFwcBCfTZIhm-b1rdh9TFh7KFr3E,4754
 openstef/tasks/split_forecast.py,sha256=AF_AwFcD6BqOrfvNLhIm_8gb7SpyKxEx60mymoxohPg,9323
-openstef/tasks/train_model.py,sha256=o8QVPReJ71BZVCOL6Rs3PFD9Zg4LT16dPcbf87xnXpA,8494
+openstef/tasks/train_model.py,sha256=PLkZMMZppsHuc2miCKDeVKoPhrwZi68xXL0axH2cDoA,8518
 openstef/tasks/utils/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/tasks/utils/dependencies.py,sha256=Jy9dtV_G7lTEa5Cdy--wvMxJuAb0adb3R0X4QDjVteM,3077
 openstef/tasks/utils/predictionjobloop.py,sha256=Ysy3zF5lzPMz_asYDKeF5m0qgVT3tCtwSPihqMjnI5Q,9580
 openstef/tasks/utils/taskcontext.py,sha256=L9K14ycwgVxbIVUjH2DIn_QWbnu-OfxcGtQ1K9T6sus,5630
 openstef/validation/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
-openstef/validation/validation.py,sha256=HVgreHvcZvPazfwC3NNE8_3lsMsZEd_42osCAg1_6W4,11128
-openstef-3.4.53.dist-info/LICENSE,sha256=7Pm2fWFFHHUG5lDHed1vl5CjzxObIXQglnYsEdtjo_k,14907
-openstef-3.4.53.dist-info/METADATA,sha256=MCAC7JnwfKToob21A5_a-eFE4CkG7PJ3Wov0BIDJ6Uc,8305
-openstef-3.4.53.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-openstef-3.4.53.dist-info/top_level.txt,sha256=kD0H4PqrQoncZ957FvqwfBxa89kTrun4Z_RAPs_HhLs,9
-openstef-3.4.53.dist-info/RECORD,,
+openstef/validation/validation.py,sha256=6FY-mD7bWxM7NpM9y-RcGZJt-kyyOmPl8QSemYRY11w,11200
+openstef-3.4.55.dist-info/LICENSE,sha256=7Pm2fWFFHHUG5lDHed1vl5CjzxObIXQglnYsEdtjo_k,14907
+openstef-3.4.55.dist-info/METADATA,sha256=c9Fc6xo3AyhI00vDWHrEzuFsCXfJbdPKAAgMRJ8DYcY,8305
+openstef-3.4.55.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+openstef-3.4.55.dist-info/top_level.txt,sha256=kD0H4PqrQoncZ957FvqwfBxa89kTrun4Z_RAPs_HhLs,9
+openstef-3.4.55.dist-info/RECORD,,