openstef 3.4.63__py3-none-any.whl → 3.4.65__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -96,6 +96,10 @@ class PredictionJobDataClass(BaseModel):
96
96
  1440,
97
97
  description="Number of minutes that the load has to be constant to detect a flatliner.",
98
98
  )
99
+ detect_non_zero_flatliner: bool = Field(
100
+ False,
101
+ description="If True, flatliners are also detected on non-zero values (median of the load).",
102
+ )
99
103
  data_balancing_ratio: Optional[float] = Field(
100
104
  None,
101
105
  description="If data balancing is enabled, the data will be balanced with data from 1 year ago in the future.",
openstef/exceptions.py CHANGED
@@ -44,8 +44,8 @@ class InputDataWrongColumnOrderError(InputDataInvalidError):
44
44
  """Wrong column order input data."""
45
45
 
46
46
 
47
- class InputDataOngoingZeroFlatlinerError(InputDataInvalidError):
48
- """All recent load measurements are zero."""
47
+ class InputDataOngoingFlatlinerError(InputDataInvalidError):
48
+ """All recent load measurements are constant."""
49
49
 
50
50
 
51
51
  class OldModelHigherScoreError(Exception):
@@ -8,7 +8,7 @@ import pandas as pd
8
8
  import structlog
9
9
 
10
10
  from openstef.data_classes.prediction_job import PredictionJobDataClass
11
- from openstef.exceptions import InputDataOngoingZeroFlatlinerError, NoRealisedLoadError
11
+ from openstef.exceptions import NoRealisedLoadError
12
12
  from openstef.feature_engineering.feature_applicator import (
13
13
  OperationalPredictFeatureApplicator,
14
14
  )
@@ -58,12 +58,12 @@ def create_basecase_forecast_pipeline(
58
58
  if not isinstance(input_data.index, pd.DatetimeIndex):
59
59
  raise ValueError("Input dataframe does not have a datetime index.")
60
60
 
61
- zero_flatliner_ongoing = validation.detect_ongoing_zero_flatliner(
61
+ flatliner_ongoing = validation.detect_ongoing_flatliner(
62
62
  load=input_data.iloc[:, 0],
63
63
  duration_threshold_minutes=pj.flatliner_threshold_minutes,
64
64
  )
65
65
 
66
- if zero_flatliner_ongoing:
66
+ if flatliner_ongoing:
67
67
  # Set historic load to zero to force the basecase forecasts to be zero.
68
68
  input_data.loc[input_data.index < forecast_start, "load"] = 0
69
69
 
@@ -45,7 +45,7 @@ def create_forecast_pipeline(
45
45
  DataFrame with the forecast
46
46
 
47
47
  Raises:
48
- InputDataOngoingZeroFlatlinerError: When all recent load measurements are zero.
48
+ InputDataOngoingFlatlinerError: When all recent load measurements are constant.
49
49
  LookupError: When no model is found for the given prediction job in MLflow.
50
50
 
51
51
  """
@@ -85,7 +85,7 @@ def create_forecast_pipeline_core(
85
85
  Forecast
86
86
 
87
87
  Raises:
88
- InputDataOngoingZeroFlatlinerError: When all recent load measurements are zero.
88
+ InputDataOngoingFlatlinerError: When all recent load measurements are constant.
89
89
 
90
90
  """
91
91
  structlog.configure(
@@ -103,6 +103,7 @@ def create_forecast_pipeline_core(
103
103
  input_data,
104
104
  pj["flatliner_threshold_minutes"],
105
105
  pj["resolution_minutes"],
106
+ detect_non_zero_flatliner=pj["detect_non_zero_flatliner"],
106
107
  )
107
108
 
108
109
  # Custom data prep or legacy behavior
@@ -132,7 +132,7 @@ def optimize_hyperparameters_pipeline_core(
132
132
  InputDataInsufficientError: If the input dataframe is empty.
133
133
  InputDataWrongColumnOrderError: If the load column is missing in the input dataframe.
134
134
  OldModelHigherScoreError: When old model is better than new model.
135
- InputDataOngoingZeroFlatlinerError: When all recent load measurements are zero.
135
+ InputDataOngoingFlatlinerError: If all recent load measurements are constant.
136
136
 
137
137
  Returns:
138
138
  - Best model,
@@ -157,6 +157,7 @@ def optimize_hyperparameters_pipeline_core(
157
157
  input_data,
158
158
  pj["flatliner_threshold_minutes"],
159
159
  pj["resolution_minutes"],
160
+ detect_non_zero_flatliner=pj["detect_non_zero_flatliner"],
160
161
  )
161
162
  )
162
163
 
@@ -60,7 +60,7 @@ def train_model_and_forecast_back_test(
60
60
  InputDataInsufficientError: when input data is insufficient.
61
61
  InputDataWrongColumnOrderError: when input data has a invalid column order.
62
62
  ValueError: when the horizon is a string and the corresponding column in not in the input data
63
- InputDataOngoingZeroFlatlinerError: when all recent load measurements are zero.
63
+ InputDataOngoingFlatlinerError: If all recent load measurements are constant.
64
64
 
65
65
  """
66
66
  if pj.backtest_split_func is None:
@@ -177,7 +177,7 @@ def train_model_pipeline_core(
177
177
  InputDataInsufficientError: when input data is insufficient.
178
178
  InputDataWrongColumnOrderError: when input data has a invalid column order.
179
179
  OldModelHigherScoreError: When old model is better than new model.
180
- InputDataOngoingZeroFlatlinerError: when all recent load measurements are zero.
180
+ InputDataOngoingFlatlinerError: If all recent load measurements are constant.
181
181
 
182
182
  Returns:
183
183
  - Fitted_model (OpenstfRegressor)
@@ -272,7 +272,7 @@ def train_pipeline_common(
272
272
  InputDataInsufficientError: when input data is insufficient.
273
273
  InputDataWrongColumnOrderError: when input data has a invalid column order.
274
274
  'load' column should be first and 'horizon' column last.
275
- InputDataOngoingZeroFlatlinerError: when all recent load measurements are zero.
275
+ InputDataOngoingFlatlinerError: If all recent load measurements are constant.
276
276
 
277
277
  """
278
278
  data_with_features = train_pipeline_step_compute_features(
@@ -363,7 +363,7 @@ def train_pipeline_step_compute_features(
363
363
  InputDataInsufficientError: when input data is insufficient.
364
364
  InputDataWrongColumnOrderError: when input data has a invalid column order.
365
365
  ValueError: when the horizon is a string and the corresponding column in not in the input data
366
- InputDataOngoingZeroFlatlinerError: when all recent load measurements are zero.
366
+ InputDataOngoingFlatlinerError: If all recent load measurements are constant.
367
367
 
368
368
  """
369
369
  if input_data.empty:
@@ -389,6 +389,7 @@ def train_pipeline_step_compute_features(
389
389
  input_data,
390
390
  pj["flatliner_threshold_minutes"],
391
391
  pj["resolution_minutes"],
392
+ detect_non_zero_flatliner=pj["detect_non_zero_flatliner"],
392
393
  )
393
394
  )
394
395
  # Check if sufficient data is left after cleaning
@@ -0,0 +1,3 @@
1
+ # SPDX-FileCopyrightText: 2017-2025 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
2
+ #
3
+ # SPDX-License-Identifier: MPL-2.0
@@ -0,0 +1,216 @@
1
+ # SPDX-FileCopyrightText: 2017-2025 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
2
+ #
3
+ # SPDX-License-Identifier: MPL-2.0
4
+
5
+ from typing import Tuple, Optional
6
+
7
+ import numpy as np
8
+ from pydantic import BaseModel
9
+ import pandas as pd
10
+ import plotly.graph_objects as go
11
+ import plotly.express as px
12
+
13
+
14
+ class LoadForecastPlotter(BaseModel):
15
+ colormap: str = "Blues"
16
+ colormap_range: Tuple[float, float] = (0.2, 0.8)
17
+
18
+ fill_opacity: float = 0.5
19
+ stroke_opacity: float = 0.8
20
+ stroke_width: float = 1.5
21
+
22
+ def _get_color_by_value(self, value: float) -> str:
23
+ """Maps a normalized value to a color using the specified colormap.
24
+
25
+ Args:
26
+ value (float): A value between 0 and 1 to be mapped to a color.
27
+
28
+ Returns:
29
+ str: A color in the rgba format.
30
+ """
31
+ rescaled = self.colormap_range[0] + value * (
32
+ self.colormap_range[1] - self.colormap_range[0]
33
+ )
34
+ rescaled = min(1.0, max(0.0, rescaled))
35
+
36
+ return px.colors.sample_colorscale(
37
+ colorscale=self.colormap, samplepoints=[rescaled]
38
+ )[0]
39
+
40
+ def _get_quantile_colors(self, quantile: float) -> Tuple[str, str]:
41
+ """Generate fill and stroke colors for a given quantile using a colorscale.
42
+
43
+ Colors are determined based on the distance from the median (50th percentile).
44
+
45
+ Args:
46
+ quantile (float): The quantile value (0-100) to generate colors for.
47
+
48
+ Returns:
49
+ Tuple[str, str]: A tuple containing (fill_color, stroke_color).
50
+ """
51
+ fill_value = 1 - abs(quantile - 50.0) / 50.0
52
+ stroke_value = 1 - abs(quantile + 5.0 - 50.0) / 50.0
53
+ return (
54
+ self._get_color_by_value(fill_value),
55
+ self._get_color_by_value(stroke_value),
56
+ )
57
+
58
+ def _add_quantile_band(
59
+ self,
60
+ figure: go.Figure,
61
+ lower_quantile_data: pd.Series,
62
+ lower_quantile: float,
63
+ upper_quantile_data: pd.Series,
64
+ upper_quantile: float,
65
+ ):
66
+ """Add a quantile band to the plotly figure.
67
+
68
+ Creates a filled polygon representing the area between lower and upper quantiles,
69
+ and adds it to the provided figure along with hover information.
70
+
71
+ Args:
72
+ figure (go.Figure): The plotly figure to add the quantile band to.
73
+ lower_quantile_data (pd.Series): Series with data for the lower quantile.
74
+ lower_quantile (float): The percentile value of the lower quantile.
75
+ upper_quantile_data (pd.Series): Series with data for the upper quantile.
76
+ upper_quantile (float): The percentile value of the upper quantile.
77
+
78
+ Returns:
79
+ None: The figure is modified in place.
80
+ """
81
+ # Create polygon shape for the quantile band in counterclockwise order
82
+ x = list(lower_quantile_data.index) + list(upper_quantile_data.index[::-1])
83
+ y = list(lower_quantile_data) + list(upper_quantile_data[::-1])
84
+
85
+ # Get colors for the band
86
+ fill_color, stroke_color = self._get_quantile_colors(lower_quantile)
87
+
88
+ # Group traces by quantile range
89
+ legendgroup = f"quantile_{lower_quantile}_{upper_quantile}"
90
+
91
+ # Add a single trace that forms a filled polygon
92
+ figure.add_trace(
93
+ go.Scatter(
94
+ x=x,
95
+ y=y,
96
+ fill="toself",
97
+ fillcolor=f"rgba{fill_color[3:-1]}, {self.fill_opacity})",
98
+ line=dict(
99
+ color=f"rgba{stroke_color[3:-1]}, {self.stroke_opacity})",
100
+ width=self.stroke_width,
101
+ ),
102
+ name=f"{lower_quantile}%-{upper_quantile}%",
103
+ showlegend=True,
104
+ hoverinfo="skip",
105
+ legendgroup=legendgroup,
106
+ )
107
+ )
108
+
109
+ # Add an (invisible) line around the filled area to make quantile
110
+ # values selectable/hover-able.
111
+ # Hovering on filled area values is not supported by plotly.
112
+ figure.add_trace(
113
+ go.Scatter(
114
+ x=lower_quantile_data.index,
115
+ y=lower_quantile_data.values,
116
+ mode="lines",
117
+ line=dict(
118
+ width=self.stroke_width,
119
+ color=f"rgba{stroke_color[3:-1]}, {self.stroke_opacity})",
120
+ ),
121
+ customdata=np.column_stack(
122
+ (lower_quantile_data.values, upper_quantile_data.values)
123
+ ),
124
+ hovertemplate=(
125
+ f"{lower_quantile}%: %{{customdata[0]:,.4s}}<br>"
126
+ f"{upper_quantile}%: %{{customdata[1]:,.4s}}"
127
+ "<extra></extra>"
128
+ ),
129
+ name=f"{lower_quantile}%-{upper_quantile}% Hover Info",
130
+ showlegend=False,
131
+ legendgroup=legendgroup,
132
+ )
133
+ )
134
+
135
+ def plot(
136
+ self,
137
+ realized: Optional[pd.Series] = None,
138
+ forecast: Optional[pd.Series] = None,
139
+ quantiles: Optional[pd.DataFrame] = None,
140
+ ):
141
+ """Create a plot showing forecast quantiles and realized values.
142
+
143
+ Generates an interactive plotly figure displaying the forecast distribution
144
+ through quantile bands, the median forecast, and the actual realized values.
145
+
146
+ Args:
147
+ realized (pd.Series): Time series of realized (actual) values.
148
+ forecast (pd.Series): Time series of forecast values (typically the median).
149
+ quantiles (pd.DataFrame): DataFrame containing quantile predictions.
150
+ Column names should follow the format 'quantile_P{percentile:02d}',
151
+ e.g., 'quantile_P10', 'quantile_P90'.
152
+
153
+ Returns:
154
+ go.Figure: A plotly figure object with the configured visualization.
155
+ """
156
+ figure = go.Figure()
157
+
158
+ if quantiles is not None:
159
+ # Extract and sort quantile percentages
160
+ quantile_cols = [
161
+ col for col in quantiles.columns if col.startswith("quantile_P")
162
+ ]
163
+ percentiles = sorted([int(col.split("P")[1]) for col in quantile_cols])
164
+
165
+ # Add quantile bands from widest to narrowest
166
+ for i in range(len(percentiles) // 2):
167
+ lower_quantile, upper_quantile = percentiles[i], percentiles[-(i + 1)]
168
+ if float(lower_quantile) == 50.0:
169
+ continue
170
+
171
+ self._add_quantile_band(
172
+ figure=figure,
173
+ lower_quantile_data=quantiles[f"quantile_P{lower_quantile:02d}"],
174
+ lower_quantile=lower_quantile,
175
+ upper_quantile_data=quantiles[f"quantile_P{upper_quantile:02d}"],
176
+ upper_quantile=upper_quantile,
177
+ )
178
+
179
+ if forecast is not None:
180
+ # Add forecast line (50th percentile)
181
+ figure.add_trace(
182
+ go.Scatter(
183
+ x=forecast.index,
184
+ y=forecast,
185
+ mode="lines",
186
+ line=dict(color="blue", width=self.stroke_width),
187
+ name="Forecast (50th)",
188
+ customdata=forecast.values,
189
+ hovertemplate="Forecast (50th): %{customdata:,.4s}<extra></extra>",
190
+ )
191
+ )
192
+
193
+ if realized is not None:
194
+ # Add realized values on top
195
+ figure.add_trace(
196
+ go.Scatter(
197
+ x=realized.index,
198
+ y=realized,
199
+ mode="lines",
200
+ line=dict(color="red", width=self.stroke_width),
201
+ customdata=realized.values,
202
+ hovertemplate="Realized: %{customdata:,.4s}<extra></extra>",
203
+ name="Realized",
204
+ )
205
+ )
206
+
207
+ # Styling configuration
208
+ figure.update_layout(
209
+ title=f"Load Forecast vs Actual",
210
+ xaxis_title="Datetime [UTC]",
211
+ yaxis_title="Load [W]",
212
+ template="plotly_white",
213
+ hovermode="x unified",
214
+ )
215
+
216
+ return figure
@@ -25,11 +25,11 @@ from pathlib import Path
25
25
 
26
26
  from openstef.data_classes.prediction_job import PredictionJobDataClass
27
27
  from openstef.enums import BiddingZone, ModelType, PipelineType
28
- from openstef.exceptions import InputDataOngoingZeroFlatlinerError
28
+ from openstef.exceptions import InputDataOngoingFlatlinerError
29
29
  from openstef.pipeline.create_forecast import create_forecast_pipeline
30
30
  from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
31
31
  from openstef.tasks.utils.taskcontext import TaskContext
32
- from openstef.validation.validation import detect_ongoing_zero_flatliner
32
+ from openstef.validation.validation import detect_ongoing_flatliner
33
33
 
34
34
  T_BEHIND_DAYS: int = 14
35
35
 
@@ -94,7 +94,7 @@ def create_forecast_task(
94
94
  forecast = create_forecast_pipeline(
95
95
  pj, input_data, mlflow_tracking_uri=mlflow_tracking_uri
96
96
  )
97
- except (InputDataOngoingZeroFlatlinerError, LookupError) as e:
97
+ except (InputDataOngoingFlatlinerError, LookupError) as e:
98
98
  if (
99
99
  context.config.known_zero_flatliners
100
100
  and pj.id in context.config.known_zero_flatliners
@@ -103,18 +103,18 @@ def create_forecast_task(
103
103
  "No forecasts were made for this known zero flatliner prediction job. No forecasts need to be made either, since the fallback forecasts are sufficient."
104
104
  )
105
105
  return
106
- elif isinstance(e, InputDataOngoingZeroFlatlinerError):
107
- raise InputDataOngoingZeroFlatlinerError(
108
- 'All recent load measurements are zero. Check the load profile of this pid as well as related/neighbouring prediction jobs. Afterwards, consider adding this pid to the "known_zero_flatliners" app_setting and possibly removing other pids from the same app_setting.'
106
+ elif isinstance(e, InputDataOngoingFlatlinerError):
107
+ raise InputDataOngoingFlatlinerError(
108
+ 'All recent load measurements are constant. Check the load profile of this pid as well as related/neighbouring prediction jobs. Afterwards, consider adding this pid to the "known_zero_flatliners" app_setting and possibly removing other pids from the same app_setting.'
109
109
  ) from e
110
110
  elif isinstance(e, LookupError):
111
- zero_flatliner_ongoing = detect_ongoing_zero_flatliner(
111
+ zero_flatliner_ongoing = detect_ongoing_flatliner(
112
112
  load=input_data.iloc[:, 0],
113
113
  duration_threshold_minutes=pj.flatliner_threshold_minutes,
114
114
  )
115
115
  if zero_flatliner_ongoing:
116
116
  raise LookupError(
117
- 'Model not found. Consider checking for a zero flatliner and adding this pid to the "known_zero_flatliners" app_setting. For zero flatliners, no model can be trained.'
117
+ 'Model not found. Consider checking for a flatliner and adding this pid to the "known_zero_flatliners" app_setting. For flatliners, no model can be trained.'
118
118
  ) from e
119
119
  else:
120
120
  raise e
@@ -27,7 +27,7 @@ import pandas as pd
27
27
  from openstef.data_classes.prediction_job import PredictionJobDataClass
28
28
  from openstef.enums import ModelType, PipelineType
29
29
  from openstef.exceptions import (
30
- InputDataOngoingZeroFlatlinerError,
30
+ InputDataOngoingFlatlinerError,
31
31
  SkipSaveTrainingForecasts,
32
32
  )
33
33
  from openstef.model.serializer import MLflowSerializer
@@ -67,7 +67,7 @@ def train_model_task(
67
67
 
68
68
  Raises:
69
69
  SkipSaveTrainingForecasts: If old model is better or too young, you don't need to save the traing forcast.
70
- InputDataOngoingZeroFlatlinerError: If all recent load measurements are zero.
70
+ InputDataOngoingFlatlinerError: If all recent load measurements are constant.
71
71
 
72
72
  """
73
73
  # Check pipeline types
@@ -187,18 +187,18 @@ def train_model_task(
187
187
  context.logger.debug("Saved Forecasts from trained model on datasets")
188
188
  except SkipSaveTrainingForecasts:
189
189
  context.logger.debug("Skip saving forecasts")
190
- except InputDataOngoingZeroFlatlinerError:
190
+ except InputDataOngoingFlatlinerError:
191
191
  if (
192
192
  context.config.known_zero_flatliners
193
193
  and pj.id in context.config.known_zero_flatliners
194
194
  ):
195
195
  context.logger.info(
196
- "No model was trained for this known zero flatliner. No model needs to be trained either, since the fallback forecasts are sufficient."
196
+ "No model was trained for this known flatliner. No model needs to be trained either, since the fallback forecasts are sufficient."
197
197
  )
198
198
  return
199
199
  else:
200
- raise InputDataOngoingZeroFlatlinerError(
201
- 'All recent load measurements are zero. Check the load profile of this pid as well as related/neighbouring prediction jobs. Afterwards, consider adding this pid to the "known_zero_flatliners" app_setting and possibly removing other pids from the same app_setting.'
200
+ raise InputDataOngoingFlatlinerError(
201
+ 'All recent load measurements are constant. Check the load profile of this pid as well as related/neighbouring prediction jobs. Afterwards, consider adding this pid to the "known_zero_flatliners" app_setting and possibly removing other pids from the same app_setting.'
202
202
  )
203
203
 
204
204
 
@@ -10,7 +10,8 @@ import numpy as np
10
10
  import pandas as pd
11
11
  import structlog
12
12
 
13
- from openstef.exceptions import InputDataOngoingZeroFlatlinerError
13
+ from openstef.data_classes.prediction_job import PredictionJobDataClass
14
+ from openstef.exceptions import InputDataOngoingFlatlinerError
14
15
  from openstef.model.regressors.regressor import OpenstfRegressor
15
16
  from openstef.preprocessing.preprocessing import replace_repeated_values_with_nan
16
17
  from openstef.settings import Settings
@@ -21,12 +22,15 @@ def validate(
21
22
  data: pd.DataFrame,
22
23
  flatliner_threshold_minutes: Union[int, None],
23
24
  resolution_minutes: int,
25
+ *,
26
+ detect_non_zero_flatliner: bool = False,
24
27
  ) -> pd.DataFrame:
25
28
  """Validate prediction job and timeseries data.
26
29
 
27
30
  Steps:
28
31
  1. Check if input dataframe has a datetime index.
29
- 1. Check if a zero flatliner pattern is ongoing (i.e. all recent measurements are zero).
32
+ 1. Check if a flatliner pattern is ongoing (i.e. all recent measurements are constant,
33
+ 0 in case detect_non_zero_flatliner = True).
30
34
  2. Replace repeated values for longer than flatliner_threshold_minutes with NaN.
31
35
 
32
36
  Args:
@@ -35,12 +39,14 @@ def validate(
35
39
  flatliner_threshold_minutes: int indicating the number of minutes after which constant load is considered a flatline.
36
40
  if None, the validation is effectively skipped
37
41
  resolution_minutes: The forecasting resolution in minutes.
42
+ detect_non_zero_flatliner: If True, a flatliner is detected for non-zero values. If False,
43
+ a flatliner is detected for zero values only.
38
44
 
39
45
  Returns:
40
46
  Dataframe where repeated values are set to None
41
47
 
42
48
  Raises:
43
- InputDataOngoingZeroFlatlinerError: If all recent load measurements are zero.
49
+ InputDataOngoingFlatlinerError: If all recent load measurements are constant.
44
50
 
45
51
  """
46
52
  structlog.configure(
@@ -57,13 +63,15 @@ def validate(
57
63
  logger.info("Skipping validation of input data", pj_id=pj_id)
58
64
  return data
59
65
 
60
- zero_flatliner_ongoing = detect_ongoing_zero_flatliner(
61
- load=data.iloc[:, 0], duration_threshold_minutes=flatliner_threshold_minutes
66
+ flatliner_ongoing = detect_ongoing_flatliner(
67
+ load=data.iloc[:, 0],
68
+ duration_threshold_minutes=flatliner_threshold_minutes,
69
+ detect_non_zero_flatliner=detect_non_zero_flatliner,
62
70
  )
63
71
 
64
- if zero_flatliner_ongoing:
65
- raise InputDataOngoingZeroFlatlinerError(
66
- "All recent load measurements are zero."
72
+ if flatliner_ongoing:
73
+ raise InputDataOngoingFlatlinerError(
74
+ "All recent load measurements are constant."
67
75
  )
68
76
 
69
77
  flatliner_threshold_repetitions = math.ceil(
@@ -228,18 +236,22 @@ def calc_completeness_features(
228
236
  return completeness
229
237
 
230
238
 
231
- def detect_ongoing_zero_flatliner(
239
+ def detect_ongoing_flatliner(
232
240
  load: pd.Series,
233
241
  duration_threshold_minutes: int,
242
+ *,
243
+ detect_non_zero_flatliner: bool = False,
234
244
  ) -> bool:
235
- """Detects if the latest measurements follow a zero flatliner pattern.
245
+ """Detects if the latest measurements follow a flatliner pattern.
236
246
 
237
247
  Args:
238
248
  load (pd.Series): A timeseries of measured load with a datetime index.
239
- duration_threshold_minutes (int): A zero flatliner is only detected if it exceeds the threshold duration.
249
+ duration_threshold_minutes (int): A flatliner is only detected if it exceeds the threshold duration.
250
+ detect_non_zero_flatliner (bool): If True, a flatliner is detected for non-zero values. If False,
251
+ a flatliner is detected for zero values only.
240
252
 
241
253
  Returns:
242
- bool: Indicating whether or not there is a zero flatliner ongoing for the given load.
254
+ bool: Indicating whether or not there is a flatliner ongoing for the given load.
243
255
 
244
256
  """
245
257
  # remove all timestamps in the future
@@ -249,7 +261,18 @@ def detect_ongoing_zero_flatliner(
249
261
  latest_measurement_time - timedelta(minutes=duration_threshold_minutes) :
250
262
  ].dropna()
251
263
 
252
- return (latest_measurements == 0).all() & (not latest_measurements.empty)
264
+ flatliner_value = latest_measurements.median() if detect_non_zero_flatliner else 0
265
+
266
+ # check if all values are within a relative tolerance of each other
267
+ flatline_condition = np.isclose(
268
+ latest_measurements,
269
+ flatliner_value,
270
+ atol=0,
271
+ rtol=1e-5,
272
+ ).all()
273
+ non_empty_condition = not latest_measurements.empty
274
+
275
+ return flatline_condition & non_empty_condition
253
276
 
254
277
 
255
278
  def calc_completeness_dataframe(
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: openstef
3
- Version: 3.4.63
3
+ Version: 3.4.65
4
4
  Summary: Open short term energy forecaster
5
5
  Home-page: https://github.com/OpenSTEF/openstef
6
6
  Author: Alliander N.V
@@ -33,7 +33,11 @@ Requires-Dist: scikit-learn<1.6,>=1.3
33
33
  Requires-Dist: scipy~=1.10
34
34
  Requires-Dist: statsmodels<1.0.0,>=0.13.5
35
35
  Requires-Dist: structlog<25,>=23.1
36
- Requires-Dist: xgboost~=2.0
36
+ Requires-Dist: xgboost~=2.0; extra == "gpu"
37
+ Provides-Extra: cpu
38
+ Requires-Dist: xgboost-cpu~=2.0; extra == "cpu"
39
+ Provides-Extra: gpu
40
+ Requires-Dist: xgboost; extra == "gpu"
37
41
  Dynamic: author
38
42
  Dynamic: author-email
39
43
  Dynamic: classifier
@@ -42,6 +46,7 @@ Dynamic: description-content-type
42
46
  Dynamic: home-page
43
47
  Dynamic: keywords
44
48
  Dynamic: license
49
+ Dynamic: provides-extra
45
50
  Dynamic: requires-dist
46
51
  Dynamic: requires-python
47
52
  Dynamic: summary
@@ -53,11 +58,15 @@ SPDX-License-Identifier: MPL-2.0
53
58
  -->
54
59
 
55
60
  # OpenSTEF
61
+
56
62
  <!-- Badges -->
63
+
57
64
  [![Downloads](https://static.pepy.tech/badge/openstef)](https://pepy.tech/project/openstef)
58
65
  [![Downloads](https://static.pepy.tech/badge/openstef/month)](https://pepy.tech/project/openstef)
59
66
  [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/5585/badge)](https://bestpractices.coreinfrastructure.org/projects/5585)
67
+
60
68
  <!-- SonarCloud badges -->
69
+
61
70
  [![Bugs](https://sonarcloud.io/api/project_badges/measure?project=OpenSTEF_openstef&metric=bugs)](https://sonarcloud.io/dashboard?id=OpenSTEF_openstef)
62
71
  [![Code Smells](https://sonarcloud.io/api/project_badges/measure?project=OpenSTEF_openstef&metric=code_smells)](https://sonarcloud.io/dashboard?id=OpenSTEF_openstef)
63
72
  [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=OpenSTEF_openstef&metric=coverage)](https://sonarcloud.io/dashboard?id=OpenSTEF_openstef)
@@ -71,19 +80,21 @@ SPDX-License-Identifier: MPL-2.0
71
80
  OpenSTEF is a Python package designed for generating short-term forecasts in the energy sector. The repository includes all the essential components required for machine learning pipelines that facilitate the forecasting process. To utilize the package, users are required to furnish their own data storage and retrieval interface.
72
81
 
73
82
  # Table of contents
83
+
74
84
  - [OpenSTEF](#openstef)
75
85
  - [Table of contents](#table-of-contents)
76
86
  - [External information sources](#external-information-sources)
77
87
  - [Installation](#installation)
78
88
  - [Usage](#usage)
79
- - [Example notebooks](#example-notebooks)
80
- - [Reference Implementation](#reference-implementation)
81
- - [Database connector for OpenSTEF](#database-connector-for-openstef)
89
+ - [Example notebooks](#example-notebooks)
90
+ - [Reference Implementation](#reference-implementation)
91
+ - [Database connector for OpenSTEF](#database-connector-for-openstef)
82
92
  - [License](license)
83
93
  - [Contributing](#contributing)
84
94
  - [Contact](#contact)
85
95
 
86
96
  # External information sources
97
+
87
98
  - [Documentation website](https://openstef.github.io/openstef/index.html);
88
99
  - [Python package](https://pypi.org/project/openstef/);
89
100
  - [Linux Foundation project page](https://www.lfenergy.org/projects/openstef/);
@@ -101,9 +112,11 @@ pip install openstef
101
112
  ### Remark regarding installation within a **conda environment on Windows**
102
113
 
103
114
  A version of the pywin32 package will be installed as a secondary dependency along with the installation of the openstef package. Since conda relies on an old version of pywin32, the new installation can break conda's functionality. The following command can solve this issue:
115
+
104
116
  ```shell
105
117
  pip install pywin32==300
106
118
  ```
119
+
107
120
  For more information on this issue see the [readme of pywin32](https://github.com/mhammond/pywin32#installing-via-pip) or [this Github issue](https://github.com/mhammond/pywin32/issues/1865#issue-1212752696).
108
121
 
109
122
  ## Remark regarding installation on Apple Silicon
@@ -112,19 +125,31 @@ If you want to install the `openstef` package on Apple Silicon (Mac with M1-chip
112
125
 
113
126
  1. Run `brew install libomp` (if you haven’t installed Homebrew: [follow instructions here](https://brew.sh/))
114
127
  2. If your interpreter can not find the `libomp` installation in `/usr/local/bin`, it is probably in `/opt/brew/Cellar`. Run:
128
+
115
129
  ```sh
116
130
  mkdir -p /usr/local/opt/libomp/
117
131
  ln -s /opt/brew/Cellar/libomp/{your_version}/lib /usr/local/opt/libomp/lib
118
132
  ```
133
+
119
134
  3. Uninstall `xgboost` with `pip` (`pip uninstall xgboost`) and install with `conda-forge` (`conda install -c conda-forge xgboost`)
120
135
  4. If you encounter similar issues with `lightgbm`: uninstall `lightgbm` with `pip` (`pip uninstall lightgbm`) and install later version with `conda-forge` (`conda install -c conda-forge 'lightgbm>=4.2.0'`)
121
136
 
137
+ ### Remark regarding installation with minimal XGBoost dependency
138
+
139
+ It is possible to install openSTEF with a minimal XGBoost (CPU-only) package. This only works on x86_64 (amd64) Linux and Windows platforms. Advantage is that significantly smaller dependencies are installed. In that case run:
140
+
141
+ ```shell
142
+ pip install openstef[cpu]
143
+ ```
144
+
122
145
  # Usage
123
146
 
124
147
  ## Example notebooks
148
+
125
149
  To help you get started, a set of fundamental example notebooks has been created. You can access these offline examples [here](https://github.com/OpenSTEF/openstef-offline-example).
126
150
 
127
151
  ## Reference Implementation
152
+
128
153
  A complete implementation including databases, user interface, example data, etc. is available at: https://github.com/OpenSTEF/openstef-reference
129
154
 
130
155
  ![screenshot](https://user-images.githubusercontent.com/60883372/146760483-29af3ac7-62af-4f13-98c7-982a79c517d1.jpg)
@@ -138,17 +163,21 @@ python -m openstef task <task_name>
138
163
  ```
139
164
 
140
165
  ## Database connector for openstef
166
+
141
167
  This repository provides an interface to OpenSTEF (reference) databases. The repository can be found [here](https://github.com/OpenSTEF/openstef-dbc).
142
168
 
143
169
  # License
170
+
144
171
  This project is licensed under the Mozilla Public License, version 2.0 - see LICENSE for details.
145
172
 
146
173
  ## Licenses third-party libraries
174
+
147
175
  This project includes third-party libraries, which are licensed under their own respective Open-Source licenses. SPDX-License-Identifier headers are used to show which license is applicable. The concerning license files can be found in the LICENSES directory.
148
176
 
149
177
  # Contributing
178
+
150
179
  Please read [CODE_OF_CONDUCT.md](https://github.com/OpenSTEF/.github/blob/main/CODE_OF_CONDUCT.md), [CONTRIBUTING.md](https://github.com/OpenSTEF/.github/blob/main/CONTRIBUTING.md) and [PROJECT_GOVERNANCE.md](https://github.com/OpenSTEF/.github/blob/main/PROJECT_GOVERNANCE.md) for details on the process for submitting pull requests to us.
151
180
 
152
181
  # Contact
182
+
153
183
  Please read [SUPPORT.md](https://github.com/OpenSTEF/.github/blob/main/SUPPORT.md) for how to connect and get into contact with the OpenSTEF project
154
-
@@ -2,7 +2,7 @@ openstef/__init__.py,sha256=93UM6m0LLQhO69-mSqLuUy73jgs4W7Iuxfo3Lm8c98g,419
2
2
  openstef/__main__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
3
3
  openstef/app_settings.py,sha256=EJTDtimctFQQ-3f7ZcOQaRYohpZk3JD6aZBWPFYM2_A,582
4
4
  openstef/enums.py,sha256=FrP0m_Tk0kV7gSZ2hTY_8iD45KIKnexHrjNufhpKXpE,2829
5
- openstef/exceptions.py,sha256=U4u2LTcdT6cmzpipT2Jh7kq9nCjT_-6gntn8yjuhGU0,1993
5
+ openstef/exceptions.py,sha256=dgnvZe5WWuJWCZm_GES6suEATbusPlwbiEUfNQKeExY,1993
6
6
  openstef/settings.py,sha256=nSgkBqFxuqB3w7Rwo60i8j37c5ngDbt6vpjHS6QtJXQ,354
7
7
  openstef/data/NL_terrestrial_radiation.csv,sha256=A4kbW56GDzWi4tWUwY2C-4PiOvcKJCwkWQQtdg4ekPE,820246
8
8
  openstef/data/NL_terrestrial_radiation.csv.license,sha256=AxxHusqwIXU5RHl5ZMU65LyXmgtbj6QlcnFaOEN4kEE,145
@@ -17,7 +17,7 @@ openstef/data/dazls_model_3.4.24/dazls_stored_3.4.24_model_card.md.license,sha25
17
17
  openstef/data_classes/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
18
18
  openstef/data_classes/data_prep.py,sha256=sANgFjfwmSWhLCfmLjfqXQnczuvVZfk2765jZd7LwuE,3691
19
19
  openstef/data_classes/model_specifications.py,sha256=PZeBLfH_MrP9-QorL1r0Hklp0befE8Nw05vNhTX9Y20,1338
20
- openstef/data_classes/prediction_job.py,sha256=pbvffqje7X7UGx2AC1LX2bvChiEd4n9brbxtS6-4_iE,6553
20
+ openstef/data_classes/prediction_job.py,sha256=e6_PFAovNd31tjzoTQJvqRNQyVM-M0XHffclAG9Ez8A,6721
21
21
  openstef/data_classes/split_function.py,sha256=K8y1dsQC5exeIDh37f7UwJ11tV71_uVSNbnKmwXpnOM,3435
22
22
  openstef/feature_engineering/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
23
23
  openstef/feature_engineering/apply_features.py,sha256=9Yzg61Whd4n0osQBfrcW8cI0gaUiv7u8KnQIQPR40fY,5327
@@ -68,13 +68,15 @@ openstef/monitoring/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4
68
68
  openstef/monitoring/performance_meter.py,sha256=6aCGjJFXFq-7qwaJyBkF3MLqjgVK6FMFVcO-bcLLUb4,2803
69
69
  openstef/monitoring/teams.py,sha256=A-tlZeuAgolxFHjgT3gGjraxzW2dmuB-UAOz4xgYNIQ,6668
70
70
  openstef/pipeline/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
71
- openstef/pipeline/create_basecase_forecast.py,sha256=8pyUHauJEFXMFxAg-lMy8B6oaZbsXMpdHXuNS6o_3QM,4630
71
+ openstef/pipeline/create_basecase_forecast.py,sha256=ChIh8iQSRL9n2pc7l3Cw3RWRONkp2e7MOoUnpY9VT_s,4579
72
72
  openstef/pipeline/create_component_forecast.py,sha256=U2v_R-FSOXWVbWeknsJbkulN1YK56fL7-bB1h2B1yzw,6021
73
- openstef/pipeline/create_forecast.py,sha256=_WKoi-FRdODUmJh-6X82CbbFiHrbGwvoTPfLf60thco,5696
74
- openstef/pipeline/optimize_hyperparameters.py,sha256=3SLkcLR7XC4IeN48C-XT_lxlfCqW_D0NoMpZcrB9UUM,11045
75
- openstef/pipeline/train_create_forecast_backtest.py,sha256=-kZqCWal5zYLL0k0Sapks1zTmU5unNAooVPaPos1_7E,6050
76
- openstef/pipeline/train_model.py,sha256=ThZwPo5otikVqVe6NdXkYcxkVFh-kegRVxMsQg1lbFc,19743
73
+ openstef/pipeline/create_forecast.py,sha256=uvp5mQqGSOx-ANY-9o5reiBYNNby0npm-0lt4w9EQ18,5763
74
+ openstef/pipeline/optimize_hyperparameters.py,sha256=uwXkzRA_fTSFt0yBuvvEoY5-4dMv42FPdS4hZocL-N8,11114
75
+ openstef/pipeline/train_create_forecast_backtest.py,sha256=hBJPxfDkbrmFSSGZrRH1vTiIVqJP-SWe0ibVpHT_8Qg,6048
76
+ openstef/pipeline/train_model.py,sha256=zFQS_XSqN4VQWEpvC1dvGN1rcI2tGOuFOTaN_dnKZSA,19808
77
77
  openstef/pipeline/utils.py,sha256=23mB31p19FoGWelLJzxNmqlzGwEr3fCDBEA37V2kpYY,2167
78
+ openstef/plotting/__init__.py,sha256=KQjXzyafCt1bE7XDrSeV4TDUIO7MkwN_Br4ASOcNI2g,163
79
+ openstef/plotting/load_forecast_plotter.py,sha256=n-dB2dQnqjWCvV3kBjnOZYQ03J-9jSIHVovJy3nGSnQ,8129
78
80
  openstef/postprocessing/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
79
81
  openstef/postprocessing/postprocessing.py,sha256=6x_2ZcZaHEKMg_kxBAuKUlA_dDEs-KaO5SgGqGWHK14,8997
80
82
  openstef/preprocessing/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
@@ -83,20 +85,20 @@ openstef/tasks/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,16
83
85
  openstef/tasks/calculate_kpi.py,sha256=tcW_G0JRMA2tBcb8JN5eUbFFV9UcTsqHXQ1x3f-8Biw,11881
84
86
  openstef/tasks/create_basecase_forecast.py,sha256=_4Ry7AQmXNAKq19J1qmVyG-94atygXePLxVCejCfGPw,4227
85
87
  openstef/tasks/create_components_forecast.py,sha256=8LINqAHt7SnVsQAQMOuve5K-3bLJW-tK_dXTqzlh5Mw,6140
86
- openstef/tasks/create_forecast.py,sha256=xASSfHehdcxS64--alYoA6oElx_1Sy4S0tfxvWucVRw,6107
88
+ openstef/tasks/create_forecast.py,sha256=CVUZDG-obMb78ytJ79Hf6LYhMCbqaDvX_vc7fkt9VXI,6075
87
89
  openstef/tasks/create_solar_forecast.py,sha256=HDrJrvTPCM8GS7EQwNr9uJNamf-nH2pu0o4d_xo4w4E,15062
88
90
  openstef/tasks/create_wind_forecast.py,sha256=RhshkmNSyFWx4Y6yQn02GzHjWTREbN5A5GAeWv0JpcE,2907
89
91
  openstef/tasks/optimize_hyperparameters.py,sha256=3NT0KFgim8wAzWPJ0S-GULM3zoshyj63Ivp-g1_oPDw,4765
90
92
  openstef/tasks/split_forecast.py,sha256=X1D3MnnMdAb9wzDWubAJwfMkWpNGdRUPDvPAbJApNhg,9277
91
- openstef/tasks/train_model.py,sha256=gbKRB3F5qFNfTt0HQnxOjwGS721MEmF110_-FMVlYh4,8527
93
+ openstef/tasks/train_model.py,sha256=ioEOMpKMveEdY63aRXSh4IsioVFTqj-EDggFmJ7nfiw,8514
92
94
  openstef/tasks/utils/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
93
95
  openstef/tasks/utils/dependencies.py,sha256=Jy9dtV_G7lTEa5Cdy--wvMxJuAb0adb3R0X4QDjVteM,3077
94
96
  openstef/tasks/utils/predictionjobloop.py,sha256=Ysy3zF5lzPMz_asYDKeF5m0qgVT3tCtwSPihqMjnI5Q,9580
95
97
  openstef/tasks/utils/taskcontext.py,sha256=L9K14ycwgVxbIVUjH2DIn_QWbnu-OfxcGtQ1K9T6sus,5630
96
98
  openstef/validation/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
97
- openstef/validation/validation.py,sha256=DfnT7f29n9AbduJy9I6mXYQSnjt241Pn36Fp9SGehR0,11225
98
- openstef-3.4.63.dist-info/LICENSE,sha256=7Pm2fWFFHHUG5lDHed1vl5CjzxObIXQglnYsEdtjo_k,14907
99
- openstef-3.4.63.dist-info/METADATA,sha256=9kJWSv45Y-61ZmpZfIJWRzZYIogkP1lZrxGawsvDsEQ,8305
100
- openstef-3.4.63.dist-info/WHEEL,sha256=nn6H5-ilmfVryoAQl3ZQ2l8SH5imPWFpm1A5FgEuFV4,91
101
- openstef-3.4.63.dist-info/top_level.txt,sha256=kD0H4PqrQoncZ957FvqwfBxa89kTrun4Z_RAPs_HhLs,9
102
- openstef-3.4.63.dist-info/RECORD,,
99
+ openstef/validation/validation.py,sha256=24GEzLyjVqaE2a-MppbFS-YQT5n739BxD7fH3LK5LEE,12133
100
+ openstef-3.4.65.dist-info/LICENSE,sha256=7Pm2fWFFHHUG5lDHed1vl5CjzxObIXQglnYsEdtjo_k,14907
101
+ openstef-3.4.65.dist-info/METADATA,sha256=YRXl2VGYnkq3BNtZ90rovW5__cBKVf5XuV8KrV7mhLI,8816
102
+ openstef-3.4.65.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
103
+ openstef-3.4.65.dist-info/top_level.txt,sha256=kD0H4PqrQoncZ957FvqwfBxa89kTrun4Z_RAPs_HhLs,9
104
+ openstef-3.4.65.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (75.8.1)
2
+ Generator: setuptools (76.0.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5