openstef 3.4.64__py3-none-any.whl → 3.4.66__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openstef/data_classes/prediction_job.py +4 -0
- openstef/exceptions.py +2 -2
- openstef/pipeline/create_basecase_forecast.py +3 -3
- openstef/pipeline/create_forecast.py +3 -2
- openstef/pipeline/optimize_hyperparameters.py +2 -1
- openstef/pipeline/train_create_forecast_backtest.py +1 -1
- openstef/pipeline/train_model.py +28 -17
- openstef/tasks/create_forecast.py +8 -8
- openstef/tasks/train_model.py +17 -10
- openstef/validation/validation.py +36 -13
- {openstef-3.4.64.dist-info → openstef-3.4.66.dist-info}/METADATA +1 -1
- {openstef-3.4.64.dist-info → openstef-3.4.66.dist-info}/RECORD +15 -15
- {openstef-3.4.64.dist-info → openstef-3.4.66.dist-info}/WHEEL +1 -1
- {openstef-3.4.64.dist-info → openstef-3.4.66.dist-info}/LICENSE +0 -0
- {openstef-3.4.64.dist-info → openstef-3.4.66.dist-info}/top_level.txt +0 -0
@@ -96,6 +96,10 @@ class PredictionJobDataClass(BaseModel):
|
|
96
96
|
1440,
|
97
97
|
description="Number of minutes that the load has to be constant to detect a flatliner.",
|
98
98
|
)
|
99
|
+
detect_non_zero_flatliner: bool = Field(
|
100
|
+
False,
|
101
|
+
description="If True, flatliners are also detected on non-zero values (median of the load).",
|
102
|
+
)
|
99
103
|
data_balancing_ratio: Optional[float] = Field(
|
100
104
|
None,
|
101
105
|
description="If data balancing is enabled, the data will be balanced with data from 1 year ago in the future.",
|
openstef/exceptions.py
CHANGED
@@ -44,8 +44,8 @@ class InputDataWrongColumnOrderError(InputDataInvalidError):
|
|
44
44
|
"""Wrong column order input data."""
|
45
45
|
|
46
46
|
|
47
|
-
class
|
48
|
-
"""All recent load measurements are
|
47
|
+
class InputDataOngoingFlatlinerError(InputDataInvalidError):
|
48
|
+
"""All recent load measurements are constant."""
|
49
49
|
|
50
50
|
|
51
51
|
class OldModelHigherScoreError(Exception):
|
@@ -8,7 +8,7 @@ import pandas as pd
|
|
8
8
|
import structlog
|
9
9
|
|
10
10
|
from openstef.data_classes.prediction_job import PredictionJobDataClass
|
11
|
-
from openstef.exceptions import
|
11
|
+
from openstef.exceptions import NoRealisedLoadError
|
12
12
|
from openstef.feature_engineering.feature_applicator import (
|
13
13
|
OperationalPredictFeatureApplicator,
|
14
14
|
)
|
@@ -58,12 +58,12 @@ def create_basecase_forecast_pipeline(
|
|
58
58
|
if not isinstance(input_data.index, pd.DatetimeIndex):
|
59
59
|
raise ValueError("Input dataframe does not have a datetime index.")
|
60
60
|
|
61
|
-
|
61
|
+
flatliner_ongoing = validation.detect_ongoing_flatliner(
|
62
62
|
load=input_data.iloc[:, 0],
|
63
63
|
duration_threshold_minutes=pj.flatliner_threshold_minutes,
|
64
64
|
)
|
65
65
|
|
66
|
-
if
|
66
|
+
if flatliner_ongoing:
|
67
67
|
# Set historic load to zero to force the basecase forecasts to be zero.
|
68
68
|
input_data.loc[input_data.index < forecast_start, "load"] = 0
|
69
69
|
|
@@ -45,7 +45,7 @@ def create_forecast_pipeline(
|
|
45
45
|
DataFrame with the forecast
|
46
46
|
|
47
47
|
Raises:
|
48
|
-
|
48
|
+
InputDataOngoingFlatlinerError: When all recent load measurements are constant.
|
49
49
|
LookupError: When no model is found for the given prediction job in MLflow.
|
50
50
|
|
51
51
|
"""
|
@@ -85,7 +85,7 @@ def create_forecast_pipeline_core(
|
|
85
85
|
Forecast
|
86
86
|
|
87
87
|
Raises:
|
88
|
-
|
88
|
+
InputDataOngoingFlatlinerError: When all recent load measurements are constant.
|
89
89
|
|
90
90
|
"""
|
91
91
|
structlog.configure(
|
@@ -103,6 +103,7 @@ def create_forecast_pipeline_core(
|
|
103
103
|
input_data,
|
104
104
|
pj["flatliner_threshold_minutes"],
|
105
105
|
pj["resolution_minutes"],
|
106
|
+
detect_non_zero_flatliner=pj["detect_non_zero_flatliner"],
|
106
107
|
)
|
107
108
|
|
108
109
|
# Custom data prep or legacy behavior
|
@@ -132,7 +132,7 @@ def optimize_hyperparameters_pipeline_core(
|
|
132
132
|
InputDataInsufficientError: If the input dataframe is empty.
|
133
133
|
InputDataWrongColumnOrderError: If the load column is missing in the input dataframe.
|
134
134
|
OldModelHigherScoreError: When old model is better than new model.
|
135
|
-
|
135
|
+
InputDataOngoingFlatlinerError: If all recent load measurements are constant.
|
136
136
|
|
137
137
|
Returns:
|
138
138
|
- Best model,
|
@@ -157,6 +157,7 @@ def optimize_hyperparameters_pipeline_core(
|
|
157
157
|
input_data,
|
158
158
|
pj["flatliner_threshold_minutes"],
|
159
159
|
pj["resolution_minutes"],
|
160
|
+
detect_non_zero_flatliner=pj["detect_non_zero_flatliner"],
|
160
161
|
)
|
161
162
|
)
|
162
163
|
|
@@ -60,7 +60,7 @@ def train_model_and_forecast_back_test(
|
|
60
60
|
InputDataInsufficientError: when input data is insufficient.
|
61
61
|
InputDataWrongColumnOrderError: when input data has a invalid column order.
|
62
62
|
ValueError: when the horizon is a string and the corresponding column in not in the input data
|
63
|
-
|
63
|
+
InputDataOngoingFlatlinerError: If all recent load measurements are constant.
|
64
64
|
|
65
65
|
"""
|
66
66
|
if pj.backtest_split_func is None:
|
openstef/pipeline/train_model.py
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
# SPDX-License-Identifier: MPL-2.0
|
4
4
|
import logging
|
5
5
|
import os
|
6
|
-
from typing import Optional,
|
6
|
+
from typing import Optional, Tuple, Union
|
7
7
|
|
8
8
|
import pandas as pd
|
9
9
|
import structlog
|
@@ -46,6 +46,7 @@ def train_model_pipeline(
|
|
46
46
|
check_old_model_age: bool,
|
47
47
|
mlflow_tracking_uri: str,
|
48
48
|
artifact_folder: str,
|
49
|
+
ignore_existing_models: bool = False,
|
49
50
|
) -> Optional[tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]]:
|
50
51
|
"""Middle level pipeline that takes care of all persistent storage dependencies.
|
51
52
|
|
@@ -79,7 +80,7 @@ def train_model_pipeline(
|
|
79
80
|
|
80
81
|
# Get old model and age
|
81
82
|
old_model, model_specs, old_model_age = train_pipeline_step_load_model(
|
82
|
-
pj, serializer
|
83
|
+
pj, serializer, ignore_existing_models
|
83
84
|
)
|
84
85
|
|
85
86
|
# Check old model age and continue yes/no
|
@@ -106,6 +107,7 @@ def train_model_pipeline(
|
|
106
107
|
input_data,
|
107
108
|
old_model,
|
108
109
|
horizons=horizons,
|
110
|
+
ignore_existing_models=ignore_existing_models,
|
109
111
|
)
|
110
112
|
except OldModelHigherScoreError as OMHSE:
|
111
113
|
logger.error("Old model is better than new model", pid=pj["id"], exc_info=OMHSE)
|
@@ -155,6 +157,7 @@ def train_model_pipeline_core(
|
|
155
157
|
input_data: pd.DataFrame,
|
156
158
|
old_model: OpenstfRegressor = None,
|
157
159
|
horizons: list[float] = DEFAULT_TRAIN_HORIZONS_HOURS,
|
160
|
+
ignore_existing_models: bool = False,
|
158
161
|
) -> Tuple[
|
159
162
|
OpenstfRegressor,
|
160
163
|
Report,
|
@@ -177,7 +180,7 @@ def train_model_pipeline_core(
|
|
177
180
|
InputDataInsufficientError: when input data is insufficient.
|
178
181
|
InputDataWrongColumnOrderError: when input data has a invalid column order.
|
179
182
|
OldModelHigherScoreError: When old model is better than new model.
|
180
|
-
|
183
|
+
InputDataOngoingFlatlinerError: If all recent load measurements are constant.
|
181
184
|
|
182
185
|
Returns:
|
183
186
|
- Fitted_model (OpenstfRegressor)
|
@@ -203,7 +206,7 @@ def train_model_pipeline_core(
|
|
203
206
|
model_specs.feature_names = list(train_data.columns)
|
204
207
|
|
205
208
|
# Check if new model is better than old model
|
206
|
-
if old_model:
|
209
|
+
if old_model and not ignore_existing_models:
|
207
210
|
combined = pd.concat([train_data, validation_data])
|
208
211
|
# skip the forecast column added at the end of dataframes
|
209
212
|
if pj.save_train_forecasts:
|
@@ -220,6 +223,7 @@ def train_model_pipeline_core(
|
|
220
223
|
# Try to compare new model to old model.
|
221
224
|
# If this does not success, for example since the feature names of the
|
222
225
|
# old model differ from the new model, the new model is considered better
|
226
|
+
|
223
227
|
try:
|
224
228
|
score_old_model = old_model.score(x_data, y_data)
|
225
229
|
|
@@ -272,7 +276,7 @@ def train_pipeline_common(
|
|
272
276
|
InputDataInsufficientError: when input data is insufficient.
|
273
277
|
InputDataWrongColumnOrderError: when input data has a invalid column order.
|
274
278
|
'load' column should be first and 'horizon' column last.
|
275
|
-
|
279
|
+
InputDataOngoingFlatlinerError: If all recent load measurements are constant.
|
276
280
|
|
277
281
|
"""
|
278
282
|
data_with_features = train_pipeline_step_compute_features(
|
@@ -315,25 +319,31 @@ def train_pipeline_common(
|
|
315
319
|
|
316
320
|
|
317
321
|
def train_pipeline_step_load_model(
|
318
|
-
pj: PredictionJobDataClass,
|
322
|
+
pj: PredictionJobDataClass,
|
323
|
+
serializer: MLflowSerializer,
|
324
|
+
ignore_existing_models: bool = False,
|
319
325
|
) -> Tuple[OpenstfRegressor, ModelSpecificationDataClass, Union[int, float]]:
|
320
326
|
old_model: Optional[OpenstfRegressor]
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
327
|
+
|
328
|
+
if not ignore_existing_models:
|
329
|
+
try:
|
330
|
+
old_model, model_specs = serializer.load_model(experiment_name=str(pj.id))
|
331
|
+
old_model_age = old_model.age # Age attribute is openstef specific
|
332
|
+
return old_model, model_specs, old_model_age
|
333
|
+
except (AttributeError, FileNotFoundError, LookupError):
|
334
|
+
logger.warning("No old model found, training new model", pid=pj.id)
|
335
|
+
except Exception:
|
336
|
+
logger.exception(
|
337
|
+
"Old model could not be loaded, training new model", pid=pj.id
|
338
|
+
)
|
339
|
+
|
329
340
|
old_model = None
|
330
341
|
old_model_age = float("inf")
|
331
342
|
if pj["default_modelspecs"] is not None:
|
332
343
|
model_specs = pj["default_modelspecs"]
|
333
344
|
if model_specs.id != pj.id:
|
334
345
|
raise RuntimeError(
|
335
|
-
"The id of the prediction job and its default model_specs do not"
|
336
|
-
" match."
|
346
|
+
"The id of the prediction job and its default model_specs do not match."
|
337
347
|
)
|
338
348
|
else:
|
339
349
|
# create basic model_specs
|
@@ -363,7 +373,7 @@ def train_pipeline_step_compute_features(
|
|
363
373
|
InputDataInsufficientError: when input data is insufficient.
|
364
374
|
InputDataWrongColumnOrderError: when input data has a invalid column order.
|
365
375
|
ValueError: when the horizon is a string and the corresponding column in not in the input data
|
366
|
-
|
376
|
+
InputDataOngoingFlatlinerError: If all recent load measurements are constant.
|
367
377
|
|
368
378
|
"""
|
369
379
|
if input_data.empty:
|
@@ -389,6 +399,7 @@ def train_pipeline_step_compute_features(
|
|
389
399
|
input_data,
|
390
400
|
pj["flatliner_threshold_minutes"],
|
391
401
|
pj["resolution_minutes"],
|
402
|
+
detect_non_zero_flatliner=pj["detect_non_zero_flatliner"],
|
392
403
|
)
|
393
404
|
)
|
394
405
|
# Check if sufficient data is left after cleaning
|
@@ -25,11 +25,11 @@ from pathlib import Path
|
|
25
25
|
|
26
26
|
from openstef.data_classes.prediction_job import PredictionJobDataClass
|
27
27
|
from openstef.enums import BiddingZone, ModelType, PipelineType
|
28
|
-
from openstef.exceptions import
|
28
|
+
from openstef.exceptions import InputDataOngoingFlatlinerError
|
29
29
|
from openstef.pipeline.create_forecast import create_forecast_pipeline
|
30
30
|
from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
|
31
31
|
from openstef.tasks.utils.taskcontext import TaskContext
|
32
|
-
from openstef.validation.validation import
|
32
|
+
from openstef.validation.validation import detect_ongoing_flatliner
|
33
33
|
|
34
34
|
T_BEHIND_DAYS: int = 14
|
35
35
|
|
@@ -94,7 +94,7 @@ def create_forecast_task(
|
|
94
94
|
forecast = create_forecast_pipeline(
|
95
95
|
pj, input_data, mlflow_tracking_uri=mlflow_tracking_uri
|
96
96
|
)
|
97
|
-
except (
|
97
|
+
except (InputDataOngoingFlatlinerError, LookupError) as e:
|
98
98
|
if (
|
99
99
|
context.config.known_zero_flatliners
|
100
100
|
and pj.id in context.config.known_zero_flatliners
|
@@ -103,18 +103,18 @@ def create_forecast_task(
|
|
103
103
|
"No forecasts were made for this known zero flatliner prediction job. No forecasts need to be made either, since the fallback forecasts are sufficient."
|
104
104
|
)
|
105
105
|
return
|
106
|
-
elif isinstance(e,
|
107
|
-
raise
|
108
|
-
'All recent load measurements are
|
106
|
+
elif isinstance(e, InputDataOngoingFlatlinerError):
|
107
|
+
raise InputDataOngoingFlatlinerError(
|
108
|
+
'All recent load measurements are constant. Check the load profile of this pid as well as related/neighbouring prediction jobs. Afterwards, consider adding this pid to the "known_zero_flatliners" app_setting and possibly removing other pids from the same app_setting.'
|
109
109
|
) from e
|
110
110
|
elif isinstance(e, LookupError):
|
111
|
-
zero_flatliner_ongoing =
|
111
|
+
zero_flatliner_ongoing = detect_ongoing_flatliner(
|
112
112
|
load=input_data.iloc[:, 0],
|
113
113
|
duration_threshold_minutes=pj.flatliner_threshold_minutes,
|
114
114
|
)
|
115
115
|
if zero_flatliner_ongoing:
|
116
116
|
raise LookupError(
|
117
|
-
'Model not found. Consider checking for a
|
117
|
+
'Model not found. Consider checking for a flatliner and adding this pid to the "known_zero_flatliners" app_setting. For flatliners, no model can be trained.'
|
118
118
|
) from e
|
119
119
|
else:
|
120
120
|
raise e
|
openstef/tasks/train_model.py
CHANGED
@@ -19,15 +19,17 @@ Example:
|
|
19
19
|
$ python model_train.py
|
20
20
|
|
21
21
|
"""
|
22
|
-
|
22
|
+
|
23
|
+
from datetime import UTC, datetime, timedelta
|
23
24
|
from pathlib import Path
|
25
|
+
from typing import Optional
|
24
26
|
|
25
27
|
import pandas as pd
|
26
28
|
|
27
29
|
from openstef.data_classes.prediction_job import PredictionJobDataClass
|
28
30
|
from openstef.enums import ModelType, PipelineType
|
29
31
|
from openstef.exceptions import (
|
30
|
-
|
32
|
+
InputDataOngoingFlatlinerError,
|
31
33
|
SkipSaveTrainingForecasts,
|
32
34
|
)
|
33
35
|
from openstef.model.serializer import MLflowSerializer
|
@@ -41,14 +43,16 @@ from openstef.tasks.utils.taskcontext import TaskContext
|
|
41
43
|
|
42
44
|
TRAINING_PERIOD_DAYS: int = 120
|
43
45
|
DEFAULT_CHECK_MODEL_AGE: bool = True
|
46
|
+
DEFAULT_IGNORE_EXISTING_MODELS: bool = False
|
44
47
|
|
45
48
|
|
46
49
|
def train_model_task(
|
47
50
|
pj: PredictionJobDataClass,
|
48
51
|
context: TaskContext,
|
49
52
|
check_old_model_age: bool = DEFAULT_CHECK_MODEL_AGE,
|
50
|
-
datetime_start: datetime = None,
|
51
|
-
datetime_end: datetime = None,
|
53
|
+
datetime_start: Optional[datetime] = None,
|
54
|
+
datetime_end: Optional[datetime] = None,
|
55
|
+
ignore_existing_models: bool = DEFAULT_IGNORE_EXISTING_MODELS,
|
52
56
|
) -> None:
|
53
57
|
"""Train model task.
|
54
58
|
|
@@ -67,7 +71,7 @@ def train_model_task(
|
|
67
71
|
|
68
72
|
Raises:
|
69
73
|
SkipSaveTrainingForecasts: If old model is better or too young, you don't need to save the traing forcast.
|
70
|
-
|
74
|
+
InputDataOngoingFlatlinerError: If all recent load measurements are constant.
|
71
75
|
|
72
76
|
"""
|
73
77
|
# Check pipeline types
|
@@ -104,7 +108,9 @@ def train_model_task(
|
|
104
108
|
serializer = MLflowSerializer(mlflow_tracking_uri=mlflow_tracking_uri)
|
105
109
|
|
106
110
|
# Get old model and age
|
107
|
-
_, _, old_model_age = train_pipeline_step_load_model(
|
111
|
+
_, _, old_model_age = train_pipeline_step_load_model(
|
112
|
+
pj, serializer, ignore_existing_models
|
113
|
+
)
|
108
114
|
|
109
115
|
# Check old model age and continue yes/no
|
110
116
|
if (old_model_age < MAXIMUM_MODEL_AGE) and check_old_model_age:
|
@@ -168,6 +174,7 @@ def train_model_task(
|
|
168
174
|
check_old_model_age=check_old_model_age,
|
169
175
|
mlflow_tracking_uri=mlflow_tracking_uri,
|
170
176
|
artifact_folder=artifact_folder,
|
177
|
+
ignore_existing_models=ignore_existing_models,
|
171
178
|
)
|
172
179
|
|
173
180
|
if data_sets:
|
@@ -187,18 +194,18 @@ def train_model_task(
|
|
187
194
|
context.logger.debug("Saved Forecasts from trained model on datasets")
|
188
195
|
except SkipSaveTrainingForecasts:
|
189
196
|
context.logger.debug("Skip saving forecasts")
|
190
|
-
except
|
197
|
+
except InputDataOngoingFlatlinerError:
|
191
198
|
if (
|
192
199
|
context.config.known_zero_flatliners
|
193
200
|
and pj.id in context.config.known_zero_flatliners
|
194
201
|
):
|
195
202
|
context.logger.info(
|
196
|
-
"No model was trained for this known
|
203
|
+
"No model was trained for this known flatliner. No model needs to be trained either, since the fallback forecasts are sufficient."
|
197
204
|
)
|
198
205
|
return
|
199
206
|
else:
|
200
|
-
raise
|
201
|
-
'All recent load measurements are
|
207
|
+
raise InputDataOngoingFlatlinerError(
|
208
|
+
'All recent load measurements are constant. Check the load profile of this pid as well as related/neighbouring prediction jobs. Afterwards, consider adding this pid to the "known_zero_flatliners" app_setting and possibly removing other pids from the same app_setting.'
|
202
209
|
)
|
203
210
|
|
204
211
|
|
@@ -10,7 +10,8 @@ import numpy as np
|
|
10
10
|
import pandas as pd
|
11
11
|
import structlog
|
12
12
|
|
13
|
-
from openstef.
|
13
|
+
from openstef.data_classes.prediction_job import PredictionJobDataClass
|
14
|
+
from openstef.exceptions import InputDataOngoingFlatlinerError
|
14
15
|
from openstef.model.regressors.regressor import OpenstfRegressor
|
15
16
|
from openstef.preprocessing.preprocessing import replace_repeated_values_with_nan
|
16
17
|
from openstef.settings import Settings
|
@@ -21,12 +22,15 @@ def validate(
|
|
21
22
|
data: pd.DataFrame,
|
22
23
|
flatliner_threshold_minutes: Union[int, None],
|
23
24
|
resolution_minutes: int,
|
25
|
+
*,
|
26
|
+
detect_non_zero_flatliner: bool = False,
|
24
27
|
) -> pd.DataFrame:
|
25
28
|
"""Validate prediction job and timeseries data.
|
26
29
|
|
27
30
|
Steps:
|
28
31
|
1. Check if input dataframe has a datetime index.
|
29
|
-
1. Check if a
|
32
|
+
1. Check if a flatliner pattern is ongoing (i.e. all recent measurements are constant,
|
33
|
+
0 in case detect_non_zero_flatliner = True).
|
30
34
|
2. Replace repeated values for longer than flatliner_threshold_minutes with NaN.
|
31
35
|
|
32
36
|
Args:
|
@@ -35,12 +39,14 @@ def validate(
|
|
35
39
|
flatliner_threshold_minutes: int indicating the number of minutes after which constant load is considered a flatline.
|
36
40
|
if None, the validation is effectively skipped
|
37
41
|
resolution_minutes: The forecasting resolution in minutes.
|
42
|
+
detect_non_zero_flatliner: If True, a flatliner is detected for non-zero values. If False,
|
43
|
+
a flatliner is detected for zero values only.
|
38
44
|
|
39
45
|
Returns:
|
40
46
|
Dataframe where repeated values are set to None
|
41
47
|
|
42
48
|
Raises:
|
43
|
-
|
49
|
+
InputDataOngoingFlatlinerError: If all recent load measurements are constant.
|
44
50
|
|
45
51
|
"""
|
46
52
|
structlog.configure(
|
@@ -57,13 +63,15 @@ def validate(
|
|
57
63
|
logger.info("Skipping validation of input data", pj_id=pj_id)
|
58
64
|
return data
|
59
65
|
|
60
|
-
|
61
|
-
load=data.iloc[:, 0],
|
66
|
+
flatliner_ongoing = detect_ongoing_flatliner(
|
67
|
+
load=data.iloc[:, 0],
|
68
|
+
duration_threshold_minutes=flatliner_threshold_minutes,
|
69
|
+
detect_non_zero_flatliner=detect_non_zero_flatliner,
|
62
70
|
)
|
63
71
|
|
64
|
-
if
|
65
|
-
raise
|
66
|
-
"All recent load measurements are
|
72
|
+
if flatliner_ongoing:
|
73
|
+
raise InputDataOngoingFlatlinerError(
|
74
|
+
"All recent load measurements are constant."
|
67
75
|
)
|
68
76
|
|
69
77
|
flatliner_threshold_repetitions = math.ceil(
|
@@ -228,18 +236,22 @@ def calc_completeness_features(
|
|
228
236
|
return completeness
|
229
237
|
|
230
238
|
|
231
|
-
def
|
239
|
+
def detect_ongoing_flatliner(
|
232
240
|
load: pd.Series,
|
233
241
|
duration_threshold_minutes: int,
|
242
|
+
*,
|
243
|
+
detect_non_zero_flatliner: bool = False,
|
234
244
|
) -> bool:
|
235
|
-
"""Detects if the latest measurements follow a
|
245
|
+
"""Detects if the latest measurements follow a flatliner pattern.
|
236
246
|
|
237
247
|
Args:
|
238
248
|
load (pd.Series): A timeseries of measured load with a datetime index.
|
239
|
-
duration_threshold_minutes (int): A
|
249
|
+
duration_threshold_minutes (int): A flatliner is only detected if it exceeds the threshold duration.
|
250
|
+
detect_non_zero_flatliner (bool): If True, a flatliner is detected for non-zero values. If False,
|
251
|
+
a flatliner is detected for zero values only.
|
240
252
|
|
241
253
|
Returns:
|
242
|
-
bool: Indicating whether or not there is a
|
254
|
+
bool: Indicating whether or not there is a flatliner ongoing for the given load.
|
243
255
|
|
244
256
|
"""
|
245
257
|
# remove all timestamps in the future
|
@@ -249,7 +261,18 @@ def detect_ongoing_zero_flatliner(
|
|
249
261
|
latest_measurement_time - timedelta(minutes=duration_threshold_minutes) :
|
250
262
|
].dropna()
|
251
263
|
|
252
|
-
|
264
|
+
flatliner_value = latest_measurements.median() if detect_non_zero_flatliner else 0
|
265
|
+
|
266
|
+
# check if all values are within a relative tolerance of each other
|
267
|
+
flatline_condition = np.isclose(
|
268
|
+
latest_measurements,
|
269
|
+
flatliner_value,
|
270
|
+
atol=0,
|
271
|
+
rtol=1e-5,
|
272
|
+
).all()
|
273
|
+
non_empty_condition = not latest_measurements.empty
|
274
|
+
|
275
|
+
return flatline_condition & non_empty_condition
|
253
276
|
|
254
277
|
|
255
278
|
def calc_completeness_dataframe(
|
@@ -2,7 +2,7 @@ openstef/__init__.py,sha256=93UM6m0LLQhO69-mSqLuUy73jgs4W7Iuxfo3Lm8c98g,419
|
|
2
2
|
openstef/__main__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
3
3
|
openstef/app_settings.py,sha256=EJTDtimctFQQ-3f7ZcOQaRYohpZk3JD6aZBWPFYM2_A,582
|
4
4
|
openstef/enums.py,sha256=FrP0m_Tk0kV7gSZ2hTY_8iD45KIKnexHrjNufhpKXpE,2829
|
5
|
-
openstef/exceptions.py,sha256=
|
5
|
+
openstef/exceptions.py,sha256=dgnvZe5WWuJWCZm_GES6suEATbusPlwbiEUfNQKeExY,1993
|
6
6
|
openstef/settings.py,sha256=nSgkBqFxuqB3w7Rwo60i8j37c5ngDbt6vpjHS6QtJXQ,354
|
7
7
|
openstef/data/NL_terrestrial_radiation.csv,sha256=A4kbW56GDzWi4tWUwY2C-4PiOvcKJCwkWQQtdg4ekPE,820246
|
8
8
|
openstef/data/NL_terrestrial_radiation.csv.license,sha256=AxxHusqwIXU5RHl5ZMU65LyXmgtbj6QlcnFaOEN4kEE,145
|
@@ -17,7 +17,7 @@ openstef/data/dazls_model_3.4.24/dazls_stored_3.4.24_model_card.md.license,sha25
|
|
17
17
|
openstef/data_classes/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
18
18
|
openstef/data_classes/data_prep.py,sha256=sANgFjfwmSWhLCfmLjfqXQnczuvVZfk2765jZd7LwuE,3691
|
19
19
|
openstef/data_classes/model_specifications.py,sha256=PZeBLfH_MrP9-QorL1r0Hklp0befE8Nw05vNhTX9Y20,1338
|
20
|
-
openstef/data_classes/prediction_job.py,sha256=
|
20
|
+
openstef/data_classes/prediction_job.py,sha256=e6_PFAovNd31tjzoTQJvqRNQyVM-M0XHffclAG9Ez8A,6721
|
21
21
|
openstef/data_classes/split_function.py,sha256=K8y1dsQC5exeIDh37f7UwJ11tV71_uVSNbnKmwXpnOM,3435
|
22
22
|
openstef/feature_engineering/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
23
23
|
openstef/feature_engineering/apply_features.py,sha256=9Yzg61Whd4n0osQBfrcW8cI0gaUiv7u8KnQIQPR40fY,5327
|
@@ -68,12 +68,12 @@ openstef/monitoring/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4
|
|
68
68
|
openstef/monitoring/performance_meter.py,sha256=6aCGjJFXFq-7qwaJyBkF3MLqjgVK6FMFVcO-bcLLUb4,2803
|
69
69
|
openstef/monitoring/teams.py,sha256=A-tlZeuAgolxFHjgT3gGjraxzW2dmuB-UAOz4xgYNIQ,6668
|
70
70
|
openstef/pipeline/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
71
|
-
openstef/pipeline/create_basecase_forecast.py,sha256=
|
71
|
+
openstef/pipeline/create_basecase_forecast.py,sha256=ChIh8iQSRL9n2pc7l3Cw3RWRONkp2e7MOoUnpY9VT_s,4579
|
72
72
|
openstef/pipeline/create_component_forecast.py,sha256=U2v_R-FSOXWVbWeknsJbkulN1YK56fL7-bB1h2B1yzw,6021
|
73
|
-
openstef/pipeline/create_forecast.py,sha256=
|
74
|
-
openstef/pipeline/optimize_hyperparameters.py,sha256=
|
75
|
-
openstef/pipeline/train_create_forecast_backtest.py,sha256
|
76
|
-
openstef/pipeline/train_model.py,sha256=
|
73
|
+
openstef/pipeline/create_forecast.py,sha256=uvp5mQqGSOx-ANY-9o5reiBYNNby0npm-0lt4w9EQ18,5763
|
74
|
+
openstef/pipeline/optimize_hyperparameters.py,sha256=uwXkzRA_fTSFt0yBuvvEoY5-4dMv42FPdS4hZocL-N8,11114
|
75
|
+
openstef/pipeline/train_create_forecast_backtest.py,sha256=hBJPxfDkbrmFSSGZrRH1vTiIVqJP-SWe0ibVpHT_8Qg,6048
|
76
|
+
openstef/pipeline/train_model.py,sha256=8tqJcfqjT9gsXoOSBJxf3i-N_3BPmxbUqt_Ygd7Oao0,20134
|
77
77
|
openstef/pipeline/utils.py,sha256=23mB31p19FoGWelLJzxNmqlzGwEr3fCDBEA37V2kpYY,2167
|
78
78
|
openstef/plotting/__init__.py,sha256=KQjXzyafCt1bE7XDrSeV4TDUIO7MkwN_Br4ASOcNI2g,163
|
79
79
|
openstef/plotting/load_forecast_plotter.py,sha256=n-dB2dQnqjWCvV3kBjnOZYQ03J-9jSIHVovJy3nGSnQ,8129
|
@@ -85,20 +85,20 @@ openstef/tasks/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,16
|
|
85
85
|
openstef/tasks/calculate_kpi.py,sha256=tcW_G0JRMA2tBcb8JN5eUbFFV9UcTsqHXQ1x3f-8Biw,11881
|
86
86
|
openstef/tasks/create_basecase_forecast.py,sha256=_4Ry7AQmXNAKq19J1qmVyG-94atygXePLxVCejCfGPw,4227
|
87
87
|
openstef/tasks/create_components_forecast.py,sha256=8LINqAHt7SnVsQAQMOuve5K-3bLJW-tK_dXTqzlh5Mw,6140
|
88
|
-
openstef/tasks/create_forecast.py,sha256=
|
88
|
+
openstef/tasks/create_forecast.py,sha256=CVUZDG-obMb78ytJ79Hf6LYhMCbqaDvX_vc7fkt9VXI,6075
|
89
89
|
openstef/tasks/create_solar_forecast.py,sha256=HDrJrvTPCM8GS7EQwNr9uJNamf-nH2pu0o4d_xo4w4E,15062
|
90
90
|
openstef/tasks/create_wind_forecast.py,sha256=RhshkmNSyFWx4Y6yQn02GzHjWTREbN5A5GAeWv0JpcE,2907
|
91
91
|
openstef/tasks/optimize_hyperparameters.py,sha256=3NT0KFgim8wAzWPJ0S-GULM3zoshyj63Ivp-g1_oPDw,4765
|
92
92
|
openstef/tasks/split_forecast.py,sha256=X1D3MnnMdAb9wzDWubAJwfMkWpNGdRUPDvPAbJApNhg,9277
|
93
|
-
openstef/tasks/train_model.py,sha256
|
93
|
+
openstef/tasks/train_model.py,sha256=-d1VewDAaZV2B_JAnwl02Y3hONq7cPZrpH6X87_IOKA,8772
|
94
94
|
openstef/tasks/utils/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
95
95
|
openstef/tasks/utils/dependencies.py,sha256=Jy9dtV_G7lTEa5Cdy--wvMxJuAb0adb3R0X4QDjVteM,3077
|
96
96
|
openstef/tasks/utils/predictionjobloop.py,sha256=Ysy3zF5lzPMz_asYDKeF5m0qgVT3tCtwSPihqMjnI5Q,9580
|
97
97
|
openstef/tasks/utils/taskcontext.py,sha256=L9K14ycwgVxbIVUjH2DIn_QWbnu-OfxcGtQ1K9T6sus,5630
|
98
98
|
openstef/validation/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
99
|
-
openstef/validation/validation.py,sha256=
|
100
|
-
openstef-3.4.
|
101
|
-
openstef-3.4.
|
102
|
-
openstef-3.4.
|
103
|
-
openstef-3.4.
|
104
|
-
openstef-3.4.
|
99
|
+
openstef/validation/validation.py,sha256=24GEzLyjVqaE2a-MppbFS-YQT5n739BxD7fH3LK5LEE,12133
|
100
|
+
openstef-3.4.66.dist-info/LICENSE,sha256=7Pm2fWFFHHUG5lDHed1vl5CjzxObIXQglnYsEdtjo_k,14907
|
101
|
+
openstef-3.4.66.dist-info/METADATA,sha256=L8J4MBiz55-LU8iettxkpAP4Nj5UF4kR7wi8WBlFvtY,8816
|
102
|
+
openstef-3.4.66.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
|
103
|
+
openstef-3.4.66.dist-info/top_level.txt,sha256=kD0H4PqrQoncZ957FvqwfBxa89kTrun4Z_RAPs_HhLs,9
|
104
|
+
openstef-3.4.66.dist-info/RECORD,,
|
File without changes
|
File without changes
|