openstef 3.4.10__py3-none-any.whl → 3.4.44__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openstef/app_settings.py +19 -0
- openstef/data/dazls_model_3.4.24/dazls_stored_3.4.24_baseline_model.z +0 -0
- openstef/data/dazls_model_3.4.24/dazls_stored_3.4.24_model_card.md +18 -0
- openstef/data/dutch_holidays.csv +1759 -0
- openstef/data_classes/data_prep.py +1 -1
- openstef/data_classes/prediction_job.py +15 -9
- openstef/enums.py +108 -9
- openstef/exceptions.py +1 -1
- openstef/feature_engineering/apply_features.py +25 -6
- openstef/feature_engineering/bidding_zone_to_country_mapping.py +106 -0
- openstef/feature_engineering/cyclic_features.py +102 -0
- openstef/feature_engineering/data_preparation.py +12 -5
- openstef/feature_engineering/feature_applicator.py +1 -5
- openstef/feature_engineering/general.py +14 -0
- openstef/feature_engineering/holiday_features.py +35 -26
- openstef/feature_engineering/missing_values_transformer.py +141 -0
- openstef/feature_engineering/weather_features.py +7 -0
- openstef/metrics/figure.py +3 -0
- openstef/metrics/metrics.py +58 -1
- openstef/metrics/reporter.py +7 -0
- openstef/model/confidence_interval_applicator.py +28 -3
- openstef/model/model_creator.py +54 -41
- openstef/model/objective.py +17 -34
- openstef/model/objective_creator.py +13 -12
- openstef/model/regressors/arima.py +1 -1
- openstef/model/regressors/dazls.py +35 -96
- openstef/model/regressors/flatliner.py +95 -0
- openstef/model/regressors/linear_quantile.py +296 -0
- openstef/model/regressors/xgb.py +23 -0
- openstef/model/regressors/xgb_multioutput_quantile.py +261 -0
- openstef/model/regressors/xgb_quantile.py +3 -0
- openstef/model/serializer.py +10 -0
- openstef/model_selection/model_selection.py +4 -1
- openstef/monitoring/performance_meter.py +1 -2
- openstef/monitoring/teams.py +11 -0
- openstef/pipeline/create_basecase_forecast.py +11 -1
- openstef/pipeline/create_component_forecast.py +24 -28
- openstef/pipeline/create_forecast.py +20 -1
- openstef/pipeline/optimize_hyperparameters.py +18 -16
- openstef/pipeline/train_create_forecast_backtest.py +11 -1
- openstef/pipeline/train_model.py +31 -12
- openstef/pipeline/utils.py +3 -0
- openstef/postprocessing/postprocessing.py +29 -0
- openstef/settings.py +15 -0
- openstef/tasks/calculate_kpi.py +23 -20
- openstef/tasks/create_basecase_forecast.py +15 -7
- openstef/tasks/create_components_forecast.py +24 -8
- openstef/tasks/create_forecast.py +9 -6
- openstef/tasks/create_solar_forecast.py +4 -4
- openstef/tasks/optimize_hyperparameters.py +2 -2
- openstef/tasks/split_forecast.py +9 -2
- openstef/tasks/train_model.py +9 -7
- openstef/tasks/utils/taskcontext.py +7 -0
- openstef/validation/validation.py +28 -3
- {openstef-3.4.10.dist-info → openstef-3.4.44.dist-info}/METADATA +65 -57
- openstef-3.4.44.dist-info/RECORD +97 -0
- {openstef-3.4.10.dist-info → openstef-3.4.44.dist-info}/WHEEL +1 -1
- openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_adaptation_model.z +0 -0
- openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_adaptation_model_features.z +0 -0
- openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_adaptation_model_scaler.z +0 -0
- openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_domain_model.z +0 -0
- openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_domain_model.z.license +0 -3
- openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_domain_model_features.z +0 -2
- openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_domain_model_features.z.license +0 -3
- openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_domain_model_scaler.z +0 -0
- openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_domain_model_scaler.z.license +0 -3
- openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_target.z +0 -0
- openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_target.z.license +0 -3
- openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_target_scaler.z +0 -6
- openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_target_scaler.z.license +0 -3
- openstef/data/dutch_holidays_2020-2022.csv +0 -831
- openstef/data/dutch_holidays_2020-2022.csv.license +0 -3
- openstef/feature_engineering/historic_features.py +0 -40
- openstef/model/regressors/proloaf.py +0 -281
- openstef/tasks/run_tracy.py +0 -145
- openstef-3.4.10.dist-info/RECORD +0 -104
- /openstef/data/{dazls_model_3.4.0/dazls_stored_3.4.0_adaptation_model.z.license → dazls_model_3.4.24/dazls_stored_3.4.24_baseline_model.z.license} +0 -0
- /openstef/data/{dazls_model_3.4.0/dazls_stored_3.4.0_adaptation_model_features.z.license → dazls_model_3.4.24/dazls_stored_3.4.24_model_card.md.license} +0 -0
- /openstef/data/{dazls_model_3.4.0/dazls_stored_3.4.0_adaptation_model_scaler.z.license → dutch_holidays.csv.license} +0 -0
- {openstef-3.4.10.dist-info → openstef-3.4.44.dist-info}/LICENSE +0 -0
- {openstef-3.4.10.dist-info → openstef-3.4.44.dist-info}/top_level.txt +0 -0
@@ -1,40 +0,0 @@
|
|
1
|
-
# SPDX-FileCopyrightText: 2017-2023 Alliander N.V. <korte.termijn.prognoses@alliander.com> # noqa E501>
|
2
|
-
#
|
3
|
-
# SPDX-License-Identifier: MPL-2.0
|
4
|
-
|
5
|
-
"""This module contains historic load related functions used for feature engineering."""
|
6
|
-
|
7
|
-
import pandas as pd
|
8
|
-
import structlog
|
9
|
-
|
10
|
-
from openstef.data_classes.prediction_job import PredictionJobDataClass
|
11
|
-
|
12
|
-
|
13
|
-
def add_historic_load_as_a_feature(
|
14
|
-
data: pd.DataFrame,
|
15
|
-
pj: PredictionJobDataClass = None,
|
16
|
-
) -> pd.DataFrame:
|
17
|
-
"""Adds additional proloaf features to the input data, historic_load (equal to the load).
|
18
|
-
|
19
|
-
Args:
|
20
|
-
data: Dataframe to which the wind features have to be added
|
21
|
-
pj: Prediction job.
|
22
|
-
|
23
|
-
Returns:
|
24
|
-
DataFrame that is the same as input dataframe with extra columns for the added proloaf features
|
25
|
-
|
26
|
-
"""
|
27
|
-
logger = structlog.get_logger(__name__)
|
28
|
-
|
29
|
-
if pj is None:
|
30
|
-
pj = {}
|
31
|
-
|
32
|
-
if pj.get("model") == "proloaf":
|
33
|
-
data["historic_load"] = data["load"]
|
34
|
-
logger.warning(
|
35
|
-
"The historic_load is added to the data, this is a copy of the load. Adding"
|
36
|
-
" this feature is in most of the cases not a good idea, it is designed for"
|
37
|
-
" the proloaf model."
|
38
|
-
)
|
39
|
-
|
40
|
-
return data
|
@@ -1,281 +0,0 @@
|
|
1
|
-
# SPDX-FileCopyrightText: 2017-2023 Alliander N.V. <korte.termijn.prognoses@alliander.com> # noqa E501>
|
2
|
-
#
|
3
|
-
# SPDX-License-Identifier: MPL-2.0
|
4
|
-
from typing import Any, Union
|
5
|
-
|
6
|
-
import numpy as np
|
7
|
-
import pandas as pd
|
8
|
-
|
9
|
-
# These imports will require the proloaf optional dependencies to be installed
|
10
|
-
import proloaf.datahandler as dh
|
11
|
-
import torch
|
12
|
-
from proloaf.modelhandler import ModelWrapper
|
13
|
-
|
14
|
-
from openstef.model.regressors.regressor import OpenstfRegressor
|
15
|
-
|
16
|
-
# TODO: implement the hyperparameter optimalisation via optuna
|
17
|
-
# TODO: set the default for hyperparameters in the init of OpenstfProloafRegressor
|
18
|
-
# TODO: implement function for defining encoder and decoder features
|
19
|
-
|
20
|
-
|
21
|
-
def divide_scaling_groups(x: pd.DataFrame) -> tuple[list[str], list[str], list[str]]:
|
22
|
-
"""Divides the column names over different type of scaling groups.
|
23
|
-
|
24
|
-
Args:
|
25
|
-
x: Dataframe from which columns have to be divided
|
26
|
-
|
27
|
-
Returns:
|
28
|
-
List of all the grouped features for scaling (three groups)
|
29
|
-
|
30
|
-
"""
|
31
|
-
minmax_scale_features = []
|
32
|
-
oh_scale_features = []
|
33
|
-
no_scale_features = []
|
34
|
-
|
35
|
-
for column in x.columns:
|
36
|
-
if (x[column].min() <= -1) or (x[column].max() >= 1):
|
37
|
-
minmax_scale_features.append(column)
|
38
|
-
elif x[column].dtype == "bool":
|
39
|
-
oh_scale_features.append(column)
|
40
|
-
else:
|
41
|
-
no_scale_features.append(column)
|
42
|
-
|
43
|
-
return minmax_scale_features, oh_scale_features, no_scale_features
|
44
|
-
|
45
|
-
|
46
|
-
def apply_scaling(
|
47
|
-
scaler_features: tuple[list[str], list[str], list[str]],
|
48
|
-
data: pd.DataFrame,
|
49
|
-
scalers=None,
|
50
|
-
):
|
51
|
-
"""Applies different scaling methods to a certain dataframe (minmax, one hot, or no scaling).
|
52
|
-
|
53
|
-
Args:
|
54
|
-
scaler_features: Three different lists with features for each scaling
|
55
|
-
x: Dataframe from which columns have to be divided
|
56
|
-
scalers: scalers resulting from the previous scaling
|
57
|
-
|
58
|
-
Returns:
|
59
|
-
Dataframe with all the scaled features
|
60
|
-
|
61
|
-
"""
|
62
|
-
selected_features, scalers = dh.scale_all(
|
63
|
-
data,
|
64
|
-
scalers=scalers,
|
65
|
-
feature_groups=[
|
66
|
-
{
|
67
|
-
"name": "main",
|
68
|
-
"scaler": ["minmax", -1.0, 1.0],
|
69
|
-
"features": scaler_features[0],
|
70
|
-
},
|
71
|
-
{"name": "aux", "scaler": None, "features": scaler_features[2]},
|
72
|
-
],
|
73
|
-
)
|
74
|
-
|
75
|
-
# One hot encoding certain features
|
76
|
-
onehot_feature_groups = [
|
77
|
-
{
|
78
|
-
"name": "main",
|
79
|
-
"scaler": [
|
80
|
-
"onehot",
|
81
|
-
],
|
82
|
-
"features": scaler_features[1],
|
83
|
-
}
|
84
|
-
]
|
85
|
-
for group in onehot_feature_groups:
|
86
|
-
df_onehot = data.filter(group["features"])
|
87
|
-
result_oh_scale = np.transpose(np.array(df_onehot.iloc[:, :], dtype=np.int))
|
88
|
-
df_onehot.iloc[:, :] = result_oh_scale.T
|
89
|
-
|
90
|
-
if not df_onehot.columns.empty:
|
91
|
-
selected_features = pd.concat([selected_features, df_onehot], axis=1)
|
92
|
-
data = selected_features.iloc[:, :].replace(np.nan, 0)
|
93
|
-
|
94
|
-
return data, scalers
|
95
|
-
|
96
|
-
|
97
|
-
class OpenstfProloafRegressor(OpenstfRegressor, ModelWrapper):
|
98
|
-
def __init__(
|
99
|
-
self,
|
100
|
-
name: str = "model",
|
101
|
-
core_net: str = "torch.nn.LSTM",
|
102
|
-
relu_leak: float = 0.1,
|
103
|
-
encoder_features: list[str] = [
|
104
|
-
"historic_load",
|
105
|
-
], # make sure historic load is present, TODO: implement so you can use None
|
106
|
-
decoder_features: list[str] = [
|
107
|
-
"air_density"
|
108
|
-
], # TODO: implement so you can use None
|
109
|
-
core_layers: int = 1,
|
110
|
-
rel_linear_hidden_size: float = 1.0,
|
111
|
-
rel_core_hidden_size: float = 1.0,
|
112
|
-
dropout_fc: float = 0.4,
|
113
|
-
dropout_core: float = 0.3,
|
114
|
-
training_metric: str = "nllgauss",
|
115
|
-
metric_options: dict[str, Any] = {},
|
116
|
-
optimizer_name: str = "adam",
|
117
|
-
early_stopping_patience: int = 7,
|
118
|
-
early_stopping_margin: float = 0,
|
119
|
-
learning_rate: float = 1e-3,
|
120
|
-
max_epochs: int = 100,
|
121
|
-
device: Union[str, int] = "cpu", # "cuda" or "cpu"
|
122
|
-
batch_size: int = 6,
|
123
|
-
history_horizon: int = 24,
|
124
|
-
horizon_minutes: int = 2880, # 2 days in minutes,
|
125
|
-
):
|
126
|
-
self.device = device
|
127
|
-
self.batch_size = batch_size
|
128
|
-
self.history_horizon = history_horizon
|
129
|
-
self.forecast_horizon = int(horizon_minutes / 60)
|
130
|
-
ModelWrapper.__init__(
|
131
|
-
self,
|
132
|
-
name=name,
|
133
|
-
core_net=core_net,
|
134
|
-
relu_leak=relu_leak,
|
135
|
-
encoder_features=encoder_features,
|
136
|
-
decoder_features=decoder_features,
|
137
|
-
core_layers=core_layers,
|
138
|
-
rel_linear_hidden_size=rel_linear_hidden_size,
|
139
|
-
rel_core_hidden_size=rel_core_hidden_size,
|
140
|
-
dropout_fc=dropout_fc,
|
141
|
-
dropout_core=dropout_core,
|
142
|
-
training_metric=training_metric,
|
143
|
-
metric_options=metric_options,
|
144
|
-
optimizer_name=optimizer_name,
|
145
|
-
early_stopping_patience=early_stopping_patience,
|
146
|
-
early_stopping_margin=early_stopping_margin,
|
147
|
-
learning_rate=learning_rate,
|
148
|
-
max_epochs=max_epochs,
|
149
|
-
)
|
150
|
-
self.to(device)
|
151
|
-
|
152
|
-
@property
|
153
|
-
def feature_names(self):
|
154
|
-
return (
|
155
|
-
["load"] + self.encoder_features + self.decoder_features
|
156
|
-
) # TODO: gehele range, of een enkele feature
|
157
|
-
|
158
|
-
@property
|
159
|
-
def can_predict_quantiles(self):
|
160
|
-
return False
|
161
|
-
|
162
|
-
def predict(self, x: pd.DataFrame) -> np.ndarray:
|
163
|
-
x = x[list(self.feature_names)[1:]]
|
164
|
-
# Apply scaling and interpolation for NaN values
|
165
|
-
x = dh.fill_if_missing(x, periodicity=24)
|
166
|
-
x, _ = apply_scaling(
|
167
|
-
[
|
168
|
-
self.minmax_scale_features,
|
169
|
-
self.oh_scale_features,
|
170
|
-
self.no_scale_features,
|
171
|
-
],
|
172
|
-
x,
|
173
|
-
self.scalers,
|
174
|
-
)
|
175
|
-
|
176
|
-
inputs_enc = torch.tensor(
|
177
|
-
x[self.encoder_features].to_numpy(), dtype=torch.float
|
178
|
-
).unsqueeze(dim=0)
|
179
|
-
inputs_dec = torch.tensor(
|
180
|
-
x[self.decoder_features].to_numpy(), dtype=torch.float
|
181
|
-
).unsqueeze(dim=0)
|
182
|
-
prediction = (
|
183
|
-
ModelWrapper.predict(self, inputs_enc, inputs_dec)[:, :, 0]
|
184
|
-
.squeeze()
|
185
|
-
.detach()
|
186
|
-
.numpy()
|
187
|
-
)
|
188
|
-
|
189
|
-
return prediction
|
190
|
-
|
191
|
-
def fit(
|
192
|
-
self,
|
193
|
-
x: pd.DataFrame,
|
194
|
-
y: pd.DataFrame,
|
195
|
-
eval_set: tuple = None,
|
196
|
-
early_stopping_rounds: int = None,
|
197
|
-
verbose: bool = False,
|
198
|
-
**kwargs,
|
199
|
-
) -> ModelWrapper:
|
200
|
-
x = x[list(self.feature_names)[1:]]
|
201
|
-
# Apply scaling and interpolation for NaN values
|
202
|
-
x = dh.fill_if_missing(x, periodicity=24)
|
203
|
-
(
|
204
|
-
self.minmax_scale_features,
|
205
|
-
self.oh_scale_features,
|
206
|
-
self.no_scale_features,
|
207
|
-
) = divide_scaling_groups(x)
|
208
|
-
x, self.scalers = apply_scaling(
|
209
|
-
[
|
210
|
-
self.minmax_scale_features,
|
211
|
-
self.oh_scale_features,
|
212
|
-
self.no_scale_features,
|
213
|
-
],
|
214
|
-
x,
|
215
|
-
self.scalers,
|
216
|
-
)
|
217
|
-
y = y.to_frame()
|
218
|
-
self.target_id = [y.columns[0]]
|
219
|
-
|
220
|
-
df_train = pd.concat([x, y], axis="columns", verify_integrity=True)
|
221
|
-
|
222
|
-
print(f"{self.encoder_features = }")
|
223
|
-
print(f"{self.decoder_features = }")
|
224
|
-
|
225
|
-
train_dl, _, _ = dh.transform(
|
226
|
-
df=df_train,
|
227
|
-
encoder_features=self.encoder_features,
|
228
|
-
decoder_features=self.decoder_features,
|
229
|
-
batch_size=self.batch_size,
|
230
|
-
history_horizon=self.history_horizon,
|
231
|
-
forecast_horizon=self.forecast_horizon,
|
232
|
-
target_id=self.target_id,
|
233
|
-
train_split=1.0,
|
234
|
-
validation_split=1.0,
|
235
|
-
device=self.device,
|
236
|
-
)
|
237
|
-
|
238
|
-
val_x, val_y = eval_set[1][0], eval_set[1][1]
|
239
|
-
val_x = dh.fill_if_missing(val_x, periodicity=24)
|
240
|
-
val_x, _ = apply_scaling(
|
241
|
-
[
|
242
|
-
self.minmax_scale_features,
|
243
|
-
self.oh_scale_features,
|
244
|
-
self.no_scale_features,
|
245
|
-
],
|
246
|
-
val_x,
|
247
|
-
self.scalers,
|
248
|
-
)
|
249
|
-
|
250
|
-
df_val = pd.concat([val_x, val_y], axis="columns", verify_integrity=True)
|
251
|
-
_, validation_dl, _ = dh.transform(
|
252
|
-
df=df_val,
|
253
|
-
encoder_features=self.encoder_features,
|
254
|
-
decoder_features=self.decoder_features,
|
255
|
-
batch_size=self.batch_size,
|
256
|
-
history_horizon=self.history_horizon,
|
257
|
-
forecast_horizon=self.forecast_horizon,
|
258
|
-
target_id=self.target_id,
|
259
|
-
train_split=0.0,
|
260
|
-
validation_split=1.0,
|
261
|
-
device=self.device,
|
262
|
-
)
|
263
|
-
self.to(self.device)
|
264
|
-
self.init_model()
|
265
|
-
self.is_fitted_ = True
|
266
|
-
|
267
|
-
return self.run_training(train_dl, validation_dl)
|
268
|
-
|
269
|
-
def get_params(self, deep=True):
|
270
|
-
model_params = self.get_model_config()
|
271
|
-
training_params = self.get_training_config()
|
272
|
-
return {**model_params, **training_params}
|
273
|
-
|
274
|
-
def set_params(self, **params):
|
275
|
-
self.update(**params)
|
276
|
-
return self
|
277
|
-
|
278
|
-
|
279
|
-
if __name__ == "__main__":
|
280
|
-
test = OpenstfProloafRegressor()
|
281
|
-
print(test)
|
openstef/tasks/run_tracy.py
DELETED
@@ -1,145 +0,0 @@
|
|
1
|
-
# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
|
2
|
-
#
|
3
|
-
# SPDX-License-Identifier: MPL-2.0
|
4
|
-
"""Tracy checks the mysql todolist and tries her best to execute the functions with desired inputs.
|
5
|
-
|
6
|
-
This scripts works as follows:
|
7
|
-
1. Checks the mysql table 'todolist' for jobs (which are not already in progress and
|
8
|
-
which are not already failed)
|
9
|
-
2. Set all newly acquired jobs to 'in progress'
|
10
|
-
For each job:
|
11
|
-
3. Convert input arguments to a dict with 'args' and 'kwargs'
|
12
|
-
4. Interpret the given function and arguments
|
13
|
-
5. Execute the job
|
14
|
-
6. Post result to Teams
|
15
|
-
7. Remove job from mysql table
|
16
|
-
If job fails, set in progress to 2
|
17
|
-
All functions that tracy is able to execute need to be imported and defined in the
|
18
|
-
available_functions.
|
19
|
-
|
20
|
-
Example:
|
21
|
-
This module is meant to be called directly from a CRON job.
|
22
|
-
|
23
|
-
Alternatively this code can be run directly by running::
|
24
|
-
$ python run_tracy.py
|
25
|
-
|
26
|
-
"""
|
27
|
-
# sql to create the Tracy jobs table (todolist)
|
28
|
-
|
29
|
-
# CREATE TABLE IF NOT EXISTS `tst_icarus`.`todolist` (
|
30
|
-
# `id` INT NOT NULL AUTO_INCREMENT ,
|
31
|
-
# `created` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ,
|
32
|
-
# `function` VARCHAR(200) NOT NULL ,
|
33
|
-
# `args` VARCHAR(200) NOT NULL ,
|
34
|
-
# `inprogress` BOOLEAN NULL DEFAULT NULL ,
|
35
|
-
# PRIMARY KEY (`id`), UNIQUE `id` (`id`))
|
36
|
-
# ENGINE = InnoDB;
|
37
|
-
from pathlib import Path
|
38
|
-
|
39
|
-
from openstef.data_classes.prediction_job import PredictionJobDataClass
|
40
|
-
from openstef.enums import TracyJobResult
|
41
|
-
from openstef.monitoring import teams
|
42
|
-
from openstef.tasks.optimize_hyperparameters import optimize_hyperparameters_task
|
43
|
-
from openstef.tasks.train_model import train_model_task
|
44
|
-
from openstef.tasks.utils.taskcontext import TaskContext
|
45
|
-
|
46
|
-
|
47
|
-
def run_tracy(context: TaskContext) -> None:
|
48
|
-
# Get all Tracy jobs
|
49
|
-
tracy_jobs = context.database.ktp_api.get_all_tracy_jobs(inprogress=0)
|
50
|
-
num_jobs = len(tracy_jobs)
|
51
|
-
|
52
|
-
if num_jobs == 0:
|
53
|
-
context.logger.warning(f"Number of tracy jobs is {num_jobs}, exit task")
|
54
|
-
return
|
55
|
-
|
56
|
-
context.logger.info("Start processing Tracy jobs", num_jobs=num_jobs)
|
57
|
-
|
58
|
-
for i, job in enumerate(tracy_jobs):
|
59
|
-
# get a new logger with bound job
|
60
|
-
logger = context.logger.bind(job=job)
|
61
|
-
logger.info("Process job", job_counter=i, total_jobs=num_jobs)
|
62
|
-
|
63
|
-
# Set all retrieved items of the todolist to inprogress
|
64
|
-
job["inprogress"] = 1
|
65
|
-
context.database.ktp_api.update_tracy_job(job)
|
66
|
-
|
67
|
-
pid = int(job["args"])
|
68
|
-
pj = context.database.get_prediction_job(pid)
|
69
|
-
result, exc = run_tracy_job(job, pj, context)
|
70
|
-
# job processing was succefull
|
71
|
-
if result is TracyJobResult.SUCCESS:
|
72
|
-
logger.info("Succesfully processed Tracy job")
|
73
|
-
# Delete job when succesfull
|
74
|
-
context.database.ktp_api.delete_tracy_job(job)
|
75
|
-
logger.info("Delete Tracy job")
|
76
|
-
|
77
|
-
# job was unknown
|
78
|
-
elif result is TracyJobResult.UNKNOWN:
|
79
|
-
logger.error(f"Unkown Tracy job {job['function']}")
|
80
|
-
|
81
|
-
# job processing failed / raised an exception
|
82
|
-
elif result is TracyJobResult.FAILED:
|
83
|
-
job["inprogress"] = 2
|
84
|
-
context.database.ktp_api.update_tracy_job(job)
|
85
|
-
msg = "Exception occured while processing Tracy job"
|
86
|
-
logger.error(msg, exc_info=exc)
|
87
|
-
teams.post_teams(teams.format_message(title=msg, params=job))
|
88
|
-
|
89
|
-
context.logger.info("Finished processing all Tracy jobs - Tracy out!")
|
90
|
-
|
91
|
-
|
92
|
-
def run_tracy_job(
|
93
|
-
job: dict, pj: PredictionJobDataClass, context: TaskContext
|
94
|
-
) -> TracyJobResult:
|
95
|
-
"""Run tracy job.
|
96
|
-
|
97
|
-
Args:
|
98
|
-
job: Tracy jon
|
99
|
-
pj: Prediction job
|
100
|
-
context: Task context.
|
101
|
-
|
102
|
-
Returns:
|
103
|
-
Result of the Tracy job.
|
104
|
-
|
105
|
-
"""
|
106
|
-
# Try to process Tracy job
|
107
|
-
try:
|
108
|
-
# If train model job (TODO remove old name when jobs are done)
|
109
|
-
if job["function"] in ["train_model", "train_specific_model"]:
|
110
|
-
train_model_task(pj, context, check_old_model_age=False)
|
111
|
-
|
112
|
-
# If optimize hyperparameters job (TODO remove old name when jobs are done)
|
113
|
-
elif job["function"] in [
|
114
|
-
"optimize_hyperparameters",
|
115
|
-
"optimize_hyperparameters_for_specific_pid",
|
116
|
-
]:
|
117
|
-
optimize_hyperparameters_task(pj, context, check_hyper_param_age=False)
|
118
|
-
|
119
|
-
# Else unknown job
|
120
|
-
else:
|
121
|
-
return TracyJobResult.UNKNOWN, None
|
122
|
-
|
123
|
-
# job processing was succesfull
|
124
|
-
return TracyJobResult.SUCCESS, None
|
125
|
-
|
126
|
-
# Processing of Tracy job failed
|
127
|
-
except Exception as e:
|
128
|
-
return TracyJobResult.FAILED, e
|
129
|
-
|
130
|
-
|
131
|
-
def main(config=None, database=None):
|
132
|
-
taskname = Path(__file__).name.replace(".py", "")
|
133
|
-
|
134
|
-
if database is None or config is None:
|
135
|
-
raise RuntimeError(
|
136
|
-
"Please specifiy a config object and/or database connection object. These"
|
137
|
-
" can be found in the openstef-dbc package."
|
138
|
-
)
|
139
|
-
|
140
|
-
with TaskContext(taskname, config, database) as context:
|
141
|
-
run_tracy(context)
|
142
|
-
|
143
|
-
|
144
|
-
if __name__ == "__main__":
|
145
|
-
main()
|
openstef-3.4.10.dist-info/RECORD
DELETED
@@ -1,104 +0,0 @@
|
|
1
|
-
openstef/__init__.py,sha256=93UM6m0LLQhO69-mSqLuUy73jgs4W7Iuxfo3Lm8c98g,419
|
2
|
-
openstef/__main__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
3
|
-
openstef/enums.py,sha256=dFpGFySauUt-m_bsPNcmc5WVG33ZXcWsbJeQp_4iEBE,753
|
4
|
-
openstef/exceptions.py,sha256=fVqjyrVMBiSGxcoZ3JfTcgZjIur1cPennZpfwqgc9qY,1992
|
5
|
-
openstef/data/dutch_holidays_2020-2022.csv,sha256=pS-CjE0igYXd-2dG-MlqyvR2fgYgXkbNmgCKyTjmwxs,23704
|
6
|
-
openstef/data/dutch_holidays_2020-2022.csv.license,sha256=AxxHusqwIXU5RHl5ZMU65LyXmgtbj6QlcnFaOEN4kEE,145
|
7
|
-
openstef/data/pv_single_coefs.csv,sha256=jadIEYdHvl1lnV_06X_FASkJZ6C3Hecs5xZnH1gPMvI,24779
|
8
|
-
openstef/data/pv_single_coefs.csv.license,sha256=AxxHusqwIXU5RHl5ZMU65LyXmgtbj6QlcnFaOEN4kEE,145
|
9
|
-
openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_adaptation_model.z,sha256=M2S-3OBz-CA7iePcP0dt5dHQhlYOh6gW14jQQ3P_Fjw,29570256
|
10
|
-
openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_adaptation_model.z.license,sha256=AxxHusqwIXU5RHl5ZMU65LyXmgtbj6QlcnFaOEN4kEE,145
|
11
|
-
openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_adaptation_model_features.z,sha256=J9mF1mqfVwrZoyzJ8VoWPehKIoGtRakfe1a1m5lZt5I,88
|
12
|
-
openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_adaptation_model_features.z.license,sha256=AxxHusqwIXU5RHl5ZMU65LyXmgtbj6QlcnFaOEN4kEE,145
|
13
|
-
openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_adaptation_model_scaler.z,sha256=AJTlZ3M4fwB1XOkat4RuYX6sGJKYWO6mWuoSPGrqtLc,883
|
14
|
-
openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_adaptation_model_scaler.z.license,sha256=AxxHusqwIXU5RHl5ZMU65LyXmgtbj6QlcnFaOEN4kEE,145
|
15
|
-
openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_domain_model.z,sha256=n-F4Iv1XsQ42-IXjNYDjy6bcQadWhtkmCxK360A2_wE,32794824
|
16
|
-
openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_domain_model.z.license,sha256=AxxHusqwIXU5RHl5ZMU65LyXmgtbj6QlcnFaOEN4kEE,145
|
17
|
-
openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_domain_model_features.z,sha256=UUwyRZcUl4S884R86bL-8g9llaDHINmW5-WRoMJF0Do,123
|
18
|
-
openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_domain_model_features.z.license,sha256=AxxHusqwIXU5RHl5ZMU65LyXmgtbj6QlcnFaOEN4kEE,145
|
19
|
-
openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_domain_model_scaler.z,sha256=yrXVWBgKXp5uQBh--n4db3aaB3SWrQcyZ_MJLsmCJV4,1077
|
20
|
-
openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_domain_model_scaler.z.license,sha256=AxxHusqwIXU5RHl5ZMU65LyXmgtbj6QlcnFaOEN4kEE,145
|
21
|
-
openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_target.z,sha256=7GkwwyQPJosqmGa0LslpfYIf8qgLDMW9Krx8CM_YO10,40
|
22
|
-
openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_target.z.license,sha256=AxxHusqwIXU5RHl5ZMU65LyXmgtbj6QlcnFaOEN4kEE,145
|
23
|
-
openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_target_scaler.z,sha256=Lh4EL1vuKr0rkvXq9xUr-4twVpow2vtjEUunXvh-XDM,683
|
24
|
-
openstef/data/dazls_model_3.4.0/dazls_stored_3.4.0_target_scaler.z.license,sha256=AxxHusqwIXU5RHl5ZMU65LyXmgtbj6QlcnFaOEN4kEE,145
|
25
|
-
openstef/data_classes/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
26
|
-
openstef/data_classes/data_prep.py,sha256=3SLvHDXlL-fBw8IFGBP_pKXTfWjQsKjgVxDyYlgic1o,3417
|
27
|
-
openstef/data_classes/model_specifications.py,sha256=Uod1W3QzhRqVLb6zvXwxh9wRL3EHCzSvX0oDNd28cFk,1197
|
28
|
-
openstef/data_classes/prediction_job.py,sha256=YrPaUACW7u09yabmwXkXu3fRpu-M90FPgprd7vjSP0A,4856
|
29
|
-
openstef/data_classes/split_function.py,sha256=ljQIQQu1t1Y_CVWGAy25jrM6wG9odIVVQVimrT1n-1s,3358
|
30
|
-
openstef/feature_engineering/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
31
|
-
openstef/feature_engineering/apply_features.py,sha256=pKvTl2c-L5OUuRPWy9mr_OcGExSNbwDRhTuLmNzGFqE,4114
|
32
|
-
openstef/feature_engineering/data_preparation.py,sha256=vFdCdXrX7RWWa4uj1D8rVkHymacbfhvwoLiEnUk-v6g,5399
|
33
|
-
openstef/feature_engineering/feature_adder.py,sha256=aSqDl_gUrB3H2TD3cNvU5JniY_KOb4u4a2A6J7zB2BQ,6835
|
34
|
-
openstef/feature_engineering/feature_applicator.py,sha256=FpEEirbOTuOKZipoGHYliKvY0r_XE5f2z-r8NiAx4oU,7712
|
35
|
-
openstef/feature_engineering/general.py,sha256=igAPyejYN5d09-a_c53C79a0A4cHjvMCa47KC4IfgCo,4072
|
36
|
-
openstef/feature_engineering/historic_features.py,sha256=aDxoS42ndUmLmwqpII8rZ_-L86NFer-Gw_fGf0TAobU,1215
|
37
|
-
openstef/feature_engineering/holiday_features.py,sha256=3Ff4Lkm26h8wJVoBplUewt4HfsvOUS9zj0x0MxewIm8,7842
|
38
|
-
openstef/feature_engineering/lag_features.py,sha256=Dr6qS8UhdgEHPZZSe-w6ibtjl_lcbcQohhqdZN9fqEU,5652
|
39
|
-
openstef/feature_engineering/weather_features.py,sha256=wy3KFXUIIwSydFJZpiejsJMwURtDpv9l0HBHu-uLAGQ,15561
|
40
|
-
openstef/metrics/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
41
|
-
openstef/metrics/figure.py,sha256=NPJGI4FygjSnOQuL8qCbB87-T31q6EkewkbVmpLwmnk,9657
|
42
|
-
openstef/metrics/metrics.py,sha256=c6HGQubArT5G4YxF0KY9HCP19PRHaVfXQ8KEkSwrt0w,13164
|
43
|
-
openstef/metrics/reporter.py,sha256=V6pa4IUOzVcZ8OY632g5KoF8hr2MT2ySexrjZCjnuwY,7668
|
44
|
-
openstef/model/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
45
|
-
openstef/model/basecase.py,sha256=caI6Q-8y0ymlxGK9Js_H3Vh0q6ruNHlGD5RG0_kE5M0,2878
|
46
|
-
openstef/model/confidence_interval_applicator.py,sha256=7E1_JFLZ4-hyEhleacMvp5szdmYZS4tpKAjfhGvXXvg,8602
|
47
|
-
openstef/model/fallback.py,sha256=VV9ehgnoMZtWzqKk9H1t8wnERFh5CyC4TvDIuRP_ZDI,2861
|
48
|
-
openstef/model/model_creator.py,sha256=U1Lw4HFyajfxQ2o5lEnCxnmRC62DEu5PBHXrm_jnEJU,5582
|
49
|
-
openstef/model/objective.py,sha256=qBZGij4P2uZXFt-1x4MxCkIRmpVY3TJoo4Mycxu9dO8,15433
|
50
|
-
openstef/model/objective_creator.py,sha256=OiPPFSiSu7z9K_983ib5iqhhu6_9tt7iyTyKNZ2Iz68,2057
|
51
|
-
openstef/model/serializer.py,sha256=Q7pKkA-K1bszER_tkOWhvwTQsj6-qZoX5zaXIemRKGs,16934
|
52
|
-
openstef/model/standard_deviation_generator.py,sha256=Od9bzXi2TLb1v8Nz-VhBMZHSopWH6ssaDe8gYLlqO1I,2911
|
53
|
-
openstef/model/metamodels/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
54
|
-
openstef/model/metamodels/grouped_regressor.py,sha256=yMN_a6TnQSyFaqlB_6Nifq-ydpb5hs6w_b97IaBbHj4,8337
|
55
|
-
openstef/model/metamodels/missing_values_handler.py,sha256=veyvYZHhKvlYZxaUpxRQ7XoE033_3Lcg9LrbuKchlOk,5241
|
56
|
-
openstef/model/regressors/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
57
|
-
openstef/model/regressors/arima.py,sha256=yATxKKz5kl-irr3CBNVHpkFY45aY3RILu0nUFfEFMgg,7587
|
58
|
-
openstef/model/regressors/custom_regressor.py,sha256=Hsmxahc9nfSWD0aEZ6cm4pxW2noQ8B1SujS17_fmxcU,1768
|
59
|
-
openstef/model/regressors/dazls.py,sha256=V8g88LhgsAZka_TFvUDI_sXlvuqSiGIbEUrO2hkAPVM,7216
|
60
|
-
openstef/model/regressors/lgbm.py,sha256=zCdn1euEdSFxYJzH8XqQFFnb6R4JVUnmineKjX_Gy-g,800
|
61
|
-
openstef/model/regressors/linear.py,sha256=uOvZMLGZH_9nXfmS5honCMfyVeyGXP1Cza9A_BdXlVw,3665
|
62
|
-
openstef/model/regressors/proloaf.py,sha256=HjyfEQRiFMFzyfYSrFL3yPhduhL1igGr1FTO6d0ET7I,9081
|
63
|
-
openstef/model/regressors/regressor.py,sha256=uJcx59AyCPE9f_yPcAQ59h2ZS7eNsDpIHJrladKvHIw,3461
|
64
|
-
openstef/model/regressors/xgb.py,sha256=HggA1U10srzdysjV560BMMX66kfaxCKAnOZB3JyyT_Y,808
|
65
|
-
openstef/model/regressors/xgb_quantile.py,sha256=Oenhe0cMLAEXlinHZF588zNTjwR-hB7NB0anQZPwPKU,7710
|
66
|
-
openstef/model_selection/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
67
|
-
openstef/model_selection/model_selection.py,sha256=oGloQBP_FPdNyCs9wzS3l8zFNJxMs1P5XPjVN9qUOsw,11081
|
68
|
-
openstef/monitoring/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
69
|
-
openstef/monitoring/performance_meter.py,sha256=mMQKpDNv_-RcNYdEvEFPvB76lkG8V9gJOKYQqnH5BX4,2851
|
70
|
-
openstef/monitoring/teams.py,sha256=fnZScPD55z9yC0q3YavWj40GEZmL7tsSGhWzG_sMPws,6401
|
71
|
-
openstef/pipeline/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
72
|
-
openstef/pipeline/create_basecase_forecast.py,sha256=7jig8lt-e3xwIWDSk-bMwUC45y2mwKe9-zDZYHiiQMY,4324
|
73
|
-
openstef/pipeline/create_component_forecast.py,sha256=ahtFKykD5Ff758fowil0StQa7zK8AjsqyMGCz5iRoEE,6255
|
74
|
-
openstef/pipeline/create_forecast.py,sha256=El9xXKN8DeIADy_76-V22wBY8AMSa4KEfYuW00jw85k,5083
|
75
|
-
openstef/pipeline/optimize_hyperparameters.py,sha256=BnahfVHlE1z9so0gxPyd75OKCXSZvtFvuqicH6skm5U,10892
|
76
|
-
openstef/pipeline/train_create_forecast_backtest.py,sha256=upuoiE01vjjxUu_sY0tANPqdOtpGKrQQ3azhVDnBJdc,5512
|
77
|
-
openstef/pipeline/train_model.py,sha256=cPTXDUjsQK5o5ygMNnmH-DfXins1khAnvt8gVF5eK4Q,18677
|
78
|
-
openstef/pipeline/utils.py,sha256=fkc-oNirJ-JiyuOAL08RFrnPYPwudWal_N-BO6Cw980,2086
|
79
|
-
openstef/postprocessing/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
80
|
-
openstef/postprocessing/postprocessing.py,sha256=nehd0tDpkdIaWFJggQ-fDizIKdfmqJ3IOGfk0sDnrzk,8409
|
81
|
-
openstef/preprocessing/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
82
|
-
openstef/preprocessing/preprocessing.py,sha256=bM_cSSSb2vGTD79RGzUrI6KoELbzlCyJwc7jqQGNEsE,1454
|
83
|
-
openstef/tasks/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
84
|
-
openstef/tasks/calculate_kpi.py,sha256=pC8CJ0UqhySpVVewMN0GIe0ELEmYOf1Wc9xElUe0Q5M,11985
|
85
|
-
openstef/tasks/create_basecase_forecast.py,sha256=Hk9fDljXvo5TfeS3nWHrerWi7y-lQzoJEaqWbqaxHOs,3852
|
86
|
-
openstef/tasks/create_components_forecast.py,sha256=zLJQ-5w-L3sqxkGw6KaaXVkmzLqjLJoU5mT6QtvkME0,5520
|
87
|
-
openstef/tasks/create_forecast.py,sha256=FPILsCqt2lT2QIjseXyKjViZG6SVRoGCxoj9tPiozIg,5575
|
88
|
-
openstef/tasks/create_solar_forecast.py,sha256=bTr7NThTF6Yj405qAqRaJmlBUrL7HATqVVzsi9hMdMw,15049
|
89
|
-
openstef/tasks/create_wind_forecast.py,sha256=RhshkmNSyFWx4Y6yQn02GzHjWTREbN5A5GAeWv0JpcE,2907
|
90
|
-
openstef/tasks/optimize_hyperparameters.py,sha256=s-z8YQJF6Lf3DdYgKHEpAdlbFJ3a-0Gj0Ahsqj1DErc,4758
|
91
|
-
openstef/tasks/run_tracy.py,sha256=mWRg5u74iSaUGHRQzIa-2Weyg6ChuW5w3JBL-7MrNBc,5036
|
92
|
-
openstef/tasks/split_forecast.py,sha256=hBRoIlZ_DK4747EtMpY-HVh_tmTdGa65oYOtrjHRUxQ,9118
|
93
|
-
openstef/tasks/train_model.py,sha256=-6JzNQIDdBC6j-nrp10KF_eFkTjkadcOGS5S5iG-JXM,7263
|
94
|
-
openstef/tasks/utils/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
95
|
-
openstef/tasks/utils/dependencies.py,sha256=Jy9dtV_G7lTEa5Cdy--wvMxJuAb0adb3R0X4QDjVteM,3077
|
96
|
-
openstef/tasks/utils/predictionjobloop.py,sha256=Ysy3zF5lzPMz_asYDKeF5m0qgVT3tCtwSPihqMjnI5Q,9580
|
97
|
-
openstef/tasks/utils/taskcontext.py,sha256=yI6TntOkZcW8JiNVuw4uJIigEBL0_iIrkPklF4ZeCX4,5401
|
98
|
-
openstef/validation/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
99
|
-
openstef/validation/validation.py,sha256=SaI-Mff9UOHQPnQ2jodXzZAVZilc-2AXZsPpSjDRqAg,10346
|
100
|
-
openstef-3.4.10.dist-info/LICENSE,sha256=7Pm2fWFFHHUG5lDHed1vl5CjzxObIXQglnYsEdtjo_k,14907
|
101
|
-
openstef-3.4.10.dist-info/METADATA,sha256=s2gQqhwp9LXyxL0nt0jOhzguXJjHREZp5XHYJ_hj11o,7872
|
102
|
-
openstef-3.4.10.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
103
|
-
openstef-3.4.10.dist-info/top_level.txt,sha256=kD0H4PqrQoncZ957FvqwfBxa89kTrun4Z_RAPs_HhLs,9
|
104
|
-
openstef-3.4.10.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|