openstef 3.4.67__py3-none-any.whl → 3.4.69__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openstef/app_settings.py +7 -0
- openstef/feature_engineering/apply_features.py +8 -9
- openstef/feature_engineering/cyclic_features.py +2 -11
- openstef/feature_engineering/data_preparation.py +2 -9
- openstef/feature_engineering/general.py +3 -15
- openstef/feature_engineering/missing_values_transformer.py +1 -1
- openstef/feature_engineering/rolling_features.py +1 -1
- openstef/feature_engineering/weather_features.py +2 -9
- openstef/logging/__init__.py +3 -0
- openstef/logging/base_logger.py +34 -0
- openstef/logging/logger_factory.py +17 -0
- openstef/logging/logger_types.py +10 -0
- openstef/logging/standard_logger.py +33 -0
- openstef/logging/structlog_logger.py +38 -0
- openstef/metrics/reporter.py +2 -9
- openstef/model/confidence_interval_applicator.py +3 -10
- openstef/model/fallback.py +1 -1
- openstef/model/metamodels/feature_clipper.py +3 -2
- openstef/model/model_creator.py +4 -12
- openstef/model/objective_creator.py +1 -1
- openstef/model/regressors/gblinear_quantile.py +2 -2
- openstef/model/regressors/linear_quantile.py +1 -1
- openstef/model/regressors/xgb.py +0 -1
- openstef/model/serializer.py +3 -8
- openstef/monitoring/teams.py +2 -8
- openstef/pipeline/create_basecase_forecast.py +2 -10
- openstef/pipeline/create_component_forecast.py +2 -10
- openstef/pipeline/create_forecast.py +2 -9
- openstef/pipeline/optimize_hyperparameters.py +2 -9
- openstef/pipeline/train_model.py +4 -11
- openstef/plotting/load_forecast_plotter.py +3 -3
- openstef/postprocessing/postprocessing.py +2 -9
- openstef/tasks/calculate_kpi.py +3 -10
- openstef/tasks/create_basecase_forecast.py +1 -1
- openstef/tasks/create_components_forecast.py +3 -10
- openstef/tasks/create_forecast.py +1 -1
- openstef/tasks/create_solar_forecast.py +1 -1
- openstef/tasks/optimize_hyperparameters.py +1 -1
- openstef/tasks/split_forecast.py +3 -10
- openstef/tasks/utils/taskcontext.py +2 -10
- openstef/validation/validation.py +7 -29
- {openstef-3.4.67.dist-info → openstef-3.4.69.dist-info}/METADATA +1 -1
- {openstef-3.4.67.dist-info → openstef-3.4.69.dist-info}/RECORD +46 -40
- {openstef-3.4.67.dist-info → openstef-3.4.69.dist-info}/WHEEL +0 -0
- {openstef-3.4.67.dist-info → openstef-3.4.69.dist-info}/licenses/LICENSE +0 -0
- {openstef-3.4.67.dist-info → openstef-3.4.69.dist-info}/top_level.txt +0 -0
@@ -2,19 +2,16 @@
|
|
2
2
|
#
|
3
3
|
# SPDX-License-Identifier: MPL-2.0
|
4
4
|
|
5
|
-
import logging
|
6
|
-
|
7
5
|
import joblib
|
8
6
|
import numpy as np
|
9
7
|
import pandas as pd
|
10
|
-
import structlog
|
11
8
|
|
12
9
|
import openstef.postprocessing.postprocessing as postprocessing
|
13
10
|
from openstef import PROJECT_ROOT
|
14
11
|
from openstef.data_classes.prediction_job import PredictionJobDataClass
|
15
12
|
from openstef.enums import ForecastType
|
13
|
+
from openstef.logging.logger_factory import get_logger
|
16
14
|
from openstef.model.regressors.dazls import Dazls
|
17
|
-
from openstef.settings import Settings
|
18
15
|
|
19
16
|
# Set the path for the Dazls stored model
|
20
17
|
DAZLS_STORED = str(
|
@@ -98,12 +95,7 @@ def create_components_forecast_pipeline(
|
|
98
95
|
"algtype"
|
99
96
|
|
100
97
|
"""
|
101
|
-
|
102
|
-
wrapper_class=structlog.make_filtering_bound_logger(
|
103
|
-
logging.getLevelName(Settings.log_level)
|
104
|
-
)
|
105
|
-
)
|
106
|
-
logger = structlog.get_logger(__name__)
|
98
|
+
logger = get_logger(__name__)
|
107
99
|
logger.info("Make components prediction", pid=pj["id"])
|
108
100
|
|
109
101
|
# Make component forecasts
|
@@ -1,16 +1,15 @@
|
|
1
1
|
# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
|
2
2
|
#
|
3
3
|
# SPDX-License-Identifier: MPL-2.0
|
4
|
-
import logging
|
5
4
|
|
6
5
|
import pandas as pd
|
7
|
-
import structlog
|
8
6
|
|
9
7
|
from openstef.data_classes.model_specifications import ModelSpecificationDataClass
|
10
8
|
from openstef.data_classes.prediction_job import PredictionJobDataClass
|
11
9
|
from openstef.feature_engineering.feature_applicator import (
|
12
10
|
OperationalPredictFeatureApplicator,
|
13
11
|
)
|
12
|
+
from openstef.logging.logger_factory import get_logger
|
14
13
|
from openstef.model.confidence_interval_applicator import ConfidenceIntervalApplicator
|
15
14
|
from openstef.model.fallback import generate_fallback
|
16
15
|
from openstef.model.regressors.regressor import OpenstfRegressor
|
@@ -20,7 +19,6 @@ from openstef.postprocessing.postprocessing import (
|
|
20
19
|
add_prediction_job_properties_to_forecast,
|
21
20
|
sort_quantiles,
|
22
21
|
)
|
23
|
-
from openstef.settings import Settings
|
24
22
|
from openstef.validation import validation
|
25
23
|
|
26
24
|
|
@@ -88,12 +86,7 @@ def create_forecast_pipeline_core(
|
|
88
86
|
InputDataOngoingFlatlinerError: When all recent load measurements are constant.
|
89
87
|
|
90
88
|
"""
|
91
|
-
|
92
|
-
wrapper_class=structlog.make_filtering_bound_logger(
|
93
|
-
logging.getLevelName(Settings.log_level)
|
94
|
-
)
|
95
|
-
)
|
96
|
-
logger = structlog.get_logger(__name__)
|
89
|
+
logger = get_logger(__name__)
|
97
90
|
|
98
91
|
fallback_strategy = "extreme_day" # this can later be expanded
|
99
92
|
|
@@ -1,13 +1,11 @@
|
|
1
1
|
# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
|
2
2
|
#
|
3
3
|
# SPDX-License-Identifier: MPL-2.0
|
4
|
-
import logging
|
5
4
|
import os
|
6
5
|
from typing import Any
|
7
6
|
|
8
7
|
import optuna
|
9
8
|
import pandas as pd
|
10
|
-
import structlog
|
11
9
|
|
12
10
|
from openstef.data_classes.model_specifications import ModelSpecificationDataClass
|
13
11
|
from openstef.data_classes.prediction_job import PredictionJobDataClass
|
@@ -16,6 +14,7 @@ from openstef.exceptions import (
|
|
16
14
|
InputDataWrongColumnOrderError,
|
17
15
|
)
|
18
16
|
from openstef.feature_engineering.feature_applicator import TrainFeatureApplicator
|
17
|
+
from openstef.logging.logger_factory import get_logger
|
19
18
|
from openstef.metrics.reporter import Report, Reporter
|
20
19
|
from openstef.model.model_creator import ModelCreator
|
21
20
|
from openstef.model.objective import RegressorObjective
|
@@ -27,18 +26,12 @@ from openstef.pipeline.train_model import (
|
|
27
26
|
DEFAULT_TRAIN_HORIZONS_HOURS,
|
28
27
|
train_model_pipeline_core,
|
29
28
|
)
|
30
|
-
from openstef.settings import Settings
|
31
29
|
from openstef.validation import validation
|
32
30
|
|
33
31
|
optuna.logging.enable_propagation() # Propagate logs to the root logger.
|
34
32
|
optuna.logging.disable_default_handler() # Stop showing logs in sys.stderr.
|
35
33
|
|
36
|
-
|
37
|
-
wrapper_class=structlog.make_filtering_bound_logger(
|
38
|
-
logging.getLevelName(Settings.log_level)
|
39
|
-
)
|
40
|
-
)
|
41
|
-
logger = structlog.get_logger(__name__)
|
34
|
+
logger = get_logger(__name__)
|
42
35
|
|
43
36
|
# See https://optuna.readthedocs.io/en/stable/reference/generated/optuna.study.Study.html#optuna.study.Study.optimize
|
44
37
|
N_TRIALS: int = 100 # The number of trials.
|
openstef/pipeline/train_model.py
CHANGED
@@ -1,12 +1,10 @@
|
|
1
1
|
# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
|
2
2
|
#
|
3
3
|
# SPDX-License-Identifier: MPL-2.0
|
4
|
-
import logging
|
5
4
|
import os
|
6
5
|
from typing import Optional, Tuple, Union
|
7
6
|
|
8
7
|
import pandas as pd
|
9
|
-
import structlog
|
10
8
|
|
11
9
|
from openstef.data_classes.model_specifications import ModelSpecificationDataClass
|
12
10
|
from openstef.data_classes.prediction_job import PredictionJobDataClass
|
@@ -17,13 +15,13 @@ from openstef.exceptions import (
|
|
17
15
|
SkipSaveTrainingForecasts,
|
18
16
|
)
|
19
17
|
from openstef.feature_engineering.feature_applicator import TrainFeatureApplicator
|
18
|
+
from openstef.logging.logger_factory import get_logger
|
20
19
|
from openstef.metrics.reporter import Report, Reporter
|
21
20
|
from openstef.model.model_creator import ModelCreator
|
22
21
|
from openstef.model.regressors.regressor import OpenstfRegressor
|
23
22
|
from openstef.model.serializer import MLflowSerializer
|
24
23
|
from openstef.model.standard_deviation_generator import StandardDeviationGenerator
|
25
24
|
from openstef.model_selection.model_selection import split_data_train_validation_test
|
26
|
-
from openstef.settings import Settings
|
27
25
|
from openstef.validation import validation
|
28
26
|
|
29
27
|
DEFAULT_TRAIN_HORIZONS_HOURS: list[float] = [0.25, 47.0]
|
@@ -32,12 +30,7 @@ MAXIMUM_MODEL_AGE: int = 7
|
|
32
30
|
DEFAULT_EARLY_STOPPING_ROUNDS: int = 10
|
33
31
|
PENALTY_FACTOR_OLD_MODEL: float = 1.2
|
34
32
|
|
35
|
-
|
36
|
-
wrapper_class=structlog.make_filtering_bound_logger(
|
37
|
-
logging.getLevelName(Settings.log_level)
|
38
|
-
)
|
39
|
-
)
|
40
|
-
logger = structlog.get_logger(__name__)
|
33
|
+
logger = get_logger(__name__)
|
41
34
|
|
42
35
|
|
43
36
|
def train_model_pipeline(
|
@@ -495,7 +488,7 @@ def train_pipeline_step_train_model(
|
|
495
488
|
# Temporary fix to allow xgboost version upgrade -> set n_estimators if present and None
|
496
489
|
if not valid_hyper_parameters.get("n_estimators", True):
|
497
490
|
valid_hyper_parameters.update(dict(n_estimators=100))
|
498
|
-
|
491
|
+
logger.info("Deprecation warning: n_estimators=None found, overwriting.")
|
499
492
|
|
500
493
|
model.set_params(**valid_hyper_parameters)
|
501
494
|
model.fit(
|
@@ -507,7 +500,7 @@ def train_pipeline_step_train_model(
|
|
507
500
|
# Gets the feature importance df or None if we don't have feature importance
|
508
501
|
model.feature_importance_dataframe = model.set_feature_importance()
|
509
502
|
|
510
|
-
|
503
|
+
logger.info("Fitted a new model, not yet stored")
|
511
504
|
|
512
505
|
# Do confidence interval determination
|
513
506
|
model = StandardDeviationGenerator(
|
@@ -2,13 +2,13 @@
|
|
2
2
|
#
|
3
3
|
# SPDX-License-Identifier: MPL-2.0
|
4
4
|
|
5
|
-
from typing import
|
5
|
+
from typing import Optional, Tuple
|
6
6
|
|
7
7
|
import numpy as np
|
8
|
-
from pydantic import BaseModel
|
9
8
|
import pandas as pd
|
10
|
-
import plotly.graph_objects as go
|
11
9
|
import plotly.express as px
|
10
|
+
import plotly.graph_objects as go
|
11
|
+
from pydantic import BaseModel
|
12
12
|
|
13
13
|
|
14
14
|
class LoadForecastPlotter(BaseModel):
|
@@ -1,17 +1,15 @@
|
|
1
1
|
# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
|
2
2
|
#
|
3
3
|
# SPDX-License-Identifier: MPL-2.0
|
4
|
-
import logging
|
5
4
|
from enum import Enum
|
6
5
|
|
7
6
|
import numpy as np
|
8
7
|
import pandas as pd
|
9
|
-
import structlog
|
10
8
|
|
11
9
|
from openstef.data_classes.prediction_job import PredictionJobDataClass
|
12
10
|
from openstef.enums import ForecastType
|
13
11
|
from openstef.feature_engineering import weather_features
|
14
|
-
from openstef.
|
12
|
+
from openstef.logging.logger_factory import get_logger
|
15
13
|
|
16
14
|
# this is the default for "Lagerwey100"
|
17
15
|
TURBINE_DATA = {
|
@@ -221,12 +219,7 @@ def add_prediction_job_properties_to_forecast(
|
|
221
219
|
Dataframe with added metadata.
|
222
220
|
|
223
221
|
"""
|
224
|
-
|
225
|
-
wrapper_class=structlog.make_filtering_bound_logger(
|
226
|
-
logging.getLevelName(Settings.log_level)
|
227
|
-
)
|
228
|
-
)
|
229
|
-
logger = structlog.get_logger(__name__)
|
222
|
+
logger = get_logger(__name__)
|
230
223
|
|
231
224
|
logger.info("Postproces in preparation of storing")
|
232
225
|
if forecast_type is None:
|
openstef/tasks/calculate_kpi.py
CHANGED
@@ -18,21 +18,19 @@ Example:
|
|
18
18
|
$ python calculate_kpi.py
|
19
19
|
|
20
20
|
"""
|
21
|
-
import logging
|
22
21
|
|
23
22
|
# Import builtins
|
24
|
-
from datetime import datetime, timedelta
|
23
|
+
from datetime import UTC, datetime, timedelta
|
25
24
|
from pathlib import Path
|
26
25
|
|
27
26
|
import numpy as np
|
28
27
|
import pandas as pd
|
29
|
-
import structlog
|
30
28
|
|
31
29
|
from openstef.data_classes.prediction_job import PredictionJobDataClass
|
32
30
|
from openstef.enums import ModelType
|
33
31
|
from openstef.exceptions import NoPredictedLoadError, NoRealisedLoadError
|
32
|
+
from openstef.logging.logger_factory import get_logger
|
34
33
|
from openstef.metrics import metrics
|
35
|
-
from openstef.settings import Settings
|
36
34
|
from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
|
37
35
|
from openstef.tasks.utils.taskcontext import TaskContext
|
38
36
|
from openstef.validation import validation
|
@@ -158,12 +156,7 @@ def calc_kpi_for_specific_pid(
|
|
158
156
|
COMPLETENESS_REALISED_THRESHOLDS = 0.7
|
159
157
|
COMPLETENESS_PREDICTED_LOAD_THRESHOLD = 0.7
|
160
158
|
|
161
|
-
|
162
|
-
wrapper_class=structlog.make_filtering_bound_logger(
|
163
|
-
logging.getLevelName(Settings.log_level)
|
164
|
-
)
|
165
|
-
)
|
166
|
-
logger = structlog.get_logger(__name__)
|
159
|
+
logger = get_logger(__name__)
|
167
160
|
|
168
161
|
# If predicted is empty
|
169
162
|
if len(predicted_load) == 0:
|
@@ -21,20 +21,18 @@ Example:
|
|
21
21
|
$ python create_components_forecast.py
|
22
22
|
|
23
23
|
"""
|
24
|
-
import
|
25
|
-
from datetime import datetime, timedelta, UTC
|
24
|
+
from datetime import UTC, datetime, timedelta
|
26
25
|
from pathlib import Path
|
27
26
|
|
28
27
|
import pandas as pd
|
29
|
-
import structlog
|
30
28
|
|
31
29
|
from openstef.data_classes.prediction_job import PredictionJobDataClass
|
32
30
|
from openstef.enums import ModelType
|
33
31
|
from openstef.exceptions import ComponentForecastTooShortHorizonError
|
32
|
+
from openstef.logging.logger_factory import get_logger
|
34
33
|
from openstef.pipeline.create_component_forecast import (
|
35
34
|
create_components_forecast_pipeline,
|
36
35
|
)
|
37
|
-
from openstef.settings import Settings
|
38
36
|
from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
|
39
37
|
from openstef.tasks.utils.taskcontext import TaskContext
|
40
38
|
|
@@ -63,12 +61,7 @@ def create_components_forecast_task(
|
|
63
61
|
(less than 30 minutes in advance)
|
64
62
|
|
65
63
|
"""
|
66
|
-
|
67
|
-
wrapper_class=structlog.make_filtering_bound_logger(
|
68
|
-
logging.getLevelName(Settings.log_level)
|
69
|
-
)
|
70
|
-
)
|
71
|
-
logger = structlog.get_logger(__name__)
|
64
|
+
logger = get_logger(__name__)
|
72
65
|
if pj["train_components"] == 0:
|
73
66
|
context.logger.info(
|
74
67
|
"Skip prediction job", train_components=pj["train_components"]
|
@@ -16,7 +16,7 @@ Example:
|
|
16
16
|
$ python optimize_hyperparameters.py
|
17
17
|
|
18
18
|
"""
|
19
|
-
from datetime import datetime, timedelta
|
19
|
+
from datetime import UTC, datetime, timedelta
|
20
20
|
from pathlib import Path
|
21
21
|
|
22
22
|
from openstef.data_classes.prediction_job import PredictionJobDataClass
|
openstef/tasks/split_forecast.py
CHANGED
@@ -22,19 +22,17 @@ Example:
|
|
22
22
|
$ python split_forecast.py
|
23
23
|
|
24
24
|
"""
|
25
|
-
import
|
26
|
-
from datetime import datetime, UTC
|
25
|
+
from datetime import UTC, datetime
|
27
26
|
from pathlib import Path
|
28
27
|
|
29
28
|
import numpy as np
|
30
29
|
import pandas as pd
|
31
30
|
import scipy.optimize
|
32
|
-
import structlog
|
33
31
|
|
34
32
|
import openstef.monitoring.teams as monitoring
|
35
33
|
from openstef.data_classes.prediction_job import PredictionJobDataClass
|
36
34
|
from openstef.enums import ModelType
|
37
|
-
from openstef.
|
35
|
+
from openstef.logging.logger_factory import get_logger
|
38
36
|
from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
|
39
37
|
from openstef.tasks.utils.taskcontext import TaskContext
|
40
38
|
|
@@ -72,12 +70,7 @@ def split_forecast_task(
|
|
72
70
|
Energy splitting coefficients.
|
73
71
|
|
74
72
|
"""
|
75
|
-
|
76
|
-
wrapper_class=structlog.make_filtering_bound_logger(
|
77
|
-
logging.getLevelName(Settings.log_level)
|
78
|
-
)
|
79
|
-
)
|
80
|
-
logger = structlog.get_logger(__name__)
|
73
|
+
logger = get_logger(__name__)
|
81
74
|
|
82
75
|
logger.info("Start splitting energy", pid=pj["id"])
|
83
76
|
|
@@ -1,16 +1,13 @@
|
|
1
1
|
# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
|
2
2
|
#
|
3
3
|
# SPDX-License-Identifier: MPL-2.0
|
4
|
-
import logging
|
5
4
|
import traceback
|
6
5
|
from typing import Callable
|
7
6
|
|
8
|
-
import structlog
|
9
|
-
|
10
7
|
from openstef.exceptions import PredictionJobException
|
8
|
+
from openstef.logging.logger_factory import get_logger
|
11
9
|
from openstef.monitoring.performance_meter import PerformanceMeter
|
12
10
|
from openstef.monitoring.teams import post_teams
|
13
|
-
from openstef.settings import Settings
|
14
11
|
|
15
12
|
|
16
13
|
class TaskContext:
|
@@ -64,12 +61,7 @@ class TaskContext:
|
|
64
61
|
self.database = database
|
65
62
|
|
66
63
|
def __enter__(self):
|
67
|
-
|
68
|
-
wrapper_class=structlog.make_filtering_bound_logger(
|
69
|
-
logging.getLevelName(Settings.log_level)
|
70
|
-
)
|
71
|
-
)
|
72
|
-
self.logger = structlog.get_logger(__name__).bind(task=self.name)
|
64
|
+
self.logger = get_logger(__name__)
|
73
65
|
|
74
66
|
self.perf_meter = PerformanceMeter(self.logger)
|
75
67
|
|
@@ -1,20 +1,18 @@
|
|
1
1
|
# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
|
2
2
|
#
|
3
3
|
# SPDX-License-Identifier: MPL-2.0
|
4
|
-
import logging
|
5
4
|
import math
|
6
|
-
from datetime import datetime, timedelta
|
5
|
+
from datetime import UTC, datetime, timedelta
|
7
6
|
from typing import Union
|
8
7
|
|
9
8
|
import numpy as np
|
10
9
|
import pandas as pd
|
11
|
-
import structlog
|
12
10
|
|
13
11
|
from openstef.data_classes.prediction_job import PredictionJobDataClass
|
14
12
|
from openstef.exceptions import InputDataOngoingFlatlinerError
|
13
|
+
from openstef.logging.logger_factory import get_logger
|
15
14
|
from openstef.model.regressors.regressor import OpenstfRegressor
|
16
15
|
from openstef.preprocessing.preprocessing import replace_repeated_values_with_nan
|
17
|
-
from openstef.settings import Settings
|
18
16
|
|
19
17
|
|
20
18
|
def validate(
|
@@ -49,12 +47,7 @@ def validate(
|
|
49
47
|
InputDataOngoingFlatlinerError: If all recent load measurements are constant.
|
50
48
|
|
51
49
|
"""
|
52
|
-
|
53
|
-
wrapper_class=structlog.make_filtering_bound_logger(
|
54
|
-
logging.getLevelName(Settings.log_level)
|
55
|
-
)
|
56
|
-
)
|
57
|
-
logger = structlog.get_logger(__name__)
|
50
|
+
logger = get_logger(__name__)
|
58
51
|
|
59
52
|
if not isinstance(data.index, pd.DatetimeIndex):
|
60
53
|
raise ValueError("Input dataframe does not have a datetime index.")
|
@@ -99,12 +92,7 @@ def validate(
|
|
99
92
|
|
100
93
|
|
101
94
|
def drop_target_na(data: pd.DataFrame) -> pd.DataFrame:
|
102
|
-
|
103
|
-
wrapper_class=structlog.make_filtering_bound_logger(
|
104
|
-
logging.getLevelName(Settings.log_level)
|
105
|
-
)
|
106
|
-
)
|
107
|
-
logger = structlog.get_logger(__name__)
|
95
|
+
logger = get_logger(__name__)
|
108
96
|
len_original = len(data)
|
109
97
|
# Remove where load is NA, NaN features are preserved
|
110
98
|
data = data.loc[np.isnan(data.iloc[:, 0]) != True, :] # noqa E712
|
@@ -142,12 +130,7 @@ def is_data_sufficient(
|
|
142
130
|
else:
|
143
131
|
weights = model.feature_importance_dataframe
|
144
132
|
|
145
|
-
|
146
|
-
wrapper_class=structlog.make_filtering_bound_logger(
|
147
|
-
logging.getLevelName(Settings.log_level)
|
148
|
-
)
|
149
|
-
)
|
150
|
-
logger = structlog.get_logger(__name__)
|
133
|
+
logger = get_logger(__name__)
|
151
134
|
# Set output variable
|
152
135
|
is_sufficient = True
|
153
136
|
|
@@ -294,18 +277,13 @@ def calc_completeness_dataframe(
|
|
294
277
|
Dataframe with fraction of completeness per column
|
295
278
|
|
296
279
|
"""
|
297
|
-
|
298
|
-
wrapper_class=structlog.make_filtering_bound_logger(
|
299
|
-
logging.getLevelName(Settings.log_level)
|
300
|
-
)
|
301
|
-
)
|
302
|
-
logger = structlog.get_logger(__name__)
|
280
|
+
logger = get_logger(__name__)
|
303
281
|
|
304
282
|
if homogenise and isinstance(df.index, pd.DatetimeIndex) and len(df) > 0:
|
305
283
|
median_timediff = int(
|
306
284
|
df.reset_index().iloc[:, 0].diff().median().total_seconds() / 60.0
|
307
285
|
)
|
308
|
-
df = df.resample("{:d}
|
286
|
+
df = df.resample("{:d}min".format(median_timediff)).mean()
|
309
287
|
|
310
288
|
if time_delayed is False:
|
311
289
|
# Calculate completeness
|
@@ -1,6 +1,6 @@
|
|
1
1
|
openstef/__init__.py,sha256=93UM6m0LLQhO69-mSqLuUy73jgs4W7Iuxfo3Lm8c98g,419
|
2
2
|
openstef/__main__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
3
|
-
openstef/app_settings.py,sha256=
|
3
|
+
openstef/app_settings.py,sha256=4SiEoPfPmM_Vc9gwWleT_IRxHIk36bZaYRCdNf3ewoo,762
|
4
4
|
openstef/enums.py,sha256=FrP0m_Tk0kV7gSZ2hTY_8iD45KIKnexHrjNufhpKXpE,2829
|
5
5
|
openstef/exceptions.py,sha256=dgnvZe5WWuJWCZm_GES6suEATbusPlwbiEUfNQKeExY,1993
|
6
6
|
openstef/settings.py,sha256=nSgkBqFxuqB3w7Rwo60i8j37c5ngDbt6vpjHS6QtJXQ,354
|
@@ -20,33 +20,39 @@ openstef/data_classes/model_specifications.py,sha256=PZeBLfH_MrP9-QorL1r0Hklp0be
|
|
20
20
|
openstef/data_classes/prediction_job.py,sha256=e6_PFAovNd31tjzoTQJvqRNQyVM-M0XHffclAG9Ez8A,6721
|
21
21
|
openstef/data_classes/split_function.py,sha256=K8y1dsQC5exeIDh37f7UwJ11tV71_uVSNbnKmwXpnOM,3435
|
22
22
|
openstef/feature_engineering/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
23
|
-
openstef/feature_engineering/apply_features.py,sha256=
|
23
|
+
openstef/feature_engineering/apply_features.py,sha256=pro4eUmOFexX_9g9kJtDcbrQ1hWKzXjVpiJBmmBi89o,5326
|
24
24
|
openstef/feature_engineering/bidding_zone_to_country_mapping.py,sha256=u9aabjFDImydkO6_cXiaQxBT4gb5zy0gGTg2EoIUO_Y,2106
|
25
|
-
openstef/feature_engineering/cyclic_features.py,sha256=
|
26
|
-
openstef/feature_engineering/data_preparation.py,sha256=
|
25
|
+
openstef/feature_engineering/cyclic_features.py,sha256=ris-pYn26kJxR0T0KFl4gTDX5k4xRAiPKdssCdtwLjQ,5799
|
26
|
+
openstef/feature_engineering/data_preparation.py,sha256=TXAPTtSmBRC_LZP7o5Jlmj7Ju2mfGUQH7VXg9eNG60o,5439
|
27
27
|
openstef/feature_engineering/feature_adder.py,sha256=aSqDl_gUrB3H2TD3cNvU5JniY_KOb4u4a2A6J7zB2BQ,6835
|
28
28
|
openstef/feature_engineering/feature_applicator.py,sha256=bU1Pu5V1fxMCQCwh6HG66nmctBjrNa7gHUYqOqPmLTU,7501
|
29
|
-
openstef/feature_engineering/general.py,sha256=
|
29
|
+
openstef/feature_engineering/general.py,sha256=PdvnDqkze31FggUuWHQ1ysroh_uDOa1hZ7NftMYH2_U,4130
|
30
30
|
openstef/feature_engineering/holiday_features.py,sha256=CbolIP5bfiQkqDct-9TbD828-lhC48bfeNQ2-VFnsJA,8274
|
31
31
|
openstef/feature_engineering/lag_features.py,sha256=Dr6qS8UhdgEHPZZSe-w6ibtjl_lcbcQohhqdZN9fqEU,5652
|
32
|
-
openstef/feature_engineering/missing_values_transformer.py,sha256=
|
33
|
-
openstef/feature_engineering/rolling_features.py,sha256=
|
34
|
-
openstef/feature_engineering/weather_features.py,sha256=
|
32
|
+
openstef/feature_engineering/missing_values_transformer.py,sha256=U8pdA61k8CRosO3yR2IsCy5C4Ka3c8BWCimDLIB4LCQ,5010
|
33
|
+
openstef/feature_engineering/rolling_features.py,sha256=V-UulqWKuSksFQAASyVSQim1stEA4TmtHNULCrrdgjo,2160
|
34
|
+
openstef/feature_engineering/weather_features.py,sha256=3BIE_DDKJdef1a_FVXO81dfhDeUKsX0P1vBK6RBn6j0,15547
|
35
|
+
openstef/logging/__init__.py,sha256=KQjXzyafCt1bE7XDrSeV4TDUIO7MkwN_Br4ASOcNI2g,163
|
36
|
+
openstef/logging/base_logger.py,sha256=CQF0uJ3lQ9n7seAaL0r3Pulg3fS18sGKsHlAaaYBq-0,839
|
37
|
+
openstef/logging/logger_factory.py,sha256=_6HF0MHfwj90FNpHP58nUySbKsaKyWgvEEn3pZAJOqg,684
|
38
|
+
openstef/logging/logger_types.py,sha256=wnPKw_uvpZFD59RGbFt_6LXQkRDr9tirqSydOf-Ou9Q,271
|
39
|
+
openstef/logging/standard_logger.py,sha256=BoXFzb9yhARz0l6P_QIwSx9Fgs2-vPK5M1h1GwltMgk,1027
|
40
|
+
openstef/logging/structlog_logger.py,sha256=nCxDLizanKLLwUcQEs-3x0JG-Pj6VKJJSpLHsdzU3Oo,1123
|
35
41
|
openstef/metrics/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
36
42
|
openstef/metrics/figure.py,sha256=KDoezYem9wdS13kUx7M7FOy-4u88Sg3OX1DuhNT6kgQ,9751
|
37
43
|
openstef/metrics/metrics.py,sha256=qV_EdzjKNiqEGKYUp4DL0KgsnCjTf4P9FqKcccFNF-o,15515
|
38
|
-
openstef/metrics/reporter.py,sha256=
|
44
|
+
openstef/metrics/reporter.py,sha256=2F1uRmh2MC-JH8Lsr1xGLxUFYDGfQ0Q85Pcjcc31TP0,7696
|
39
45
|
openstef/model/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
40
46
|
openstef/model/basecase.py,sha256=caI6Q-8y0ymlxGK9Js_H3Vh0q6ruNHlGD5RG0_kE5M0,2878
|
41
|
-
openstef/model/confidence_interval_applicator.py,sha256=
|
42
|
-
openstef/model/fallback.py,sha256=
|
43
|
-
openstef/model/model_creator.py,sha256=
|
47
|
+
openstef/model/confidence_interval_applicator.py,sha256=VOdHsDJhfeyaq_cnk9QMUaZ2IumbiBBoW1zo8AuqDg0,9559
|
48
|
+
openstef/model/fallback.py,sha256=g6TEuEhV4w07SCGkR_AvPf2up9f0ixGKQojYC-Ewl6Y,2812
|
49
|
+
openstef/model/model_creator.py,sha256=lbWtarr9_PlMfG9u9Qbnzyw5GNcYIExt4-V3PevTRn8,6506
|
44
50
|
openstef/model/objective.py,sha256=qJdI6GAzv8l5Mxd8G7BIqQnfdJNM7aOlg9DMzMGjWqA,14558
|
45
|
-
openstef/model/objective_creator.py,sha256=
|
46
|
-
openstef/model/serializer.py,sha256=
|
51
|
+
openstef/model/objective_creator.py,sha256=3jJgcmY1sm-Yoe3SfjKrJukrsqtYyloUFaPbBWqswhQ,2208
|
52
|
+
openstef/model/serializer.py,sha256=k5GY8eRJdlii8mEY7Qheu4yb5USyIyxw77EYkSQJGYk,17034
|
47
53
|
openstef/model/standard_deviation_generator.py,sha256=OorRvX2wRScU7f4SIBoiT24yJeeM50sETP3xC6m5IG4,2865
|
48
54
|
openstef/model/metamodels/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
49
|
-
openstef/model/metamodels/feature_clipper.py,sha256=
|
55
|
+
openstef/model/metamodels/feature_clipper.py,sha256=DNsyYdjUT7ZNimJJIyTvv1nmwTwDUk5fX9EDgV9FbUQ,2862
|
50
56
|
openstef/model/metamodels/grouped_regressor.py,sha256=yMN_a6TnQSyFaqlB_6Nifq-ydpb5hs6w_b97IaBbHj4,8337
|
51
57
|
openstef/model/metamodels/missing_values_handler.py,sha256=glgAlkeubLZFWbD8trTYBik7_qOJi4GCPGl1sSybSkQ,5257
|
52
58
|
openstef/model/regressors/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
@@ -54,51 +60,51 @@ openstef/model/regressors/arima.py,sha256=wt7FVykjSvljpl7vjtliq61SiyjQ7KKtw8PF9x
|
|
54
60
|
openstef/model/regressors/custom_regressor.py,sha256=T4JdJ-oTTt1PHQV0DdIEIhALvEEh07WCFlWxl8EFih0,1765
|
55
61
|
openstef/model/regressors/dazls.py,sha256=Xt89yFHjkwpIUTkkhPmPZ74F8_tht_XV88INuP5GU2E,3994
|
56
62
|
openstef/model/regressors/flatliner.py,sha256=T9u-ukhqFcatQmlgUtBL_G-1b_wQzgdVRq0ac64GnjQ,2789
|
57
|
-
openstef/model/regressors/gblinear_quantile.py,sha256=
|
63
|
+
openstef/model/regressors/gblinear_quantile.py,sha256=PKQL_TAXa3Kw9oZrKC6Uvo_n2NSAPvAHsst9dwr7gd4,11263
|
58
64
|
openstef/model/regressors/lgbm.py,sha256=zCdn1euEdSFxYJzH8XqQFFnb6R4JVUnmineKjX_Gy-g,800
|
59
65
|
openstef/model/regressors/linear.py,sha256=uOvZMLGZH_9nXfmS5honCMfyVeyGXP1Cza9A_BdXlVw,3665
|
60
|
-
openstef/model/regressors/linear_quantile.py,sha256=
|
66
|
+
openstef/model/regressors/linear_quantile.py,sha256=zIpGo9deMeTZdwFWoZ3FstX74mYdlAhfg-YOsPRFl0k,10534
|
61
67
|
openstef/model/regressors/regressor.py,sha256=uJcx59AyCPE9f_yPcAQ59h2ZS7eNsDpIHJrladKvHIw,3461
|
62
|
-
openstef/model/regressors/xgb.py,sha256=
|
68
|
+
openstef/model/regressors/xgb.py,sha256=mYQpfVVQQbmNA-esDr_PGbYXVqdEzHdkHIJ0CoUyA2c,1466
|
63
69
|
openstef/model/regressors/xgb_multioutput_quantile.py,sha256=xWzA7tymC_o-F1OS3I7vUKf9zP6RR1ZglEeY4NAgjU0,9146
|
64
70
|
openstef/model/regressors/xgb_quantile.py,sha256=PzKIxqN_CnEPFmzXACNuzLSmZSHbooTuiJ5ckJ9vh_E,7805
|
65
71
|
openstef/model_selection/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
66
72
|
openstef/model_selection/model_selection.py,sha256=ZTykej6aL5TY2oZ5XTZc7fTrTNrgxAUDRqu_rKhIyYg,11233
|
67
73
|
openstef/monitoring/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
68
74
|
openstef/monitoring/performance_meter.py,sha256=6aCGjJFXFq-7qwaJyBkF3MLqjgVK6FMFVcO-bcLLUb4,2803
|
69
|
-
openstef/monitoring/teams.py,sha256=
|
75
|
+
openstef/monitoring/teams.py,sha256=klN7Ge-0VktJbZ_I-K8MJIc3LWgdNy0MGL8b2TdoUR8,6526
|
70
76
|
openstef/pipeline/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
71
|
-
openstef/pipeline/create_basecase_forecast.py,sha256=
|
72
|
-
openstef/pipeline/create_component_forecast.py,sha256=
|
73
|
-
openstef/pipeline/create_forecast.py,sha256=
|
74
|
-
openstef/pipeline/optimize_hyperparameters.py,sha256=
|
77
|
+
openstef/pipeline/create_basecase_forecast.py,sha256=7IShIjEmjkzpNzWzQVKmYQvy0q_uwCGO-E0mSRmGdhw,4397
|
78
|
+
openstef/pipeline/create_component_forecast.py,sha256=40fYKajdj4F9K7fzmL3euyvwTr0v-oO_5cXpya87A0c,5839
|
79
|
+
openstef/pipeline/create_forecast.py,sha256=mVbbu7jM31NEwfaDeQPqF3Okps9H1oLfjhPPiJRL4zg,5582
|
80
|
+
openstef/pipeline/optimize_hyperparameters.py,sha256=w5LpZhW3KVklCJzaogNzyHfpMJfNqeRAnvyV4vi35wg,10953
|
75
81
|
openstef/pipeline/train_create_forecast_backtest.py,sha256=hBJPxfDkbrmFSSGZrRH1vTiIVqJP-SWe0ibVpHT_8Qg,6048
|
76
|
-
openstef/pipeline/train_model.py,sha256=
|
82
|
+
openstef/pipeline/train_model.py,sha256=4ooyPve2uBvL6sWzdLX47kiMpYmaXH7agc4qdTEXtSg,19971
|
77
83
|
openstef/pipeline/utils.py,sha256=23mB31p19FoGWelLJzxNmqlzGwEr3fCDBEA37V2kpYY,2167
|
78
84
|
openstef/plotting/__init__.py,sha256=KQjXzyafCt1bE7XDrSeV4TDUIO7MkwN_Br4ASOcNI2g,163
|
79
|
-
openstef/plotting/load_forecast_plotter.py,sha256=
|
85
|
+
openstef/plotting/load_forecast_plotter.py,sha256=GWHVmUB2YosNj7TnSrMnxYAfM2Z1mNg5oRV9A_lJmQY,8129
|
80
86
|
openstef/postprocessing/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
81
|
-
openstef/postprocessing/postprocessing.py,sha256=
|
87
|
+
openstef/postprocessing/postprocessing.py,sha256=lQz642SN7Stul7A95nFbn34dksVAcOPI8ktawyqOzbc,8816
|
82
88
|
openstef/preprocessing/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
83
89
|
openstef/preprocessing/preprocessing.py,sha256=bM_cSSSb2vGTD79RGzUrI6KoELbzlCyJwc7jqQGNEsE,1454
|
84
90
|
openstef/tasks/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
85
|
-
openstef/tasks/calculate_kpi.py,sha256=
|
86
|
-
openstef/tasks/create_basecase_forecast.py,sha256=
|
87
|
-
openstef/tasks/create_components_forecast.py,sha256=
|
88
|
-
openstef/tasks/create_forecast.py,sha256=
|
89
|
-
openstef/tasks/create_solar_forecast.py,sha256=
|
91
|
+
openstef/tasks/calculate_kpi.py,sha256=ncLnpqci_9EUGjaCLbGS5f9l5GjNxjkH-tyQ2R4fkRA,11700
|
92
|
+
openstef/tasks/create_basecase_forecast.py,sha256=K5Fa8AYWYGwF7uv_Ala0RCqEiKMRoFKlA9U65rMGggc,4227
|
93
|
+
openstef/tasks/create_components_forecast.py,sha256=V05KJ5h4OVB6jMyliLtegohGDf6kygn5FPTDT8tQzDs,5959
|
94
|
+
openstef/tasks/create_forecast.py,sha256=n31s2fIq7oP3Rg0lfWv6eoXxxZJrgoQyR9_fIxc9VaA,6075
|
95
|
+
openstef/tasks/create_solar_forecast.py,sha256=Np6-BKbmEiYvkPjQEHprDGzK5gRWqLa2deWT7G54Qd0,15062
|
90
96
|
openstef/tasks/create_wind_forecast.py,sha256=RhshkmNSyFWx4Y6yQn02GzHjWTREbN5A5GAeWv0JpcE,2907
|
91
|
-
openstef/tasks/optimize_hyperparameters.py,sha256=
|
92
|
-
openstef/tasks/split_forecast.py,sha256=
|
97
|
+
openstef/tasks/optimize_hyperparameters.py,sha256=W2Ynkbiw_j7aGk0NmU_g-zIgwdGf7-WTqjDr_0bhSyY,4765
|
98
|
+
openstef/tasks/split_forecast.py,sha256=RG-VW4-jTUEgXZFFo9eA5cW5i_3XtZaQhuMvgi0VyNc,9096
|
93
99
|
openstef/tasks/train_model.py,sha256=-d1VewDAaZV2B_JAnwl02Y3hONq7cPZrpH6X87_IOKA,8772
|
94
100
|
openstef/tasks/utils/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
95
101
|
openstef/tasks/utils/dependencies.py,sha256=Jy9dtV_G7lTEa5Cdy--wvMxJuAb0adb3R0X4QDjVteM,3077
|
96
102
|
openstef/tasks/utils/predictionjobloop.py,sha256=Ysy3zF5lzPMz_asYDKeF5m0qgVT3tCtwSPihqMjnI5Q,9580
|
97
|
-
openstef/tasks/utils/taskcontext.py,sha256=
|
103
|
+
openstef/tasks/utils/taskcontext.py,sha256=O-LZ_wHEl5vbT8oB7EYtOeMkvk6EqCnI1-KiyER7Eu4,5407
|
98
104
|
openstef/validation/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
|
99
|
-
openstef/validation/validation.py,sha256=
|
100
|
-
openstef-3.4.
|
101
|
-
openstef-3.4.
|
102
|
-
openstef-3.4.
|
103
|
-
openstef-3.4.
|
104
|
-
openstef-3.4.
|
105
|
+
openstef/validation/validation.py,sha256=r6UqkdH5TMjsGfn8Ta07K1jkqmrVmwcPGfyQvMmZyO4,11459
|
106
|
+
openstef-3.4.69.dist-info/licenses/LICENSE,sha256=7Pm2fWFFHHUG5lDHed1vl5CjzxObIXQglnYsEdtjo_k,14907
|
107
|
+
openstef-3.4.69.dist-info/METADATA,sha256=6Uhp4aSLz8DGU3MMJJ9EojzQMl_YXMFVYy7tout5Kpg,8838
|
108
|
+
openstef-3.4.69.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
|
109
|
+
openstef-3.4.69.dist-info/top_level.txt,sha256=kD0H4PqrQoncZ957FvqwfBxa89kTrun4Z_RAPs_HhLs,9
|
110
|
+
openstef-3.4.69.dist-info/RECORD,,
|
File without changes
|
File without changes
|