rtc-tools-interface 0.10.2a2__tar.gz → 0.10.2a3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rtc_tools_interface-0.10.2a3/PKG-INFO +21 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/README.md +8 -0
- rtc_tools_interface-0.10.2a3/rtc_tools_interface.egg-info/PKG-INFO +21 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/_version.py +3 -3
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/closed_loop/config.py +4 -4
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/closed_loop/optimization_ranges.py +25 -19
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/closed_loop/results_construction.py +27 -14
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/closed_loop/runner.py +53 -24
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/closed_loop/time_series_handler.py +24 -19
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/optimization/base_goal.py +30 -12
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/optimization/base_optimization_problem.py +1 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/optimization/goal_generator_mixin.py +18 -7
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/optimization/goal_performance_metrics.py +33 -14
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/optimization/goal_table_schema.py +16 -9
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/optimization/helpers/statistics_mixin.py +10 -10
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/optimization/plot_goals_mixin.py +4 -1
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/optimization/plot_mixin.py +14 -5
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/optimization/read_goals.py +31 -13
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/plotting/plot_tools.py +60 -25
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/plotting/subplot_classes.py +65 -31
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/simulation/base_simulation_problem.py +1 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/simulation/plot_mixin.py +14 -7
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/utils/plot_table_schema.py +8 -5
- rtc_tools_interface-0.10.2a3/rtctools_interface/utils/read_goals_mixin.py +55 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/utils/read_plot_table.py +27 -9
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/utils/results_collection.py +39 -20
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/utils/serialization.py +7 -2
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/utils/type_definitions.py +15 -13
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/setup.py +0 -1
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/tests/closed_loop/test_optimization_ranges.py +20 -18
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/tests/closed_loop/test_read_xml.py +1 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/tests/closed_loop/test_run_optization_problem_closed_loop.py +31 -38
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/tests/optimization/test_base_optimization_problem.py +1 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/tests/optimization/test_plot_goals_mixin.py +19 -1
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/tests/optimization/test_read_goals.py +7 -1
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/tests/simulation/test_plot_mixin.py +10 -5
- rtc_tools_interface-0.10.2a2/PKG-INFO +0 -14
- rtc_tools_interface-0.10.2a2/rtc_tools_interface.egg-info/PKG-INFO +0 -14
- rtc_tools_interface-0.10.2a2/rtctools_interface/utils/read_goals_mixin.py +0 -31
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/COPYING.LESSER +0 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtc_tools_interface.egg-info/SOURCES.txt +0 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtc_tools_interface.egg-info/dependency_links.txt +0 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtc_tools_interface.egg-info/requires.txt +0 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtc_tools_interface.egg-info/top_level.txt +0 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/__init__.py +0 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/closed_loop/__init__.py +0 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/optimization/__init__.py +0 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/optimization/helpers/__init__.py +0 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/plotting/__init__.py +0 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/simulation/__init__.py +0 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/utils/__init__.py +0 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/setup.cfg +0 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/tests/__init__.py +0 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/tests/closed_loop/__init__.py +0 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/tests/optimization/__init__.py +0 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/tests/optimization/test_passing_goals_directly.py +1 -1
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/tests/simulation/__init__.py +0 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/tests/simulation/test_base_simulation_problem.py +1 -1
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/tests/utils/__init__.py +0 -0
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/tests/utils/get_test.py +1 -1
- {rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/versioneer.py +0 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: rtc_tools_interface
|
|
3
|
+
Version: 0.10.2a3
|
|
4
|
+
Summary: Toolbox for user interfaces for RTC-Tools
|
|
5
|
+
Author: Deltares
|
|
6
|
+
Maintainer: Deltares
|
|
7
|
+
Requires-Python: >=3.9
|
|
8
|
+
License-File: COPYING.LESSER
|
|
9
|
+
Requires-Dist: matplotlib
|
|
10
|
+
Requires-Dist: numpy
|
|
11
|
+
Requires-Dist: pandas
|
|
12
|
+
Requires-Dist: plotly
|
|
13
|
+
Requires-Dist: pydantic
|
|
14
|
+
Requires-Dist: casadi!=3.6.6
|
|
15
|
+
Requires-Dist: rtc-tools>=2.7.0a3
|
|
16
|
+
Dynamic: author
|
|
17
|
+
Dynamic: license-file
|
|
18
|
+
Dynamic: maintainer
|
|
19
|
+
Dynamic: requires-dist
|
|
20
|
+
Dynamic: requires-python
|
|
21
|
+
Dynamic: summary
|
|
@@ -255,3 +255,11 @@ Note that this happens for:
|
|
|
255
255
|
- Currently, only the initial values of the first time step in a given time range are set.
|
|
256
256
|
- The closed_loop runner only works in combination with the CSVMixin or the PIMixin.
|
|
257
257
|
The CDFMixin is not supported.
|
|
258
|
+
|
|
259
|
+
### Matplotlib in Headless Environments
|
|
260
|
+
|
|
261
|
+
For CI or servers without a display, set `MPLBACKEND=Agg` to ensure Matplotlib uses a non-interactive backend:
|
|
262
|
+
|
|
263
|
+
```bash
|
|
264
|
+
export MPLBACKEND=Agg
|
|
265
|
+
```
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: rtc_tools_interface
|
|
3
|
+
Version: 0.10.2a3
|
|
4
|
+
Summary: Toolbox for user interfaces for RTC-Tools
|
|
5
|
+
Author: Deltares
|
|
6
|
+
Maintainer: Deltares
|
|
7
|
+
Requires-Python: >=3.9
|
|
8
|
+
License-File: COPYING.LESSER
|
|
9
|
+
Requires-Dist: matplotlib
|
|
10
|
+
Requires-Dist: numpy
|
|
11
|
+
Requires-Dist: pandas
|
|
12
|
+
Requires-Dist: plotly
|
|
13
|
+
Requires-Dist: pydantic
|
|
14
|
+
Requires-Dist: casadi!=3.6.6
|
|
15
|
+
Requires-Dist: rtc-tools>=2.7.0a3
|
|
16
|
+
Dynamic: author
|
|
17
|
+
Dynamic: license-file
|
|
18
|
+
Dynamic: maintainer
|
|
19
|
+
Dynamic: requires-dist
|
|
20
|
+
Dynamic: requires-python
|
|
21
|
+
Dynamic: summary
|
{rtc_tools_interface-0.10.2a2 → rtc_tools_interface-0.10.2a3}/rtctools_interface/_version.py
RENAMED
|
@@ -8,11 +8,11 @@ import json
|
|
|
8
8
|
|
|
9
9
|
version_json = '''
|
|
10
10
|
{
|
|
11
|
-
"date": "
|
|
11
|
+
"date": "2026-01-19T15:25:58+0100",
|
|
12
12
|
"dirty": false,
|
|
13
13
|
"error": null,
|
|
14
|
-
"full-revisionid": "
|
|
15
|
-
"version": "0.10.
|
|
14
|
+
"full-revisionid": "b1cd2bed75dbdf57577a1474771953e8983e405d",
|
|
15
|
+
"version": "0.10.2a3"
|
|
16
16
|
}
|
|
17
17
|
''' # END VERSION_JSON
|
|
18
18
|
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
"""Module for configuring a closed-loop optimization problem."""
|
|
2
|
+
|
|
2
3
|
from datetime import timedelta
|
|
3
4
|
from pathlib import Path
|
|
4
|
-
from typing import Optional
|
|
5
5
|
|
|
6
6
|
|
|
7
|
-
class ClosedLoopConfig
|
|
7
|
+
class ClosedLoopConfig:
|
|
8
8
|
"""Configuration of a closed-loop optimization problem."""
|
|
9
9
|
|
|
10
10
|
def __init__(
|
|
@@ -30,8 +30,8 @@ class ClosedLoopConfig():
|
|
|
30
30
|
if file is not None:
|
|
31
31
|
file = Path(file).resolve()
|
|
32
32
|
self._file = file
|
|
33
|
-
self._forecast_timestep:
|
|
34
|
-
self._optimization_period:
|
|
33
|
+
self._forecast_timestep: timedelta | None = None
|
|
34
|
+
self._optimization_period: timedelta | None = None
|
|
35
35
|
self.round_to_dates = round_to_dates
|
|
36
36
|
|
|
37
37
|
@classmethod
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""Module for calculating optimization periods."""
|
|
2
|
+
|
|
2
3
|
import bisect
|
|
3
4
|
import datetime
|
|
4
5
|
from pathlib import Path
|
|
@@ -12,30 +13,40 @@ def get_optimization_ranges_from_file(
|
|
|
12
13
|
"""Read horizon config from a csv file"""
|
|
13
14
|
if not file_path.exists():
|
|
14
15
|
raise FileNotFoundError(
|
|
15
|
-
|
|
16
|
+
"The closed_loop_dates csv does not exist. Please create a horizon "
|
|
17
|
+
f"config file in {file_path}."
|
|
16
18
|
)
|
|
17
19
|
try:
|
|
18
20
|
closed_loop_dates = pd.read_csv(file_path)
|
|
19
21
|
except pd.errors.EmptyDataError:
|
|
20
22
|
raise ValueError(
|
|
21
|
-
"The closed_loop_dates csv is empty. Please provide a valid file with
|
|
23
|
+
"The closed_loop_dates csv is empty. Please provide a valid file with "
|
|
24
|
+
"start_date and end_date column."
|
|
22
25
|
)
|
|
23
26
|
closed_loop_dates.columns = closed_loop_dates.columns.str.replace(" ", "")
|
|
24
|
-
if not all(
|
|
25
|
-
raise ValueError(
|
|
27
|
+
if not all(col in closed_loop_dates.columns for col in ["start_date", "end_date"]):
|
|
28
|
+
raise ValueError(
|
|
29
|
+
"The closed_loop_dates csv should have both 'start_date' and 'end_date' columns."
|
|
30
|
+
)
|
|
26
31
|
closed_loop_dates["start_date"] = pd.to_datetime(closed_loop_dates["start_date"])
|
|
27
32
|
closed_loop_dates["end_date"] = pd.to_datetime(closed_loop_dates["end_date"])
|
|
28
33
|
for i in range(1, len(closed_loop_dates)):
|
|
29
34
|
if closed_loop_dates["start_date"].iloc[i] > closed_loop_dates["end_date"].iloc[i - 1]:
|
|
30
|
-
raise ValueError(
|
|
35
|
+
raise ValueError(
|
|
36
|
+
f"Closed loop date table: Start date at row {i} is later than previous end date."
|
|
37
|
+
)
|
|
31
38
|
if any(closed_loop_dates["start_date"] < closed_loop_dates["start_date"].shift(1)):
|
|
32
39
|
raise ValueError("Closed loop date table: The start dates are not in ascending order.")
|
|
33
40
|
if any(closed_loop_dates["end_date"] < closed_loop_dates["end_date"].shift(1)):
|
|
34
41
|
raise ValueError("Closed loop date table: The end dates are not in ascending order.")
|
|
35
42
|
if any(closed_loop_dates["end_date"] < closed_loop_dates["start_date"]):
|
|
36
|
-
raise ValueError(
|
|
43
|
+
raise ValueError(
|
|
44
|
+
"Closed loop date table: For one or more rows the end date is before the start date."
|
|
45
|
+
)
|
|
37
46
|
if any(closed_loop_dates["start_date"] > closed_loop_dates["end_date"]):
|
|
38
|
-
raise ValueError(
|
|
47
|
+
raise ValueError(
|
|
48
|
+
"Closed loop date table: For one or more rows the start date is after the end date."
|
|
49
|
+
)
|
|
39
50
|
if (
|
|
40
51
|
any(closed_loop_dates["start_date"].dt.hour != 0)
|
|
41
52
|
or any(closed_loop_dates["start_date"].dt.minute != 0)
|
|
@@ -43,17 +54,14 @@ def get_optimization_ranges_from_file(
|
|
|
43
54
|
or any(closed_loop_dates["end_date"].dt.minute != 0)
|
|
44
55
|
):
|
|
45
56
|
raise ValueError(
|
|
46
|
-
"Closed loop date table: Currently, the date ranges can only be specific
|
|
57
|
+
"Closed loop date table: Currently, the date ranges can only be specific "
|
|
58
|
+
"up to the level of days."
|
|
47
59
|
)
|
|
48
|
-
assert (
|
|
49
|
-
min(closed_loop_dates["start_date"]).date() == model_time_range[0].date()
|
|
50
|
-
), (
|
|
60
|
+
assert min(closed_loop_dates["start_date"]).date() == model_time_range[0].date(), (
|
|
51
61
|
"The start day of the first optimization run is not equal"
|
|
52
62
|
" to the start day of the forecast date (or first timestep)."
|
|
53
63
|
)
|
|
54
|
-
assert (
|
|
55
|
-
max(closed_loop_dates["end_date"]).date() <= model_time_range[1].date()
|
|
56
|
-
), (
|
|
64
|
+
assert max(closed_loop_dates["end_date"]).date() <= model_time_range[1].date(), (
|
|
57
65
|
"The end date of one or more optimization runs is later"
|
|
58
66
|
" than the end date of the timeseries import."
|
|
59
67
|
)
|
|
@@ -65,9 +73,7 @@ def get_optimization_ranges_from_file(
|
|
|
65
73
|
|
|
66
74
|
|
|
67
75
|
def _get_next_time_index(
|
|
68
|
-
times: list[datetime.date],
|
|
69
|
-
i_current: int,
|
|
70
|
-
timestep_size: datetime.timedelta
|
|
76
|
+
times: list[datetime.date], i_current: int, timestep_size: datetime.timedelta
|
|
71
77
|
) -> int:
|
|
72
78
|
"""
|
|
73
79
|
Get the next timestep index.
|
|
@@ -91,7 +97,7 @@ def get_optimization_ranges(
|
|
|
91
97
|
model_times: list[datetime.date],
|
|
92
98
|
start_time: datetime.datetime,
|
|
93
99
|
forecast_timestep: datetime.timedelta,
|
|
94
|
-
optimization_period: datetime.timedelta
|
|
100
|
+
optimization_period: datetime.timedelta,
|
|
95
101
|
) -> list[tuple[datetime.datetime, datetime.datetime]]:
|
|
96
102
|
"""Calculate a list of optimization periods."""
|
|
97
103
|
if forecast_timestep > optimization_period:
|
|
@@ -114,7 +120,7 @@ def get_optimization_ranges(
|
|
|
114
120
|
|
|
115
121
|
|
|
116
122
|
def round_datetime_ranges_to_days(
|
|
117
|
-
datetime_ranges: list[tuple[datetime.datetime, datetime.datetime]]
|
|
123
|
+
datetime_ranges: list[tuple[datetime.datetime, datetime.datetime]],
|
|
118
124
|
) -> list[tuple[datetime.datetime, datetime.datetime]]:
|
|
119
125
|
"""Round datetimes to dats in datetime ranges.
|
|
120
126
|
|
|
@@ -2,10 +2,9 @@ import copy
|
|
|
2
2
|
import logging
|
|
3
3
|
import os
|
|
4
4
|
from pathlib import Path
|
|
5
|
-
|
|
5
|
+
|
|
6
6
|
import pandas as pd
|
|
7
|
-
from rtctools.data import rtc
|
|
8
|
-
from rtctools.data import pi
|
|
7
|
+
from rtctools.data import pi, rtc
|
|
9
8
|
|
|
10
9
|
logger = logging.getLogger("rtctools")
|
|
11
10
|
|
|
@@ -22,7 +21,9 @@ def _get_variables_from_pi(data_config: rtc.DataConfig, timeseries: pi.Timeserie
|
|
|
22
21
|
return variables
|
|
23
22
|
|
|
24
23
|
|
|
25
|
-
def combine_xml_exports(
|
|
24
|
+
def combine_xml_exports(
|
|
25
|
+
output_base_path: Path, original_input_timeseries_path: Path, write_csv_out: bool = False
|
|
26
|
+
):
|
|
26
27
|
"""Combine the xml exports of multiple periods into a single xml file."""
|
|
27
28
|
logger.info("Combining XML exports.")
|
|
28
29
|
dataconfig = rtc.DataConfig(folder=original_input_timeseries_path)
|
|
@@ -34,14 +35,22 @@ def combine_xml_exports(output_base_path: Path, original_input_timeseries_path:
|
|
|
34
35
|
binary=False,
|
|
35
36
|
)
|
|
36
37
|
if ts_import_orig.forecast_datetime > ts_import_orig.start_datetime:
|
|
37
|
-
logger.info(
|
|
38
|
+
logger.info(
|
|
39
|
+
"Timeseries export will start at original forecast date, "
|
|
40
|
+
"disregarding data before forecast date."
|
|
41
|
+
)
|
|
38
42
|
ts_import_orig.resize(ts_import_orig.forecast_datetime, ts_import_orig.end_datetime)
|
|
39
|
-
ts_import_orig.times = ts_import_orig.times[
|
|
43
|
+
ts_import_orig.times = ts_import_orig.times[
|
|
44
|
+
ts_import_orig.times.index(ts_import_orig.forecast_datetime) :
|
|
45
|
+
]
|
|
40
46
|
orig_start_datetime = ts_import_orig.start_datetime
|
|
41
47
|
orig_end_datetime = ts_import_orig.end_datetime
|
|
42
48
|
|
|
43
49
|
ts_export = pi.Timeseries(
|
|
44
|
-
data_config=dataconfig,
|
|
50
|
+
data_config=dataconfig,
|
|
51
|
+
folder=output_base_path / "period_0",
|
|
52
|
+
basename="timeseries_export",
|
|
53
|
+
binary=False,
|
|
45
54
|
) # Use the first timeseries export as a starting point for the combined timeseries export.
|
|
46
55
|
ts_export.resize(orig_start_datetime, orig_end_datetime)
|
|
47
56
|
|
|
@@ -63,7 +72,7 @@ def combine_xml_exports(output_base_path: Path, original_input_timeseries_path:
|
|
|
63
72
|
current_values = ts_export.get(loc_par)
|
|
64
73
|
new_values = ts_export_step.get(loc_par)
|
|
65
74
|
except KeyError:
|
|
66
|
-
logger.debug("Variable {} not found in output of model horizon: {}"
|
|
75
|
+
logger.debug(f"Variable {loc_par} not found in output of model horizon: {i}")
|
|
67
76
|
continue
|
|
68
77
|
new_times = ts_export_step.times
|
|
69
78
|
try:
|
|
@@ -77,7 +86,9 @@ def combine_xml_exports(output_base_path: Path, original_input_timeseries_path:
|
|
|
77
86
|
+ "with the end of the previous."
|
|
78
87
|
)
|
|
79
88
|
combined_values = copy.deepcopy(current_values)
|
|
80
|
-
combined_values[start_new_data_index : start_new_data_index + len(new_values)] =
|
|
89
|
+
combined_values[start_new_data_index : start_new_data_index + len(new_values)] = (
|
|
90
|
+
new_values # noqa
|
|
91
|
+
)
|
|
81
92
|
ts_export.set(loc_par, combined_values)
|
|
82
93
|
i += 1
|
|
83
94
|
ts_export.write(output_folder=output_base_path.parent, output_filename="timeseries_export")
|
|
@@ -90,16 +101,16 @@ def combine_xml_exports(output_base_path: Path, original_input_timeseries_path:
|
|
|
90
101
|
values = ts_export.get(timeseries_id)
|
|
91
102
|
new_columns.append(pd.Series(values, name=timeseries_id))
|
|
92
103
|
except KeyError:
|
|
93
|
-
logger.debug("Variable {} not found in output of model horizon: {}"
|
|
104
|
+
logger.debug(f"Variable {timeseries_id} not found in output of model horizon: {i}")
|
|
94
105
|
continue
|
|
95
106
|
data = pd.concat([data] + new_columns, axis=1)
|
|
96
107
|
data.round(6).to_csv(output_base_path.parent / "timeseries_export.csv", index=False)
|
|
97
108
|
|
|
98
109
|
|
|
99
|
-
def combine_dataframes(dfs:
|
|
110
|
+
def combine_dataframes(dfs: list[pd.DataFrame], index_col: str = "time"):
|
|
100
111
|
"""Combine multiple dataframes with the same index column.
|
|
101
|
-
The dataframes are combined in the order they are passed,
|
|
102
|
-
in case of overlapping indices."""
|
|
112
|
+
The dataframes are combined in the order they are passed,
|
|
113
|
+
with the last dataframe taking precedence in case of overlapping indices."""
|
|
103
114
|
combined_df = pd.DataFrame()
|
|
104
115
|
for df in dfs:
|
|
105
116
|
df.set_index(index_col, inplace=True)
|
|
@@ -125,5 +136,7 @@ if __name__ == "__main__":
|
|
|
125
136
|
output_base_path = closed_loop_test_folder / Path(
|
|
126
137
|
r"test_models\goal_programming_xml\output\output_modelling_periods_reference"
|
|
127
138
|
)
|
|
128
|
-
original_input_timeseries_path = closed_loop_test_folder / Path(
|
|
139
|
+
original_input_timeseries_path = closed_loop_test_folder / Path(
|
|
140
|
+
r"test_models\goal_programming_xml\input"
|
|
141
|
+
)
|
|
129
142
|
combine_xml_exports(output_base_path, original_input_timeseries_path, True)
|
|
@@ -6,28 +6,38 @@ import os
|
|
|
6
6
|
import shutil
|
|
7
7
|
import sys
|
|
8
8
|
from pathlib import Path
|
|
9
|
-
from typing import List, Optional
|
|
10
9
|
|
|
11
10
|
from rtctools.data.pi import DiagHandler
|
|
12
|
-
from rtctools.optimization.pi_mixin import PIMixin
|
|
13
11
|
from rtctools.optimization.csv_mixin import CSVMixin
|
|
14
|
-
from rtctools.
|
|
15
|
-
from
|
|
12
|
+
from rtctools.optimization.pi_mixin import PIMixin
|
|
13
|
+
from rtctools.util import _resolve_folder, run_optimization_problem
|
|
14
|
+
|
|
16
15
|
import rtctools_interface.closed_loop.optimization_ranges as opt_ranges
|
|
17
|
-
from rtctools_interface.closed_loop.
|
|
18
|
-
from rtctools_interface.closed_loop.
|
|
16
|
+
from rtctools_interface.closed_loop.config import ClosedLoopConfig
|
|
17
|
+
from rtctools_interface.closed_loop.results_construction import (
|
|
18
|
+
combine_csv_exports,
|
|
19
|
+
combine_xml_exports,
|
|
20
|
+
)
|
|
21
|
+
from rtctools_interface.closed_loop.time_series_handler import (
|
|
22
|
+
CSVTimeSeriesFile,
|
|
23
|
+
TimeSeriesHandler,
|
|
24
|
+
XMLTimeSeriesFile,
|
|
25
|
+
)
|
|
19
26
|
|
|
20
27
|
logger = logging.getLogger("rtctools")
|
|
21
28
|
|
|
22
29
|
|
|
23
30
|
def set_initial_values_from_previous_run(
|
|
24
|
-
results_previous_run:
|
|
31
|
+
results_previous_run: dict | None,
|
|
25
32
|
timeseries: TimeSeriesHandler,
|
|
26
|
-
previous_run_datetimes:
|
|
33
|
+
previous_run_datetimes: list[datetime.datetime],
|
|
27
34
|
) -> None:
|
|
28
|
-
"""Modifies the initial values of `timeseries` based on the results of the previous run
|
|
35
|
+
"""Modifies the initial values of `timeseries` based on the results of the previous run
|
|
36
|
+
(if any)"""
|
|
29
37
|
if results_previous_run is not None:
|
|
30
|
-
variables_to_set = {
|
|
38
|
+
variables_to_set = {
|
|
39
|
+
key: value for key, value in results_previous_run.items() if not timeseries.is_set(key)
|
|
40
|
+
}
|
|
31
41
|
if timeseries.forecast_date:
|
|
32
42
|
index_of_initial_value = previous_run_datetimes.index(timeseries.forecast_date)
|
|
33
43
|
else:
|
|
@@ -63,7 +73,9 @@ def _get_optimization_ranges(
|
|
|
63
73
|
"""Return a list of optimization periods."""
|
|
64
74
|
if config.file is not None:
|
|
65
75
|
datetime_range = input_timeseries.get_datetime_range()
|
|
66
|
-
optimization_ranges = opt_ranges.get_optimization_ranges_from_file(
|
|
76
|
+
optimization_ranges = opt_ranges.get_optimization_ranges_from_file(
|
|
77
|
+
config.file, datetime_range
|
|
78
|
+
)
|
|
67
79
|
elif config.optimization_period is not None:
|
|
68
80
|
datetimes = input_timeseries.get_datetimes()
|
|
69
81
|
optimization_ranges = opt_ranges.get_optimization_ranges(
|
|
@@ -73,7 +85,9 @@ def _get_optimization_ranges(
|
|
|
73
85
|
optimization_period=config.optimization_period,
|
|
74
86
|
)
|
|
75
87
|
else:
|
|
76
|
-
raise ValueError(
|
|
88
|
+
raise ValueError(
|
|
89
|
+
"The closed-loop configuration should have either a file or optimization_period set."
|
|
90
|
+
)
|
|
77
91
|
if config.round_to_dates:
|
|
78
92
|
optimization_ranges = opt_ranges.round_datetime_ranges_to_days(optimization_ranges)
|
|
79
93
|
return optimization_ranges
|
|
@@ -84,8 +98,8 @@ def run_optimization_problem_closed_loop(
|
|
|
84
98
|
base_folder="..",
|
|
85
99
|
log_level=logging.INFO,
|
|
86
100
|
profile=False,
|
|
87
|
-
config:
|
|
88
|
-
modelling_period_input_folder:
|
|
101
|
+
config: ClosedLoopConfig | None = None,
|
|
102
|
+
modelling_period_input_folder: str | None = None,
|
|
89
103
|
**kwargs,
|
|
90
104
|
) -> dict:
|
|
91
105
|
"""
|
|
@@ -111,7 +125,7 @@ def run_optimization_problem_closed_loop(
|
|
|
111
125
|
handler.setFormatter(formatter)
|
|
112
126
|
logger.addHandler(handler)
|
|
113
127
|
if issubclass(optimization_problem_class, PIMixin):
|
|
114
|
-
if not any(
|
|
128
|
+
if not any(isinstance(h, DiagHandler) for h in logger.handlers):
|
|
115
129
|
diag_handler = DiagHandler(original_output_folder)
|
|
116
130
|
logger.addHandler(diag_handler)
|
|
117
131
|
logger.setLevel(log_level)
|
|
@@ -129,11 +143,14 @@ def run_optimization_problem_closed_loop(
|
|
|
129
143
|
if not fixed_input_config_file.exists():
|
|
130
144
|
raise FileNotFoundError(
|
|
131
145
|
f"Could not find fixed inputs configuration file: {fixed_input_config_file}"
|
|
132
|
-
"Create a file with a list of strings that represent the fixed inputs
|
|
146
|
+
"Create a file with a list of strings that represent the fixed inputs"
|
|
147
|
+
" (can be an empty list)."
|
|
133
148
|
)
|
|
134
|
-
with open(fixed_input_config_file
|
|
149
|
+
with open(fixed_input_config_file) as file:
|
|
135
150
|
fixed_input_series = json.load(file)
|
|
136
|
-
if not isinstance(fixed_input_series, list) and all(
|
|
151
|
+
if not isinstance(fixed_input_series, list) and all(
|
|
152
|
+
isinstance(item, str) for item in fixed_input_series
|
|
153
|
+
):
|
|
137
154
|
raise ValueError("Fixed input config file should be a list of strings (or an empty list).")
|
|
138
155
|
|
|
139
156
|
if modelling_period_input_folder is None:
|
|
@@ -157,13 +174,17 @@ def run_optimization_problem_closed_loop(
|
|
|
157
174
|
|
|
158
175
|
timeseries_import.select_time_range(start_date=start_time, end_date=end_time)
|
|
159
176
|
|
|
160
|
-
set_initial_values_from_previous_run(
|
|
177
|
+
set_initial_values_from_previous_run(
|
|
178
|
+
results_previous_run, timeseries_import, previous_run_datetimes
|
|
179
|
+
)
|
|
161
180
|
|
|
162
181
|
modelling_period_name = f"period_{i}"
|
|
163
182
|
modelling_period_output_folder_i = modelling_periods_output_folder / modelling_period_name
|
|
164
183
|
modelling_period_output_folder_i.mkdir(exist_ok=True)
|
|
165
184
|
modelling_period_input_folder_i = modelling_period_input_folder / modelling_period_name
|
|
166
|
-
write_input_folder(
|
|
185
|
+
write_input_folder(
|
|
186
|
+
modelling_period_input_folder_i, original_input_folder, timeseries_import
|
|
187
|
+
)
|
|
167
188
|
|
|
168
189
|
logger.info(f"Running optimization for period {i}: {(str(start_time), str(end_time))}.")
|
|
169
190
|
result = run_optimization_problem(
|
|
@@ -179,22 +200,30 @@ def run_optimization_problem_closed_loop(
|
|
|
179
200
|
if result.solver_stats["success"]:
|
|
180
201
|
logger.info(f"Successful optimization for {period}.")
|
|
181
202
|
else:
|
|
182
|
-
message =
|
|
203
|
+
message = (
|
|
204
|
+
f"Failed optimization for {period} with status"
|
|
205
|
+
+ f"{result.solver_stats['return_status']}'."
|
|
206
|
+
)
|
|
183
207
|
logger.error(message)
|
|
184
208
|
raise Exception(message)
|
|
185
209
|
|
|
186
210
|
results_previous_run = {
|
|
187
|
-
key: result.extract_results().get(key)
|
|
211
|
+
key: result.extract_results().get(key)
|
|
212
|
+
for key in variables_in_import
|
|
213
|
+
if key not in fixed_input_series
|
|
188
214
|
}
|
|
189
215
|
previous_run_datetimes = result.io.datetimes
|
|
190
216
|
|
|
191
217
|
logger.info("Finished all optimization runs.")
|
|
192
218
|
if issubclass(optimization_problem_class, PIMixin):
|
|
193
|
-
combine_xml_exports(
|
|
219
|
+
combine_xml_exports(
|
|
220
|
+
modelling_periods_output_folder, original_input_folder, write_csv_out=True
|
|
221
|
+
)
|
|
194
222
|
elif issubclass(optimization_problem_class, CSVMixin):
|
|
195
223
|
combine_csv_exports(modelling_periods_output_folder)
|
|
196
224
|
else:
|
|
197
225
|
logger.warning(
|
|
198
|
-
"Could not combine exports because the optimization problem class is not
|
|
226
|
+
"Could not combine exports because the optimization problem class is not "
|
|
227
|
+
"derived from PIMixin or CSVMixin."
|
|
199
228
|
)
|
|
200
229
|
return result.solver_stats
|
|
@@ -3,14 +3,10 @@ import datetime
|
|
|
3
3
|
import logging
|
|
4
4
|
from abc import ABC, abstractmethod
|
|
5
5
|
from pathlib import Path
|
|
6
|
-
from typing import List, Optional, Tuple
|
|
7
6
|
|
|
8
|
-
import pandas as pd
|
|
9
7
|
import numpy as np
|
|
10
|
-
|
|
11
|
-
from rtctools.data import pi
|
|
12
|
-
from rtctools.data import rtc
|
|
13
|
-
from rtctools.data import csv
|
|
8
|
+
import pandas as pd
|
|
9
|
+
from rtctools.data import csv, pi, rtc
|
|
14
10
|
|
|
15
11
|
ns = {"fews": "http://www.wldelft.nl/fews", "pi": "http://www.wldelft.nl/fews/PI"}
|
|
16
12
|
|
|
@@ -21,7 +17,7 @@ class TimeSeriesHandler(ABC):
|
|
|
21
17
|
"""ABC for handling timeseries data."""
|
|
22
18
|
|
|
23
19
|
# The forecast date determines at which date the optimization starts.
|
|
24
|
-
forecast_date:
|
|
20
|
+
forecast_date: datetime.datetime | None = None
|
|
25
21
|
|
|
26
22
|
@abstractmethod
|
|
27
23
|
def read(self, file_name: str) -> None:
|
|
@@ -41,15 +37,15 @@ class TimeSeriesHandler(ABC):
|
|
|
41
37
|
"""Get the timestep of the timeseries data."""
|
|
42
38
|
|
|
43
39
|
@abstractmethod
|
|
44
|
-
def get_datetimes(self) ->
|
|
40
|
+
def get_datetimes(self) -> list[datetime.datetime]:
|
|
45
41
|
"""Get the dates of the timeseries."""
|
|
46
42
|
|
|
47
43
|
@abstractmethod
|
|
48
|
-
def get_datetime_range(self) ->
|
|
44
|
+
def get_datetime_range(self) -> tuple[datetime.datetime, datetime.datetime]:
|
|
49
45
|
"""Get the date range of the timeseries data (min, max)."""
|
|
50
46
|
|
|
51
47
|
@abstractmethod
|
|
52
|
-
def get_all_internal_ids(self) ->
|
|
48
|
+
def get_all_internal_ids(self) -> list[str]:
|
|
53
49
|
"""Get all internal id's of the timeseries data."""
|
|
54
50
|
|
|
55
51
|
@abstractmethod
|
|
@@ -58,7 +54,8 @@ class TimeSeriesHandler(ABC):
|
|
|
58
54
|
|
|
59
55
|
@abstractmethod
|
|
60
56
|
def is_set(self, internal_id: str) -> bool:
|
|
61
|
-
"""Check whether the variable exists in the timeseries data
|
|
57
|
+
"""Check whether the variable exists in the timeseries data
|
|
58
|
+
and whether it has a least one non-nan value"""
|
|
62
59
|
|
|
63
60
|
|
|
64
61
|
class CSVTimeSeriesFile(TimeSeriesHandler):
|
|
@@ -98,7 +95,7 @@ class CSVTimeSeriesFile(TimeSeriesHandler):
|
|
|
98
95
|
delimiter=self.csv_delimiter,
|
|
99
96
|
with_time=False,
|
|
100
97
|
)
|
|
101
|
-
self.initial_state:
|
|
98
|
+
self.initial_state: dict | None = {
|
|
102
99
|
field: float(initial_state[field]) for field in initial_state.dtype.names
|
|
103
100
|
}
|
|
104
101
|
else:
|
|
@@ -123,7 +120,9 @@ class CSVTimeSeriesFile(TimeSeriesHandler):
|
|
|
123
120
|
def write_initial_state(self, file_path: Path, file_name: str = "initial_state"):
|
|
124
121
|
if self.initial_state is not None:
|
|
125
122
|
initial_state = pd.DataFrame(self.initial_state, index=[0])
|
|
126
|
-
initial_state.to_csv(
|
|
123
|
+
initial_state.to_csv(
|
|
124
|
+
(file_path / file_name).with_suffix(".csv"), header=True, index=False
|
|
125
|
+
)
|
|
127
126
|
|
|
128
127
|
def get_timestep(self):
|
|
129
128
|
return self.data[self.date_col].diff().min()
|
|
@@ -171,7 +170,9 @@ class XMLTimeSeriesFile(TimeSeriesHandler):
|
|
|
171
170
|
self.read(timeseries_import_basename)
|
|
172
171
|
|
|
173
172
|
if self.get_datetime_range()[0] < self.forecast_date:
|
|
174
|
-
logger.warning(
|
|
173
|
+
logger.warning(
|
|
174
|
+
"Currently, the closed loop runner does support data before the forecast date."
|
|
175
|
+
)
|
|
175
176
|
logger.warning("Removing data before forecast date.")
|
|
176
177
|
self.select_time_range(self.forecast_date, self.pi_timeseries.times[-1])
|
|
177
178
|
|
|
@@ -189,8 +190,9 @@ class XMLTimeSeriesFile(TimeSeriesHandler):
|
|
|
189
190
|
self.forecast_date = self.pi_timeseries.forecast_datetime
|
|
190
191
|
|
|
191
192
|
def is_set(self, internal_id):
|
|
192
|
-
"""Check whether the variable exists in the timeseries data
|
|
193
|
-
|
|
193
|
+
"""Check whether the variable exists in the timeseries data.
|
|
194
|
+
|
|
195
|
+
and whether it has a value at at least one of time steps."""
|
|
194
196
|
try:
|
|
195
197
|
var: np.ndarray = self.pi_timeseries.get(variable=internal_id)
|
|
196
198
|
return not np.isnan(var).all()
|
|
@@ -206,7 +208,9 @@ class XMLTimeSeriesFile(TimeSeriesHandler):
|
|
|
206
208
|
return False
|
|
207
209
|
|
|
208
210
|
def get_all_internal_ids(self):
|
|
209
|
-
"""Get all internal id's of the timeseries data.
|
|
211
|
+
"""Get all internal id's of the timeseries data.
|
|
212
|
+
|
|
213
|
+
Only returns the id's that are also in the dataconfig."""
|
|
210
214
|
all_ids = [var for var, _ in self.pi_timeseries.items() if self._is_in_dataconfig(var)]
|
|
211
215
|
return all_ids
|
|
212
216
|
|
|
@@ -215,12 +219,13 @@ class XMLTimeSeriesFile(TimeSeriesHandler):
|
|
|
215
219
|
i_start = bisect.bisect_left(times, start_date)
|
|
216
220
|
i_end = bisect.bisect_right(times, end_date) - 1
|
|
217
221
|
self.pi_timeseries.resize(start_datetime=times[i_start], end_datetime=times[i_end])
|
|
218
|
-
self.pi_timeseries.times = times[i_start:i_end + 1]
|
|
222
|
+
self.pi_timeseries.times = times[i_start : i_end + 1]
|
|
219
223
|
self.pi_timeseries.forecast_datetime = times[i_start]
|
|
220
224
|
self.forecast_date = times[i_start]
|
|
221
225
|
|
|
222
226
|
def write(self, file_path: Path, file_name: str = "timeseries_import"):
|
|
223
|
-
# By setting make_new_file headers will be recreated,
|
|
227
|
+
# By setting make_new_file headers will be recreated,
|
|
228
|
+
# neceesary for writing new forecast date
|
|
224
229
|
self.pi_timeseries.make_new_file = True
|
|
225
230
|
self.pi_timeseries.write(output_folder=file_path, output_filename=file_name)
|
|
226
231
|
|