rtc-tools-interface 0.8.0b4__tar.gz → 0.9.0__tar.gz

{rtc-tools-interface-0.8.0b4 → rtc-tools-interface-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rtc-tools-interface
-Version: 0.8.0b4
+Version: 0.9.0
 Summary: Toolbox for user interfaces for RTC-Tools
 Home-page: UNKNOWN
 Author: Deltares

{rtc-tools-interface-0.8.0b4 → rtc-tools-interface-0.9.0}/README.md

@@ -172,30 +172,72 @@ The table could thus look like:
 After running the model, in your output folder the folder `figures` containing the figures is created.
 
 ## Closed loop runner
-To run a closed loop experiment one can use the `run_optimization_problem_closed_loop` function from `run_closed_loop`.
-This function is a drop-in replacement for the `run_optimization_problem` of rtc-tools. The user only needs to specify the `closed_loop_dates.csv` in the input folder of the optimization problem.
+
+To run a closed loop experiment
+one can use the `run_optimization_problem_closed_loop` function
+from `rtctools_interface.closed_loop.runner`.
+This function is a drop-in replacement for the `run_optimization_problem` of rtc-tools.
+The user needs to specify a `ClosedLoopConfig` configuration
+from `rtctools_interface.closed_loop.config`
+to specify the time ranges for which to subsequentially solve the optimization problem.
 
 ### Setup
-Import `run_optimization_problem_closed_loop` with:
+
+Import `ClosedLoopConfig` and `run_optimization_problem_closed_loop` with:
+
+```python
+from rtctools_interface.closed_loop.config import ClosedLoopConfig
+from rtctools_interface.closed_loop.runner import run_optimization_problem_closed_loop
+```
+
+A `ClosedLoopConfig` configuration can be created from a csv file or
+from a given forecast timestep (time between each time range)
+and optimization period (duration of each time range).
+An option `round_to_dates` can be used to round the start and end time of each range to a date,
+i.e. the start time is rounded to the start of the day
+and the end time is rounded to the end of a day.
+Examples of creating a configuration are given below.
+
 ```python
-from rtctools_interface import run_optimization_problem_closed_loop
+from datetime import timedelta
+
+from rtctools_interface.closed_loop.config import ClosedLoopConfig
+
+config_from_file = ClosedLoopConfig(
+    file="path/to/closed_loop_dates.csv",
+    round_to_dates=True
+)
+
+config_from_fixed_periods = ClosedLoopConfig.from_fixed_periods(
+    optimization_period=timedelta(days=3),
+    forecast_timestep=timedelta(days=2)
+)
 ```
-Add a table named `closed_loop_dates.csv` to the input folder of your optimization problem. The table should contain two columns: `start_date` and `end_date`.
-Each row of the table corresponds to one modelling period. 
 
-Example table `closed_loop_dates.csv`:
+The CSV file `closed_loop_dates.csv` has two columns `start_date` and `end_date`
+and looks as follows.
+
 ```
 start_date, end_date
 2024-05-19, 2024-05-23
 2024-05-23, 2024-05-25
 ```
-With this table rtc-tools will run two optimization problems (modelling periods): one with the data from 2024-05-19 upto and including 2024-05-23 and one from 2024-05-23 upto and including 2024-05-25. 
-The `run_optimization_problem_closed_loop` will automatically set the final results from the previous as initial conditions of the next run. Note that this happens for:
-- All variables available at the first time step in original timeseries_import, but not available at any timestep in the modelling period.
+
+The `run_optimization_problem_closed_loop` will solve the optimization problem
+for each time range subseqentually.
+It will use the final results from the previous run to set the initial values of the next run.
+Note that this happens for:
+
+- All variables available at the first time step in original timeseries_import,
+    but not available at any timestep in the modelling period.
 - All variables in the `initial_state.csv` (if the csv_mixin is used).
 
 ### Notes
-- The first start_date in your `closed_loop_dates.csv` should be equal to the start_date of your timeseries_import.
-- The different horizons should overlap with at least one day (to allow retrieving and setting initial values). An overlap of more days is allowed.
-- Currently only a single timestep is copied as an initial value.
-- The closed_loop runner only works in combination with the CSVMixin or the PIMixin. The CDFMixin is not supported.
+
+- The start time of the first time range should coincide
+    with the start time of the input timeseries.
+- The start time of the next time range should be less or equal
+    to the end time of the current time range.
+- Currently, only the initial values of the first time step in a given time range are set.
+- The closed_loop runner only works in combination with the CSVMixin or the PIMixin.
+    The CDFMixin is not supported.

{rtc-tools-interface-0.8.0b4 → rtc-tools-interface-0.9.0}/rtc_tools_interface.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rtc-tools-interface
-Version: 0.8.0b4
+Version: 0.9.0
 Summary: Toolbox for user interfaces for RTC-Tools
 Home-page: UNKNOWN
 Author: Deltares

{rtc-tools-interface-0.8.0b4 → rtc-tools-interface-0.9.0}/rtc_tools_interface.egg-info/SOURCES.txt

@@ -11,7 +11,8 @@ rtc_tools_interface.egg-info/top_level.txt
 rtctools_interface/__init__.py
 rtctools_interface/_version.py
 rtctools_interface/closed_loop/__init__.py
-rtctools_interface/closed_loop/closed_loop_dates.py
+rtctools_interface/closed_loop/config.py
+rtctools_interface/closed_loop/optimization_ranges.py
 rtctools_interface/closed_loop/results_construction.py
 rtctools_interface/closed_loop/runner.py
 rtctools_interface/closed_loop/time_series_handler.py
@@ -41,6 +42,7 @@ rtctools_interface/utils/serialization.py
 rtctools_interface/utils/type_definitions.py
 tests/__init__.py
 tests/closed_loop/__init__.py
+tests/closed_loop/test_optimization_ranges.py
 tests/closed_loop/test_read_xml.py
 tests/closed_loop/test_run_optization_problem_closed_loop.py
 tests/optimization/__init__.py

{rtc-tools-interface-0.8.0b4 → rtc-tools-interface-0.9.0}/rtc_tools_interface.egg-info/requires.txt

@@ -1,6 +1,7 @@
-pandas
-rtc-tools>=2.5.0
 matplotlib
-plotly
 numpy
+pandas
+plotly
 pydantic
+casadi!=3.6.6
+rtc-tools>=2.7.0a3

{rtc-tools-interface-0.8.0b4 → rtc-tools-interface-0.9.0}/rtctools_interface/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-06-11T11:21:50+0200",
+ "date": "2024-08-13T11:30:09+0200",
  "dirty": false,
  "error": null,
- "full-revisionid": "6bc6e859ff0f928c2f4c1127a2aca08ce9720051",
- "version": "0.8.0b4"
+ "full-revisionid": "a1e33a6c949d514b395d8a3a5bcd341f6c7cc7ad",
+ "version": "0.9.0"
 }
 '''  # END VERSION_JSON
 

rtc-tools-interface-0.9.0/rtctools_interface/closed_loop/config.py

@@ -0,0 +1,73 @@
+"""Module for configuring a closed-loop optimization problem."""
+from pathlib import Path
+from datetime import timedelta
+
+
+class ClosedLoopConfig():
+    """Configuration of a closed-loop optimization problem."""
+
+    def __init__(
+        self,
+        file: Path = None,
+        round_to_dates: bool = False,
+    ):
+        """
+        Create a configuration for closed-loop optimization.
+
+        :param file: CSV file with two columns 'start_date' and 'end_date'.
+            Each row indicates a time range for which to optimize a given problem.
+            Note:
+            * The start time of the first time range should coincide with the
+              start time of the input timeseries.
+            * The start time of the next time range should be less or equal
+              to the end time of the current time range.
+        :param round_to_dates: If true, the start time and end time of each time range
+            is rounded to just the date.
+            In particular, the start time is rounded to the start of the day
+            and the end time is rounded to the end of the day.
+        """
+        if file is not None:
+            file = Path(file).resolve()
+        self._file = file
+        self._forecast_timestep = None
+        self._optimization_period = None
+        self.round_to_dates = round_to_dates
+
+    @classmethod
+    def from_fixed_periods(
+        cls,
+        forecast_timestep: timedelta,
+        optimization_period: timedelta,
+        round_to_dates: bool = False,
+    ):
+        """
+        Create a closed loop configuration based on fixed periods.
+
+        :param forecast_timestep: The time between the start of each optimization time range.
+        :param optimization_period: the duration of each optimization time range
+        """
+        if forecast_timestep > optimization_period:
+            raise ValueError(
+                f"The forecast timestep ({forecast_timestep}) should be less than or equal to"
+                f" the optimization period ({optimization_period})."
+            )
+        config = cls()
+        config._forecast_timestep = forecast_timestep
+        config._optimization_period = optimization_period
+        config.round_to_dates = round_to_dates
+        return config
+
+    @property
+    def file(self):
+        """Get the file that defines the closed-loop periods."""
+        return self._file
+
+    @property
+    def forecast_timestep(self):
+        """Get the forecast timestep of the closed-loop periods."""
+        return self._forecast_timestep
+
+    @property
+    def optimization_period(self):
+        """Get the optimization period of the closed-loop periods."""
+        return self._optimization_period

rtc-tools-interface-0.9.0/rtctools_interface/closed_loop/optimization_ranges.py

@@ -0,0 +1,131 @@
+"""Module for calculating optimization periods."""
+import bisect
+import datetime
+from pathlib import Path
+
+import pandas as pd
+
+
+def get_optimization_ranges_from_file(
+    file_path: Path, model_time_range: tuple[datetime.datetime, datetime.datetime]
+):
+    """Read horizon config from a csv file"""
+    if not file_path.exists():
+        raise FileNotFoundError(
+            f"The closed_loop_dates csv does not exist. Please create a horizon config file in {file_path}."
+        )
+    try:
+        closed_loop_dates = pd.read_csv(file_path)
+    except pd.errors.EmptyDataError:
+        raise ValueError(
+            "The closed_loop_dates csv is empty. Please provide a valid file with start_date and end_date column."
+        )
+    closed_loop_dates.columns = closed_loop_dates.columns.str.replace(" ", "")
+    if not all([col in closed_loop_dates.columns for col in ["start_date", "end_date"]]):
+        raise ValueError("The closed_loop_dates csv should have both 'start_date' and 'end_date' columns.")
+    closed_loop_dates["start_date"] = pd.to_datetime(closed_loop_dates["start_date"])
+    closed_loop_dates["end_date"] = pd.to_datetime(closed_loop_dates["end_date"])
+    for i in range(1, len(closed_loop_dates)):
+        if closed_loop_dates["start_date"].iloc[i] > closed_loop_dates["end_date"].iloc[i - 1]:
+            raise ValueError(f"Closed loop date table: Start date at row {i} is later than the previous end date. ")
+    if any(closed_loop_dates["start_date"] < closed_loop_dates["start_date"].shift(1)):
+        raise ValueError("Closed loop date table: The start dates are not in ascending order.")
+    if any(closed_loop_dates["end_date"] < closed_loop_dates["end_date"].shift(1)):
+        raise ValueError("Closed loop date table: The end dates are not in ascending order.")
+    if any(closed_loop_dates["end_date"] < closed_loop_dates["start_date"]):
+        raise ValueError("Closed loop date table: For one or more rows the end date is before the start date.")
+    if any(closed_loop_dates["start_date"] > closed_loop_dates["end_date"]):
+        raise ValueError("Closed loop date table: For one or more rows the start date is after the end date.")
+    if (
+        any(closed_loop_dates["start_date"].dt.hour != 0)
+        or any(closed_loop_dates["start_date"].dt.minute != 0)
+        or any(closed_loop_dates["end_date"].dt.hour != 0)
+        or any(closed_loop_dates["end_date"].dt.minute != 0)
+    ):
+        raise ValueError(
+            "Closed loop date table: Currently, the date ranges can only be specific up to the level of days."
+        )
+    assert (
+        min(closed_loop_dates["start_date"]).date() == model_time_range[0].date()
+    ), (
+        "The start day of the first optimization run is not equal"
+        " to the start day of the timeseries import."
+    )
+    assert (
+        max(closed_loop_dates["end_date"]).date() <= model_time_range[1].date()
+    ), (
+        "The end date of one or more optimization runs is later"
+        " than the end date of the timeseries import."
+    )
+    closed_loop_dates = [
+        (optimization_range["start_date"], optimization_range["end_date"])
+        for _, optimization_range in closed_loop_dates.iterrows()
+    ]
+    return closed_loop_dates
+
+
+def _get_next_time_index(
+    times: list[datetime.date],
+    i_current: int,
+    timestep_size: datetime.timedelta
+) -> int:
+    """
+    Get the next timestep index.
+
+    The next timestep index i_next is the highest index such that:
+    * times[i_next] > times[i_current]
+    * times[i_next] <= times[i_current] + timestep_size
+    In case i_current >= i_max, i_next will be set to i_max, where i_max = len(times) - 1.
+    """
+    i_max = len(times) - 1
+    if i_current >= i_max:
+        return i_max
+    current_time = times[i_current]
+    next_time = current_time + timestep_size
+    i_next = bisect.bisect_right(times, next_time) - 1
+    i_next = max(i_current + 1, i_next)
+    return i_next
+
+
+def get_optimization_ranges(
+    model_times: list[datetime.date],
+    start_time: datetime.datetime,
+    forecast_timestep: datetime.timedelta,
+    optimization_period: datetime.timedelta
+) -> list[tuple[datetime.datetime, datetime.datetime]]:
+    """Calculate a list of optimization periods."""
+    if forecast_timestep > optimization_period:
+        raise ValueError(
+            f"Forecast timestep {forecast_timestep} cannot be larger than"
+            f" the optimization period {optimization_period}."
+        )
+    if start_time not in model_times:
+        raise ValueError(f"Start time {start_time} is not in the given model times.")
+    i_max = len(model_times) - 1
+    i_start = model_times.index(start_time)
+    i_end = _get_next_time_index(model_times, i_start, optimization_period)
+    optimization_periods = []
+    optimization_periods.append((model_times[i_start], model_times[i_end]))
+    while i_end < i_max:
+        i_start = _get_next_time_index(model_times, i_start, forecast_timestep)
+        i_end = _get_next_time_index(model_times, i_start, optimization_period)
+        optimization_periods.append((model_times[i_start], model_times[i_end]))
+    return optimization_periods
+
+
+def round_datetime_ranges_to_days(
+    datetime_ranges: list[tuple[datetime.datetime, datetime.datetime]]
+) -> list[tuple[datetime.datetime, datetime.datetime]]:
+    """Round datetimes to dats in datetime ranges.
+
+    The start of the range is rounded to the start of the day
+    and the end of the range is rounded to the end of the day.
+    """
+    rounded_ranges = []
+    for start, end in datetime_ranges:
+        if start.date() == end.date():
+            continue
+        start = datetime.datetime.combine(start.date(), datetime.time.min)
+        end = datetime.datetime.combine(end.date(), datetime.time.max)
+        rounded_ranges.append((start, end))
+    return rounded_ranges

{rtc-tools-interface-0.8.0b4 → rtc-tools-interface-0.9.0}/rtctools_interface/closed_loop/results_construction.py

@@ -10,6 +10,18 @@ from rtctools.data import pi
 logger = logging.getLogger("rtctools")
 
 
+def _get_variables_from_pi(data_config: rtc.DataConfig, timeseries: pi.Timeseries):
+    """Get all variables of a PI timeseries that are in the data configuration."""
+    variables = []
+    for var, _ in timeseries.items():
+        try:
+            data_config.pi_variable_ids(var)
+            variables.append(var)
+        except KeyError:
+            pass
+    return variables
+
+
 def combine_xml_exports(output_base_path: Path, original_input_timeseries_path: Path, write_csv_out: bool = False):
     """Combine the xml exports of multiple periods into a single xml file."""
     logger.info("Combining XML exports.")
@@ -24,9 +36,10 @@ def combine_xml_exports(output_base_path: Path, original_input_timeseries_path:
     ts_export = pi.Timeseries(
         data_config=dataconfig, folder=output_base_path / "period_0", basename="timeseries_export", binary=False
     )  # Use the first timeseries export as a starting point for the combined timeseries export.
-    ts_export._Timeseries__path_xml = os.path.join(output_base_path.parent, "timeseries_export.xml")
     ts_export.resize(orig_start_datetime, orig_end_datetime)
 
+    variables = _get_variables_from_pi(data_config=dataconfig, timeseries=ts_export)
+
     i = 0
     while os.path.isfile(os.path.join(output_base_path, f"period_{i}", "timeseries_export.xml")):
         ts_export_step = pi.Timeseries(
@@ -38,7 +51,7 @@ def combine_xml_exports(output_base_path: Path, original_input_timeseries_path:
         all_times = ts_import_orig.times  # Workaround to map indices to times, as ts_export does
         # not contain all times. TODO Check whether the assumption that these times map to
         # the correct indices for ts_export always holds.
-        for loc_par in dataconfig._DataConfig__location_parameter_ids:
+        for loc_par in variables:
             try:
                 current_values = ts_export.get(loc_par)
                 new_values = ts_export_step.get(loc_par)
@@ -60,12 +73,12 @@ def combine_xml_exports(output_base_path: Path, original_input_timeseries_path:
             combined_values[start_new_data_index : start_new_data_index + len(new_values)] = new_values  # noqa
             ts_export.set(loc_par, combined_values)
         i += 1
-    ts_export.write()
+    ts_export.write(output_folder=output_base_path.parent, output_filename="timeseries_export")
 
     if write_csv_out:
         data = pd.DataFrame()
         data["date"] = all_times
-        for timeseries_id in dataconfig._DataConfig__location_parameter_ids:
+        for timeseries_id in variables:
             try:
                 values = ts_export.get(timeseries_id)
             except KeyError:

{rtc-tools-interface-0.8.0b4 → rtc-tools-interface-0.9.0}/rtctools_interface/closed_loop/runner.py

@@ -9,7 +9,8 @@ from rtctools.data.pi import DiagHandler
 from rtctools.optimization.pi_mixin import PIMixin
 from rtctools.optimization.csv_mixin import CSVMixin
 from rtctools.util import run_optimization_problem, _resolve_folder
-from rtctools_interface.closed_loop.closed_loop_dates import read_closed_loop_dates
+from rtctools_interface.closed_loop.config import ClosedLoopConfig
+import rtctools_interface.closed_loop.optimization_ranges as opt_ranges
 from rtctools_interface.closed_loop.results_construction import combine_csv_exports, combine_xml_exports
 from rtctools_interface.closed_loop.time_series_handler import XMLTimeSeriesFile, CSVTimeSeriesFile, TimeSeriesHandler
 import logging
@@ -53,26 +54,49 @@ def write_input_folder(
     timeseries_import.write(modelling_period_input_folder_i)
 
 
+def _get_optimization_ranges(
+    config: ClosedLoopConfig,
+    input_timeseries: TimeSeriesHandler,
+) -> list[tuple[datetime.datetime, datetime.datetime]]:
+    """Return a list of optimization periods."""
+    if config.file is not None:
+        datetime_range = input_timeseries.get_datetime_range()
+        optimization_ranges = opt_ranges.get_optimization_ranges_from_file(
+            config.file, datetime_range
+        )
+    elif config.optimization_period is not None:
+        datetimes = input_timeseries.get_datetimes()
+        optimization_ranges = opt_ranges.get_optimization_ranges(
+            model_times=datetimes,
+            start_time=datetimes[0],
+            forecast_timestep=config.forecast_timestep,
+            optimization_period=config.optimization_period,
+        )
+    else:
+        raise ValueError(
+            "The closed-loop configuration should have either a file or optimization_period set."
+        )
+    if config.round_to_dates:
+        optimization_ranges = opt_ranges.round_datetime_ranges_to_days(optimization_ranges)
+    return optimization_ranges
+
+
 def run_optimization_problem_closed_loop(
     optimization_problem_class,
     base_folder="..",
     log_level=logging.INFO,
     profile=False,
+    config: ClosedLoopConfig = None,
     **kwargs,
 ):
-    """Runs an optimization problem in closed loop mode.
-    This function is a drop-in replacement for the run_optimization_problem of rtc-tools. The user
-    needs to specify a closed_loop_dates.csv in the input folder of the optimization problem. This
-    CSV file should contain two columns: start_date and end_date, each row corresponding to one
-    modelling period. The function will run optimization problems for each modelling period specified
-    in the CSV file, setting the final results from the previous run as initial conditions for the next.
-    See the readme.md for more details.
-
-    Notes:
-        - The first start_date in your closed_loop_dates.csv should be equal to the start_date of
-          your timeseries_import.
-        - The different horizons should overlap with at least one day to allow retrieving and setting
-          initial values. More days of overlap are allowed.
+    """
+    Runs an optimization problem in closed loop mode.
+
+    This function is a drop-in replacement for the run_optimization_problem of rtc-tools.
+    The user needs to specify a closed-loop configuration that describes the time ranges
+    for which to subsequentially solve the optimization problem.
+    The results from the previous run will be used to set the initial values of the next run.
+    See README.md for more details.
     """
     base_folder = Path(base_folder)
     if not os.path.isabs(base_folder):
@@ -91,10 +115,10 @@ def run_optimization_problem_closed_loop(
         if not any((isinstance(h, DiagHandler) for h in logger.handlers)):
             diag_handler = DiagHandler(original_output_folder)
             logger.addHandler(diag_handler)
+    logger.setLevel(log_level)
 
     if issubclass(optimization_problem_class, PIMixin):
         original_import = XMLTimeSeriesFile(original_input_folder)
-        original_import.set_reference_data(copy.deepcopy(original_import))
     elif issubclass(optimization_problem_class, CSVMixin):
         original_import = CSVTimeSeriesFile(original_input_folder)
     else:
@@ -112,24 +136,15 @@ def run_optimization_problem_closed_loop(
         shutil.rmtree(modelling_periods_output_folder)
     modelling_periods_output_folder.mkdir(exist_ok=True)
 
-    original_date_range = original_import.get_datetime_range()
-    closed_loop_dates = read_closed_loop_dates(original_input_folder / "closed_loop_dates.csv")
-    assert (
-        min(closed_loop_dates["start_date"]).date() == original_date_range[0].date()
-    ), "The start day of the first optimization run is not equal to the start day of the timeseries import."
-    assert (
-        max(closed_loop_dates["end_date"]).date() <= original_date_range[1].date()
-    ), "The end date of one or more optimization runs is later than the end date of the timeseries import."
-
+    if config is None:
+        config = ClosedLoopConfig(original_input_folder / "closed_loop_dates.csv")
+    optimization_ranges = _get_optimization_ranges(config, original_import)
     results_previous_run = None
     previous_run_datetimes = None
-    for i, date_range in closed_loop_dates.iterrows():
+    for i, (start_time, end_time) in enumerate(optimization_ranges):
         timeseries_import = copy.deepcopy(original_import)
 
-        timeseries_import.select_time_range(
-            start_date=date_range["start_date"],
-            end_date=date_range["end_date"] + datetime.timedelta(days=1) - datetime.timedelta(seconds=1),
-        )
+        timeseries_import.select_time_range(start_date=start_time, end_date=end_time)
 
         set_initial_values_from_previous_run(results_previous_run, timeseries_import, previous_run_datetimes)
 
@@ -139,6 +154,7 @@ def run_optimization_problem_closed_loop(
         modelling_period_input_folder_i = modelling_period_input_folder / modelling_period_name
         write_input_folder(modelling_period_input_folder_i, original_input_folder, timeseries_import)
 
+        logger.info(f"Running optimization for period {i}: {(str(start_time), str(end_time))}.")
         result = run_optimization_problem(
             optimization_problem_class,
             base_folder,
@@ -148,7 +164,13 @@ def run_optimization_problem_closed_loop(
             output_folder=modelling_period_output_folder_i,
             **kwargs,
         )
-        logger.info(f"Finished optimization run {i} with result {result}")
+        period = f"period {i} {(str(start_time), str(end_time))}"
+        if result.solver_stats["success"]:
+            logger.info(f"Successful optimization for {period}.")
+        else:
+            message = f"Failed optimization for {period} with status '{result.solver_stats['return_status']}'."
+            logger.error(message)
+            raise Exception(message)
 
         results_previous_run = {key: result.extract_results().get(key) for key in variables_in_import}
         previous_run_datetimes = result.io.datetimes