rtc-tools-interface 0.7.6__tar.gz → 0.8.0b2__tar.gz

{rtc-tools-interface-0.7.6 → rtc-tools-interface-0.8.0b2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rtc-tools-interface
-Version: 0.7.6
+Version: 0.8.0b2
 Summary: Toolbox for user interfaces for RTC-Tools
 Home-page: UNKNOWN
 Author: Deltares

{rtc-tools-interface-0.7.6 → rtc-tools-interface-0.8.0b2}/README.md

@@ -8,6 +8,12 @@ This is rtc-tools-interface, a toolbox for user-interfaces for [rtc-tools](https
 pip install rtc-tools-interface
 ```
 
+## Table of Contents
+1. [Goal generator](#goal-generator)
+2. [Goal performance metrics](#goal-performance-metrics)
+3. [Automatic plotting of results](#automatic-plotting-of-results)
+4. [Closed loop runner](#closed-loop-runner)
+
 ## Goal generator
 The `goal generator` can be used to automatically add goals based on a csv file. Currently, the following goal types are supported:
 - range (default order is 2)
@@ -164,3 +170,32 @@ The table could thus look like:
 
 
 After running the model, in your output folder the folder `figures` containing the figures is created.
+
+## Closed loop runner
+To run a closed loop experiment one can use the `run_optimization_problem_closed_loop` function from `run_closed_loop`.
+This function is a drop-in replacement for the `run_optimization_problem` of rtc-tools. The user only needs to specify the `closed_loop_dates.csv` in the input folder of the optimization problem.
+
+### Setup
+Import `run_optimization_problem_closed_loop` with:
+```python
+from rtctools_interface import run_optimization_problem_closed_loop
+```
+Add a table named `closed_loop_dates.csv` to the input folder of your optimization problem. The table should contain two columns: `start_date` and `end_date`.
+Each row of the table corresponds to one modelling period. 
+
+Example table `closed_loop_dates.csv`:
+```
+start_date, end_date
+2024-05-19, 2024-05-23
+2024-05-23, 2024-05-25
+```
+With this table rtc-tools will run two optimization problems (modelling periods): one with the data from 2024-05-19 upto and including 2024-05-23 and one from 2024-05-23 upto and including 2024-05-25. 
+The `run_optimization_problem_closed_loop` will automatically set the final results from the previous as initial conditions of the next run. Note that this happens for:
+- All variables available at the first time step in original timeseries_import, but not available at any timestep in the modelling period.
+- All variables in the `initial_state.csv` (if the csv_mixin is used).
+
+### Notes
+- The first start_date in your `closed_loop_dates.csv` should be equal to the start_date of your timeseries_import.
+- The different horizons should overlap with at least one day (to allow retrieving and setting initial values). An overlap of more days is allowed.
+- Currently only a single timestep is copied as an initial value.
+- The closed_loop runner only works in combination with the CSVMixin or the PIMixin. The CDFMixin is not supported.

{rtc-tools-interface-0.7.6 → rtc-tools-interface-0.8.0b2}/rtc_tools_interface.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rtc-tools-interface
-Version: 0.7.6
+Version: 0.8.0b2
 Summary: Toolbox for user interfaces for RTC-Tools
 Home-page: UNKNOWN
 Author: Deltares

{rtc-tools-interface-0.7.6 → rtc-tools-interface-0.8.0b2}/rtc_tools_interface.egg-info/SOURCES.txt

@@ -10,6 +10,11 @@ rtc_tools_interface.egg-info/requires.txt
 rtc_tools_interface.egg-info/top_level.txt
 rtctools_interface/__init__.py
 rtctools_interface/_version.py
+rtctools_interface/closed_loop/__init__.py
+rtctools_interface/closed_loop/closed_loop_dates.py
+rtctools_interface/closed_loop/results_construction.py
+rtctools_interface/closed_loop/runner.py
+rtctools_interface/closed_loop/time_series_handler.py
 rtctools_interface/optimization/__init__.py
 rtctools_interface/optimization/base_goal.py
 rtctools_interface/optimization/base_optimization_problem.py
@@ -35,6 +40,9 @@ rtctools_interface/utils/results_collection.py
 rtctools_interface/utils/serialization.py
 rtctools_interface/utils/type_definitions.py
 tests/__init__.py
+tests/closed_loop/__init__.py
+tests/closed_loop/test_read_xml.py
+tests/closed_loop/test_run_optization_problem_closed_loop.py
 tests/optimization/__init__.py
 tests/optimization/test_base_optimization_problem.py
 tests/optimization/test_passing_goals_directly.py

rtc-tools-interface-0.8.0b2/rtctools_interface/__init__.py

@@ -0,0 +1,7 @@
+from . import _version
+
+__version__ = _version.get_versions()["version"]
+
+from rtctools_interface.closed_loop.runner import run_optimization_problem_closed_loop
+
+__all__ = ["run_optimization_problem_closed_loop"]

{rtc-tools-interface-0.7.6 → rtc-tools-interface-0.8.0b2}/rtctools_interface/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-05-07T17:02:33+0200",
+ "date": "2024-05-28T11:08:07+0200",
  "dirty": false,
  "error": null,
- "full-revisionid": "0a20a376b41a3ca0e27f8d19c46f6b6d91536766",
- "version": "0.7.6"
+ "full-revisionid": "706b384c4fcf2b2621e9370973ec8ecc78765fb8",
+ "version": "0.8.0b2"
 }
 '''  # END VERSION_JSON
 

rtc-tools-interface-0.8.0b2/rtctools_interface/closed_loop/closed_loop_dates.py

@@ -0,0 +1,43 @@
+from pathlib import Path
+
+import pandas as pd
+
+
+def read_closed_loop_dates(file_path: Path):
+    """Read horizon config from a csv file"""
+    if not file_path.exists():
+        raise FileNotFoundError(
+            f"The closed_loop_dates csv does not exist. Please create a horizon config file in {file_path}."
+        )
+    try:
+        closed_loop_dates = pd.read_csv(file_path)
+    except pd.errors.EmptyDataError:
+        raise ValueError(
+            "The closed_loop_dates csv is empty. Please provide a valid file with start_date and end_date column."
+        )
+    closed_loop_dates.columns = closed_loop_dates.columns.str.replace(" ", "")
+    if not all([col in closed_loop_dates.columns for col in ["start_date", "end_date"]]):
+        raise ValueError("The closed_loop_dates csv should have both 'start_date' and 'end_date' columns.")
+    closed_loop_dates["start_date"] = pd.to_datetime(closed_loop_dates["start_date"])
+    closed_loop_dates["end_date"] = pd.to_datetime(closed_loop_dates["end_date"])
+    for i in range(1, len(closed_loop_dates)):
+        if closed_loop_dates["start_date"].iloc[i] > closed_loop_dates["end_date"].iloc[i - 1]:
+            raise ValueError(f"Closed loop date table: Start date at row {i} is later than the previous end date. ")
+    if any(closed_loop_dates["start_date"] < closed_loop_dates["start_date"].shift(1)):
+        raise ValueError("Closed loop date table: The start dates are not in ascending order.")
+    if any(closed_loop_dates["end_date"] < closed_loop_dates["end_date"].shift(1)):
+        raise ValueError("Closed loop date table: The end dates are not in ascending order.")
+    if any(closed_loop_dates["end_date"] < closed_loop_dates["start_date"]):
+        raise ValueError("Closed loop date table: For one or more rows the end date is before the start date.")
+    if any(closed_loop_dates["start_date"] > closed_loop_dates["end_date"]):
+        raise ValueError("Closed loop date table: For one or more rows the start date is after the end date.")
+    if (
+        any(closed_loop_dates["start_date"].dt.hour != 0)
+        or any(closed_loop_dates["start_date"].dt.minute != 0)
+        or any(closed_loop_dates["end_date"].dt.hour != 0)
+        or any(closed_loop_dates["end_date"].dt.minute != 0)
+    ):
+        raise ValueError(
+            "Closed loop date table: Currently, the date ranges can only be specific up to the level of days."
+        )
+    return closed_loop_dates

rtc-tools-interface-0.8.0b2/rtctools_interface/closed_loop/results_construction.py

@@ -0,0 +1,108 @@
+import copy
+import logging
+import os
+from pathlib import Path
+from typing import List
+import pandas as pd
+from rtctools.data import rtc
+from rtctools.data import pi
+
+logger = logging.getLogger("rtctools")
+
+
+def combine_xml_exports(output_base_path: Path, original_input_timeseries_path: Path, write_csv_out: bool = False):
+    """Combine the xml exports of multiple periods into a single xml file."""
+    logger.info("Combining XML exports.")
+    dataconfig = rtc.DataConfig(folder=original_input_timeseries_path)
+
+    ts_import_orig = pi.Timeseries(
+        data_config=dataconfig, folder=original_input_timeseries_path, basename="timeseries_import", binary=False
+    )
+    orig_start_datetime = ts_import_orig.start_datetime
+    orig_end_datetime = ts_import_orig.end_datetime
+
+    ts_export = pi.Timeseries(
+        data_config=dataconfig, folder=output_base_path / "period_0", basename="timeseries_export", binary=False
+    )  # Use the first timeseries export as a starting point for the combined timeseries export.
+    ts_export._Timeseries__path_xml = os.path.join(output_base_path.parent, "timeseries_export.xml")
+    ts_export.resize(orig_start_datetime, orig_end_datetime)
+
+    i = 0
+    while os.path.isfile(os.path.join(output_base_path, f"period_{i}", "timeseries_export.xml")):
+        ts_export_step = pi.Timeseries(
+            data_config=dataconfig,
+            folder=os.path.join(output_base_path, f"period_{i}"),
+            basename="timeseries_export",
+            binary=False,
+        )
+        all_times = ts_import_orig.times  # Workaround to map indices to times, as ts_export does
+        # not contain all times. TODO Check whether the assumption that these times map to
+        # the correct indices for ts_export always holds.
+        for loc_par in dataconfig._DataConfig__location_parameter_ids:
+            try:
+                current_values = ts_export.get(loc_par)
+                new_values = ts_export_step.get(loc_par)
+            except KeyError:
+                logger.debug("Variable {} not found in output of model horizon: {}".format(loc_par, i))
+                continue
+            new_times = ts_export_step.times
+            try:
+                start_new_data_index = all_times.index(new_times[0])
+            except ValueError:
+                if all_times[-1] + ts_export.dt == new_times[0]:
+                    start_new_data_index = len(all_times)
+                else:
+                    raise ValueError(
+                        "Could not match the start data of the timeseries export file "
+                        + "with the end of the previous."
+                    )
+            combined_values = copy.deepcopy(current_values)
+            combined_values[start_new_data_index : start_new_data_index + len(new_values)] = new_values  # noqa
+            ts_export.set(loc_par, combined_values)
+        i += 1
+    ts_export.write()
+
+    if write_csv_out:
+        data = pd.DataFrame()
+        data["date"] = all_times
+        for timeseries_id in dataconfig._DataConfig__location_parameter_ids:
+            try:
+                values = ts_export.get(timeseries_id)
+            except KeyError:
+                logger.debug("Variable {} not found in output of model horizon: {}".format(timeseries_id, i))
+                continue
+            data[timeseries_id] = values
+        data.round(6).to_csv(output_base_path.parent / "timeseries_export.csv", index=False)
+
+
+def combine_dataframes(dfs: List[pd.DataFrame], index_col: str = "time"):
+    """Combine multiple dataframes with the same index column.
+    The dataframes are combined in the order they are passed, with the last dataframe taking precedence
+    in case of overlapping indices."""
+    combined_df = pd.DataFrame()
+    for df in dfs:
+        df.set_index(index_col, inplace=True)
+        combined_df = df.combine_first(combined_df)
+    combined_df.reset_index(inplace=True)
+    return combined_df
+
+
+def combine_csv_exports(output_base_path: Path):
+    """Combine the csv exports of multiple periods into a single csv file."""
+    i = 0
+    dfs = []
+    while os.path.isfile(os.path.join(output_base_path, f"period_{i}", "timeseries_export.csv")):
+        df = pd.read_csv(os.path.join(output_base_path, f"period_{i}", "timeseries_export.csv"))
+        dfs.append(df)
+        i += 1
+    combined_df = combine_dataframes(dfs)
+    combined_df.round(6).to_csv(output_base_path.parent / "timeseries_export.csv", index=False)
+
+
+if __name__ == "__main__":
+    closed_loop_test_folder = Path(__file__).parents[2] / "tests" / "closed_loop"
+    output_base_path = closed_loop_test_folder / Path(
+        r"test_models\goal_programming_xml\output\output_modelling_periods_reference"
+    )
+    original_input_timeseries_path = closed_loop_test_folder / Path(r"test_models\goal_programming_xml\input")
+    combine_xml_exports(output_base_path, original_input_timeseries_path, True)

rtc-tools-interface-0.8.0b2/rtctools_interface/closed_loop/runner.py

@@ -0,0 +1,156 @@
+import datetime
+import os
+import copy
+from pathlib import Path
+import shutil
+import sys
+from typing import List, Optional
+from rtctools.optimization.pi_mixin import PIMixin
+from rtctools.optimization.csv_mixin import CSVMixin
+from rtctools.util import run_optimization_problem, _resolve_folder
+from rtctools_interface.closed_loop.closed_loop_dates import read_closed_loop_dates
+from rtctools_interface.closed_loop.results_construction import combine_csv_exports, combine_xml_exports
+from rtctools_interface.closed_loop.time_series_handler import XMLTimeSeriesFile, CSVTimeSeriesFile, TimeSeriesHandler
+import logging
+
+logger = logging.getLogger("rtctools")
+
+
+def set_initial_values_from_previous_run(
+    results_previous_run: Optional[dict],
+    timeseries: TimeSeriesHandler,
+    previous_run_datetimes: List[datetime.datetime],
+) -> None:
+    """Modifies the initial values of `timeseries` based on the results of the previous run (if any)"""
+    if results_previous_run is not None:
+        variables_to_set = {key: value for key, value in results_previous_run.items() if not timeseries.is_set(key)}
+        if timeseries.forecast_date:
+            index_of_initial_value = previous_run_datetimes.index(timeseries.forecast_date)
+        else:
+            raise ValueError("Could not find forecast date in timeseries import.")
+        for key, values in variables_to_set.items():
+            if values is not None:
+                timeseries.set_initial_value(key, values[index_of_initial_value])
+            else:
+                raise ValueError(f"Could not find initial value for {key}.")
+
+
+def write_input_folder(
+    modelling_period_input_folder_i: Path,
+    original_input_folder: Path,
+    timeseries_import: TimeSeriesHandler,
+) -> None:
+    """Write the input folder for the current modelling period.
+    Copies the original input folder to the modelling period input folder and writes the new
+    timeseries_import."""
+    modelling_period_input_folder_i.mkdir(exist_ok=True)
+    for file in original_input_folder.iterdir():
+        if file.is_file():
+            shutil.copy(file, modelling_period_input_folder_i / file.name)
+        elif file.is_dir():
+            shutil.copytree(file, modelling_period_input_folder_i / file.name)
+    timeseries_import.write(modelling_period_input_folder_i)
+
+
+def run_optimization_problem_closed_loop(
+    optimization_problem_class,
+    base_folder="..",
+    log_level=logging.INFO,
+    profile=False,
+    **kwargs,
+):
+    """Runs an optimization problem in closed loop mode.
+    This function is a drop-in replacement for the run_optimization_problem of rtc-tools. The user
+    needs to specify a closed_loop_dates.csv in the input folder of the optimization problem. This
+    CSV file should contain two columns: start_date and end_date, each row corresponding to one
+    modelling period. The function will run optimization problems for each modelling period specified
+    in the CSV file, setting the final results from the previous run as initial conditions for the next.
+    See the readme.md for more details.
+
+    Notes:
+        - The first start_date in your closed_loop_dates.csv should be equal to the start_date of
+          your timeseries_import.
+        - The different horizons should overlap with at least one day to allow retrieving and setting
+          initial values. More days of overlap are allowed.
+    """
+    base_folder = Path(base_folder)
+    if not os.path.isabs(base_folder):
+        base_folder = Path(sys.path[0]) / base_folder
+    original_input_folder = Path(_resolve_folder(kwargs, base_folder, "input_folder", "input"))
+    original_output_folder = Path(_resolve_folder(kwargs, base_folder, "output_folder", "output"))
+    original_output_folder.mkdir(exist_ok=True)
+
+    if issubclass(optimization_problem_class, PIMixin):
+        original_import = XMLTimeSeriesFile(original_input_folder)
+        original_import.set_reference_data(copy.deepcopy(original_import))
+    elif issubclass(optimization_problem_class, CSVMixin):
+        original_import = CSVTimeSeriesFile(original_input_folder)
+    else:
+        raise ValueError("Optimization problem class must be derived from PIMixin or CSVMixin.")
+
+    variables_in_import = original_import.get_all_internal_ids()
+
+    modelling_period_input_folder = base_folder / "input_modelling_periods"
+    if modelling_period_input_folder.exists():
+        shutil.rmtree(modelling_period_input_folder)
+    modelling_period_input_folder.mkdir(exist_ok=True)
+
+    modelling_periods_output_folder = original_output_folder / "output_modelling_periods"
+    if modelling_periods_output_folder.exists():
+        shutil.rmtree(modelling_periods_output_folder)
+    modelling_periods_output_folder.mkdir(exist_ok=True)
+
+    original_date_range = original_import.get_datetime_range()
+    closed_loop_dates = read_closed_loop_dates(original_input_folder / "closed_loop_dates.csv")
+    assert (
+        min(closed_loop_dates["start_date"]).date() == original_date_range[0].date()
+    ), "The start day of the first optimization run is not equal to the start day of the timeseries import."
+    assert (
+        max(closed_loop_dates["end_date"]).date() <= original_date_range[1].date()
+    ), "The end date of one or more optimization runs is later than the end date of the timeseries import."
+
+    results_previous_run = None
+    previous_run_datetimes = None
+    for i, date_range in closed_loop_dates.iterrows():
+        timeseries_import = copy.deepcopy(original_import)
+
+        timeseries_import.select_time_range(
+            start_date=date_range["start_date"],
+            end_date=date_range["end_date"] + datetime.timedelta(days=1) - datetime.timedelta(seconds=1),
+        )
+
+        set_initial_values_from_previous_run(results_previous_run, timeseries_import, previous_run_datetimes)
+
+        modelling_period_name = f"period_{i}"
+        modelling_period_output_folder_i = modelling_periods_output_folder / modelling_period_name
+        modelling_period_output_folder_i.mkdir(exist_ok=True)
+        modelling_period_input_folder_i = modelling_period_input_folder / modelling_period_name
+        write_input_folder(modelling_period_input_folder_i, original_input_folder, timeseries_import)
+
+        result = run_optimization_problem(
+            optimization_problem_class,
+            base_folder,
+            log_level,
+            profile,
+            input_folder=modelling_period_input_folder_i,
+            output_folder=modelling_period_output_folder_i,
+            **kwargs,
+        )
+        logger.info(f"Finished optimization run {i} with result {result}")
+
+        results_previous_run = {key: result.extract_results().get(key) for key in variables_in_import}
+        previous_run_datetimes = result.io.datetimes
+        if len(results_previous_run) != len(variables_in_import):
+            logger.warning("Could not find the results for all input variables.")
+            logger.warning("Missing variables: " + str(set(variables_in_import) - set(results_previous_run.keys())))
+            raise ValueError("Could not find the results for all input variables.")
+
+    logger.info("Finished all optimization runs.")
+    if issubclass(optimization_problem_class, PIMixin):
+        combine_xml_exports(modelling_periods_output_folder, original_input_folder, write_csv_out=True)
+    elif issubclass(optimization_problem_class, CSVMixin):
+        combine_csv_exports(modelling_periods_output_folder)
+    else:
+        logger.warning(
+            "Could not combine exports because the optimization problem class is not derived from PIMixin or CSVMixin."
+        )

rtc-tools-interface-0.8.0b2/rtctools_interface/closed_loop/time_series_handler.py

@@ -0,0 +1,430 @@
+import copy
+import datetime
+import logging
+from abc import ABC, abstractmethod
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import List, Optional, Tuple
+import pandas as pd
+from rtctools.data import pi
+from rtctools.data import rtc
+from rtctools.data import csv
+
+ns = {"fews": "http://www.wldelft.nl/fews", "pi": "http://www.wldelft.nl/fews/PI"}
+
+logger = logging.getLogger("rtctools")
+
+
+class TimeSeriesHandler(ABC):
+    """ABC for handling timeseries data."""
+
+    # The forecast date determines at which date the optimization starts.
+    forecast_date: Optional[datetime.datetime] = None
+
+    @abstractmethod
+    def read(self, file_name: str) -> None:
+        """Read the timeseries."""
+
+    @abstractmethod
+    def select_time_range(self, start_date: datetime.datetime, end_date: datetime.datetime) -> None:
+        """Select a time range from the timeseries data. Removes data outside the interval.
+        The specified range is inclusive on both sides."""
+
+    @abstractmethod
+    def write(self, file_path: Path) -> None:
+        """Write the timeseries data to a file."""
+
+    def set_reference_data(self, reference_data: "TimeSeriesHandler"):
+        """Set the reference TimeSeriesHandler. Only relevant for XMLTimeSeriesFile.
+        Required when setting initial values for variables that are not in the modelling period data range."""
+
+    @abstractmethod
+    def get_timestep(self) -> datetime.timedelta:
+        """Get the timestep of the timeseries data."""
+
+    @abstractmethod
+    def get_datetime_range(self) -> Tuple[datetime.datetime, datetime.datetime]:
+        """Get the date range of the timeseries data (min, max)."""
+
+    @abstractmethod
+    def get_all_internal_ids(self) -> List[str]:
+        """Get all internal id's of the timeseries data."""
+
+    @abstractmethod
+    def set_initial_value(self, internal_id: str, value: float) -> None:
+        """Set the initial value of a variable in the timeseries data."""
+
+    @abstractmethod
+    def is_set(self, internal_id: str) -> bool:
+        """Check whether the variable exists in the timeseries data and whether it has a least one non-nan value"""
+
+
+class CSVTimeSeriesFile(TimeSeriesHandler):
+    """Timeseries handler for csv files."""
+
+    def __init__(
+        self,
+        input_folder: Path,
+        timeseries_import_basename: str = "timeseries_import",
+        csv_delimiter=",",
+        initial_state_base_name: str = "initial_state",
+    ):
+        self.data = pd.DataFrame()
+        self.input_folder = input_folder
+        self.csv_delimiter = csv_delimiter
+        self.read(timeseries_import_basename, initial_state_base_name)
+
+    def read(self, file_name: str, initial_state_base_name=None):
+        timeseries = csv.load(
+            (self.input_folder / file_name).with_suffix(".csv"),
+            delimiter=self.csv_delimiter,
+            with_time=True,
+        )
+        self.data = pd.DataFrame(timeseries)
+        if self.data is not None:
+            self.date_col = self.data.columns[0]
+            self.forecast_date = self.data[self.date_col].iloc[0]
+        else:
+            raise ValueError("No data to read.")
+        if initial_state_base_name is not None:
+            initial_state_file = self.input_folder / initial_state_base_name
+            if initial_state_file.with_suffix(".csv").exists():
+                initial_state = csv.load(
+                    initial_state_file.with_suffix(".csv"),
+                    delimiter=self.csv_delimiter,
+                    with_time=False,
+                )
+                self.initial_state: Optional[dict] = {
+                    field: float(initial_state[field]) for field in initial_state.dtype.names
+                }
+        else:
+            self.initial_state = None
+
+    def select_time_range(self, start_date: datetime.datetime, end_date: datetime.datetime):
+        mask = (self.data[self.date_col] >= start_date) & (self.data[self.date_col] <= end_date)
+        self.data = self.data.loc[mask]
+        self.forecast_date = start_date
+
+    def write(self, file_path: Path):
+        self.write_timeseries(file_path)
+        self.write_initial_state(file_path)
+
+    def write_timeseries(self, file_path: Path, file_name: str = "timeseries_import"):
+        self.data.to_csv(
+            (file_path / file_name).with_suffix(".csv"),
+            index=False,
+            date_format="%Y-%m-%d %H:%M:%S",
+        )
+
+    def write_initial_state(self, file_path: Path, file_name: str = "initial_state"):
+        if self.initial_state is not None:
+            initial_state = pd.DataFrame(self.initial_state, index=[0])
+            initial_state.to_csv((file_path / file_name).with_suffix(".csv"), header=True, index=False)
+
+    def get_timestep(self):
+        return self.data[self.date_col].diff().min()
+
+    def get_datetime_range(self):
+        return self.data[self.date_col].min(), self.data[self.date_col].max()
+
+    def get_all_internal_ids(self):
+        ids = list(self.data.columns[1:])
+        if self.initial_state is not None:
+            ids.extend(list(self.initial_state.keys()))
+        return ids
+
+    def set_initial_value(self, internal_id, value):
+        if self.initial_state is None or internal_id not in self.initial_state:
+            self.data[internal_id].iloc[0] = value
+        else:
+            self.initial_state[internal_id] = value
+
+    def is_set(self, internal_id):
+        val_is_set = False
+        if internal_id in self.data.columns:
+            val_is_set = not self.data[internal_id].isna().all()
+        if self.initial_state is not None and internal_id in self.initial_state:
+            val_is_set = False
+        return val_is_set
+
+
+class XMLTimeSeriesFile(TimeSeriesHandler):
+    """ "Timeseries handler for xml files"""
+
+    # Whether the timeseries data has a forecast date in the header.
+    forecast_date_in_header = False
+
+    def __parse_date_time(self, el):
+        return datetime.datetime.strptime(el.get("date") + " " + el.get("time"), "%Y-%m-%d %H:%M:%S")
+
+    def __init__(
+        self,
+        input_folder: Path,
+        timeseries_import_basename: str = "timeseries_import",
+    ):
+        self.input_folder = input_folder
+        self.pi_binary_timeseries = False
+        self.pi_validate_timeseries = True
+        self.read(timeseries_import_basename)
+
+    def read(self, file_name: str):
+        """Read the timeseries data from a file."""
+        timeseries_import_basename = file_name
+        self.data_config = rtc.DataConfig(self.input_folder)
+        self.pi_timeseries = pi.Timeseries(
+            self.data_config,
+            self.input_folder,
+            timeseries_import_basename,
+            binary=self.pi_binary_timeseries,
+            pi_validate_times=self.pi_validate_timeseries,
+        )
+        tree = self.pi_timeseries._Timeseries__tree
+        self.root = tree.getroot()
+        if self.root is None:
+            raise ValueError("No data to read.")
+        self.set_forecast_date()
+
+    def set_forecast_date(self):
+        """Set the internal attribute `forecast_date` of the timeseries data.
+
+        The forecast date is set to the first event date of the first series
+        if no forecast date is present in the header."""
+        first_series = self.root.find("pi:series", ns)
+        first_header = first_series.find("pi:header", ns)
+        forecast_date_element = first_header.find("pi:forecastDate", ns)
+        if forecast_date_element is not None:
+            self.forecast_date = self.__parse_date_time(forecast_date_element)
+            self.forecast_date_in_header = True
+        else:
+            first_event = first_series.find("pi:event", ns)
+            self.forecast_date = self.__parse_date_time(first_event)
+            self.forecast_date_in_header = False
+
+    def is_set(self, internal_id):
+        """Check whether the variable exists in the timeseries data and whether it has a value at at least
+        one of time steps."""
+        location_id, parameter_id, qualifier_id = self.get_external_id_from_internal_id(internal_id)
+        for series in self.root.findall("pi:series", ns):
+            if (
+                location_id,
+                parameter_id,
+                qualifier_id,
+            ) == self.get_external_id_from_series(series):
+                events = series.findall("pi:event", ns)
+                for event in events:
+                    if event.get("value") is not None:
+                        return True
+        return False
+
+    def get_internal_id(self, series):
+        """Get the internal id of a series element."""
+        pi_header = series.find("pi:header", ns)
+        return self.data_config.variable(pi_header)
+
+    def get_external_id_from_internal_id(self, internal_id):
+        """Get the external id of a series element. Returns a tuple with three elements: location, parameter
+        and qualifier ID's."""
+        return self.data_config.pi_variable_ids(internal_id)
+
+    def get_all_internal_ids(self):
+        """Get all internal id's of the timeseries data. Only returns the id's that are also in the dataconfig."""
+        all_ids = [self.get_internal_id(series) for series in self.root.findall("pi:series", ns)]
+        return [id for id in all_ids if ":" not in id]  # Variables that contain ":"  are not in the dataconfig.
+
+    def set_new_forecast_date(self, forecast_date: datetime.datetime):
+        for series in self.root.findall("pi:series", ns):
+            header = series.find("pi:header", ns)
+            start_date, end_date = self.get_single_date_range_from_series(series)
+            if start_date <= forecast_date <= end_date:
+                header.find("pi:forecastDate", ns).attrib = {
+                    "date": forecast_date.strftime("%Y-%m-%d"),
+                    "time": forecast_date.strftime("%H:%M:%S"),
+                }
+            else:
+                raise ValueError("Forecast date is not within the date range of the timeseries data.")
+
+    def select_time_range(self, start_date: datetime.datetime, end_date: datetime.datetime):
+        assert isinstance(start_date, datetime.datetime) and isinstance(
+            end_date, datetime.datetime
+        ), "Dates must be datetime objects"
+        for series in self.root.findall("pi:series", ns):
+            new_start_date, new_end_date = datetime.datetime.max, datetime.datetime.min
+            events = series.findall("pi:event", ns)
+            for event in events:
+                event_datetime = self.__parse_date_time(event)
+                if event_datetime < start_date or event_datetime > end_date:
+                    series.remove(event)
+                    continue
+                if event_datetime < new_start_date:
+                    new_start_date = event_datetime
+                if event_datetime > new_end_date:
+                    new_end_date = event_datetime
+            start_date_attrib = {
+                "date": new_start_date.strftime("%Y-%m-%d"),
+                "time": new_start_date.strftime("%H:%M:%S"),
+            }
+            end_date_attrib = {
+                "date": new_end_date.strftime("%Y-%m-%d"),
+                "time": new_end_date.strftime("%H:%M:%S"),
+            }
+            series.find("pi:header", ns).find("pi:startDate", ns).attrib = start_date_attrib
+            series.find("pi:header", ns).find("pi:endDate", ns).attrib = end_date_attrib
+            if self.forecast_date_in_header:
+                series.find("pi:header", ns).find("pi:forecastDate", ns).attrib = start_date_attrib
+            if not series.findall("pi:event", ns):
+                self.root.remove(series)
+                logger.warning("Removed series with no events.")
+            self.set_forecast_date()
+
+    def write(self, file_path: Path, file_name: str = "timeseries_import"):
+        tree = ET.ElementTree(self.root)
+        tree.write((file_path / file_name).with_suffix(".xml"))
+
+    def get_external_id_from_series(self, series) -> Tuple[str, str, List[str]]:
+        header = series.find("pi:header", ns)
+        locationId = header.find("pi:locationId", ns).text
+        parameterId = header.find("pi:parameterId", ns).text
+        qualifier_ids = []
+        qualifiers_els = header.findall("pi:qualifierId", ns)
+        for qualifier in qualifiers_els:
+            qualifier_ids.append(qualifier.text)
+        return locationId, parameterId, qualifier_ids
+
+    def get_single_date_range(self, locationId, paramterId, qualifier_ids):
+        """Get the date range of the timeseries data for a single location and parameter"""
+        min_date = datetime.datetime.max
+        max_date = datetime.datetime.min
+        for series in self.root.findall("pi:series", ns):
+            if (
+                locationId,
+                paramterId,
+                qualifier_ids,
+            ) == self.get_external_id_from_series(series):
+                events = series.findall("pi:event", ns)
+                for event in events:
+                    event_datetime = self.__parse_date_time(event)
+                    if event_datetime < min_date:
+                        min_date = event_datetime
+                    if event_datetime > max_date:
+                        max_date = event_datetime
+        if min_date == datetime.datetime.max:
+            raise ValueError(
+                "No data for locationId {},  parameterId {} and qualifierIds {}".format(
+                    locationId, paramterId, qualifier_ids
+                )
+            )
+        return min_date, max_date
+
+    def get_single_date_range_from_series(self, series):
+        (
+            location_id,
+            parameter_id,
+            qualifier_ids,
+        ) = self.get_external_id_from_series(series)
+        return self.get_single_date_range(location_id, parameter_id, qualifier_ids)
+
+    def get_datetime_range(self):
+        """Get the date range of the timeseries data, minimum and maximum over all series"""
+        min_date = datetime.datetime.max
+        max_date = datetime.datetime.min
+        for series in self.root.findall("pi:series", ns):
+            events = series.findall("pi:event", ns)
+            for event in events:
+                event_datetime = self.__parse_date_time(event)
+                if event_datetime < min_date:
+                    min_date = event_datetime
+                if event_datetime > max_date:
+                    max_date = event_datetime
+        return min_date, max_date
+
+    def get_timestep(self):
+        """Get the timestep of the timeseries data, raise error if different stepsizes"""
+        timestep = None
+        for series in self.root.findall("pi:series", ns):
+            events = series.findall("pi:event", ns)
+            for i, event in enumerate(events):
+                if i == 0:
+                    continue
+                event_datetime = self.__parse_date_time(event)
+                previous_event_datetime = self.__parse_date_time(events[i - 1])
+                if timestep is None:
+                    timestep = event_datetime - previous_event_datetime
+                elif timestep != event_datetime - previous_event_datetime:
+                    raise ValueError("Different timesteps in timeseries data.")
+        return timestep
+
+    def set_reference_data(self, reference_data):
+        """Set the the reference XMLTimeSeriesFile object to use for creating new series."""
+        self._reference_data = reference_data
+
+    def get_series(
+        self,
+        location_id: str,
+        parameter_id: str,
+        qualifier_ids: List[str],
+        from_reference=False,
+    ) -> ET.Element:
+        """Get the XML series element for a location and parameter.
+        If from_reference is True, the series is taken from the reference data."""
+        if from_reference:
+            if hasattr(self, "_reference_data"):
+                root = self._reference_data.root
+            else:
+                raise ValueError("No reference data available.")
+        elif hasattr(self, "root") and self.root is not None:
+            root = self.root
+        else:
+            raise ValueError("No data to select from, use read first.")
+        for series in root.findall("pi:series", ns):
+            if (
+                location_id,
+                parameter_id,
+                qualifier_ids,
+            ) == self.get_external_id_from_series(series):
+                return series
+        raise ValueError("No series found for locationId {} and parameterId {}".format(location_id, parameter_id))
+
+    def set_initial_value(self, internal_id: str, value: float):
+        location_id, parameter_id, qualifier_ids = self.get_external_id_from_internal_id(internal_id)
+        for series in self.root.findall("pi:series", ns):
+            if (
+                location_id,
+                parameter_id,
+                qualifier_ids,
+            ) == self.get_external_id_from_series(series):
+                logger.info(
+                    "Overwriting initial value for locationId {} and parameterId {} and qualifier ids {}".format(
+                        location_id, parameter_id, qualifier_ids
+                    )
+                )
+                event = series.find("pi:event", ns)
+                event.attrib["value"] = str(value)
+                return
+        else:
+            # if no series found, create a new series with just one event (based on the reference data)
+            reference_series = self.get_series(location_id, parameter_id, qualifier_ids, from_reference=True)
+            new_series = copy.deepcopy(reference_series)
+            start_date, end_date = self.get_datetime_range()
+            first_event = new_series.find("pi:event", ns)
+            if first_event is None:
+                raise ValueError("No event found in reference data.")
+            first_event.attrib["value"] = str(value)
+            first_event.attrib["date"] = start_date.strftime("%Y-%m-%d")
+            first_event.attrib["time"] = start_date.strftime("%H:%M:%S")
+            header = new_series.find("pi:header", ns)
+            if header is None:
+                raise ValueError("No header found in reference data.")
+            header.find("pi:startDate", ns).attrib = {
+                "date": start_date.strftime("%Y-%m-%d"),
+                "time": start_date.strftime("%H:%M:%S"),
+            }
+            header.find("pi:endDate", ns).attrib = {
+                "date": end_date.strftime("%Y-%m-%d"),
+                "time": end_date.strftime("%H:%M:%S"),
+            }
+            if self.forecast_date_in_header:
+                header.find("pi:forecastDate", ns).attrib = {
+                    "date": start_date.strftime("%Y-%m-%d"),
+                    "time": start_date.strftime("%H:%M:%S"),
+                }
+            self.root.append(new_series)

{rtc-tools-interface-0.7.6 → rtc-tools-interface-0.8.0b2}/rtctools_interface/optimization/base_goal.py

@@ -124,9 +124,9 @@ class BaseGoal(Goal):
     ):
         """Set function bounds either by user specified value or calculated"""
         state_range = self._get_state_range(optimization_problem, self.state)
-        if (not np.isfinite(function_min) and not np.isfinite(state_range[0])) or (
-            not np.isfinite(function_max) and not np.isfinite(state_range[1])
-        ):
+        if (~np.isfinite(function_min) & ~np.isfinite(state_range[0])).any() or (
+            ~np.isfinite(function_max) & ~np.isfinite(state_range[1])
+        ).any():
             raise ValueError(
                 f"The upper/lower bound for state {self.state} for goal with id={self.goal_id} is not specified"
                 + " so the function range should be specified!"
@@ -215,8 +215,8 @@ class BaseGoal(Goal):
             "goal_id": self.goal_id,
             "state": self.state,
             "goal_type": self.goal_type,
-            "function_min": self.function_range[0] if np.isfinite(self.function_range[0]) else None,
-            "function_max": self.function_range[1] if np.isfinite(self.function_range[1]) else None,
+            "function_min": self.function_range[0] if np.any(np.isfinite(self.function_range[0])) else None,
+            "function_max": self.function_range[1] if np.any(np.isfinite(self.function_range[1])) else None,
             "function_nominal": self.function_nominal if np.isfinite(self.function_nominal) else None,
             "target_min_series": None,
             "target_max_series": None,

{rtc-tools-interface-0.7.6 → rtc-tools-interface-0.8.0b2}/rtctools_interface/optimization/goal_performance_metrics.py

@@ -25,12 +25,16 @@ def get_mean_absolute_percentual_difference(timeseries: np.ndarray) -> float:
     nonzero_indices = np.nonzero(timeseries)
     timeseries = timeseries[nonzero_indices]
     differences = np.diff(timeseries)
+    if len(timeseries) <= 1:
+        return 0
     mapd = np.mean(np.abs(differences / timeseries[:-1]))
     return mapd
 
 
 def get_absolute_sum_difference(timeseries: np.ndarray) -> float:
     """Calculate the mean of absolute first-order difference."""
+    if len(timeseries) <= 1:
+        return 0
     mad = np.mean(np.abs(np.diff(timeseries)))
     return mad
 

{rtc-tools-interface-0.7.6 → rtc-tools-interface-0.8.0b2}/rtctools_interface/optimization/helpers/statistics_mixin.py

@@ -53,7 +53,8 @@ class StatisticsMixin:
                 target_max = goal.target_max
             return target_min, target_max
 
-        if goal.goal_type in ["range", "range_rate_of_change"]:
+        supported_goal_types = ["range", "range_rate_of_change"]
+        if goal.goal_type in supported_goal_types:
             if goal.target_data_type == "parameter":
                 target_min, target_max = get_parameter_ranges(goal)
             elif goal.target_data_type == "value":
@@ -64,5 +65,9 @@ class StatisticsMixin:
                 message = "Target type {} not known for goal {}.".format(goal.target_data_type, goal.goal_id)
                 logger.error(message)
                 raise ValueError(message)
-            target_dict: TargetDict = {"target_min": target_min, "target_max": target_max}
+        else:
+            message = "Goal type {} not supported for target collection.".format(goal.goal_type)
+            logger.error(message)
+            raise ValueError(message)
+        target_dict: TargetDict = {"target_min": target_min, "target_max": target_max}
         return target_dict

rtc-tools-interface-0.8.0b2/tests/closed_loop/test_read_xml.py

@@ -0,0 +1,9 @@
+"""Tests for the base optimization problem class."""
+import unittest
+
+
+class TestReadXml(unittest.TestCase):
+    """Placeholder for testing reading XML files."""
+
+    def test_read_xml(self):
+        """TODO"""

rtc-tools-interface-0.8.0b2/tests/closed_loop/test_run_optization_problem_closed_loop.py

@@ -0,0 +1,98 @@
+"""Test the closed loop runner"""
+import math
+import xml.etree.ElementTree as ET
+from unittest import TestCase
+from pathlib import Path
+import pandas as pd
+from rtctools_interface.closed_loop.runner import run_optimization_problem_closed_loop
+from .test_models.goal_programming_xml.src.example import Example as ExampleXml
+from .test_models.goal_programming_csv.src.example import Example as ExampleCsv
+
+ns = {"fews": "http://www.wldelft.nl/fews", "pi": "http://www.wldelft.nl/fews/PI"}
+
+# Elementwise comparisons are practially disabled.
+A_TOL = 0.1
+R_TOL = 0.1
+
+
+def compare_xml_file(file_result: Path, file_ref: Path):
+    """Compare two timeseries_export files elementwise."""	
+    tree_result = ET.parse(file_result)
+    tree_ref = ET.parse(file_ref)
+    series_result = tree_result.findall("pi:series", ns)
+    series_ref = tree_ref.findall("pi:series", ns)
+    assert len(series_result) == len(series_ref), "Different number of series found in exports."
+    for serie_result, serie_ref in zip(series_result, series_ref):
+        for event_result, event_ref in zip(serie_result.findall("pi:event", ns), serie_ref.findall("pi:event", ns)):
+            value_result = float(event_result.attrib["value"])
+            value_ref = float(event_ref.attrib["value"])
+            assert math.isclose(
+                value_result, value_ref, rel_tol=R_TOL, abs_tol=A_TOL
+            ), f"Difference found in event: {value_result} != {value_ref}"
+
+
+def compare_xml_files(output_modelling_period_folder: Path, reference_folder: Path):
+    """Compare the timeseries_export.xml files in the output and reference folders."""
+    for folder in output_modelling_period_folder.iterdir():
+        if not folder.is_dir():
+            continue
+        file_name = "timeseries_export.xml"
+        file_result = folder / file_name
+        file_ref = reference_folder / folder.name / file_name
+        compare_xml_file(file_result, file_ref)
+
+
+class TestClosedLoop(TestCase):
+    """
+    Class for testing closed loop runner.
+    """
+
+    def test_running_closed_loop_csv(self):
+        """
+        Check if test model runs without problems and generates same results.
+        """
+        base_folder = Path(__file__).parent / "test_models" / "goal_programming_csv"
+        run_optimization_problem_closed_loop(ExampleCsv, base_folder=base_folder)
+
+        output_modelling_period_folder = base_folder / "output" / "output_modelling_periods"
+        self.assertTrue(output_modelling_period_folder.exists(), "Output modelling period folder should be created.")
+        self.assertEqual(
+            len(list(output_modelling_period_folder.iterdir())), 3, "Three modelling periods should be created."
+        )
+        for folder in output_modelling_period_folder.iterdir():
+            self.assertTrue((folder / "timeseries_export.csv").exists())
+        reference_folder = base_folder / "output" / "output_modelling_periods_reference"
+        for folder in output_modelling_period_folder.iterdir():
+            reference_folder_i = reference_folder / folder.name
+            for file in folder.iterdir():
+                df_result = pd.read_csv(file)
+                df_ref = pd.read_csv(reference_folder_i / file.name)
+                pd.testing.assert_frame_equal(df_result, df_ref, atol=A_TOL, rtol=R_TOL)
+        # Also compare the combined timeseries_export.csv
+        df_result = pd.read_csv(base_folder / "output" / "timeseries_export.csv")
+        df_ref = pd.read_csv(base_folder / "output" / "timeseries_export_reference.csv")
+        pd.testing.assert_frame_equal(df_result, df_ref, atol=A_TOL, rtol=R_TOL)
+
+    def test_running_closed_loop_xml(self):
+        """
+        Check if test model runs without problems and generates same results.
+        """
+        base_folder = Path(__file__).parent / "test_models" / "goal_programming_xml"
+        run_optimization_problem_closed_loop(ExampleXml, base_folder=base_folder)
+
+        output_modelling_period_folder = base_folder / "output" / "output_modelling_periods"
+        self.assertTrue(output_modelling_period_folder.exists(), "Output modelling period folder should be created.")
+        self.assertEqual(
+            len([f for f in output_modelling_period_folder.iterdir() if f.is_dir()]),
+            3,
+            "Three modelling periods should be created.",
+        )
+        for folder in output_modelling_period_folder.iterdir():
+            if folder.is_dir():
+                self.assertTrue((folder / "timeseries_export.xml").exists())
+        reference_folder = base_folder / "output" / "output_modelling_periods_reference"
+        compare_xml_files(output_modelling_period_folder, reference_folder)
+        # Also compare the combined timeseries_export
+        compare_xml_file(
+            base_folder / "output" / "timeseries_export.xml", base_folder / "output" / "timeseries_export_reference.xml"
+        )

rtc-tools-interface-0.7.6/rtctools_interface/__init__.py

@@ -1,3 +0,0 @@
-from . import _version
-
-__version__ = _version.get_versions()["version"]

{rtc-tools-interface-0.7.6 → rtc-tools-interface-0.8.0b2}/tests/optimization/test_base_optimization_problem.py

@@ -5,9 +5,9 @@ from rtctools_interface.optimization.base_optimization_problem import BaseOptimi
 from tests.utils.get_test import get_test_data
 
 
-
 class TestBaseOptimizationProblem(unittest.TestCase):
     """Test for the base optimization problem class."""
+
     def run_test(self, test):
         """Solve an optimization problem."""
         test_data = get_test_data(test, optimization=True)