rtc-tools-interface 0.9.0a2__tar.gz → 0.10.1__tar.gz

{rtc-tools-interface-0.9.0a2 → rtc-tools-interface-0.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rtc-tools-interface
-Version: 0.9.0a2
+Version: 0.10.1
 Summary: Toolbox for user interfaces for RTC-Tools
 Home-page: UNKNOWN
 Author: Deltares

{rtc-tools-interface-0.9.0a2 → rtc-tools-interface-0.10.1}/README.md

@@ -189,7 +189,19 @@ Import `ClosedLoopConfig` and `run_optimization_problem_closed_loop` with:
 from rtctools_interface.closed_loop.config import ClosedLoopConfig
 from rtctools_interface.closed_loop.runner import run_optimization_problem_closed_loop
 ```
+#### Fixed inputs
+Create the file `fixed_inputs.json`, in which you specify which variables in your
+timeseries import are what we call 'fixed_inputs'. They are timeseries that the closed
+loop runner should simply copy as they are, even if they contain only NaNs in a modelling
+period.
 
+The variables that are not mentioned in this list of fixed_inputs, and have only NaN's
+in a modelling period, are considered being 'initial values'. The closed loop runner will set 
+the first timestep of each modelling period with the corresponding calculated value from the 
+previous modelling period.
+
+
+#### Closed loop config
 A `ClosedLoopConfig` configuration can be created from a csv file or
 from a given forecast timestep (time between each time range)
 and optimization period (duration of each time range).

{rtc-tools-interface-0.9.0a2 → rtc-tools-interface-0.10.1}/rtc_tools_interface.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rtc-tools-interface
-Version: 0.9.0a2
+Version: 0.10.1
 Summary: Toolbox for user interfaces for RTC-Tools
 Home-page: UNKNOWN
 Author: Deltares

{rtc-tools-interface-0.9.0a2 → rtc-tools-interface-0.10.1}/rtc_tools_interface.egg-info/requires.txt

@@ -3,4 +3,5 @@ numpy
 pandas
 plotly
 pydantic
+casadi!=3.6.6
 rtc-tools>=2.7.0a3

{rtc-tools-interface-0.9.0a2 → rtc-tools-interface-0.10.1}/rtctools_interface/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-08-01T10:56:38+0200",
+ "date": "2025-06-04T10:02:27+0200",
  "dirty": false,
  "error": null,
- "full-revisionid": "ef695c81333aa6aca2b0d62cc2b86ff937294803",
- "version": "0.9.0a2"
+ "full-revisionid": "33cba7abbd05364334aabef715ce319b398ff945",
+ "version": "0.10.1"
 }
 '''  # END VERSION_JSON
 

{rtc-tools-interface-0.9.0a2 → rtc-tools-interface-0.10.1}/rtctools_interface/closed_loop/config.py

@@ -1,6 +1,7 @@
 """Module for configuring a closed-loop optimization problem."""
-from pathlib import Path
 from datetime import timedelta
+from pathlib import Path
+from typing import Optional
 
 
 class ClosedLoopConfig():
@@ -29,8 +30,8 @@ class ClosedLoopConfig():
         if file is not None:
             file = Path(file).resolve()
         self._file = file
-        self._forecast_timestep = None
-        self._optimization_period = None
+        self._forecast_timestep: Optional[timedelta] = None
+        self._optimization_period: Optional[timedelta] = None
         self.round_to_dates = round_to_dates
 
     @classmethod

{rtc-tools-interface-0.9.0a2 → rtc-tools-interface-0.10.1}/rtctools_interface/closed_loop/optimization_ranges.py

@@ -49,7 +49,7 @@ def get_optimization_ranges_from_file(
         min(closed_loop_dates["start_date"]).date() == model_time_range[0].date()
     ), (
         "The start day of the first optimization run is not equal"
-        " to the start day of the timeseries import."
+        " to the start day of the forecast date (or first timestep)."
     )
     assert (
         max(closed_loop_dates["end_date"]).date() <= model_time_range[1].date()

{rtc-tools-interface-0.9.0a2 → rtc-tools-interface-0.10.1}/rtctools_interface/closed_loop/results_construction.py

@@ -28,8 +28,15 @@ def combine_xml_exports(output_base_path: Path, original_input_timeseries_path:
     dataconfig = rtc.DataConfig(folder=original_input_timeseries_path)
 
     ts_import_orig = pi.Timeseries(
-        data_config=dataconfig, folder=original_input_timeseries_path, basename="timeseries_import", binary=False
+        data_config=dataconfig,
+        folder=original_input_timeseries_path,
+        basename="timeseries_import",
+        binary=False,
     )
+    if ts_import_orig.forecast_datetime > ts_import_orig.start_datetime:
+        logger.info("Timeseries export will start at original forecast date, disregarding data before forecast date.")
+        ts_import_orig.resize(ts_import_orig.forecast_datetime, ts_import_orig.end_datetime)
+        ts_import_orig.times = ts_import_orig.times[ts_import_orig.times.index(ts_import_orig.forecast_datetime):]
     orig_start_datetime = ts_import_orig.start_datetime
     orig_end_datetime = ts_import_orig.end_datetime
 
@@ -76,15 +83,16 @@ def combine_xml_exports(output_base_path: Path, original_input_timeseries_path:
     ts_export.write(output_folder=output_base_path.parent, output_filename="timeseries_export")
 
     if write_csv_out:
-        data = pd.DataFrame()
-        data["date"] = all_times
+        data = pd.DataFrame({"date": all_times})
+        new_columns = []
         for timeseries_id in variables:
             try:
                 values = ts_export.get(timeseries_id)
+                new_columns.append(pd.Series(values, name=timeseries_id))
             except KeyError:
                 logger.debug("Variable {} not found in output of model horizon: {}".format(timeseries_id, i))
                 continue
-            data[timeseries_id] = values
+        data = pd.concat([data] + new_columns, axis=1)
         data.round(6).to_csv(output_base_path.parent / "timeseries_export.csv", index=False)
 
 

{rtc-tools-interface-0.9.0a2 → rtc-tools-interface-0.10.1}/rtctools_interface/closed_loop/runner.py

@@ -1,10 +1,13 @@
+import copy
 import datetime
+import json
+import logging
 import os
-import copy
-from pathlib import Path
 import shutil
 import sys
+from pathlib import Path
 from typing import List, Optional
+
 from rtctools.data.pi import DiagHandler
 from rtctools.optimization.pi_mixin import PIMixin
 from rtctools.optimization.csv_mixin import CSVMixin
@@ -13,7 +16,6 @@ from rtctools_interface.closed_loop.config import ClosedLoopConfig
 import rtctools_interface.closed_loop.optimization_ranges as opt_ranges
 from rtctools_interface.closed_loop.results_construction import combine_csv_exports, combine_xml_exports
 from rtctools_interface.closed_loop.time_series_handler import XMLTimeSeriesFile, CSVTimeSeriesFile, TimeSeriesHandler
-import logging
 
 logger = logging.getLogger("rtctools")
 
@@ -61,9 +63,7 @@ def _get_optimization_ranges(
     """Return a list of optimization periods."""
     if config.file is not None:
         datetime_range = input_timeseries.get_datetime_range()
-        optimization_ranges = opt_ranges.get_optimization_ranges_from_file(
-            config.file, datetime_range
-        )
+        optimization_ranges = opt_ranges.get_optimization_ranges_from_file(config.file, datetime_range)
     elif config.optimization_period is not None:
         datetimes = input_timeseries.get_datetimes()
         optimization_ranges = opt_ranges.get_optimization_ranges(
@@ -73,9 +73,7 @@ def _get_optimization_ranges(
             optimization_period=config.optimization_period,
         )
     else:
-        raise ValueError(
-            "The closed-loop configuration should have either a file or optimization_period set."
-        )
+        raise ValueError("The closed-loop configuration should have either a file or optimization_period set.")
     if config.round_to_dates:
         optimization_ranges = opt_ranges.round_datetime_ranges_to_days(optimization_ranges)
     return optimization_ranges
@@ -86,9 +84,10 @@ def run_optimization_problem_closed_loop(
     base_folder="..",
     log_level=logging.INFO,
     profile=False,
-    config: ClosedLoopConfig = None,
+    config: Optional[ClosedLoopConfig] = None,
+    modelling_period_input_folder: Optional[str] = None,
     **kwargs,
-):
+) -> dict:
     """
     Runs an optimization problem in closed loop mode.
 
@@ -126,7 +125,19 @@ def run_optimization_problem_closed_loop(
 
     variables_in_import = original_import.get_all_internal_ids()
 
-    modelling_period_input_folder = base_folder / "input_modelling_periods"
+    fixed_input_config_file = (original_input_folder / "fixed_inputs.json").resolve()
+    if not fixed_input_config_file.exists():
+        raise FileNotFoundError(
+            f"Could not find fixed inputs configuration file: {fixed_input_config_file}"
+            "Create a file with a list of strings that represent the fixed inputs (can be an empty list)."
+        )
+    with open(fixed_input_config_file, "r") as file:
+        fixed_input_series = json.load(file)
+    if not isinstance(fixed_input_series, list) and all(isinstance(item, str) for item in fixed_input_series):
+        raise ValueError("Fixed input config file should be a list of strings (or an empty list).")
+
+    if modelling_period_input_folder is None:
+        modelling_period_input_folder = base_folder / "input_modelling_periods"
     if modelling_period_input_folder.exists():
         shutil.rmtree(modelling_period_input_folder)
     modelling_period_input_folder.mkdir(exist_ok=True)
@@ -154,7 +165,7 @@ def run_optimization_problem_closed_loop(
         modelling_period_input_folder_i = modelling_period_input_folder / modelling_period_name
         write_input_folder(modelling_period_input_folder_i, original_input_folder, timeseries_import)
 
-        logger.info(f"Run optimization for period {i}: {(str(start_time), str(end_time))}.")
+        logger.info(f"Running optimization for period {i}: {(str(start_time), str(end_time))}.")
         result = run_optimization_problem(
             optimization_problem_class,
             base_folder,
@@ -164,14 +175,18 @@ def run_optimization_problem_closed_loop(
             output_folder=modelling_period_output_folder_i,
             **kwargs,
         )
-        logger.info(f"Finished optimization for period {i}: {(str(start_time), str(end_time))}.")
+        period = f"period {i} {(str(start_time), str(end_time))}"
+        if result.solver_stats["success"]:
+            logger.info(f"Successful optimization for {period}.")
+        else:
+            message = f"Failed optimization for {period} with status '{result.solver_stats['return_status']}'."
+            logger.error(message)
+            raise Exception(message)
 
-        results_previous_run = {key: result.extract_results().get(key) for key in variables_in_import}
+        results_previous_run = {
+            key: result.extract_results().get(key) for key in variables_in_import if key not in fixed_input_series
+        }
         previous_run_datetimes = result.io.datetimes
-        if len(results_previous_run) != len(variables_in_import):
-            logger.warning("Could not find the results for all input variables.")
-            logger.warning("Missing variables: " + str(set(variables_in_import) - set(results_previous_run.keys())))
-            raise ValueError("Could not find the results for all input variables.")
 
     logger.info("Finished all optimization runs.")
     if issubclass(optimization_problem_class, PIMixin):
@@ -182,3 +197,4 @@ def run_optimization_problem_closed_loop(
         logger.warning(
             "Could not combine exports because the optimization problem class is not derived from PIMixin or CSVMixin."
         )
+    return result.solver_stats

{rtc-tools-interface-0.9.0a2 → rtc-tools-interface-0.10.1}/rtctools_interface/closed_loop/time_series_handler.py

@@ -170,6 +170,11 @@ class XMLTimeSeriesFile(TimeSeriesHandler):
         self.pi_timeseries = None
         self.read(timeseries_import_basename)
 
+        if self.get_datetime_range()[0] < self.forecast_date:
+            logger.warning("Currently, the closed loop runner does support data before the forecast date.")
+            logger.warning("Removing data before forecast date.")
+            self.select_time_range(self.forecast_date, self.pi_timeseries.times[-1])
+
     def read(self, file_name: str):
         """Read the timeseries data from a file."""
         timeseries_import_basename = file_name
@@ -215,6 +220,8 @@ class XMLTimeSeriesFile(TimeSeriesHandler):
         self.forecast_date = times[i_start]
 
     def write(self, file_path: Path, file_name: str = "timeseries_import"):
+        # By setting make_new_file headers will be recreated, neceesary for writing new forecast date
+        self.pi_timeseries.make_new_file = True
         self.pi_timeseries.write(output_folder=file_path, output_filename=file_name)
 
     def get_datetimes(self):

{rtc-tools-interface-0.9.0a2 → rtc-tools-interface-0.10.1}/rtctools_interface/optimization/base_goal.py

@@ -33,6 +33,7 @@ class BaseGoal(Goal):
         self,
         optimization_problem: OptimizationProblem,
         state,
+        *,
         goal_type="minimization_path",
         function_min=np.nan,
         function_max=np.nan,

{rtc-tools-interface-0.9.0a2 → rtc-tools-interface-0.10.1}/rtctools_interface/simulation/plot_mixin.py

@@ -39,6 +39,16 @@ class PlotMixin(PlottingBaseMixin):
         """Tasks after optimizing."""
         super().post()
 
+        # find empty arrays in self._manual_extracted_states
+        # for these variables try to use self.get_timeseries(variable)
+        for variable in self.custom_variables:
+            if not self._manual_extracted_states[variable] or len(self._manual_extracted_states[variable]) == 0:
+                logger.debug(f"Variable {variable} has empty data collected.")
+                try:
+                    self._manual_extracted_states[variable] = self.get_timeseries(variable)
+                except KeyError:
+                    logger.warning(f"Variable {variable} not found in output of model.")
+
         timeseries_data = self.collect_timeseries_data(self.custom_variables)
         self._intermediate_results.append({"timeseries_data": timeseries_data, "priority": 0})
         current_run = self.create_plot_data_and_config([])

{rtc-tools-interface-0.9.0a2 → rtc-tools-interface-0.10.1}/setup.py

@@ -16,6 +16,7 @@ setup(
         "pandas",
         "plotly",
         "pydantic",
+        "casadi != 3.6.6",
         "rtc-tools >= 2.7.0a3",
     ],
     tests_require=["pytest", "pytest-runner"],

{rtc-tools-interface-0.9.0a2 → rtc-tools-interface-0.10.1}/tests/closed_loop/test_run_optization_problem_closed_loop.py

@@ -20,7 +20,7 @@ R_TOL = 0.1
 
 
 def compare_xml_file(file_result: Path, file_ref: Path):
-    """Compare two timeseries_export files elementwise."""	
+    """Compare two timeseries_export files elementwise."""
     tree_result = ET.parse(file_result)
     tree_ref = ET.parse(file_ref)
     series_result = tree_result.findall("pi:series", ns)
@@ -142,39 +142,90 @@ class TestClosedLoop(TestCase):
         """
         Check if test model runs without problems and generates same results.
         """
+        test_cases = [
+            {
+                "description": "without forecast date",
+                "input_folder": "input",
+            },
+            {
+                "description": "with forecast date unequal to first date",
+                "input_folder": "input_with_forecast_date",
+            },
+            {
+                "description": "with forecast date equal to first date",
+                "input_folder": "input_with_forecast_date_equal_first_date",
+            }
+        ]
+
         base_folder = Path(__file__).parent / "test_models" / "goal_programming_xml"
-        config = ClosedLoopConfig(
-            file=base_folder / "input" / "closed_loop_dates.csv",
-            round_to_dates=True
-        )
-        run_optimization_problem_closed_loop(ExampleXml, base_folder=base_folder, config=config)
-        self.compare_xml_files(
-            output_folder=base_folder / "output" / "output_modelling_periods",
-            reference_folder=base_folder / "output" / "output_modelling_periods_reference",
-            n_periods=3
-        )
-        self.compare_xml_file(
-            output_file=base_folder / "output" / "timeseries_export.xml",
-            reference_file=base_folder / "output" / "timeseries_export_reference.xml"
-        )
+
+        for case in test_cases:
+            with self.subTest(case["description"]):
+                config = ClosedLoopConfig(
+                    file=base_folder / case["input_folder"] / "closed_loop_dates.csv",
+                    round_to_dates=True
+                )
+                run_optimization_problem_closed_loop(
+                    ExampleXml,
+                    base_folder=base_folder,
+                    config=config,
+                    input_folder=case["input_folder"]
+                )
+
+                self.compare_xml_files(
+                    output_folder=base_folder / "output" / "output_modelling_periods",
+                    reference_folder=base_folder / "output" / "output_modelling_periods_reference",
+                    n_periods=3
+                )
+                self.compare_xml_file(
+                    output_file=base_folder / "output" / "timeseries_export.xml",
+                    reference_file=base_folder / "output" / "timeseries_export_reference.xml"
+                )
 
     def test_running_closed_loop_xml_fixed_periods(self):
         """
         Check if test model runs for fixed optimization periods.
         """
+        test_cases = [
+            {
+                "description": "without forecast date",
+                "output_folder": "output_fixed_periods",
+                "input_folder": "input",
+                "optimization_period": timedelta(days=3),
+                "forecast_timestep": timedelta(days=2)
+            },
+            {
+                "description": "with forecast date unequal to first date",
+                "output_folder": "output_fixed_periods",
+                "input_folder": "input_with_forecast_date",
+                "optimization_period": timedelta(days=3),
+                "forecast_timestep": timedelta(days=2)
+            },
+            {
+                "description": "with forecast date equal to first date",
+                "output_folder": "output_fixed_periods",
+                "input_folder": "input_with_forecast_date_equal_first_date",
+                "optimization_period": timedelta(days=3),
+                "forecast_timestep": timedelta(days=2)
+            }
+        ]
+
         base_folder = Path(__file__).parent / "test_models" / "goal_programming_xml"
-        output_folder = "output_fixed_periods"
-        config = ClosedLoopConfig.from_fixed_periods(
-            optimization_period=timedelta(days=3),
-            forecast_timestep=timedelta(days=2)
-        )
-        run_optimization_problem_closed_loop(
-            ExampleXml,
-            base_folder=base_folder,
-            config=config,
-            output_folder=output_folder
-        )
-        self.compare_xml_file(
-            output_file=base_folder / output_folder / "timeseries_export.xml",
-            reference_file=base_folder / "output" / "timeseries_export_reference.xml"
-        )
+
+        for case in test_cases:
+            with self.subTest(case["description"]):
+                config = ClosedLoopConfig.from_fixed_periods(
+                    optimization_period=case["optimization_period"],
+                    forecast_timestep=case["forecast_timestep"]
+                )
+                run_optimization_problem_closed_loop(
+                    ExampleXml,
+                    base_folder=base_folder,
+                    config=config,
+                    output_folder=case["output_folder"],
+                    input_folder=case["input_folder"]
+                )
+                self.compare_xml_file(
+                    output_file=base_folder / case["output_folder"] / "timeseries_export.xml",
+                    reference_file=base_folder / "output" / "timeseries_export_reference.xml"
+                )