rtc-tools 2.6.0b2__tar.gz → 2.7.0a1__tar.gz

{rtc-tools-2.6.0b2/src/rtc_tools.egg-info → rtc-tools-2.7.0a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rtc-tools
-Version: 2.6.0b2
+Version: 2.7.0a1
 Summary: Toolbox for control and optimization of water systems.
 Home-page: https://oss.deltares.nl/web/rtc-tools/home
 Author: Deltares

{rtc-tools-2.6.0b2 → rtc-tools-2.7.0a1}/setup.py

@@ -47,8 +47,9 @@ setup(
         "casadi >= 3.6.3, == 3.6.*",
         "numpy >= 1.16.0, <1.26",
         "scipy >= 1.0.0, <1.11",
-        "pymoca == 0.9.1",
+        "pymoca >= 0.9.1, == 0.9.*",
         "rtc-tools-channel-flow >= 1.1.0",
+        "defusedxml >= 0.7.0",
     ],
     tests_require=["pytest", "pytest-runner", "netCDF4"],
     extras_require={

{rtc-tools-2.6.0b2 → rtc-tools-2.7.0a1/src/rtc_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rtc-tools
-Version: 2.6.0b2
+Version: 2.7.0a1
 Summary: Toolbox for control and optimization of water systems.
 Home-page: https://oss.deltares.nl/web/rtc-tools/home
 Author: Deltares

{rtc-tools-2.6.0b2 → rtc-tools-2.7.0a1}/src/rtc_tools.egg-info/requires.txt

@@ -1,8 +1,9 @@
 casadi==3.6.*,>=3.6.3
 numpy<1.26,>=1.16.0
 scipy<1.11,>=1.0.0
-pymoca==0.9.1
+pymoca==0.9.*,>=0.9.1
 rtc-tools-channel-flow>=1.1.0
+defusedxml>=0.7.0
 
 [all]
 netCDF4

{rtc-tools-2.6.0b2 → rtc-tools-2.7.0a1}/src/rtctools/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-01-18T16:11:20+0000",
+ "date": "2024-04-25T06:42:12+0000",
  "dirty": false,
  "error": null,
- "full-revisionid": "e868661a3b37bb27f61ffcf9cdf1d53d857d5108",
- "version": "2.6.0b2"
+ "full-revisionid": "e3d59506cabe8df7e05acd089ccf574e106a577f",
+ "version": "2.7.0a1"
 }
 '''  # END VERSION_JSON
 

{rtc-tools-2.6.0b2 → rtc-tools-2.7.0a1}/src/rtctools/data/pi.py

@@ -5,6 +5,7 @@ import logging
 import os
 import xml.etree.ElementTree as ET
 
+import defusedxml.ElementTree as DefusedElementTree
 import numpy as np
 
 ns = {"fews": "http://www.wldelft.nl/fews", "pi": "http://www.wldelft.nl/fews/PI"}
@@ -30,7 +31,7 @@ class Diag:
         """
         self.__path_xml = os.path.join(folder, basename + ".xml")
 
-        self.__tree = ET.parse(self.__path_xml)
+        self.__tree = DefusedElementTree.parse(self.__path_xml)
         self.__xml_root = self.__tree.getroot()
 
     def get(self, level=ERROR_FATAL):
@@ -87,7 +88,7 @@ class DiagHandler(logging.Handler):
         self.__path_xml = os.path.join(folder, basename + ".xml")
 
         try:
-            self.__tree = ET.parse(self.__path_xml)
+            self.__tree = DefusedElementTree.parse(self.__path_xml)
             self.__xml_root = self.__tree.getroot()
         except Exception:
             self.__xml_root = ET.Element("{%s}Diag" % (ns["pi"],))
@@ -135,7 +136,7 @@ class ParameterConfig:
             basename = basename + ".xml"
         self.__path_xml = os.path.join(folder, basename)
 
-        self.__tree = ET.parse(self.__path_xml)
+        self.__tree = DefusedElementTree.parse(self.__path_xml)
         self.__xml_root = self.__tree.getroot()
 
     def get(self, group_id, parameter_id, location_id=None, model=None):
@@ -300,9 +301,8 @@ class ParameterConfig:
                 # get table contenstart_datetime
                 el_row = child.findall("pi:row", ns)
                 table = {
-                    columnId[key]: np.empty(len(el_row), columnType[key])  # initialize table
-                    for key in columnId
-                }
+                    columnId[key]: np.empty(len(el_row), columnType[key]) for key in columnId
+                }  # initialize table
 
                 i_row = 0
                 for row in el_row:
@@ -378,7 +378,7 @@ class Timeseries:
         if self.make_new_file:
             self.__reset_xml_tree()
         else:
-            self.__tree = ET.parse(self.__path_xml)
+            self.__tree = DefusedElementTree.parse(self.__path_xml)
             self.__xml_root = self.__tree.getroot()
 
         self.__values = [{}]
@@ -855,7 +855,7 @@ class Timeseries:
 
                 variable = self.__data_config.variable(header)
 
-                miss_val = float(header.find("pi:missVal", ns).text)
+                miss_val = header.find("pi:missVal", ns).text
                 values = self.__values[ensemble_member][variable]
 
                 # Update the header, which may have changed
@@ -868,11 +868,8 @@ class Timeseries:
                     self.__xml_root.remove(series)
                     continue
 
-                # Replace NaN with missing value
-                nans = np.isnan(values)
-                values[nans] = miss_val
-
                 # Write output
+                nans = np.isnan(values)
                 if self.__binary:
                     f.write(values.astype(self.__pi_dtype).tobytes())
                 else:
@@ -897,7 +894,10 @@ class Timeseries:
                         event = ET.Element("pi:event")
                         event.set("date", t.strftime("%Y-%m-%d"))
                         event.set("time", t.strftime("%H:%M:%S"))
-                        event.set("value", str(values[i]))
+                        if nans[i]:
+                            event.set("value", miss_val)
+                        else:
+                            event.set("value", str(values[i]))
                         series.append(event)
                         if self.dt:
                             t += self.dt
@@ -907,9 +907,6 @@ class Timeseries:
                         for i in range(len(values), len(events)):
                             series.remove(events[i])
 
-                # Restore NaN
-                values[nans] = np.nan
-
         if self.__binary:
             f.close()
 

{rtc-tools-2.6.0b2 → rtc-tools-2.7.0a1}/src/rtctools/data/rtc.py

@@ -1,8 +1,9 @@
 import logging
 import os
-import xml.etree.ElementTree as ET
 from collections import namedtuple
 
+import defusedxml.ElementTree as DefusedElementTree
+
 ts_ids = namedtuple("ids", "location_id parameter_id qualifier_id")
 p_ids = namedtuple("ids", "model_id location_id parameter_id")
 
@@ -31,7 +32,7 @@ class DataConfig:
 
         path = os.path.join(folder, "rtcDataConfig.xml")
         try:
-            tree = ET.parse(path)
+            tree = DefusedElementTree.parse(path)
             root = tree.getroot()
 
             timeseriess1 = root.findall("./*/fews:timeSeries", ns)

{rtc-tools-2.6.0b2 → rtc-tools-2.7.0a1}/src/rtctools/optimization/csv_lookup_table_mixin.py

@@ -2,7 +2,6 @@ import configparser
 import glob
 import logging
 import os
-import pickle
 from typing import Iterable, List, Tuple, Union
 
 import casadi as ca
@@ -323,7 +322,7 @@ class CSVLookupTableMixin(OptimizationProblem):
 
             # If tck file is newer than the csv file, first try to load the cached values from
             # the tck file
-            tck_filename = filename.replace(".csv", ".tck")
+            tck_filename = filename.replace(".csv", ".npz")
             valid_cache = False
             if os.path.exists(tck_filename):
                 if no_curvefit_options:
@@ -338,11 +337,13 @@ class CSVLookupTableMixin(OptimizationProblem):
                             output
                         )
                     )
-                    with open(tck_filename, "rb") as f:
-                        try:
-                            tck, function = pickle.load(f)
-                        except Exception:
-                            valid_cache = False
+                    try:
+                        with np.load(filename.replace(".csv", ".npz")) as data:
+                            tck = (data["arr_0"], data["arr_1"], int(data["arr_2"]))
+                        function = ca.Function.load(filename.replace(".csv", ".ca"))
+                    except Exception:
+                        valid_cache = False
+
             if not valid_cache:
                 logger.info("CSVLookupTableMixin: Recalculating tck values for {}".format(output))
 
@@ -446,11 +447,8 @@ class CSVLookupTableMixin(OptimizationProblem):
                 )
 
             if not valid_cache:
-                pickle.dump(
-                    (tck, function),
-                    open(filename.replace(".csv", ".tck"), "wb"),
-                    protocol=pickle.HIGHEST_PROTOCOL,
-                )
+                np.savez(filename.replace(".csv", ".npz"), *tck)
+                function.save(filename.replace(".csv", ".ca"))
 
     def lookup_tables(self, ensemble_member):
         # Call parent class first for default values.

{rtc-tools-2.6.0b2 → rtc-tools-2.7.0a1}/src/rtctools/optimization/modelica_mixin.py

@@ -48,9 +48,21 @@ class ModelicaMixin(OptimizationProblem):
             else:
                 model_name = self.__class__.__name__
 
-        self.__pymoca_model = pymoca.backends.casadi.api.transfer_model(
-            kwargs["model_folder"], model_name, self.compiler_options()
-        )
+        compiler_options = self.compiler_options()
+        logger.info(f"Loading/compiling model {model_name}.")
+        try:
+            self.__pymoca_model = pymoca.backends.casadi.api.transfer_model(
+                kwargs["model_folder"], model_name, compiler_options
+            )
+        except RuntimeError as error:
+            if compiler_options.get("cache", False):
+                raise error
+            compiler_options["cache"] = False
+            logger.warning(f"Loading model {model_name} using a cache file failed: {error}.")
+            logger.info(f"Compiling model {model_name}.")
+            self.__pymoca_model = pymoca.backends.casadi.api.transfer_model(
+                kwargs["model_folder"], model_name, compiler_options
+            )
 
         # Extract the CasADi MX variables used in the model
         self.__mx = {}

{rtc-tools-2.6.0b2 → rtc-tools-2.7.0a1}/src/rtctools/optimization/pi_mixin.py

@@ -277,3 +277,16 @@ class PIMixin(IOMixin):
         :class:`pi.Timeseries` object for holding the output data.
         """
         return self.__timeseries_export
+
+    def set_unit(self, variable: str, unit: str):
+        """
+        Set the unit of a time series.
+
+        :param variable:        Time series ID.
+        :param unit:            Unit.
+        """
+        assert hasattr(
+            self, "_PIMixin__timeseries_import"
+        ), "set_unit can only be called after read() in pre() has finished."
+        self.__timeseries_import.set_unit(variable, unit, 0)
+        self.__timeseries_export.set_unit(variable, unit, 0)

{rtc-tools-2.6.0b2 → rtc-tools-2.7.0a1}/src/rtctools/rtctoolsapp.py

@@ -108,7 +108,8 @@ def download_examples(*args):
 
         opener = urllib.request.build_opener()
         urllib.request.install_opener(opener)
-        local_filename, _ = urllib.request.urlretrieve(url)
+        # The security warning can be dismissed as the url variable is hardcoded to a remote.
+        local_filename, _ = urllib.request.urlretrieve(url)  # nosec
     except HTTPError:
         sys.exit("Could not found examples for RTC-Tools version {}.".format(version))
 

{rtc-tools-2.6.0b2 → rtc-tools-2.7.0a1}/src/rtctools/simulation/pi_mixin.py

@@ -240,3 +240,16 @@ class PIMixin(IOMixin):
     def get_timeseries(self, variable):
         _, values = self.io.get_timeseries(variable)
         return values
+
+    def set_unit(self, variable: str, unit: str):
+        """
+        Set the unit of a time series.
+
+        :param variable:        Time series ID.
+        :param unit:            Unit.
+        """
+        assert hasattr(
+            self, "_PIMixin__timeseries_import"
+        ), "set_unit can only be called after read() in pre() has finished."
+        self.__timeseries_import.set_unit(variable, unit, 0)
+        self.__timeseries_export.set_unit(variable, unit, 0)

{rtc-tools-2.6.0b2 → rtc-tools-2.7.0a1}/src/rtctools/simulation/simulation_problem.py

@@ -82,9 +82,21 @@ class SimulationProblem(DataStoreAccessor):
                 model_name = self.__class__.__name__
 
         # Load model from pymoca backend
-        self.__pymoca_model = pymoca.backends.casadi.api.transfer_model(
-            kwargs["model_folder"], model_name, self.compiler_options()
-        )
+        compiler_options = self.compiler_options()
+        logger.info(f"Loading/compiling model {model_name}.")
+        try:
+            self.__pymoca_model = pymoca.backends.casadi.api.transfer_model(
+                kwargs["model_folder"], model_name, compiler_options
+            )
+        except RuntimeError as error:
+            if compiler_options.get("cache", False):
+                raise error
+            compiler_options["cache"] = False
+            logger.warning(f"Loading model {model_name} using a cache file failed: {error}.")
+            logger.info(f"Compiling model {model_name}.")
+            self.__pymoca_model = pymoca.backends.casadi.api.transfer_model(
+                kwargs["model_folder"], model_name, compiler_options
+            )
 
         # Extract the CasADi MX variables used in the model
         self.__mx = {}
@@ -319,6 +331,17 @@ class SimulationProblem(DataStoreAccessor):
         """
         Initialize state vector with default values
 
+        Initial values are first read from the given Modelica files.
+        If an initial value equals zero or is not provided by a Modelica file,
+        and the variable is not marked as fixed,
+        then the initial value is tried to be set with the initial_state method.
+        When using CSVMixin, this method by default looks for initial values
+        in an initial_state.csv file.
+        Furthermore, if a variable is not marked as fixed
+        and no initial value is given by the initial_state method,
+        the initial value can be overwritten using the seed method.
+        When a variable is marked as fixed, the initial value is only read from the Modelica file.
+
         :param config_file: Path to an initialization file.
         """
         if config_file:
@@ -393,29 +416,35 @@ class SimulationProblem(DataStoreAccessor):
         for var in itertools.chain(self.__pymoca_model.states, self.__pymoca_model.alg_states):
             var_name = var.symbol.name()
             var_nominal = self.get_variable_nominal(var_name)
+            start_values = {}
 
             # Attempt to cast var.start to python type
+            start_value_is_symbolic = False
             mx_start = ca.MX(var.start)
             if mx_start.is_constant():
                 # cast var.start to python type
-                start_val = var.python_type(mx_start.to_DM())
+                start_value_pymoca = var.python_type(mx_start.to_DM())
+                if start_value_pymoca is not None and start_value_pymoca != 0:
+                    start_values["modelica"] = start_value_pymoca
             else:
-                # var.start is a symbolic expression with unknown value
-                start_val = None
+                start_value_is_symbolic = True
+                start_values["modelica"] = mx_start
 
-            if start_val == 0.0 and not var.fixed:
+            if not var.fixed:
                 # To make initialization easier, we allow setting initial states by providing
                 # timeseries with names that match a symbol in the model. We only check for this
                 # matching if the start and fixed attributes were left as default
                 try:
-                    start_val = self.initial_state()[var_name]
+                    start_values["initial_state"] = self.initial_state()[var_name]
                 except KeyError:
                     pass
                 else:
                     # An initial state was found- add it to the constrained residuals
                     logger.debug(
                         "Initialize: Added {} = {} to initial equations "
-                        "(found matching timeseries).".format(var_name, start_val)
+                        "(found matching timeseries).".format(
+                            var_name, start_values["initial_state"]
+                        )
                     )
                     # Set var to be fixed
                     var.fixed = True
@@ -425,36 +454,75 @@ class SimulationProblem(DataStoreAccessor):
                 # timeseries with names that match a symbol in the model. We only check for this
                 # matching if the start and fixed attributes were left as default
                 try:
-                    start_val = self.seed()[var_name]
+                    start_values["seed"] = self.seed()[var_name]
                 except KeyError:
                     pass
                 else:
                     # An initial state was found- add it to the constrained residuals
                     logger.debug(
                         "Initialize: Added {} = {} as initial guess "
-                        "(found matching timeseries).".format(var_name, start_val)
+                        "(found matching timeseries).".format(var_name, start_values["seed"])
+                    )
+
+            # Set the start value based on the different inputs.
+            if "seed" in start_values:
+                input_source = "seed"
+                source_description = "seed method"
+            elif "modelica" in start_values:
+                input_source = "modelica"
+                source_description = "modelica file"
+            elif "initial_state" in start_values:
+                input_source = "initial_state"
+                source_description = "initial_state method (typically reads initial_state.csv)"
+            else:
+                start_values["modelica"] = start_value_pymoca
+                input_source = "modelica"
+                source_description = "modelica file or default value"
+            start_val = start_values.get(input_source, None)
+            is_numeric = start_val is not None and not start_value_is_symbolic
+            numeric_start_val = start_val if is_numeric else 0.0
+            if len(start_values) > 1:
+                logger.warning(
+                    "Initialize: Multiple initial values for {} are provided: {}.".format(
+                        var_name, start_values
                     )
+                    + " Value from {} will be used to continue.".format(source_description)
+                )
 
             # Attempt to set start_val in the state vector. Default to zero if unknown.
             try:
-                self.set_var(var_name, start_val if start_val is not None else 0.0)
+                self.set_var(var_name, numeric_start_val)
             except KeyError:
                 logger.warning(
                     "Initialize: {} not found in state vector. "
-                    "Initial value of {} not set.".format(var_name, start_val)
+                    "Initial value of {} not set.".format(var_name, numeric_start_val)
                 )
 
             # Add a residual for the difference between the state and its starting expression
-            start_expr = start_val if start_val is not None else var.start
+            start_expr = start_val
             if var.fixed:
                 # Set bounds to be equal to each other, such that IPOPT can
                 # turn the decision variable into a parameter.
+                if var.min != -np.inf or var.max != np.inf:
+                    logger.info(
+                        "Initialize: bounds of {} will be overwritten".format(var_name)
+                        + " by the start value given by {}.".format(source_description)
+                    )
                 var.min = start_expr
                 var.max = start_expr
             else:
                 # minimize residual
                 minimized_residuals.append((var.symbol - start_expr) / var_nominal)
 
+            # Check that the start_value is in between the variable bounds.
+            if start_val is not None and is_numeric:
+                if not (var.min <= start_val and start_val <= var.max):
+                    logger.warning(
+                        "Initialize: start value {} = {}".format(var_name, start_val)
+                        + " is not in between bounds {} and {}".format(var.min, var.max)
+                        + " and will be adjusted."
+                    )
+
         # Default start var for ders is zero
         for der_var in self.__mx["derivatives"]:
             self.set_var(der_var.name(), 0.0)
@@ -610,7 +678,15 @@ class SimulationProblem(DataStoreAccessor):
         # If unsuccessful, stop.
         return_status = solver.stats()["return_status"]
         if return_status not in {"Solve_Succeeded", "Solved_To_Acceptable_Level"}:
-            raise Exception('Initialization Failed with return status "{}"'.format(return_status))
+            if return_status == "Infeasible_Problem_Detected":
+                message = (
+                    "Initialization Failed with return status: {}. ".format(return_status)
+                    + "This means no initial state could be found "
+                    + "that satisfies all equations and constraints."
+                )
+            else:
+                message = "Initialization Failed with return status: {}. ".format(return_status)
+            raise Exception(message)
 
         # Update state vector with initial conditions
         self.__state_vector[: self.__n_states] = initial_state["x"][: self.__n_states].T