rtc-tools 2.7.3__py3-none-any.whl

rtctools/optimization/homotopy_mixin.py

@@ -0,0 +1,165 @@
+import logging
+from typing import Dict, Union
+
+from .optimization_problem import OptimizationProblem
+from .timeseries import Timeseries
+
+logger = logging.getLogger("rtctools")
+
+
+class HomotopyMixin(OptimizationProblem):
+    """
+    Adds homotopy to your optimization problem.  A homotopy is a continuous transformation between
+    two optimization problems, parametrized by a single parameter :math:`\\theta \\in [0, 1]`.
+
+    Homotopy may be used to solve non-convex optimization problems, by starting with a convex
+    approximation at :math:`\\theta = 0.0` and ending with the non-convex problem at
+    :math:`\\theta = 1.0`.
+
+    .. note::
+
+        It is advised to look for convex reformulations of your problem, before resorting to a use
+        of the (potentially expensive) homotopy process.
+
+    """
+
+    def seed(self, ensemble_member):
+        seed = super().seed(ensemble_member)
+        options = self.homotopy_options()
+
+        # Overwrite the seed only when the results of the latest run are
+        # stored within this class. That is, when the GoalProgrammingMixin
+        # class is not used or at the first run of the goal programming loop.
+        if self.__theta > options["theta_start"] and getattr(self, "_gp_first_run", True):
+            for key, result in self.__results[ensemble_member].items():
+                times = self.times(key)
+                if (result.ndim == 1 and len(result) == len(times)) or (
+                    result.ndim == 2 and result.shape[0] == len(times)
+                ):
+                    # Only include seed timeseries which are consistent
+                    # with the specified time stamps.
+                    seed[key] = Timeseries(times, result)
+                elif (result.ndim == 1 and len(result) == 1) or (
+                    result.ndim == 2 and result.shape[0] == 1
+                ):
+                    seed[key] = result
+        return seed
+
+    def parameters(self, ensemble_member):
+        parameters = super().parameters(ensemble_member)
+
+        options = self.homotopy_options()
+        try:
+            # Only set the theta if we are in the optimization loop. We want
+            # to avoid accidental usage of the parameter value in e.g. pre().
+            # Note that we use a try-except here instead of hasattr, to avoid
+            # explicit name mangling.
+            parameters[options["homotopy_parameter"]] = self.__theta
+        except AttributeError:
+            pass
+
+        return parameters
+
+    def homotopy_options(self) -> Dict[str, Union[str, float]]:
+        """
+        Returns a dictionary of options controlling the homotopy process.
+
+        +------------------------+------------+---------------+
+        | Option                 | Type       | Default value |
+        +========================+============+===============+
+        | ``theta_start``        | ``float``  | ``0.0``       |
+        +------------------------+------------+---------------+
+        | ``delta_theta_0``      | ``float``  | ``1.0``       |
+        +------------------------+------------+---------------+
+        | ``delta_theta_min``    | ``float``  | ``0.01``      |
+        +------------------------+------------+---------------+
+        | ``homotopy_parameter`` | ``string`` | ``theta``     |
+        +------------------------+------------+---------------+
+
+        The homotopy process is controlled by the homotopy parameter in the model, specified by the
+        option ``homotopy_parameter``.  The homotopy parameter is initialized to ``theta_start``,
+        and increases to a value of ``1.0`` with a dynamically changing step size.  This step size
+        is initialized with the value of the option ``delta_theta_0``.  If this step size is too
+        large, i.e., if the problem with the increased homotopy parameter fails to converge, the
+        step size is halved.  The process of halving terminates when the step size falls below the
+        minimum value specified by the option ``delta_theta_min``.
+
+        :returns: A dictionary of homotopy options.
+        """
+
+        return {
+            "theta_start": 0.0,
+            "delta_theta_0": 1.0,
+            "delta_theta_min": 0.01,
+            "homotopy_parameter": "theta",
+        }
+
+    def dynamic_parameters(self):
+        dynamic_parameters = super().dynamic_parameters()
+
+        if self.__theta > 0:
+            # For theta = 0, we don't mark the homotopy parameter as being dynamic,
+            # so that the correct sparsity structure is obtained for the linear model.
+            options = self.homotopy_options()
+            dynamic_parameters.append(self.variable(options["homotopy_parameter"]))
+
+        return dynamic_parameters
+
+    def optimize(self, preprocessing=True, postprocessing=True, log_solver_failure_as_error=True):
+        # Pre-processing
+        if preprocessing:
+            self.pre()
+
+        options = self.homotopy_options()
+        delta_theta = options["delta_theta_0"]
+
+        # Homotopy loop
+        self.__theta = options["theta_start"]
+
+        while self.__theta <= 1.0:
+            logger.info("Solving with homotopy parameter theta = {}.".format(self.__theta))
+
+            success = super().optimize(
+                preprocessing=False, postprocessing=False, log_solver_failure_as_error=False
+            )
+            if success:
+                self.__results = [
+                    self.extract_results(ensemble_member)
+                    for ensemble_member in range(self.ensemble_size)
+                ]
+
+                if self.__theta == 0.0:
+                    self.check_collocation_linearity = False
+                    self.linear_collocation = False
+
+                    # Recompute the sparsity structure for the nonlinear model family.
+                    self.clear_transcription_cache()
+
+            else:
+                if self.__theta == options["theta_start"]:
+                    break
+
+                self.__theta -= delta_theta
+                delta_theta /= 2
+
+                if delta_theta < options["delta_theta_min"]:
+                    failure_message = (
+                        "Solver failed with homotopy parameter theta = {}. Theta cannot "
+                        "be decreased further, as that would violate the minimum delta "
+                        "theta of {}.".format(self.__theta, options["delta_theta_min"])
+                    )
+                    if log_solver_failure_as_error:
+                        logger.error(failure_message)
+                    else:
+                        # In this case we expect some higher level process to deal
+                        # with the solver failure, so we only log it as info here.
+                        logger.info(failure_message)
+                    break
+
+            self.__theta += delta_theta
+
+        # Post-processing
+        if postprocessing:
+            self.post()
+
+        return success

rtctools/optimization/initial_state_estimation_mixin.py

@@ -0,0 +1,89 @@
+from typing import List, Tuple, Union
+
+from .goal_programming_mixin import Goal, GoalProgrammingMixin
+
+
+class _MeasurementGoal(Goal):
+    def __init__(self, state, measurement_id, max_deviation=1.0):
+        self.__state = state
+        self.__measurement_id = measurement_id
+
+        self.function_nominal = max_deviation
+
+    def function(self, optimization_problem, ensemble_member):
+        op = optimization_problem
+        return op.state_at(self.__state, op.initial_time, ensemble_member) - op.timeseries_at(
+            self.__measurement_id, op.initial_time, ensemble_member
+        )
+
+    order = 2
+    priority = -2
+
+
+class _SmoothingGoal(Goal):
+    def __init__(self, state1, state2, max_deviation=1.0):
+        self.__state1 = state1
+        self.__state2 = state2
+
+        self.function_nominal = max_deviation
+
+    def function(self, optimization_problem, ensemble_member):
+        op = optimization_problem
+        return op.state_at(self.__state1, op.initial_time, ensemble_member) - op.state_at(
+            self.__state2, op.initial_time, ensemble_member
+        )
+
+    order = 2
+    priority = -1
+
+
+class InitialStateEstimationMixin(GoalProgrammingMixin):
+    """
+    Adds initial state estimation to your optimization problem *using goal programming*.
+
+    Before any other goals are evaluated, first, the deviation between initial
+    state measurements and their respective model states is minimized in the
+    least squares sense (1DVAR, priority -2). Secondly, the distance between
+    pairs of states is minimized, again in the least squares sense, so that
+    "smooth" initial guesses are provided for states without measurements
+    (priority -1).
+
+    .. note::
+
+        There are types of problems where, in addition to minimizing
+        differences between states and measurements, it is advisable to
+        perform a steady-state initialization using additional initial-time
+        model equations.  For hydraulic models, for instance, it is often
+        helpful to require that the time-derivative of the flow variables
+        vanishes at the initial time.
+
+    """
+
+    def initial_state_measurements(self) -> List[Union[Tuple[str, str], Tuple[str, str, float]]]:
+        """
+        List of pairs ``(state, measurement_id)`` or triples ``(state, measurement_id,
+        max_deviation)``, relating states to measurement time series IDs.
+
+        The default maximum deviation is ``1.0``.
+        """
+        return []
+
+    def initial_state_smoothing_pairs(self) -> List[Union[Tuple[str, str], Tuple[str, str, float]]]:
+        """
+        List of pairs ``(state1, state2)`` or triples ``(state1, state2, max_deviation)``, relating
+        states the distance of which is to be minimized.
+
+        The default maximum deviation is ``1.0``.
+        """
+        return []
+
+    def goals(self):
+        g = super().goals()
+
+        for measurement in self.initial_state_measurements():
+            g.append(_MeasurementGoal(*measurement))
+
+        for smoothing_pair in self.initial_state_smoothing_pairs():
+            g.append(_SmoothingGoal(*smoothing_pair))
+
+        return g

rtctools/optimization/io_mixin.py

@@ -0,0 +1,320 @@
+import bisect
+import logging
+import warnings
+from abc import ABCMeta, abstractmethod
+
+import casadi as ca
+import numpy as np
+
+from rtctools._internal.caching import cached
+from rtctools.optimization.optimization_problem import OptimizationProblem
+from rtctools.optimization.timeseries import Timeseries
+
+logger = logging.getLogger("rtctools")
+
+
+class IOMixin(OptimizationProblem, metaclass=ABCMeta):
+    """
+    Base class for all IO methods of optimization problems.
+    """
+
+    def __init__(self, **kwargs):
+        # Call parent class first for default behaviour.
+        super().__init__(**kwargs)
+
+        # Additional output variables
+        self.__output_timeseries = set()
+        self.__equidistant = False
+
+    def pre(self) -> None:
+        # Call parent class first for default behaviour.
+        super().pre()
+
+        # Call read method to read all input
+        self.read()
+
+        # Check equidistancy on read input data
+        if self.io.datetimes:
+            self.__equidistant = len(set(np.diff(self.io.datetimes))) == 1
+
+    @abstractmethod
+    def read(self) -> None:
+        """
+        Reads input data from files
+        """
+        pass
+
+    def post(self) -> None:
+        # Call parent class first for default behaviour.
+        super().post()
+
+        # Call write method to write all output
+        self.write()
+
+    @abstractmethod
+    def write(self) -> None:
+        """
+        Writes output data to files
+        """
+        pass
+
+    def times(self, variable=None) -> np.ndarray:
+        """
+        Returns the times in seconds from the reference datetime onwards.
+
+        :param variable:
+        """
+        times_sec = self.io.times_sec
+        t_idx = bisect.bisect_left(times_sec, 0.0)
+        return times_sec[t_idx:]
+
+    @property
+    def equidistant(self):
+        return self.__equidistant
+
+    @property
+    def ensemble_size(self):
+        return self.io.ensemble_size
+
+    def get_timeseries(self, variable: str, ensemble_member: int = 0) -> Timeseries:
+        return Timeseries(*self.io.get_timeseries_sec(variable, ensemble_member))
+
+    def set_timeseries(
+        self,
+        variable: str,
+        timeseries: Timeseries,
+        ensemble_member: int = 0,
+        output: bool = True,
+        check_consistency: bool = True,
+    ):
+        def stretch_values(values, t_pos):
+            # Construct a values range with preceding and possibly following nans
+            new_values = np.full(self.io.times_sec.shape, np.nan)
+            new_values[t_pos : t_pos + len(values)] = values
+            return new_values
+
+        if output:
+            self.__output_timeseries.add(variable)
+
+        if isinstance(timeseries, Timeseries):
+            if len(timeseries.values) != len(timeseries.times):
+                raise ValueError(
+                    "IOMixin: Trying to set timeseries {} with times and values that are of "
+                    "different length (lengths of {} and {}, respectively).".format(
+                        variable, len(timeseries.times), len(timeseries.values)
+                    )
+                )
+
+            timeseries_times_sec = self.io.times_sec
+
+            if not np.array_equal(timeseries_times_sec, timeseries.times):
+                if check_consistency:
+                    if not set(timeseries_times_sec).issuperset(timeseries.times):
+                        raise ValueError(
+                            "IOMixin: Trying to set timeseries {} with different times "
+                            "(in seconds) than the imported timeseries. Please make sure the "
+                            "timeseries covers all timesteps of the longest "
+                            "imported timeseries.".format(variable)
+                        )
+
+                # Determine position of first times of added timeseries within the
+                # import times. For this we assume that both time ranges are ordered,
+                # and that the times of the added series is a subset of the import
+                # times.
+                t_pos = bisect.bisect_left(timeseries_times_sec, timeseries.times[0])
+
+                # Construct a new values range with length of self.io.get_times()
+                values = stretch_values(timeseries.values, t_pos)
+            else:
+                values = timeseries.values
+
+        else:
+            timeseries_times_sec = self.io.times_sec
+
+            if check_consistency:
+                if len(self.times()) != len(timeseries):
+                    raise ValueError(
+                        "IOMixin: Trying to set values for {} with a different "
+                        "length ({}) than the forecast length ({}).".format(
+                            variable, len(timeseries), len(self.times())
+                        )
+                    )
+                elif not set(timeseries_times_sec).issuperset(self.times()):
+                    raise ValueError(
+                        "IOMixin: Trying to set timeseries {} with different times "
+                        "(in seconds) than the imported timeseries. Please make sure the "
+                        "timeseries covers all timesteps of the longest "
+                        "imported timeseries.".format(variable)
+                    )
+
+            # If times is not supplied with the timeseries, we add the
+            # forecast times range to a new Timeseries object. Hereby
+            # we assume that the supplied values stretch from T0 to end.
+            t_pos = bisect.bisect_left(timeseries_times_sec, self.initial_time)
+
+            # Construct a new values range with length of self.io.get_times()
+            values = stretch_values(timeseries, t_pos)
+
+        self.io.set_timeseries(variable, self.io.datetimes, values, ensemble_member)
+
+    def min_timeseries_id(self, variable: str) -> str:
+        """
+        Returns the name of the lower bound timeseries for the specified variable.
+
+        :param variable: Variable name.
+        """
+        return "_".join((variable, "Min"))
+
+    def max_timeseries_id(self, variable: str) -> str:
+        """
+        Returns the name of the upper bound timeseries for the specified variable.
+
+        :param variable: Variable name.
+        """
+        return "_".join((variable, "Max"))
+
+    @cached
+    def bounds(self):
+        # Call parent class first for default values.
+        bounds = super().bounds()
+
+        io_times = self.io.times_sec
+        t_pos = bisect.bisect_left(io_times, self.initial_time)
+
+        # Load bounds from timeseries
+        for variable in self.dae_variables["free_variables"]:
+            variable_name = variable.name()
+
+            m, M = None, None
+
+            timeseries_id = self.min_timeseries_id(variable_name)
+            try:
+                _, values = self.io.get_timeseries_sec(timeseries_id, 0)
+                m = values[t_pos:]
+            except KeyError:
+                pass
+            else:
+                if logger.getEffectiveLevel() == logging.DEBUG:
+                    logger.debug("Read lower bound for variable {}".format(variable_name))
+
+            timeseries_id = self.max_timeseries_id(variable_name)
+            try:
+                _, values = self.io.get_timeseries_sec(timeseries_id, 0)
+                M = values[t_pos:]
+            except KeyError:
+                pass
+            else:
+                if logger.getEffectiveLevel() == logging.DEBUG:
+                    logger.debug("Read upper bound for variable {}".format(variable_name))
+
+            # Replace NaN with +/- inf, and create Timeseries objects
+            if m is not None:
+                m[np.isnan(m)] = np.finfo(m.dtype).min
+                m = Timeseries(io_times[t_pos:], m)
+            if M is not None:
+                M[np.isnan(M)] = np.finfo(M.dtype).max
+                M = Timeseries(io_times[t_pos:], M)
+
+            # Store
+            if m is not None or M is not None:
+                bounds[variable_name] = (m, M)
+        return bounds
+
+    @cached
+    def history(self, ensemble_member):
+        # Load history
+        history = super().history(ensemble_member)
+
+        end_index = bisect.bisect_left(self.io.times_sec, self.initial_time) + 1
+
+        variable_list = (
+            self.dae_variables["states"]
+            + self.dae_variables["algebraics"]
+            + self.dae_variables["control_inputs"]
+            + self.dae_variables["constant_inputs"]
+        )
+
+        for variable in variable_list:
+            variable = variable.name()
+            try:
+                times, values = self.io.get_timeseries_sec(variable, ensemble_member)
+                history[variable] = Timeseries(times[:end_index], values[:end_index])
+            except KeyError:
+                pass
+            else:
+                if logger.getEffectiveLevel() == logging.DEBUG:
+                    logger.debug("IOMixin: Read history for state {}".format(variable))
+        return history
+
+    @cached
+    def seed(self, ensemble_member):
+        # Call parent class first for default values.
+        seed = super().seed(ensemble_member)
+
+        # Load seeds
+        for variable in self.dae_variables["free_variables"]:
+            variable = variable.name()
+            try:
+                s = Timeseries(*self.io.get_timeseries_sec(variable, ensemble_member))
+            except KeyError:
+                pass
+            else:
+                if logger.getEffectiveLevel() == logging.DEBUG:
+                    logger.debug("IOMixin: Seeded free variable {}".format(variable))
+                # A seeding of NaN means no seeding
+                s.values[np.isnan(s.values)] = 0.0
+                seed[variable] = s
+        return seed
+
+    @cached
+    def parameters(self, ensemble_member):
+        # Call parent class first for default values.
+        parameters = super().parameters(ensemble_member)
+
+        for parameter, value in self.io.parameters(ensemble_member).items():
+            parameters[parameter] = value
+
+        # Done
+        return parameters
+
+    @cached
+    def constant_inputs(self, ensemble_member):
+        # Call parent class first for default values.
+        constant_inputs = super().constant_inputs(ensemble_member)
+
+        # Load inputs from timeseries
+        for variable in self.dae_variables["constant_inputs"]:
+            variable = variable.name()
+            try:
+                timeseries = Timeseries(*self.io.get_timeseries_sec(variable, ensemble_member))
+            except KeyError:
+                pass
+            else:
+                inds = timeseries.times >= self.initial_time
+                if np.any(np.isnan(timeseries.values[inds])):
+                    raise Exception("IOMixin: Constant input {} contains NaN".format(variable))
+                constant_inputs[variable] = timeseries
+                if logger.getEffectiveLevel() == logging.DEBUG:
+                    logger.debug("IOMixin: Read constant input {}".format(variable))
+        return constant_inputs
+
+    def timeseries_at(self, variable, t, ensemble_member=0):
+        return self.interpolate(t, *self.io.get_timeseries_sec(variable, ensemble_member))
+
+    @property
+    def output_variables(self):
+        variables = super().output_variables.copy()
+        variables.extend([ca.MX.sym(variable) for variable in self.__output_timeseries])
+        return variables
+
+    def get_forecast_index(self):
+        """
+        Deprecated, use `io.reference_datetime` and `io.datetimes`, or override behavior using
+        :py:meth:`OptimizationProblem.times` and/or :py:attr:`OptimizationProblem.initial_time`.
+        """
+        warnings.warn(
+            "get_forecast_index() is deprecated and will be removed in the future",
+            FutureWarning,
+            stacklevel=1,
+        )
+        return bisect.bisect_left(self.io.datetimes, self.io.reference_datetime)

rtctools/optimization/linearization_mixin.py

@@ -0,0 +1,33 @@
+from typing import Dict
+
+from .optimization_problem import OptimizationProblem
+
+
+class LinearizationMixin(OptimizationProblem):
+    """
+    Adds linearized equation parameter bookkeeping to your optimization aproblem.
+
+    If your model contains linearized equations, this mixin will set the
+    parameters of these equations based on the t0 value of an associated
+    timeseries.
+
+    The mapping between linearization parameters and time series is provided
+    in the ``linearization_parameters`` method.
+    """
+
+    def parameters(self, ensemble_member):
+        parameters = super().parameters(ensemble_member)
+
+        for parameter, timeseries_id in self.linearization_parameters().items():
+            parameters[parameter] = self.timeseries_at(
+                timeseries_id, self.initial_time, ensemble_member
+            )
+
+        return parameters
+
+    def linearization_parameters(self) -> Dict[str, str]:
+        """
+        :returns: A dictionary of parameter names mapping to time series identifiers.
+        """
+
+        return {}