rtc-tools-interface 0.4.4__tar.gz → 0.5.0__tar.gz

{rtc-tools-interface-0.4.4 → rtc-tools-interface-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rtc-tools-interface
-Version: 0.4.4
+Version: 0.5.0
 Summary: Toolbox for user interfaces for RTC-Tools
 Home-page: UNKNOWN
 Author: Deltares

{rtc-tools-interface-0.4.4 → rtc-tools-interface-0.5.0}/README.md

@@ -96,6 +96,19 @@ See the table below for an example content of the `goal_table.csv`.
 | goal_2 | reservoir_2_waterlevel     | 1      | range        | 0            | 15           | 10               | timeseries            | "target_series"        | "target_series"       | 10       |        |       |
 | goal_3 | electricity_cost     | 1      | minimization_path |              |              |                  |                  |            |            | 20       |        |       |
 
+## Goal performance metrics
+For all goals defined with the goal generator this rtc-tools-interface module will also calculate performance metrics. By default, these performance metrics are saved to a .csv in the folder `output/perfomance_metrics`, with one csv file per goal. With the class variable `calculate_performance_metrics` this functionality can be disabled (by default it is enabled).
+
+The calculated metrics are:
+- `timeseries_sum`: The sum of the state variable over all timesteps.
+- `timeseries_min`: The minimum of the state variable.
+- `timeseries_max`: The maximum of the state variable.
+- `timeseries_avg`: The average of the state variable.
+- `max_difference`: The maximum difference in one timestep.
+- `mean_absolute_percentual_difference`: The mean of the absolute percentual difference per timestep over all timesteps (only for range goals).
+- `mean_absolute_difference`: The mean absolute difference per timestep of the state variable over all timesteps (only for range goals).
+
+
 ## Automatic plotting of results
 With the `PlotGoalsMixin` one can easily make plots of the results of rtc-tools. To utilize this functionality, import the mixin as follows:
 ```python

{rtc-tools-interface-0.4.4 → rtc-tools-interface-0.5.0}/rtc_tools_interface.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rtc-tools-interface
-Version: 0.4.4
+Version: 0.5.0
 Summary: Toolbox for user interfaces for RTC-Tools
 Home-page: UNKNOWN
 Author: Deltares

{rtc-tools-interface-0.4.4 → rtc-tools-interface-0.5.0}/rtc_tools_interface.egg-info/SOURCES.txt

@@ -14,6 +14,7 @@ rtctools_interface/optimization/__init__.py
 rtctools_interface/optimization/base_goal.py
 rtctools_interface/optimization/base_optimization_problem.py
 rtctools_interface/optimization/goal_generator_mixin.py
+rtctools_interface/optimization/goal_performance_metrics.py
 rtctools_interface/optimization/goal_table_schema.py
 rtctools_interface/optimization/plot_and_goal_schema.py
 rtctools_interface/optimization/plot_goals_mixin.py

{rtc-tools-interface-0.4.4 → rtc-tools-interface-0.5.0}/rtctools_interface/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2023-10-26T13:39:40+0000",
+ "date": "2023-10-30T13:58:13+0000",
  "dirty": false,
  "error": null,
- "full-revisionid": "efe92f22d84650bb8f0d7caa0c9357809d5b34f9",
- "version": "0.4.4"
+ "full-revisionid": "34cc7954ac465b9eedd42d7efdea2bf42d26ac19",
+ "version": "0.5.0"
 }
 '''  # END VERSION_JSON
 

rtc-tools-interface-0.5.0/rtctools_interface/optimization/goal_generator_mixin.py

@@ -0,0 +1,96 @@
+"""Module for a basic optimization problem."""
+from pathlib import Path
+from typing import Dict, Union
+import logging
+import os
+import pandas as pd
+
+from rtctools_interface.optimization.base_goal import BaseGoal
+from rtctools_interface.optimization.goal_performance_metrics import get_performance_metrics
+from rtctools_interface.optimization.helpers.statistics_mixin import StatisticsMixin
+from rtctools_interface.optimization.read_goals import read_goals
+
+logger = logging.getLogger("rtctools")
+
+
+def write_performance_metrics(performance_metrics: Dict[str, pd.DataFrame], output_path: Union[str, Path]):
+    """Write the performance metrics for each goal to a csv file."""
+    output_path = Path(output_path) / "performance_metrics"
+    output_path.mkdir(parents=True, exist_ok=True)
+    for goal_id, performance_metric_table in performance_metrics.items():
+        performance_metric_table.to_csv(output_path / f"{goal_id}.csv")
+
+
+class GoalGeneratorMixin(StatisticsMixin):
+    # TODO: remove pylint disable below once we have more public functions.
+    # pylint: disable=too-few-public-methods
+    """Add path goals as specified in the goal_table.
+
+    By default, the mixin looks for the csv in the in the default input
+    folder. One can also set the path to the goal_table_file manually
+    with the `goal_table_file` class variable.
+    """
+    calculate_performance_metrics = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.goals_to_generate = kwargs.get("goals_to_generate", [])
+        self.read_from = kwargs.get("read_goals_from", "csv_table")
+        if not hasattr(self, "goal_table_file"):
+            self.goal_table_file = os.path.join(self._input_folder, "goal_table.csv")
+
+        self._goal_generator_path_goals = read_goals(
+            self.goal_table_file, path_goal=True, read_from=self.read_from, goals_to_generate=self.goals_to_generate
+        )
+        self._goal_generator_non_path_goals = read_goals(
+            self.goal_table_file, path_goal=False, read_from=self.read_from, goals_to_generate=self.goals_to_generate
+        )
+        self._all_goal_generator_goals = self._goal_generator_path_goals + self._goal_generator_non_path_goals
+
+        if self.calculate_performance_metrics:
+            # A dataframe for each goal defined by the goal generator
+            self.performance_metrics = {}
+            for goal in self._all_goal_generator_goals:
+                self.performance_metrics[goal.goal_id] = pd.DataFrame()
+
+    def path_goals(self):
+        """Return the list of path goals."""
+        goals = super().path_goals()
+        new_goals = self._goal_generator_path_goals
+        if new_goals:
+            goals = goals + [BaseGoal(optimization_problem=self, **goal.__dict__) for goal in new_goals]
+        return goals
+
+    def goals(self):
+        """Return the list of goals."""
+        goals = super().goals()
+        new_goals = self._goal_generator_non_path_goals
+        if new_goals:
+            goals = goals + [BaseGoal(optimization_problem=self, **goal.__dict__) for goal in new_goals]
+        return goals
+
+    def store_performance_metrics(self, label):
+        """Calculate and store performance metrics."""
+        results = self.extract_results()
+        goal_generator_goals = self._all_goal_generator_goals
+        targets = self.collect_range_target_values(goal_generator_goals)
+        for goal in goal_generator_goals:
+            next_row = get_performance_metrics(results, goal, targets.get(goal.goal_id))
+            if next_row is not None:
+                next_row.rename(label, inplace=True)
+                self.performance_metrics[goal.goal_id] = pd.concat(
+                    [self.performance_metrics[goal.goal_id].T, next_row], axis=1
+                ).T
+
+    def priority_completed(self, priority):
+        """Tasks after priority optimization."""
+        super().priority_completed(priority)
+        if self.calculate_performance_metrics:
+            self.store_performance_metrics(priority)
+
+    def post(self):
+        """Tasks after all optimization steps."""
+        super().post()
+        if self.calculate_performance_metrics:
+            self.store_performance_metrics("final_results")
+            write_performance_metrics(self.performance_metrics, self._output_folder)

rtc-tools-interface-0.5.0/rtctools_interface/optimization/goal_performance_metrics.py

@@ -0,0 +1,117 @@
+"""This file contains functions to get performance metrics for the BaseGoal."""
+import logging
+from typing import Dict, Optional
+
+import pandas as pd
+import numpy as np
+
+from rtctools_interface.optimization.goal_table_schema import (
+    BaseGoalModel,
+    MaximizationGoalModel,
+    MinimizationGoalModel,
+    RangeGoalModel,
+    RangeRateOfChangeGoalModel,
+)
+from rtctools_interface.optimization.type_definitions import TargetDict
+
+
+logger = logging.getLogger("rtctools")
+
+ABS_TOL = 0.001
+
+
+def get_mean_absolute_percentual_difference(timeseries: np.ndarray) -> float:
+    """Calculate the mean absolute percentual difference."""
+    differences = np.diff(timeseries)
+    mapd = np.mean(np.abs(differences / timeseries[:-1]))
+    return mapd
+
+
+def get_absolute_sum_difference(timeseries: np.ndarray) -> float:
+    """Calculate the mean of absolute first-order difference."""
+    mad = np.mean(np.abs(np.diff(timeseries)))
+    return mad
+
+
+def get_max_difference(timeseries: np.ndarray) -> float:
+    """Get maximum one step difference"""
+    return max(np.diff(timeseries))
+
+
+def get_basic_metrics(timeseries: np.ndarray) -> dict[str, float]:
+    """Get general metrics applicable for each goal type."""
+    metrics = {
+        "timeseries_sum": sum(timeseries),
+        "timeseries_min": min(timeseries),
+        "timeseries_max": max(timeseries),
+        "timeseries_avg": np.mean(timeseries),
+        "mean_absolute_percentual_difference": get_mean_absolute_percentual_difference(timeseries),
+        "mean_absolute_difference": get_absolute_sum_difference(timeseries),
+        "max_difference": get_max_difference(timeseries),
+    }
+    return metrics
+
+
+def performance_metrics_minmaximization(results: Dict[str, np.ndarray], goal: MinimizationGoalModel) -> pd.Series:
+    """Get all relevant statistics for a min/maximization goal."""
+    state_timeseries = results[goal.state]
+    metrics = get_basic_metrics(state_timeseries)
+    return pd.Series(metrics)
+
+
+def get_range_percentual_exceedance(
+    timeseries: np.ndarray, goal: RangeGoalModel, targets: TargetDict
+) -> Optional[dict[str, float]]:
+    """Calculate percentage of timesteps in which target is exceeded"""
+    if goal.goal_type not in ["range", "range_rate_of_change"]:
+        below_target = None
+        above_target = None
+    else:
+        below_target = sum(np.where(timeseries + ABS_TOL < targets["target_min"], 1, 0)) / len(timeseries)
+        above_target = sum(np.where(timeseries - ABS_TOL > targets["target_max"], 1, 0)) / len(timeseries)
+    return {"perc_below_target": below_target, "perc_above_target": above_target}
+
+
+def get_range_total_exceedance(
+    timeseries: np.ndarray, goal: RangeGoalModel, targets: TargetDict
+) -> Optional[dict[str, float]]:
+    """Calculate sum of absolute exceedances of the target"""
+    if goal.goal_type not in ["range", "range_rate_of_change"]:
+        below_target = None
+        above_target = None
+    else:
+        below_target = sum(np.abs(np.where(timeseries < targets["target_min"], timeseries - targets["target_min"], 0)))
+        above_target = sum(np.abs(np.where(timeseries > targets["target_max"], timeseries - targets["target_max"], 0)))
+    return {"sum_below_target": below_target, "sum_above_target": above_target}
+
+
+def performance_metrics_range(results: Dict[str, np.ndarray], goal: RangeGoalModel, targets: TargetDict) -> pd.Series:
+    """Get all relevant statistics for a range goal."""
+    metrics = {}
+    state_timeseries = results[goal.state]
+    metrics = metrics | get_basic_metrics(state_timeseries)
+    metrics = metrics | get_range_percentual_exceedance(state_timeseries, goal, targets)
+    metrics = metrics | get_range_total_exceedance(state_timeseries, goal, targets)
+    return pd.Series(metrics)
+
+
+def performance_metrics_rangerateofchange(
+    results: Dict[str, np.ndarray], goal: RangeGoalModel, _targets: TargetDict
+) -> pd.Series:
+    """Get all relevant statistics for a range-rate-of-change goal."""
+    metrics = {}
+    state_timeseries = results[goal.state]
+    metrics = metrics | get_basic_metrics(state_timeseries)
+    return pd.Series(metrics)
+
+
+def get_performance_metrics(results, goal: BaseGoalModel, targets: TargetDict) -> Optional[pd.Series]:
+    """Returns a series with performance metrics for each goal."""
+    if type(goal) in [MinimizationGoalModel, MaximizationGoalModel]:  # pylint: disable=unidiomatic-typecheck
+        return performance_metrics_minmaximization(results, goal)
+    if type(goal) in [RangeGoalModel]:
+        return performance_metrics_range(results, goal, targets)
+    if type(goal) in [RangeRateOfChangeGoalModel]:
+        return performance_metrics_rangerateofchange(results, goal, targets)
+    logger.info("No performance metrics are implemented for goal of type: %s", str(type(goal)))
+    return None

{rtc-tools-interface-0.4.4 → rtc-tools-interface-0.5.0}/rtctools_interface/optimization/plot_goals_mixin.py

@@ -5,17 +5,11 @@ import copy
 from pathlib import Path
 import pickle
 import time
-from typing import Union, List, Dict, Optional
+from typing import Optional
+
+from rtctools_interface.optimization.helpers.statistics_mixin import StatisticsMixin
 
-import numpy as np
 
-from rtctools_interface.optimization.plot_and_goal_schema import (
-    MinimizationGoalCombinedModel,
-    MaximizationGoalCombinedModel,
-    RangeGoalCombinedModel,
-    RangeRateOfChangeGoalCombinedModel,
-)
-from rtctools_interface.optimization.plot_table_schema import PlotTableRow
 from rtctools_interface.optimization.plotting.plot_tools import create_plot_each_priority, create_plot_final_results
 
 from rtctools_interface.optimization.read_plot_table import get_joined_plot_config
@@ -23,7 +17,6 @@ from rtctools_interface.optimization.type_definitions import (
     PlotDataAndConfig,
     PlotOptions,
     PrioIndependentData,
-    TargetDict,
 )
 
 logger = logging.getLogger("rtctools")
@@ -75,7 +68,7 @@ def read_cache_file_from_folder(cache_folder: Path) -> Optional[PlotDataAndConfi
     return loaded_data
 
 
-class PlotGoalsMixin:
+class PlotGoalsMixin(StatisticsMixin):
     """
     Class for plotting results.
     """
@@ -131,98 +124,44 @@ class PlotGoalsMixin:
         self.intermediate_results.append(to_store)
         super().priority_completed(priority)
 
-    def collect_range_target_values(
-        self,
-        all_goals: List[
-            Union[
-                MinimizationGoalCombinedModel,
-                MaximizationGoalCombinedModel,
-                RangeGoalCombinedModel,
-                RangeRateOfChangeGoalCombinedModel,
-                PlotTableRow,
-            ]
-        ],
-    ) -> Dict[str, TargetDict]:
-        """For the goals with targets, collect the actual timeseries with these targets."""
-        t = self.times()
-
-        def get_parameter_ranges(goal):
-            try:
-                target_min = np.full_like(t, 1) * self.parameters(0)[goal.target_min]
-                target_max = np.full_like(t, 1) * self.parameters(0)[goal.target_max]
-            except TypeError:
-                target_min = np.full_like(t, 1) * self.io.get_parameter(goal.target_min)
-                target_max = np.full_like(t, 1) * self.io.get_parameter(goal.target_max)
-            return target_min, target_max
-
-        def get_value_ranges(goal):
-            target_min = np.full_like(t, 1) * float(goal.target_min)
-            target_max = np.full_like(t, 1) * float(goal.target_max)
-            return target_min, target_max
-
-        def get_timeseries_ranges(goal):
-            if isinstance(goal.target_min, str):
-                target_min = self.get_timeseries(goal.target_min).values
-            else:
-                target_min = np.full_like(t, 1) * goal.target_min
-            if isinstance(goal.target_max, str):
-                target_max = self.get_timeseries(goal.target_max).values
-            else:
-                target_max = np.full_like(t, 1) * goal.target_max
-            return target_min, target_max
-
-        target_series: Dict[str, TargetDict] = {}
-        for goal in all_goals:
-            if not goal.specified_in == "python" and goal.goal_type in ["range", "range_rate_of_change"]:
-                if goal.target_data_type == "parameter":
-                    target_min, target_max = get_parameter_ranges(goal)
-                elif goal.target_data_type == "value":
-                    target_min, target_max = get_value_ranges(goal)
-                elif goal.target_data_type == "timeseries":
-                    target_min, target_max = get_timeseries_ranges(goal)
-                else:
-                    message = "Target type {} not known.".format(goal.target_data_type)
-                    logger.error(message)
-                    raise ValueError(message)
-                target_series[goal.goal_id] = {"target_min": target_min, "target_max": target_max}
-        return target_series
-
     def post(self):
         """Tasks after optimizing. Creates a plot for for each priority."""
         super().post()
-        prio_independent_data: PrioIndependentData = {
-            "io_datetimes": self.io.datetimes,
-            "times": self.times(),
-            "target_series": self.collect_range_target_values(self.plot_config),
-            "all_goals": self.goals() + self.path_goals(),
-        }
-
-        plot_options: PlotOptions = {
-            "plot_config": self.plot_config,
-            "plot_max_rows": self.plot_max_rows,
-            "output_folder": self._output_folder,
-            "save_plot_to": self.save_plot_to,
-        }
-
-        current_run: PlotDataAndConfig = {
-            "intermediate_results": self.intermediate_results,
-            "plot_options": plot_options,
-            "prio_independent_data": prio_independent_data,
-        }
-
-        self.plot_data = {}
-        if self.plot_results_each_priority:
-            self.plot_data = self.plot_data | create_plot_each_priority(
-                current_run, plotting_library=self.plotting_library
-            )
-
-        if self.plot_final_results:
-            self.plot_data = self.plot_data | create_plot_final_results(
-                current_run, self._previous_run, plotting_library=self.plotting_library
-            )
 
-        # Cache results, such that in a next run they can be used for comparison
-        self._store_current_results(self._cache_folder, current_run)
+        if self.solver_stats["success"]:
+            prio_independent_data: PrioIndependentData = {
+                "io_datetimes": self.io.datetimes,
+                "times": self.times(),
+                "target_series": self.collect_range_target_values(self.plot_config),
+                "all_goals": self.goals() + self.path_goals(),
+            }
+
+            plot_options: PlotOptions = {
+                "plot_config": self.plot_config,
+                "plot_max_rows": self.plot_max_rows,
+                "output_folder": self._output_folder,
+                "save_plot_to": self.save_plot_to,
+            }
+
+            current_run: PlotDataAndConfig = {
+                "intermediate_results": self.intermediate_results,
+                "plot_options": plot_options,
+                "prio_independent_data": prio_independent_data,
+            }
+
+            self.plot_data = {}
+            if self.plot_results_each_priority:
+                self.plot_data = self.plot_data | create_plot_each_priority(
+                    current_run, plotting_library=self.plotting_library
+                )
+
+            if self.plot_final_results:
+                self.plot_data = self.plot_data | create_plot_final_results(
+                    current_run, self._previous_run, plotting_library=self.plotting_library
+                )
+
+            # Cache results, such that in a next run they can be used for comparison
+            self._store_current_results(self._cache_folder, current_run)
 
     def _store_current_results(self, cache_folder, results_to_store):
         write_cache_file(cache_folder, results_to_store)

rtc-tools-interface-0.4.4/rtctools_interface/optimization/goal_generator_mixin.py

@@ -1,46 +0,0 @@
-"""Module for a basic optimization problem."""
-import logging
-import os
-
-from rtctools_interface.optimization.base_goal import BaseGoal
-from rtctools_interface.optimization.read_goals import read_goals
-
-logger = logging.getLogger("rtctools")
-
-
-class GoalGeneratorMixin:
-    # TODO: remove pylint disable below once we have more public functions.
-    # pylint: disable=too-few-public-methods
-    """Add path goals as specified in the goal_table.
-
-    By default, the mixin looks for the csv in the in the default input
-    folder. One can also set the path to the goal_table_file manually
-    with the `goal_table_file` class variable.
-    """
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self.goals_to_generate = kwargs.get("goals_to_generate", [])
-        self.read_from = kwargs.get("read_goals_from", "csv_table")
-        if not hasattr(self, "goal_table_file"):
-            self.goal_table_file = os.path.join(self._input_folder, "goal_table.csv")
-
-    def path_goals(self):
-        """Return the list of path goals."""
-        goals = super().path_goals()
-        new_goals = read_goals(
-            self.goal_table_file, path_goal=True, read_from=self.read_from, goals_to_generate=self.goals_to_generate
-        )
-        if new_goals:
-            goals = goals + [BaseGoal(optimization_problem=self, **goal.__dict__) for goal in new_goals]
-        return goals
-
-    def goals(self):
-        """Return the list of goals."""
-        goals = super().goals()
-        new_goals = read_goals(
-            self.goal_table_file, path_goal=False, read_from=self.read_from, goals_to_generate=self.goals_to_generate
-        )
-        if new_goals:
-            goals = goals + [BaseGoal(optimization_problem=self, **goal.__dict__) for goal in new_goals]
-        return goals