gammasimtools 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

simtools/model/model_parameter.py

@@ -67,7 +67,6 @@ class ModelParameter:
 
         self._parameters = {}
         self._simulation_config_parameters = {"corsika": {}, "simtel": {}}
-        self._derived = None
         self.collection = collection
         self.label = label
         self.model_version = model_version
@@ -77,6 +76,9 @@ class ModelParameter:
             if array_element_name is not None
             else None
         )
+        self.design_model = self.db.get_design_model(
+            self.model_version, self.name, collection="telescopes"
+        )
         self._config_file_directory = None
         self._config_file_path = None
         self._load_parameters_from_db()
@@ -111,12 +113,8 @@ class ModelParameter:
         """
         try:
             return self._parameters[par_name]
-        except KeyError:
-            pass
-        try:
-            return self.derived[par_name]
         except (KeyError, ValueError) as e:
-            msg = f"Parameter {par_name} was not found in the model"
+            msg = f"Parameter {par_name} was not found in the model {self.name}, {self.site}."
             self._logger.error(msg)
             raise InvalidModelParameterError(msg) from e
 
@@ -180,7 +178,10 @@ class ModelParameter:
         _value = self.get_parameter_value(par_name, _parameter)
 
         try:
-            _unit = [item.strip() for item in _parameter.get("unit").split(",")]
+            if isinstance(_parameter.get("unit"), str):
+                _unit = [item.strip() for item in _parameter.get("unit").split(",")]
+            else:
+                _unit = _parameter.get("unit")
 
             # if there is only one value or the values share one unit
             if (isinstance(_value, (int | float))) or (len(_value) > len(_unit)):
@@ -241,32 +242,21 @@ class ModelParameter:
             self._logger.debug(f"Parameter {par_name} does not have a file associated with it.")
         return False
 
-    @property
-    def derived(self):
-        """Load the derived values and export them if the class instance hasn't done it yet."""
-        if self._derived is None:
-            self._load_derived_values()
-            self._export_derived_files()
-        return self._derived
-
-    def _load_derived_values(self):
-        """Load derived values from the DB."""
-        self._logger.debug("Reading derived values from DB")
-        self._derived = self.db.get_derived_values(
-            self.site,
-            self.name,
-            self.model_version,
-        )
+    def get_parameter_version(self, par_name):
+        """
+        Get version for a given parameter used in the model.
 
-    def _export_derived_files(self):
-        """Write to disk a file from the derived values DB."""
-        for par_now in self.derived.values():
-            if par_now.get("File") or par_now.get("file"):
-                self.db.export_file_db(
-                    db_name=self.db.DB_DERIVED_VALUES,
-                    dest=self.config_file_directory,
-                    file_name=(par_now.get("value") or par_now.get("Value")),
-                )
+        Parameters
+        ----------
+        par_name: str
+            Name of the parameter.
+
+        Returns
+        -------
+        str
+            parameter version used in the model (eg. '1.0.0')
+        """
+        return self._get_parameter_dict(par_name)["parameter_version"]
 
     def print_parameters(self):
         """Print parameters and their values for debugging purposes."""
@@ -310,6 +300,21 @@ class ModelParameter:
         """
         return self._simulation_config_parameters.get(simulation_software)
 
+    def has_parameter(self, par_name):
+        """Check if a parameter exists in the model.
+
+        Parameters
+        ----------
+        par_name : str
+            Name of the parameter.
+
+        Returns
+        -------
+        bool
+            True if parameter exists in the model.
+        """
+        return par_name in self._parameters
+
     def _load_simulation_software_parameter(self):
         """Read simulation software parameters from DB."""
         for simulation_software in self._simulation_config_parameters:
@@ -322,10 +327,11 @@ class ModelParameter:
                         simulation_software=simulation_software,
                     )
                 )
-            except ValueError:
+            except ValueError as exc:
                 self._logger.warning(
                     f"No {simulation_software} parameters found for "
-                    f"{self.site}, {self.name} (model version {self.model_version})."
+                    f"{self.site}, {self.name} (model version {self.model_version}). "
+                    f" (Query {exc})"
                 )
 
     def _load_parameters_from_db(self):
@@ -335,15 +341,18 @@ class ModelParameter:
 
         if self.name is not None:
             self._parameters = self.db.get_model_parameters(
-                self.site, self.name, self.model_version, self.collection, only_applicable=True
+                self.site, self.name, self.collection, self.model_version
             )
 
         if self.site is not None:
-            _site_pars = self.db.get_site_parameters(
-                self.site, self.model_version, only_applicable=True
+            self._parameters.update(
+                self.db.get_model_parameters(
+                    self.site,
+                    None,
+                    "sites",
+                    self.model_version,
+                )
             )
-            self._parameters.update(_site_pars)
-
         self._load_simulation_software_parameter()
 
     def set_extra_label(self, extra_label):
@@ -369,7 +378,7 @@ class ModelParameter:
         """Return the extra label if defined, if not return ''."""
         return self._extra_label if self._extra_label is not None else ""
 
-    def get_simtel_parameters(self, parameters=None, telescope_model=True, site_model=True):
+    def get_simtel_parameters(self, parameters=None):
         """
         Get simtel parameters as name and value pairs.
 
@@ -377,10 +386,6 @@ class ModelParameter:
         ----------
         parameters: dict
             Parameters (simtools) to be renamed (if necessary)
-        telescope_model: bool
-            If True, telescope model parameters are included.
-        site_model: bool
-            If True, site model parameters are included.
 
         Returns
         -------
@@ -394,10 +399,7 @@ class ModelParameter:
         _simtel_parameter_value = {}
         for key in parameters:
             _par_name = names.get_simulation_software_name_from_parameter_name(
-                key,
-                simulation_software="sim_telarray",
-                search_telescope_parameters=telescope_model,
-                search_site_parameters=site_model,
+                key, simulation_software="sim_telarray"
             )
             if _par_name is not None:
                 _simtel_parameter_value[_par_name] = parameters[key].get("value")
@@ -441,8 +443,7 @@ class ModelParameter:
             )
 
         self._logger.debug(
-            f"Changing parameter {par_name} "
-            f"from {self.get_parameter_value(par_name)} to {value}"
+            f"Changing parameter {par_name} from {self.get_parameter_value(par_name)} to {value}"
         )
         self._parameters[par_name]["value"] = value
 
@@ -513,7 +514,7 @@ class ModelParameter:
             for par in self._added_parameter_files:
                 pars_from_db.pop(par)
 
-        self.db.export_model_files(pars_from_db, self.config_file_directory)
+        self.db.export_model_files(parameters=pars_from_db, dest=self.config_file_directory)
         self._is_exported_model_files_up_to_date = True
 
     def get_model_file_as_table(self, par_name):
@@ -535,7 +536,7 @@ class ModelParameter:
             _par_entry[par_name] = self._parameters[par_name]
         except KeyError as exc:
             raise ValueError(f"Parameter {par_name} not found in the model.") from exc
-        self.db.export_model_files(_par_entry, self.config_file_directory)
+        self.db.export_model_files(parameters=_par_entry, dest=self.config_file_directory)
         if _par_entry[par_name]["value"].endswith("ecsv"):
             return Table.read(
                 self.config_file_directory.joinpath(_par_entry[par_name]["value"]),
@@ -620,7 +621,7 @@ class ModelParameter:
             Model directory to export the file to.
         """
         self.db.export_model_files(
-            {
+            parameters={
                 "nsb_spectrum_at_2200m": {
                     "value": self._simulation_config_parameters["simtel"][
                         "correct_nsb_spectrum_to_telescope_altitude"
@@ -628,5 +629,5 @@ class ModelParameter:
                     "file": True,
                 }
             },
-            model_directory,
+            dest=model_directory,
         )

simtools/model/site_model.py

@@ -151,11 +151,11 @@ class SiteModel(ModelParameter):
             Model directory to export the file to.
         """
         self.db.export_model_files(
-            {
+            parameters={
                 "atmospheric_transmission_file": {
                     "value": self.get_parameter_value("atmospheric_profile"),
                     "file": True,
                 }
             },
-            model_directory,
+            dest=model_directory,
         )

simtools/production_configuration/calculate_statistical_errors_grid_point.py

@@ -1,13 +1,4 @@
-"""
-Provides functionality to evaluate statistical uncertainties from DL2 MC event files.
-
-Classes
--------
-StatisticalErrorEvaluator
-    Handles error calculation for given DL2 MC event files and specified metrics.
-
-
-"""
+"""Evaluate statistical uncertainties from DL2 MC event files."""
 
 import logging
 
@@ -15,7 +6,7 @@ import numpy as np
 from astropy import units as u
 from astropy.io import fits
 
-_logger = logging.getLogger(__name__)
+__all__ = ["StatisticalErrorEvaluator"]
 
 
 class StatisticalErrorEvaluator:
@@ -41,26 +32,13 @@ class StatisticalErrorEvaluator:
         metrics: dict[str, float],
         grid_point: tuple[float, float, float, float, float] | None = None,
     ):
-        """
-        Init the evaluator with a DL2 MC event file, its type, and metrics to calculate.
-
-        Parameters
-        ----------
-        file_path : str
-            The path to the DL2 MC event file.
-        file_type : str
-            The type of the file ('point-like' or 'cone').
-        metrics : dict, optional
-            Dictionary specifying which metrics to compute and their reference values.
-        grid_point : tuple, optional
-            Tuple specifying the grid point (energy, azimuth, zenith, NSB, offset).
-        """
-        self.file_path = file_path
+        """Init the evaluator with a DL2 MC event file, its type, and metrics to calculate."""
+        self._logger = logging.getLogger(__name__)
         self.file_type = file_type
         self.metrics = metrics
         self.grid_point = grid_point
 
-        self.data = self.load_data_from_file()
+        self.data = self.load_data_from_file(file_path)
 
         self.uncertainty_effective_area = None
         self.energy_estimate = None
@@ -70,7 +48,58 @@ class StatisticalErrorEvaluator:
         self.metric_results = None
         self.energy_threshold = None
 
-    def load_data_from_file(self):
+    def _load_event_data(self, hdul, data_type):
+        """
+        Load data and units for the event and simulated data data.
+
+        Parameters
+        ----------
+        hdul : HDUList
+            The HDUList object.
+        data_type: str
+            The type of data to load ('EVENTS' or 'SIMULATED EVENTS').
+
+        Returns
+        -------
+        dict
+            Dictionary containing units for the event data.
+        """
+        _data = hdul[data_type].data  # pylint: disable=E1101
+        _header = hdul[data_type].header  # pylint: disable=E1101
+        _units = {}
+        for idx, col_name in enumerate(_data.columns.names, start=1):
+            unit_key = f"TUNIT{idx}"
+            if unit_key in _header:
+                _units[col_name] = u.Unit(_header[unit_key])
+            else:
+                _units[col_name] = None
+        return _data, _units
+
+    def _set_grid_point(self, events_data):
+        """Set azimuth/zenith angle of grid point."""
+        unique_azimuths = np.unique(events_data["PNT_AZ"]) * u.deg
+        unique_zeniths = 90 * u.deg - np.unique(events_data["PNT_ALT"]) * u.deg
+        if len(unique_azimuths) > 1 or len(unique_zeniths) > 1:
+            msg = (
+                f"Multiple values found for azimuth ({unique_azimuths}) zenith ({unique_zeniths})."
+            )
+            self._logger.error(msg)
+            raise ValueError(msg)
+        if self.grid_point is not None:
+            self._logger.warning(
+                f"Grid point already set to: {self.grid_point}. "
+                "Overwriting with new values from file."
+            )
+        self.grid_point = (
+            1 * u.TeV,
+            unique_azimuths[0],
+            unique_zeniths[0],
+            0,
+            0 * u.deg,
+        )
+        self._logger.info(f"Grid point values: {self.grid_point}")
+
+    def load_data_from_file(self, file_path):
         """
         Load data from the DL2 MC event file and return dictionaries with units.
 
@@ -81,91 +110,24 @@ class StatisticalErrorEvaluator:
         """
         data = {}
         try:
-            with fits.open(self.file_path) as hdul:
-                events_data = hdul["EVENTS"].data  # pylint: disable=E1101
-                sim_events_data = hdul["SIMULATED EVENTS"].data  # pylint: disable=E1101
-                event_units = {}
-                for idx, col_name in enumerate(events_data.columns.names, start=1):
-                    unit_key = f"TUNIT{idx}"
-                    if unit_key in hdul["EVENTS"].header:  # pylint: disable=E1101
-                        event_units[col_name] = u.Unit(
-                            hdul["EVENTS"].header[unit_key]  # pylint: disable=E1101
-                        )
-                    else:
-                        event_units[col_name] = None
-
-                sim_units = {}
-                for idx, col_name in enumerate(sim_events_data.columns.names, start=1):
-                    unit_key = f"TUNIT{idx}"
-                    if unit_key in hdul["SIMULATED EVENTS"].header:  # pylint: disable=E1101
-                        sim_units[col_name] = u.Unit(
-                            hdul["SIMULATED EVENTS"].header[unit_key]  # pylint: disable=E1101
-                        )
-                    else:
-                        sim_units[col_name] = None
-                # dl2 files are required to have units for these entries
-                event_energies_reco = events_data["ENERGY"] * event_units["ENERGY"]
-
-                event_energies_mc = events_data["MC_ENERGY"] * event_units["MC_ENERGY"]
-
-                bin_edges_low = sim_events_data["MC_ENERG_LO"] * sim_units["MC_ENERG_LO"]
-
-                bin_edges_high = sim_events_data["MC_ENERG_HI"] * sim_units["MC_ENERG_HI"]
-
-                simulated_event_histogram = sim_events_data["EVENTS"] * u.count
-
-                viewcone = hdul[3].data["viewcone"][0][1]  # pylint: disable=E1101
-                core_range = hdul[3].data["core_range"][0][1]  # pylint: disable=E1101
+            with fits.open(file_path) as hdul:
+                events_data, event_units = self._load_event_data(hdul, "EVENTS")
+                sim_events_data, sim_units = self._load_event_data(hdul, "SIMULATED EVENTS")
 
                 data = {
-                    "event_energies_reco": event_energies_reco,
-                    "event_energies_mc": event_energies_mc,
-                    "bin_edges_low": bin_edges_low,
-                    "bin_edges_high": bin_edges_high,
-                    "simulated_event_histogram": simulated_event_histogram,
-                    "viewcone": viewcone,
-                    "core_range": core_range,
+                    "event_energies_reco": events_data["ENERGY"] * event_units["ENERGY"],
+                    "event_energies_mc": events_data["MC_ENERGY"] * event_units["MC_ENERGY"],
+                    "bin_edges_low": sim_events_data["MC_ENERG_LO"] * sim_units["MC_ENERG_LO"],
+                    "bin_edges_high": sim_events_data["MC_ENERG_HI"] * sim_units["MC_ENERG_HI"],
+                    "simulated_event_histogram": sim_events_data["EVENTS"] * u.count,
+                    "viewcone": hdul[3].data["viewcone"][0][1],  # pylint: disable=E1101
+                    "core_range": hdul[3].data["core_range"][0][1],  # pylint: disable=E1101
                 }
-                unique_azimuths = np.unique(events_data["PNT_AZ"]) * u.deg
-                unique_zeniths = 90 * u.deg - np.unique(events_data["PNT_ALT"]) * u.deg
-                if self.grid_point is None:
-                    _logger.info(f"Unique azimuths: {unique_azimuths}")
-                    _logger.info(f"Unique zeniths: {unique_zeniths}")
-
-                    if len(unique_azimuths) == 1 and len(unique_zeniths) == 1:
-                        _logger.info(
-                            f"Setting initial grid point with azimuth: {unique_azimuths[0]}"
-                            f" zenith: {unique_zeniths[0]}"
-                        )
-                        self.grid_point = (
-                            1 * u.TeV,
-                            unique_azimuths[0],
-                            unique_zeniths[0],
-                            0,
-                            0 * u.deg,
-                        )  # Initialize grid point with azimuth and zenith
-                    else:
-                        msg = "Multiple unique values found for azimuth or zenith."
-                        _logger.error(msg)
-                        raise ValueError(msg)
-                else:
-                    _logger.warning(
-                        f"Grid point already set to: {self.grid_point}. "
-                        "Overwriting with new values from file."
-                    )
-
-                    self.grid_point = (
-                        1 * u.TeV,
-                        unique_azimuths[0],
-                        unique_zeniths[0],
-                        0,
-                        0 * u.deg,
-                    )
-                    _logger.info(f"New grid point values: {self.grid_point}")
+                self._set_grid_point(events_data)
 
         except FileNotFoundError as e:
-            error_message = f"Error loading file {self.file_path}: {e}"
-            _logger.error(error_message)
+            error_message = f"Error loading file {file_path}: {e}"
+            self._logger.error(error_message)
             raise FileNotFoundError(error_message) from e
         return data
 
@@ -183,9 +145,9 @@ class StatisticalErrorEvaluator:
         bin_edges = np.concatenate([bin_edges_low, [bin_edges_high[-1]]])
         return np.unique(bin_edges)
 
-    def compute_triggered_event_histogram(self, event_energies_reco, bin_edges):
+    def compute_reconstructed_event_histogram(self, event_energies_reco, bin_edges):
         """
-        Compute histogram for triggered events.
+        Compute histogram of events as function of reconstructed energy.
 
         Parameters
         ----------
@@ -196,24 +158,26 @@ class StatisticalErrorEvaluator:
 
         Returns
         -------
-        triggered_event_histogram : array
-            Histogram of triggered events.
+        reconstructed_event_histogram : array
+            Histogram of reconstructed events.
         """
         event_energies_reco = event_energies_reco.to(bin_edges.unit)
 
-        triggered_event_histogram, _ = np.histogram(event_energies_reco.value, bins=bin_edges.value)
-        return triggered_event_histogram * u.count
+        reconstructed_event_histogram, _ = np.histogram(
+            event_energies_reco.value, bins=bin_edges.value
+        )
+        return reconstructed_event_histogram * u.count
 
-    def compute_efficiency_and_errors(self, triggered_event_counts, simulated_event_counts):
+    def compute_efficiency_and_errors(self, reconstructed_event_counts, simulated_event_counts):
         """
-        Compute trigger efficiency and its statistical error using the binomial distribution.
+        Compute reconstructed event efficiency and its uncertainty assuming binomial distribution.
 
         Parameters
         ----------
-        triggered_event_counts : array with units
-            Histogram counts of the triggered events.
+        reconstructed_event_counts : array with units
+            Histogram counts of reconstructed events.
         simulated_event_counts : array with units
-            Histogram counts of the simulated events.
+            Histogram counts of simulated events.
 
         Returns
         -------
@@ -223,32 +187,39 @@ class StatisticalErrorEvaluator:
             Array of relative uncertainties.
         """
         # Ensure the inputs have compatible units
-        triggered_event_counts = triggered_event_counts.to(u.ct)
-        simulated_event_counts = simulated_event_counts.to(u.ct)
+        reconstructed_event_counts = (
+            reconstructed_event_counts.to(u.ct)
+            if isinstance(reconstructed_event_counts, u.Quantity)
+            else reconstructed_event_counts * u.ct
+        )
+        simulated_event_counts = (
+            simulated_event_counts.to(u.ct)
+            if isinstance(simulated_event_counts, u.Quantity)
+            else simulated_event_counts * u.ct
+        )
+
+        if np.any(reconstructed_event_counts > simulated_event_counts):
+            raise ValueError("Reconstructed event counts exceed simulated event counts.")
 
         # Compute efficiencies, ensuring the output is dimensionless
         efficiencies = np.divide(
-            triggered_event_counts,
+            reconstructed_event_counts,
             simulated_event_counts,
-            out=np.zeros_like(triggered_event_counts),
+            out=np.zeros_like(reconstructed_event_counts),
             where=simulated_event_counts > 0,
         ).to(u.dimensionless_unscaled)
 
         # Set up a mask for valid data with a unit-consistent threshold
-        if np.any(triggered_event_counts > simulated_event_counts):
-            raise ValueError(
-                "Triggered event counts exceed simulated event counts. Please check input data."
-            )
-        valid = (simulated_event_counts > 0 * u.ct) & (triggered_event_counts > 0 * u.ct)
+        valid = (simulated_event_counts > 0) & (reconstructed_event_counts > 0)
 
-        uncertainties = np.zeros_like(triggered_event_counts.value) * u.dimensionless_unscaled
+        uncertainties = np.zeros_like(reconstructed_event_counts.value) * u.dimensionless_unscaled
 
         if np.any(valid):
             uncertainties[valid] = np.sqrt(
                 np.maximum(
                     simulated_event_counts[valid]
-                    / triggered_event_counts[valid]
-                    * (1 - triggered_event_counts[valid] / simulated_event_counts[valid]),
+                    / reconstructed_event_counts[valid]
+                    * (1 - reconstructed_event_counts[valid] / simulated_event_counts[valid]),
                     0,
                 )
             )
@@ -273,13 +244,13 @@ class StatisticalErrorEvaluator:
             Energy threshold value.
         """
         bin_edges = self.create_bin_edges()
-        triggered_event_histogram = self.compute_triggered_event_histogram(
+        reconstructed_event_histogram = self.compute_reconstructed_event_histogram(
             self.data["event_energies_mc"], bin_edges
         )
         simulated_event_histogram = self.data["simulated_event_histogram"]
 
         efficiencies, _ = self.compute_efficiency_and_errors(
-            triggered_event_histogram, simulated_event_histogram
+            reconstructed_event_histogram, simulated_event_histogram
         )
 
         # Determine the effective area threshold (10% of max effective area)
@@ -302,12 +273,12 @@ class StatisticalErrorEvaluator:
             Dictionary with uncertainties for the file.
         """
         bin_edges = self.create_bin_edges()
-        triggered_event_histogram = self.compute_triggered_event_histogram(
+        reconstructed_event_histogram = self.compute_reconstructed_event_histogram(
             self.data["event_energies_mc"], bin_edges
         )
         simulated_event_histogram = self.data["simulated_event_histogram"]
         _, relative_errors = self.compute_efficiency_and_errors(
-            triggered_event_histogram, simulated_event_histogram
+            reconstructed_event_histogram, simulated_event_histogram
         )
         return {"relative_errors": relative_errors}
 
@@ -326,7 +297,10 @@ class StatisticalErrorEvaluator:
         event_energies_mc = self.data["event_energies_mc"]
 
         if len(event_energies_reco) != len(event_energies_mc):
-            raise ValueError(f"Mismatch in the number of energies for file {self.file_path}")
+            raise ValueError(
+                f"Mismatch in the number of energies: {len(event_energies_reco)} vs "
+                f"{len(event_energies_mc)}"
+            )
 
         energy_deviation = (event_energies_reco - event_energies_mc) / event_energies_mc
 
@@ -351,15 +325,15 @@ class StatisticalErrorEvaluator:
     def calculate_metrics(self):
         """Calculate all defined metrics as specified in self.metrics and store results."""
         if "uncertainty_effective_area" in self.metrics:
-
             self.uncertainty_effective_area = self.calculate_uncertainty_effective_area()
             if self.uncertainty_effective_area:
-                validity_range = self.metrics.get("uncertainty_effective_area", {}).get(
-                    "valid_range"
+                energy_range = self.metrics.get("uncertainty_effective_area", {}).get(
+                    "energy_range"
+                )
+                min_energy, max_energy = (
+                    energy_range["value"][0] * u.Unit(energy_range["unit"]),
+                    energy_range["value"][1] * u.Unit(energy_range["unit"]),
                 )
-                min_energy, max_energy = validity_range["value"][0] * u.Unit(
-                    validity_range["unit"]
-                ), validity_range["value"][1] * u.Unit(validity_range["unit"])
 
                 valid_errors = [
                     error
@@ -375,7 +349,7 @@ class StatisticalErrorEvaluator:
                 ref_value = self.metrics.get("uncertainty_effective_area", {}).get("target_error")[
                     "value"
                 ]
-                _logger.info(
+                self._logger.info(
                     f"Effective Area Error (max in validity range): "
                     f"{self.uncertainty_effective_area['max_error'].value:.6f}, "
                     f"Reference: {ref_value:.3f}"
@@ -386,7 +360,7 @@ class StatisticalErrorEvaluator:
                 self.calculate_energy_estimate()
             )
             ref_value = self.metrics.get("energy_estimate", {}).get("target_error")["value"]
-            _logger.info(
+            self._logger.info(
                 f"Energy Estimate Error: {self.energy_estimate:.3f}, Reference: {ref_value:.3f}"
             )
         else:
@@ -442,7 +416,7 @@ class StatisticalErrorEvaluator:
                 overall_max_errors[metric_name] = result
             else:
                 raise ValueError(f"Unsupported result type for {metric_name}: {type(result)}")
-        _logger.info(f"overall_max_errors {overall_max_errors}")
+        self._logger.info(f"overall_max_errors {overall_max_errors}")
         all_max_errors = list(overall_max_errors.values())
         if metric == "average":
             overall_metric = np.mean(all_max_errors)