gammasimtools 0.25.0__py3-none-any.whl → 0.27.0__py3-none-any.whl

simtools/model/model_parameter.py

@@ -4,13 +4,15 @@
 import logging
 import shutil
 from copy import copy, deepcopy
+from pathlib import Path
 
 import astropy.units as u
 
 import simtools.utils.general as gen
 from simtools.data_model import schema
 from simtools.db import db_handler
-from simtools.io import ascii_handler, io_handler
+from simtools.io import io_handler
+from simtools.model import legacy_model_parameter
 from simtools.simtel.simtel_config_writer import SimtelConfigWriter
 from simtools.utils import names, value_conversion
 
@@ -28,8 +30,6 @@ class ModelParameter:
 
     Parameters
     ----------
-    db_config:
-        Database configuration dictionary.
     model_version: str
         Version of the model (ex. 5.0.0).
     site: str
@@ -41,8 +41,8 @@ class ModelParameter:
         as stored under collection in the DB.
     label: str
         Instance label. Used for output file naming.
-    overwrite_model_parameters: str, optional
-        File name to overwrite model parameters from DB with provided values.
+    overwrite_model_parameter_dict: dict, optional
+        Dictionary to overwrite model parameters from DB with provided values.
         Instance label. Important for output file naming.
     ignore_software_version: bool
         If True, ignore software version checks for deprecated parameters.
@@ -51,18 +51,19 @@ class ModelParameter:
 
     def __init__(
         self,
-        db_config,
         model_version,
         site=None,
         array_element_name=None,
         collection="telescopes",
         label=None,
-        overwrite_model_parameters=None,
+        overwrite_model_parameter_dict=None,
         ignore_software_version=False,
     ):
         self._logger = logging.getLogger(__name__)
         self.io_handler = io_handler.IOHandler()
-        self.db = db_handler.DatabaseHandler(db_config=db_config)
+        self.db = db_handler.DatabaseHandler()
+        if not self.db.is_configured():
+            raise RuntimeError("Database is not configured.")
 
         self.parameters = {}
         self._simulation_config_parameters = {sw: {} for sw in names.simulation_software()}
@@ -81,7 +82,7 @@ class ModelParameter:
         )
         self._config_file_directory = None
         self._config_file_path = None
-        self.overwrite_model_parameters = overwrite_model_parameters
+        self.overwrite_model_parameter_dict = overwrite_model_parameter_dict
         self._added_parameter_files = None
         self._is_exported_model_files_up_to_date = False
 
@@ -276,8 +277,6 @@ class ModelParameter:
         )
         self._config_file_path = self.config_file_directory.joinpath(config_file_name)
 
-        self._logger.debug(f"Config file path: {self._config_file_path}")
-
     def get_simulation_software_parameters(self, simulation_software):
         """
         Get simulation software parameters.
@@ -325,37 +324,47 @@ class ModelParameter:
                     self.site, self.name, self.collection, self.model_version
                 )
             )
-            if self.overwrite_model_parameters:
-                self.overwrite_parameters_from_file(self.overwrite_model_parameters)
-            self._check_model_parameter_software_versions(self.parameters.keys())
+            self.overwrite_parameters(self.overwrite_model_parameter_dict)
+            self._check_model_parameter_versions(self.parameters)
 
         self._load_simulation_software_parameter()
         for software_name, parameters in self._simulation_config_parameters.items():
-            self._check_model_parameter_software_versions(
-                parameters.keys(), software_name=software_name
-            )
+            self._check_model_parameter_versions(parameters, software_name=software_name)
 
-    def _check_model_parameter_software_versions(self, parameter_list, software_name=None):
+    def _check_model_parameter_versions(self, parameters, software_name=None):
         """
-        Ensure that model parameters are compatible with the installed software versions.
+        Ensure parameters follow the latest schema and are compatible with installed software.
 
         Compares software versions listed in schema files with the installed software versions
         (e.g., sim_telarray, CORSIKA).
 
+        For outdated model parameter schemas, legacy update functions are called to update
+        the parameters to the latest schema version.
+
         Parameters
         ----------
-        parameter_list: list
-            List containing model parameter names.
+        parameters: dict
+            Dictionary containing model parameters.
         software_name: str
             Name of the software for which the parameters are checked.
         """
-        for par_name in parameter_list:
+        _legacy_updates = {}
+        for par_name, par_data in parameters.items():
             if par_name in (parameter_schema := names.model_parameters()):
                 schema.validate_deprecation_and_version(
                     data=parameter_schema[par_name],
                     software_name=software_name,
                     ignore_software_version=self.ignore_software_version,
                 )
+                _latest_schema_version = parameter_schema[par_name]["schema_version"]
+                if par_data["model_parameter_schema_version"] != _latest_schema_version:
+                    _legacy_updates.update(
+                        legacy_model_parameter.update_parameter(
+                            par_name, parameters, _latest_schema_version
+                        )
+                    )
+
+        legacy_model_parameter.apply_legacy_updates_to_parameters(parameters, _legacy_updates)
 
     def overwrite_model_parameter(self, par_name, value, parameter_version=None):
         """
@@ -385,22 +394,20 @@ class ModelParameter:
             raise InvalidModelParameterError(f"Parameter {par_name} not in the model")
 
         if value is None and parameter_version:
-            _para_dict = self.db.get_model_parameter(
-                parameter=par_name,
-                site=self.site,
-                array_element_name=self.name,
-                parameter_version=parameter_version,
-            )
-            if _para_dict:
-                self.parameters[par_name] = _para_dict.get(par_name)
-            self._logger.debug(
-                f"Changing parameter {par_name} to version {parameter_version} with value "
-                f"{self.parameters[par_name]['value']}"
-            )
+            self._overwrite_model_parameter_from_db(par_name, parameter_version)
         else:
-            value = gen.convert_string_to_list(value) if isinstance(value, str) else value
+            self._overwrite_model_parameter_from_value(par_name, value, parameter_version)
+
+        # In case parameter is a file, the model files will be outdated
+        if self.get_parameter_file_flag(par_name):
+            self._is_exported_model_files_up_to_date = False
+
+    def _overwrite_model_parameter_from_value(self, par_name, value, parameter_version=None):
+        """Overwrite model parameter from provided value only."""
+        value = gen.convert_string_to_list(value) if isinstance(value, str) else value
+        par_type = self.get_parameter_type(par_name)
 
-            par_type = self.get_parameter_type(par_name)
+        if par_type in ("list", "dict"):
             if not gen.validate_data_type(
                 reference_dtype=par_type,
                 value=value,
@@ -408,40 +415,40 @@ class ModelParameter:
                 allow_subtypes=True,
             ):
                 raise ValueError(f"Could not cast {value} of type {type(value)} to {par_type}.")
+        else:
+            for value_element in gen.ensure_iterable(value):
+                if not gen.validate_data_type(
+                    reference_dtype=par_type,
+                    value=value_element,
+                    dtype=None,
+                    allow_subtypes=True,
+                ):
+                    raise ValueError(
+                        f"Could not cast {value_element} of type "
+                        f"{type(value_element)} to {par_type}."
+                    )
 
-            self._logger.debug(
-                f"Changing parameter {par_name} from {self.get_parameter_value(par_name)} "
-                f"to {value}"
-            )
-            self.parameters[par_name]["value"] = value
-            if parameter_version:
-                self.parameters[par_name]["parameter_version"] = parameter_version
-
-        # In case parameter is a file, the model files will be outdated
-        if self.get_parameter_file_flag(par_name):
-            self._is_exported_model_files_up_to_date = False
-
-    def overwrite_parameters_from_file(self, file_name):
-        """
-        Overwrite parameters from a file.
-
-        File is expected to follow the format described in 'simulation_models_info.schema.yml'.
-
-        This function does not modify the DB, it affects only the current instance.
-        This feature is intended for developers and lacks validation.
-
-        Parameters
-        ----------
-        file_name: str
-            File containing the parameters to be changed.
-        """
-        changes_data = schema.validate_dict_using_schema(
-            data=ascii_handler.collect_data_from_file(file_name=file_name),
-            schema_file="simulation_models_info.schema.yml",
-        ).get("changes", {})
-
-        key_for_changes = self._get_key_for_parameter_changes(self.site, self.name, changes_data)
-        self.overwrite_parameters(changes_data.get(key_for_changes, {}) if key_for_changes else {})
+        self._logger.debug(
+            f"Changing parameter {par_name} from {self.get_parameter_value(par_name)} to {value}"
+        )
+        self.parameters[par_name]["value"] = value
+        if parameter_version:
+            self.parameters[par_name]["parameter_version"] = parameter_version
+
+    def _overwrite_model_parameter_from_db(self, par_name, parameter_version):
+        """Overwrite model parameter from DB for a specific version."""
+        _para_dict = self.db.get_model_parameter(
+            parameter=par_name,
+            site=self.site,
+            array_element_name=self.name,
+            parameter_version=parameter_version,
+        )
+        if _para_dict:
+            self.parameters[par_name] = _para_dict.get(par_name)
+        self._logger.debug(
+            f"Changing parameter {par_name} to version {parameter_version} with value "
+            f"{self.parameters[par_name]['value']}"
+        )
 
     def _get_key_for_parameter_changes(self, site, array_element_name, changes_data):
         """
@@ -483,7 +490,7 @@ class ModelParameter:
 
         return None
 
-    def overwrite_parameters(self, changes):
+    def overwrite_parameters(self, changes, flat_dict=False):
         """
         Change the value of multiple existing parameters in the model.
 
@@ -491,23 +498,43 @@ class ModelParameter:
 
         Allows for two types of 'changes' dictionary:
 
-        - simple: '{parameter_name: new_value, ...}'
-        - model repository style:
-          '{parameter_name: {"value": new_value, "version": new_version}, ...}'
+        - simple (flat_dict=True): '{parameter_name: new_value, ...}'
+        - model repository style (flat_dict=False):
+          '{array_element: {parameter_name: {"value": new_value, "version": new_version}, ...}}'
 
         Parameters
         ----------
         changes: dict
             Parameters to be changed.
         """
+        if not changes:
+            return
+        if not flat_dict:
+            key_for_changes = self._get_key_for_parameter_changes(self.site, self.name, changes)
+            changes = changes.get(key_for_changes, {})
+        if not changes:
+            return
+
+        if flat_dict:
+            self._logger.debug(f"Overwriting parameters with changes: {changes}")
+        else:
+            self._logger.debug(
+                f"Overwriting parameters for {key_for_changes} with changes: {changes}"
+            )
+
         for par_name, par_value in changes.items():
-            if par_name in self.parameters:
-                if isinstance(par_value, dict) and ("value" in par_value or "version" in par_value):
-                    self.overwrite_model_parameter(
-                        par_name, par_value.get("value"), par_value.get("version")
-                    )
-                else:
-                    self.overwrite_model_parameter(par_name, par_value)
+            if par_name not in self.parameters:
+                self._logger.warning(
+                    f"Parameter {par_name} not found in model {self.name}, cannot overwrite it."
+                )
+                continue
+
+            if isinstance(par_value, dict) and ("value" in par_value or "version" in par_value):
+                self.overwrite_model_parameter(
+                    par_name, par_value.get("value"), par_value.get("version")
+                )
+            else:
+                self.overwrite_model_parameter(par_name, par_value)
 
     def overwrite_model_file(self, par_name, file_path):
         """
@@ -555,7 +582,29 @@ class ModelParameter:
         )
         self._is_exported_model_files_up_to_date = True
 
-    def write_sim_telarray_config_file(self, additional_models=None):
+    def get_config_file_path(self, label=None):
+        """Return config file path for a given label.
+
+        Parameters
+        ----------
+        label : str or None
+            Label used for output file naming. If None, use this model's label.
+
+        Returns
+        -------
+        pathlib.Path
+            Path to the sim_telarray configuration file.
+        """
+        config_file_name = names.simtel_config_file_name(
+            self.site,
+            telescope_model_name=self.name,
+            label=self.label if label is None else label,
+        )
+        return self.config_file_directory.joinpath(config_file_name)
+
+    def write_sim_telarray_config_file(
+        self, additional_models=None, label=None, config_file_path=None
+    ):
         """
         Write the sim_telarray configuration file.
 
@@ -563,15 +612,26 @@ class ModelParameter:
         ----------
         additional_models: TelescopeModel or SiteModel
             Model object for additional parameter to be written to the config file.
+        label: str or None
+            Optional label override used for output file naming.
+        config_file_path: pathlib.Path or str or None
+            Optional explicit path of the config file. If not given, it is derived from ``label``.
         """
         self.parameters.update(self._simulation_config_parameters.get("sim_telarray", {}))
         self.export_model_files(update_if_necessary=True)
 
         self._add_additional_models(additional_models)
 
-        self._load_simtel_config_writer()
+        config_file_path = (
+            Path(config_file_path)
+            if config_file_path is not None
+            else self.get_config_file_path(label=label)
+        )
+
+        # Ensure the writer label matches the config file naming label.
+        self._load_simtel_config_writer(label=label)
         self.simtel_config_writer.write_telescope_config_file(
-            config_file_path=self.config_file_path,
+            config_file_path=config_file_path,
             parameters=self.parameters,
         )
 
@@ -602,15 +662,16 @@ class ModelParameter:
             self._set_config_file_directory_and_name()
         return self._config_file_path
 
-    def _load_simtel_config_writer(self):
+    def _load_simtel_config_writer(self, label=None):
         """Load the SimtelConfigWriter object."""
-        if self.simtel_config_writer is None:
+        desired_label = self.label if label is None else label
+        if label is not None or self.simtel_config_writer is None:
             self.simtel_config_writer = SimtelConfigWriter(
                 site=self.site,
                 telescope_model_name=self.name,
                 telescope_design_model=self.design_model,
                 model_version=self.model_version,
-                label=self.label,
+                label=desired_label,
             )
 
     def export_nsb_spectrum_to_telescope_altitude_correction_file(self, model_directory):

simtools/model/model_utils.py

@@ -3,6 +3,9 @@
 
 import math
 
+from simtools import settings
+from simtools.data_model import schema
+from simtools.io import ascii_handler
 from simtools.model.calibration_model import CalibrationModel
 from simtools.model.site_model import SiteModel
 from simtools.model.telescope_model import TelescopeModel
@@ -10,7 +13,11 @@ from simtools.utils import names
 
 
 def initialize_simulation_models(
-    label, db_config, model_version, site, telescope_name, calibration_device_name=None
+    label,
+    model_version,
+    site,
+    telescope_name,
+    calibration_device_name=None,
 ):
     """
     Initialize simulation models for a single telescope, site, and calibration device model.
@@ -19,8 +26,6 @@ def initialize_simulation_models(
     ----------
     label: str
         Label for the simulation.
-    db_config: dict
-        Database configuration.
     model_version: str
         Version of the simulation model
     site: str
@@ -35,26 +40,28 @@ def initialize_simulation_models(
     Tuple
         Tuple containing the telescope site, (optional) calibration device model.
     """
+    overwrite_model_parameter_dict = read_overwrite_model_parameter_dict()
+
     tel_model = TelescopeModel(
         site=site,
         telescope_name=telescope_name,
-        db_config=db_config,
         model_version=model_version,
         label=label,
+        overwrite_model_parameter_dict=overwrite_model_parameter_dict,
     )
     site_model = SiteModel(
         site=site,
         model_version=model_version,
-        db_config=db_config,
         label=label,
+        overwrite_model_parameter_dict=overwrite_model_parameter_dict,
     )
     if calibration_device_name is not None:
         calibration_model = CalibrationModel(
             site=site,
             calibration_device_model_name=calibration_device_name,
-            db_config=db_config,
             model_version=model_version,
             label=label,
+            overwrite_model_parameter_dict=overwrite_model_parameter_dict,
         )
     else:
         calibration_model = None
@@ -63,6 +70,33 @@ def initialize_simulation_models(
     return tel_model, site_model, calibration_model
 
 
+def read_overwrite_model_parameter_dict(overwrite_model_parameters=None):
+    """
+    Read overwrite model parameters dictionary from file.
+
+    Parameters
+    ----------
+    overwrite_model_parameters: str, optional
+        File name with overwrite model parameters.
+
+    Returns
+    -------
+    dict
+        Dictionary with model parameters to overwrite.
+    """
+    overwrite_model_parameter_dict = {}
+    overwrite_model_parameters = overwrite_model_parameters or settings.config.args.get(
+        "overwrite_model_parameters"
+    )
+    if overwrite_model_parameters is not None:
+        overwrite_model_parameter_dict = schema.validate_dict_using_schema(
+            data=ascii_handler.collect_data_from_file(file_name=overwrite_model_parameters),
+            schema_file="simulation_models_info.schema.yml",
+        ).get("changes", {})
+
+    return overwrite_model_parameter_dict
+
+
 def compute_telescope_transmission(pars: list[float], off_axis: float) -> float:
     """
     Compute telescope transmission (0 < T < 1) for a given off-axis angle.

simtools/model/site_model.py

@@ -26,14 +26,12 @@ class SiteModel(ModelParameter):
     ----------
     site: str
         Site name (e.g., South or North).
-    db_config: dict
-        Database configuration.
     model_version: str or list
         Model version or list of model versions (in which case only the first one is used).
     label: str, optional
         Instance label.
-    overwrite_model_parameters: str, optional
-        File name to overwrite model parameters from DB with provided values.
+    overwrite_model_parameter_dict: dict, optional
+        Dictionary to overwrite model parameters from DB with provided values.
     ignore_software_version: bool, optional
         If True, ignore software version checks for deprecated parameters.
     """
@@ -41,10 +39,9 @@ class SiteModel(ModelParameter):
     def __init__(
         self,
         site,
-        db_config,
         model_version,
         label=None,
-        overwrite_model_parameters=None,
+        overwrite_model_parameter_dict=None,
         ignore_software_version=False,
     ):
         """Initialize SiteModel."""
@@ -52,11 +49,10 @@ class SiteModel(ModelParameter):
         self._logger.debug("Init SiteModel for site %s", site)
         super().__init__(
             site=site,
-            db_config=db_config,
             model_version=model_version,
             label=label,
             collection="sites",
-            overwrite_model_parameters=overwrite_model_parameters,
+            overwrite_model_parameter_dict=overwrite_model_parameter_dict,
             ignore_software_version=ignore_software_version,
         )
 

simtools/model/telescope_model.py

@@ -26,14 +26,12 @@ class TelescopeModel(ModelParameter):
         Site name (e.g., South or North).
     telescope_name: str
         Telescope name (ex. LSTN-01, LSTN-design, ...).
-    db_config: dict
-        Database configuration.
     model_version: str
         Model version.
     label: str, optional
         Instance label.
-    overwrite_model_parameters: str, optional
-        File name to overwrite model parameters from DB with provided values.
+    overwrite_model_parameter_dict: dict, optional
+        Dictionary to overwrite model parameters from DB with provided values.
     ignore_software_version: bool, optional
         If True, ignore software version checks for deprecated parameters.
     """
@@ -42,25 +40,22 @@ class TelescopeModel(ModelParameter):
         self,
         site,
         telescope_name,
-        db_config,
         model_version,
         label=None,
-        overwrite_model_parameters=None,
+        overwrite_model_parameter_dict=None,
         ignore_software_version=False,
     ):
         """Initialize TelescopeModel."""
         super().__init__(
             site=site,
             array_element_name=telescope_name,
-            db_config=db_config,
             model_version=model_version,
             label=label,
-            overwrite_model_parameters=overwrite_model_parameters,
+            overwrite_model_parameter_dict=overwrite_model_parameter_dict,
             ignore_software_version=ignore_software_version,
         )
 
         self._logger = logging.getLogger(__name__)
-        self._logger.debug("Init TelescopeModel %s %s", site, telescope_name)
 
         self._single_mirror_list_file_paths = None
         self._mirrors = None
@@ -326,13 +321,12 @@ class TelescopeModel(ModelParameter):
         average_curve: astropy.table.Table
             Instance of astropy.table.Table with the averaged curve.
         """
-        weights = []
-        for angle_now in curves["Angle"]:
-            weights.append(
-                incidence_angle_dist["Fraction"][
-                    np.nanargmin(np.abs(angle_now - incidence_angle_dist["Incidence angle"].value))
-                ]
-            )
+        weights = [
+            incidence_angle_dist["Fraction"][
+                np.nanargmin(np.abs(angle_now - incidence_angle_dist["Incidence angle"].value))
+            ]
+            for angle_now in curves["Angle"]
+        ]
 
         return Table(
             [curves["Wavelength"], np.average(curves["z"], weights=weights, axis=0)],

simtools/production_configuration/derive_corsika_limits.py

@@ -10,13 +10,13 @@ from astropy.table import Column, Table
 from simtools.data_model.metadata_collector import MetadataCollector
 from simtools.io import ascii_handler, io_handler
 from simtools.layout.array_layout_utils import get_array_elements_from_db_for_layouts
-from simtools.simtel.simtel_io_event_histograms import SimtelIOEventHistograms
+from simtools.sim_events.histograms import EventDataHistograms
 from simtools.visualization import plot_simtel_event_histograms
 
 _logger = logging.getLogger(__name__)
 
 
-def generate_corsika_limits_grid(args_dict, db_config=None):
+def generate_corsika_limits_grid(args_dict):
     """
     Generate CORSIKA limits.
 
@@ -24,15 +24,12 @@ def generate_corsika_limits_grid(args_dict, db_config=None):
     ----------
     args_dict : dict
         Dictionary containing command line arguments.
-    db_config : dict, optional
-        Database configuration dictionary.
     """
     if args_dict.get("array_layout_name"):
         telescope_configs = get_array_elements_from_db_for_layouts(
             args_dict["array_layout_name"],
             args_dict.get("site"),
             args_dict.get("model_version"),
-            db_config,
         )
     else:
         telescope_configs = ascii_handler.collect_data_from_file(args_dict["telescope_ids"])[
@@ -81,9 +78,7 @@ def _process_file(file_path, array_name, telescope_ids, loss_fraction, plot_hist
     dict
         Dictionary containing the computed limits and metadata.
     """
-    histograms = SimtelIOEventHistograms(
-        file_path, array_name=array_name, telescope_list=telescope_ids
-    )
+    histograms = EventDataHistograms(file_path, array_name=array_name, telescope_list=telescope_ids)
     histograms.fill()
 
     limits = {
@@ -257,7 +252,7 @@ def compute_lower_energy_limit(histograms, loss_fraction):
 
     Parameters
     ----------
-    histograms : SimtelIOEventHistograms
+    histograms : EventDataHistograms
         Histograms.
     loss_fraction : float
         Fraction of events to be lost.
@@ -299,7 +294,7 @@ def compute_upper_radius_limit(histograms, loss_fraction):
 
     Parameters
     ----------
-    histograms : SimtelIOEventHistograms
+    histograms : EventDataHistograms
         Histograms.
     loss_fraction : float
         Fraction of events to be lost.
@@ -337,7 +332,7 @@ def compute_viewcone(histograms, loss_fraction):
 
     Parameters
     ----------
-    histograms : SimtelIOEventHistograms
+    histograms : EventDataHistograms
         Histograms.
     loss_fraction : float
         Fraction of events to be lost.