gammasimtools 0.22.0__py3-none-any.whl → 0.24.0__py3-none-any.whl

simtools/model/model_repository.py

@@ -1,8 +1,15 @@
 """Utilities for managing the simulation models repository.
 
 Simulation model parameters and production tables are managed through
-a gitlab repository ('SimulationModels'). This module provides service
+a gitlab repository ('simulation_models'). This module provides service
 functions to interact with and verify the repository.
+
+Main functionalities are:
+
+- validation of production tables against model parameters
+- generation of new production tables and model parameters based on
+  updates defined in a configuration file
+
 """
 
 import logging
@@ -18,6 +25,44 @@ from simtools.utils import names
 _logger = logging.getLogger(__name__)
 
 
+def get_production_directory(simulation_models_path, model_version=None):
+    """
+    Get the production directory for a specific model version.
+
+    Parameters
+    ----------
+    simulation_models_path : str
+        Path to the simulation models repository.
+    model_version : str, optional
+        Specific model version to get the production directory for.
+
+    Returns
+    -------
+    Path
+        Path to the production directory.
+    """
+    if model_version:
+        return Path(simulation_models_path) / "simulation-models/productions" / str(model_version)
+    return Path(simulation_models_path) / "simulation-models/productions"
+
+
+def get_model_parameter_directory(simulation_models_path):
+    """
+    Get the model parameters directory.
+
+    Parameters
+    ----------
+    simulation_models_path : str
+        Path to the simulation models repository.
+
+    Returns
+    -------
+    Path
+        Path to the model parameters directory.
+    """
+    return Path(simulation_models_path) / "simulation-models/model_parameters"
+
+
 def verify_simulation_model_production_tables(simulation_models_path):
     """
     Verify the simulation model production tables in the specified path.
@@ -35,7 +80,7 @@ def verify_simulation_model_production_tables(simulation_models_path):
     bool
         True if all parameters found, False if any missing.
     """
-    productions_path = Path(simulation_models_path) / "simulation-models" / "productions"
+    productions_path = get_production_directory(simulation_models_path)
     production_files = list(productions_path.rglob("*.json"))
 
     _logger.info(
@@ -124,9 +169,7 @@ def _get_model_parameter_file_path(
     """
     collection = names.get_collection_name_from_parameter_name(parameter_name)
     return (
-        Path(simulation_models_path)
-        / "simulation-models"
-        / "model_parameters"
+        get_model_parameter_directory(simulation_models_path)
         / (
             collection
             if collection in ("configuration_sim_telarray", "configuration_corsika")
@@ -138,39 +181,36 @@ def _get_model_parameter_file_path(
     )
 
 
-def generate_new_production(modifications, simulation_models_path):
+def generate_new_production(model_version, simulation_models_path):
     """
     Generate a new production definition (production tables and model parameters).
 
     The following steps are performed:
 
     - copy of production tables from an existing base model version
-    - update production tables with changes defined in a YAML file
+    - update production tables with changes defined in a configuration file (expected
+      to be called 'info.yml' in the target production directory)
     - generate new model parameter entries for changed parameters
     - allows for full or patch updates
 
     Parameters
     ----------
-    modifications: str
-        Path to the YAML file defining the changes to be applied.
+    model_version: str
+        Model version to be created or updated.
     simulation_models_path: str
         Path to the simulation models repository.
     """
-    modifications = ascii_handler.collect_data_from_file(modifications)
-    model_version_history = modifications.get("model_version_history", [])
-    try:
-        # oldest version is the base version
-        base_model_version = min(set(model_version_history), key=Version)
-    except ValueError as exc:
-        raise ValueError(f"Base model version not found in {modifications}") from exc
-    model_version = modifications["model_version"]
-    changes = modifications.get("changes", {})
+    modification_dict = _get_changes_dict(model_version, simulation_models_path)
+    update_type = modification_dict.get("model_update", "full_update")
+    changes, base_model_version = _get_changes_to_production(
+        modification_dict, simulation_models_path, update_type
+    )
 
     _apply_changes_to_production_tables(
         changes,
         base_model_version,
-        model_version,
-        modifications.get("model_update", "full_update"),
+        modification_dict["model_version"],
+        update_type,
         simulation_models_path,
     )
 
@@ -181,27 +221,27 @@ def _apply_changes_to_production_tables(
     changes, base_model_version, model_version, update_type, simulation_models_path
 ):
     """
-    Apply changes to production tables and write them to target directory.
+    Apply changes to or generate new production tables and write them to target directory.
 
     Parameters
     ----------
     changes: dict
-        The changes to be applied.
+        Changes to be applied.
     base_model_version: str
-        The base model version (source directory for production tables).
+        Base model version (source directory for production tables).
     model_version: str
-        The model version to be set in the JSON data.
+        Model version of the new production tables.
     update_type: str
-        Update mode, either 'full_update' or 'patch_update'.
+        Update type (e.g., 'full_update' or 'patch_update').
     simulation_models_path: Path
         Path to the simulation models repository.
     """
-    source = simulation_models_path / "productions" / base_model_version
-    target = simulation_models_path / "productions" / model_version
+    source = get_production_directory(simulation_models_path, base_model_version)
+    target = get_production_directory(simulation_models_path, model_version)
     _logger.info(f"Production tables {update_type} from {source} to {target}")
     target.mkdir(parents=True, exist_ok=True)
 
-    # load existing tables
+    # load existing tables from source
     tables = {}
     for file_path in Path(source).rglob("*.json"):
         data = ascii_handler.collect_data_from_file(file_path)
@@ -217,9 +257,10 @@ def _apply_changes_to_production_tables(
         if _apply_changes_to_production_table(
             table_name, data, changes, model_version, update_type == "patch_update"
         ):
-            _logger.info(f"Writing updated production table '{table_name}'")
+            target_file = target / f"{table_name}.json"
+            _logger.info(f"Writing updated production table '{target_file}'")
             data["production_table_name"] = table_name
-            ascii_handler.write_data_to_file(data, target / f"{table_name}.json", sort_keys=True)
+            ascii_handler.write_data_to_file(data, target_file, sort_keys=True)
 
 
 def _apply_changes_to_production_table(table_name, data, changes, model_version, patch_update):
@@ -228,12 +269,14 @@ def _apply_changes_to_production_table(table_name, data, changes, model_version,
 
     Parameters
     ----------
+    table_name: str
+        Name of the production table.
     data: dict
-        The data to be updated.
+        Data to be updated.
     changes: dict
-        The changes to be applied.
+        Changes to be applied.
     model_version: str
-        The model version to be set in the JSON data.
+        Model version of the new production tables.
     patch_update: bool
         True if patch update (modify only changed parameters), False for full update.
 
@@ -248,7 +291,7 @@ def _apply_changes_to_production_table(table_name, data, changes, model_version,
         table_parameters = {} if patch_update else data.get("parameters", {}).get(table_name, {})
         parameters, deprecated = _update_parameters_dict(table_parameters, changes, table_name)
         data["parameters"] = parameters
-        if deprecated:
+        if deprecated and patch_update:
             data["deprecated_parameters"] = deprecated
     elif patch_update:
         return False
@@ -256,6 +299,86 @@ def _apply_changes_to_production_table(table_name, data, changes, model_version,
     return True
 
 
+def _get_changes_dict(model_version, simulation_models_path):
+    """
+    Load the changes dictionary from 'info.yml' files in production directories.
+
+    Parameters
+    ----------
+    model_version: str
+        Model version of the new production tables.
+    simulation_models_path: Path
+        Path to the simulation models directory.
+
+    Returns
+    -------
+    dict
+        Changes dictionary.
+    """
+    return ascii_handler.collect_data_from_file(
+        get_production_directory(simulation_models_path, model_version) / "info.yml"
+    )
+
+
+def _get_changes_to_production(
+    modification_dict, simulation_models_path, update_type="full_update"
+):
+    """
+    Prepare changes applied to production tables.
+
+    For full updates, this includes the combination of changes to be applied
+    for all model versions in the history, starting from the base version.
+
+    Parameters
+    ----------
+    modification_dict: dict
+        Modifications dictionary.
+    simulation_models_path: Path
+        Path to the simulation models directory.
+    update_type: str
+        Update mode.
+
+    Returns
+    -------
+    dict, str
+        Changes dictionary and base model version.
+    """
+    model_version_history = modification_dict.get("model_version_history", [])
+
+    try:
+        # oldest version is the base version
+        base_model_version = min(set(model_version_history), key=Version)
+    except ValueError:
+        _logger.debug(f"Base model version not found in {model_version_history}")
+        return {}, modification_dict.get("model_version")
+
+    changes = modification_dict.get("changes", {})
+    if update_type == "patch_update":
+        return changes, base_model_version
+
+    for version_mod in reversed(model_version_history):
+        _changes_dict = _get_changes_dict(version_mod, simulation_models_path)
+        _version_changes, base_model_version = _get_changes_to_production(
+            _changes_dict, simulation_models_path, update_type="full_update"
+        )
+        changes = _update_two_levels_in_changes_dict(changes, _version_changes)
+        # stop iterative loop after reaching first full version of production tables
+        if _changes_dict.get("model_update", "full_update") == "full_update":
+            break
+
+    return changes, base_model_version
+
+
+def _update_two_levels_in_changes_dict(d, u):
+    """Update changes dict, e.g. {"LSTN-design": { "parameter_name: { ... } } }."""
+    for k, v in u.items():
+        if isinstance(v, dict) and isinstance(d.get(k), dict):
+            d[k].update(v)
+        else:
+            d[k] = v
+    return d
+
+
 def _update_parameters_dict(table_parameters, changes, table_name):
     """
     Create a new parameters dictionary for the production tables.
@@ -285,6 +408,7 @@ def _update_parameters_dict(table_parameters, changes, table_name):
         if data.get("deprecated", False):
             _logger.info(f"Removing model parameter '{table_name} - {param}'")
             deprecated_params.append(param)
+            new_params[table_name].pop(param, None)
         else:
             version = data["version"]
             _logger.info(f"Setting '{table_name} - {param}' to version {version}")
@@ -330,7 +454,7 @@ def _create_new_model_parameter_entry(telescope, param, param_data, simulation_m
     simulation_models_path: Path
         Path to the simulation models directory.
     """
-    telescope_dir = simulation_models_path / "model_parameters" / telescope
+    telescope_dir = get_model_parameter_directory(simulation_models_path) / telescope
     if not telescope_dir.exists():
         _logger.info(f"Create directory for array element '{telescope}': '{telescope_dir}'.")
         telescope_dir.mkdir(parents=True, exist_ok=True)

simtools/model/model_utils.py

@@ -8,11 +8,6 @@ from simtools.model.site_model import SiteModel
 from simtools.model.telescope_model import TelescopeModel
 from simtools.utils import names
 
-__all__ = [
-    "compute_telescope_transmission",
-    "is_two_mirror_telescope",
-]
-
 
 def initialize_simulation_models(
     label, db_config, model_version, site, telescope_name, calibration_device_name=None
@@ -43,21 +38,21 @@ def initialize_simulation_models(
     tel_model = TelescopeModel(
         site=site,
         telescope_name=telescope_name,
-        mongo_db_config=db_config,
+        db_config=db_config,
         model_version=model_version,
         label=label,
     )
     site_model = SiteModel(
         site=site,
         model_version=model_version,
-        mongo_db_config=db_config,
+        db_config=db_config,
         label=label,
     )
     if calibration_device_name is not None:
         calibration_model = CalibrationModel(
             site=site,
             calibration_device_model_name=calibration_device_name,
-            mongo_db_config=db_config,
+            db_config=db_config,
             model_version=model_version,
             label=label,
         )

simtools/model/site_model.py

@@ -9,52 +9,67 @@ from astropy import units as u
 
 from simtools.model.model_parameter import ModelParameter
 
-__all__ = ["SiteModel"]
-
 
 class SiteModel(ModelParameter):
     """
-    SiteModel represents the MC model of an observatory site.
+    Representation of an observatory site model.
+
+    The site model includes (among others):
+
+    - Reference point coordinates
+    - Array elements
+    - Geomagnetic field parameters
+    - Atmospheric parameters
+    - NSB parameters
 
     Parameters
     ----------
     site: str
         Site name (e.g., South or North).
-    mongo_db_config: dict
-        MongoDB configuration.
+    db_config: dict
+        Database configuration.
     model_version: str or list
         Model version or list of model versions (in which case only the first one is used).
     label: str, optional
-        Instance label. Important for output file naming.
+        Instance label.
+    overwrite_model_parameters: str, optional
+        File name to overwrite model parameters from DB with provided values.
+    ignore_software_version: bool, optional
+        If True, ignore software version checks for deprecated parameters.
     """
 
     def __init__(
         self,
-        site: str,
-        mongo_db_config: dict,
-        model_version: str,
-        label: str | None = None,
+        site,
+        db_config,
+        model_version,
+        label=None,
+        overwrite_model_parameters=None,
+        ignore_software_version=False,
     ):
         """Initialize SiteModel."""
         self._logger = logging.getLogger(__name__)
         self._logger.debug("Init SiteModel for site %s", site)
         super().__init__(
             site=site,
-            mongo_db_config=mongo_db_config,
+            db_config=db_config,
             model_version=model_version,
-            db=None,
             label=label,
             collection="sites",
+            overwrite_model_parameters=overwrite_model_parameters,
+            ignore_software_version=ignore_software_version,
         )
 
-    def get_reference_point(self) -> dict:
+    def get_reference_point(self):
         """
-        Get reference point coordinates as dict.
+        Get reference point coordinates.
+
+        Ground coordinates are calculated relative to this point.
 
         Returns
         -------
         dict
-            Reference point coordinates as dict
+            Reference point coordinates.
         """
         return {
             "center_altitude": self.get_parameter_value_with_unit("reference_point_altitude"),
@@ -63,13 +78,12 @@ class SiteModel(ModelParameter):
             "epsg_code": self.get_parameter_value("epsg_code"),
         }
 
-    def get_corsika_site_parameters(
-        self, config_file_style: bool = False, model_directory: Path | None = None
-    ) -> dict:
+    def get_corsika_site_parameters(self, config_file_style=False, model_directory=None):
         """
-        Get site-related CORSIKA parameters as dict.
+        Get site-related CORSIKA parameters.
 
-        Parameters are returned with units wherever possible.
+        Parameters are returned with units wherever possible ('config_file_style=False')
+        or in CORSIKA-expected coordinates.
 
         Parameters
         ----------
@@ -81,7 +95,7 @@ class SiteModel(ModelParameter):
         Returns
         -------
         dict
-            Site-related CORSIKA parameters as dict
+            Site-related CORSIKA parameters.
         """
         if config_file_style:
             model_directory = model_directory or Path()
@@ -110,7 +124,7 @@ class SiteModel(ModelParameter):
             "geomag_rotation": self.get_parameter_value_with_unit("geomag_rotation"),
         }
 
-    def get_array_elements_for_layout(self, layout_name: str) -> list:
+    def get_array_elements_for_layout(self, layout_name):
         """
         Return list of array elements for a given array layout.
 
@@ -130,7 +144,27 @@ class SiteModel(ModelParameter):
                 return layout["elements"]
         raise ValueError(f"Array layout '{layout_name}' not found in '{self.site}' site model.")
 
-    def get_list_of_array_layouts(self) -> list:
+    def get_array_elements_of_type(self, array_element_type):
+        """
+        Get all array elements of a given type.
+
+        Parameters
+        ----------
+        array_element_type : str
+            Type of the array element (e.g. LSTN, MSTS)
+
+        Returns
+        -------
+        dict
+            Dict with array elements.
+        """
+        return self.db.get_array_elements_of_type(
+            array_element_type=array_element_type,
+            model_version=self.model_version,
+            collection="telescopes",
+        )
+
+    def get_list_of_array_layouts(self):
         """
         Get list of available array layouts.
 
@@ -141,13 +175,9 @@ class SiteModel(ModelParameter):
         """
         return [layout["name"] for layout in self.get_parameter_value("array_layouts")]
 
-    def export_atmospheric_transmission_file(self, model_directory: Path):
+    def export_atmospheric_transmission_file(self, model_directory):
         """
-        Export atmospheric transmission file.
-
-        This method is needed because when CORSIKA is not piped to sim_telarray,
-        the atmospheric transmission file is not written out to the model directory.
-        This method allows to export it explicitly.
+        Export atmospheric transmission file from database to the given directory.
 
         Parameters
         ----------

simtools/model/telescope_model.py

@@ -13,8 +13,6 @@ from simtools.model.mirrors import Mirrors
 from simtools.model.model_parameter import InvalidModelParameterError, ModelParameter
 from simtools.utils import names
 
-__all__ = ["TelescopeModel"]
-
 
 class TelescopeModel(ModelParameter):
     """
@@ -28,30 +26,37 @@ class TelescopeModel(ModelParameter):
         Site name (e.g., South or North).
     telescope_name: str
         Telescope name (ex. LSTN-01, LSTN-design, ...).
-    mongo_db_config: dict
-        MongoDB configuration.
+    db_config: dict
+        Database configuration.
     model_version: str
         Model version.
     label: str, optional
-        Instance label. Important for output file naming.
+        Instance label.
+    overwrite_model_parameters: str, optional
+        File name to overwrite model parameters from DB with provided values.
+    ignore_software_version: bool, optional
+        If True, ignore software version checks for deprecated parameters.
     """
 
     def __init__(
         self,
-        site: str,
-        telescope_name: str,
-        mongo_db_config: dict,
-        model_version: str,
-        label: str | None = None,
+        site,
+        telescope_name,
+        db_config,
+        model_version,
+        label=None,
+        overwrite_model_parameters=None,
+        ignore_software_version=False,
     ):
         """Initialize TelescopeModel."""
         super().__init__(
             site=site,
             array_element_name=telescope_name,
-            mongo_db_config=mongo_db_config,
+            db_config=db_config,
             model_version=model_version,
-            db=None,
             label=label,
+            overwrite_model_parameters=overwrite_model_parameters,
+            ignore_software_version=ignore_software_version,
         )
 
         self._logger = logging.getLogger(__name__)
@@ -144,7 +149,7 @@ class TelescopeModel(ModelParameter):
         except TypeError as exc:
             raise TypeError("Undefined mirror list") from exc
 
-        self._mirrors = Mirrors(mirror_list_file, parameters=self._parameters)
+        self._mirrors = Mirrors(mirror_list_file, parameters=self.parameters)
 
     def get_telescope_effective_focal_length(
         self, unit: str = "m", return_focal_length_if_zero: bool = False
@@ -376,8 +381,9 @@ class TelescopeModel(ModelParameter):
         try:
             return self.get_parameter_value_with_unit(f"array_element_position_{coordinate_system}")
         except InvalidModelParameterError as exc:
-            self._logger.error(f"Coordinate system {coordinate_system} not found.")
-            raise exc
+            raise InvalidModelParameterError(
+                f"Coordinate system {coordinate_system} not found."
+            ) from exc
 
     def get_calibration_device_name(self, device_type):
         """

simtools/production_configuration/calculate_statistical_uncertainties_grid_point.py

@@ -6,8 +6,6 @@ import numpy as np
 from astropy import units as u
 from astropy.io import fits
 
-__all__ = ["StatisticalUncertaintyEvaluator"]
-
 
 class StatisticalUncertaintyEvaluator:
     """

simtools/production_configuration/derive_production_statistics.py

@@ -10,8 +10,6 @@ metrics and the evaluator's results.
 import astropy.units as u
 import numpy as np
 
-__all__ = ["ProductionStatisticsDerivator"]
-
 
 class ProductionStatisticsDerivator:
     """

simtools/production_configuration/interpolation_handler.py

@@ -10,8 +10,6 @@ from simtools.production_configuration.derive_production_statistics import (
     ProductionStatisticsDerivator,
 )
 
-__all__ = ["InterpolationHandler"]
-
 
 class InterpolationHandler:
     """

simtools/ray_tracing/mirror_panel_psf.py

@@ -81,10 +81,10 @@ class MirrorPanelPSF:
             mirror_list_file = gen.find_file(
                 name=self.args_dict["mirror_list"], loc=self.args_dict["model_path"]
             )
-            tel_model.change_parameter("mirror_list", self.args_dict["mirror_list"])
-            tel_model.export_parameter_file("mirror_list", mirror_list_file)
+            tel_model.overwrite_model_parameter("mirror_list", self.args_dict["mirror_list"])
+            tel_model.overwrite_model_file("mirror_list", mirror_list_file)
         if self.args_dict["random_focal_length"] is not None:
-            tel_model.change_parameter(
+            tel_model.overwrite_model_parameter(
                 "random_focal_length", str(self.args_dict["random_focal_length"])
             )
 
@@ -221,7 +221,7 @@ class MirrorPanelPSF:
         sig_d80: float
             Standard deviation of D80 in cm.
         """
-        self.telescope_model.change_parameter("mirror_reflection_random_angle", rnda)
+        self.telescope_model.overwrite_model_parameter("mirror_reflection_random_angle", rnda)
         ray = RayTracing(
             telescope_model=self.telescope_model,
             site_model=self.site_model,

simtools/ray_tracing/psf_analysis.py

@@ -19,8 +19,6 @@ import numpy as np
 
 from simtools.utils.general import collect_kwargs, set_default_kwargs
 
-__all__ = ["PSFImage"]
-
 
 class PSFImage:
     """

simtools/ray_tracing/psf_parameter_optimisation.py

@@ -95,7 +95,7 @@ def _run_ray_tracing_simulation(tel_model, site_model, args_dict, pars):
     if pars is None:
         raise ValueError("No best parameters found")
 
-    tel_model.change_multiple_parameters(**pars)
+    tel_model.overwrite_parameters(pars)
     ray = RayTracing(
         telescope_model=tel_model,
         site_model=site_model,

simtools/ray_tracing/ray_tracing.py

@@ -20,8 +20,6 @@ from simtools.simtel.simulator_ray_tracing import SimulatorRayTracing
 from simtools.utils import names
 from simtools.visualization import visualize
 
-__all__ = ["RayTracing"]
-
 INVALID_KEY_TO_PLOT = "Invalid key to plot"