gammasimtools 0.21.0__py3-none-any.whl → 0.23.0__py3-none-any.whl

simtools/model/model_repository.py

@@ -8,6 +8,10 @@ functions to interact with and verify the repository.
 import logging
 from pathlib import Path
 
+from packaging.version import Version
+from packaging.version import parse as parse_version
+
+import simtools.data_model.model_data_writer as writer
 from simtools.io import ascii_handler
 from simtools.utils import names
 
@@ -134,7 +138,7 @@ def _get_model_parameter_file_path(
     )
 
 
-def generate_new_production(args_dict):
+def generate_new_production(modifications, simulation_models_path):
     """
     Generate a new production definition (production tables and model parameters).
 
@@ -143,68 +147,84 @@ def generate_new_production(args_dict):
     - copy of production tables from an existing base model version
     - update production tables with changes defined in a YAML file
     - generate new model parameter entries for changed parameters
+    - allows for full or patch updates
 
     Parameters
     ----------
-    args_dict: dict
-        Dictionary containing the arguments for copying and updating production tables.
+    modifications: str
+        Path to the YAML file defining the changes to be applied.
+    simulation_models_path: str
+        Path to the simulation models repository.
     """
-    modifications = ascii_handler.collect_data_from_file(args_dict["modifications"])
-    changes = modifications.get("changes", {})
-    base_model_version = args_dict["base_model_version"]
+    modifications = ascii_handler.collect_data_from_file(modifications)
+    model_version_history = modifications.get("model_version_history", [])
+    try:
+        # oldest version is the base version
+        base_model_version = min(set(model_version_history), key=Version)
+    except ValueError as exc:
+        raise ValueError(f"Base model version not found in {modifications}") from exc
     model_version = modifications["model_version"]
-
-    simulation_models_path = Path(args_dict["simulation_models_path"])
-    source_path = simulation_models_path / "productions" / base_model_version
-    target_path = simulation_models_path / "productions" / model_version
-    model_parameters_dir = simulation_models_path / "model_parameters"
-    patch_update = args_dict.get("patch_update", False)
-
-    _logger.info(f"Copying production tables from {source_path} to {target_path}")
+    changes = modifications.get("changes", {})
 
     _apply_changes_to_production_tables(
-        source_path,
-        target_path,
         changes,
+        base_model_version,
         model_version,
-        patch_update,
+        modifications.get("model_update", "full_update"),
+        simulation_models_path,
     )
 
-    _apply_changes_to_model_parameters(changes, model_parameters_dir)
+    _apply_changes_to_model_parameters(changes, simulation_models_path)
 
 
 def _apply_changes_to_production_tables(
-    source_path, target_path, changes, model_version, patch_update
+    changes, base_model_version, model_version, update_type, simulation_models_path
 ):
     """
     Apply changes to production tables and write them to target directory.
 
     Parameters
     ----------
-    source_path: Path
-        Path to the source production tables.
-    target_path: Path
-        Path to the target production tables.
     changes: dict
         The changes to be applied.
+    base_model_version: str
+        The base model version (source directory for production tables).
     model_version: str
         The model version to be set in the JSON data.
-    patch_update: bool
-        Patch update, copy only tables for changed elements.
+    update_type: str
+        Update mode, either 'full_update' or 'patch_update'.
+    simulation_models_path: Path
+        Path to the simulation models repository.
     """
-    target_path.mkdir(parents=True, exist_ok=True)
-    for file_path in Path(source_path).rglob("*.json"):
+    source = simulation_models_path / "productions" / base_model_version
+    target = simulation_models_path / "productions" / model_version
+    _logger.info(f"Production tables {update_type} from {source} to {target}")
+    target.mkdir(parents=True, exist_ok=True)
+
+    # load existing tables
+    tables = {}
+    for file_path in Path(source).rglob("*.json"):
         data = ascii_handler.collect_data_from_file(file_path)
-        write_to_disk = _apply_changes_to_production_table(
-            data, changes, model_version, patch_update
-        )
-        if write_to_disk:
-            ascii_handler.write_data_to_file(data, target_path / file_path.name, sort_keys=True)
+        if not isinstance(data, dict):
+            raise TypeError(f"Unsupported data type {type(data)} in {file_path}")
+        tables[data["production_table_name"]] = data
+
+    # placeholder for new tables
+    for table_name in changes:
+        tables.setdefault(table_name, {})
 
+    for table_name, data in tables.items():
+        if _apply_changes_to_production_table(
+            table_name, data, changes, model_version, update_type == "patch_update"
+        ):
+            _logger.info(f"Writing updated production table '{table_name}'")
+            data["production_table_name"] = table_name
+            ascii_handler.write_data_to_file(data, target / f"{table_name}.json", sort_keys=True)
 
-def _apply_changes_to_production_table(data, changes, model_version, patch_update):
+
+def _apply_changes_to_production_table(table_name, data, changes, model_version, patch_update):
     """
-    Recursively apply changes to the new production tables.
+    Apply changes to a single production table.
 
     Parameters
     ----------
@@ -215,32 +235,33 @@ def _apply_changes_to_production_table(data, changes, model_version, patch_updat
     model_version: str
         The model version to be set in the JSON data.
     patch_update: bool
-        Patch update, copy only tables for changed elements.
+        True if patch update (modify only changed parameters), False for full update.
 
     Returns
     -------
     bool
-        True if data was modified and should be written to disk (patch updates) and always
-        for full updates.
+        True if data was modified and should be written to disk (patch updates);
+        always True for full updates.
     """
-    if isinstance(data, dict):
-        table_name = data["production_table_name"]
-        data["model_version"] = model_version
-        if table_name in changes:
-            data["parameters"] = _update_parameters(
-                {} if patch_update else data["parameters"].get(table_name, {}), changes, table_name
-            )
-        elif patch_update:
-            return False
-    else:
-        raise TypeError(f"Unsupported data type {type(data)} in production table update")
+    data["model_version"] = model_version
+    if table_name in changes:
+        table_parameters = {} if patch_update else data.get("parameters", {}).get(table_name, {})
+        parameters, deprecated = _update_parameters_dict(table_parameters, changes, table_name)
+        data["parameters"] = parameters
+        if deprecated:
+            data["deprecated_parameters"] = deprecated
+    elif patch_update:
+        return False
 
     return True
 
 
-def _update_parameters(table_parameters, changes, table_name):
+def _update_parameters_dict(table_parameters, changes, table_name):
     """
-    Create a new parameters dictionary containing only the parameters for the specified table.
+    Create a new parameters dictionary for the production tables.
+
+    Include only changes relevant to the specific telescope.
+    Do not include parameters if 'deprecated' flag is set to True.
 
     Parameters
     ----------
@@ -253,18 +274,26 @@ def _update_parameters(table_parameters, changes, table_name):
 
     Returns
     -------
-    dict
+    dict, list
         Dictionary containing only the new/changed parameters for the specified table.
+        List of deprecated parameters.
     """
-    updated_parameters_dict = {table_name: table_parameters}
+    new_params = {table_name: table_parameters}
+    deprecated_params = []
+
     for param, data in changes[table_name].items():
-        version = data["version"]
-        _logger.info(f"Setting '{table_name} - {param}' to version {version}")
-        updated_parameters_dict[table_name][param] = version
-    return updated_parameters_dict
+        if data.get("deprecated", False):
+            _logger.info(f"Removing model parameter '{table_name} - {param}'")
+            deprecated_params.append(param)
+        else:
+            version = data["version"]
+            _logger.info(f"Setting '{table_name} - {param}' to version {version}")
+            new_params[table_name][param] = version
+
+    return new_params, deprecated_params
 
 
-def _apply_changes_to_model_parameters(changes, model_parameters_dir):
+def _apply_changes_to_model_parameters(changes, simulation_models_path):
     """
     Apply changes to model parameters by creating new parameter entries.
 
@@ -272,18 +301,23 @@ def _apply_changes_to_model_parameters(changes, model_parameters_dir):
     ----------
     changes: dict
         The changes to be applied.
-    model_parameters_dir: str
-        Path to the model parameters directory.
+    simulation_models_path: Path
+        Path to the simulation models directory.
     """
     for telescope, parameters in changes.items():
         for param, param_data in parameters.items():
-            if param_data.get("value"):
-                _create_new_parameter_entry(telescope, param, param_data, model_parameters_dir)
+            if param_data.get("value") is not None:
+                _create_new_model_parameter_entry(
+                    telescope, param, param_data, simulation_models_path
+                )
 
 
-def _create_new_parameter_entry(telescope, param, param_data, model_parameters_dir):
+def _create_new_model_parameter_entry(telescope, param, param_data, simulation_models_path):
     """
-    Create new model parameter JSON file by copying the latest version and updating fields.
+    Create new model parameter entry in the model parameters directory.
+
+    If a model parameter files exists, copy latest version and update the fields.
+    Otherwise generate new file using the model parameter schema.
 
     Parameters
     ----------
@@ -293,43 +327,40 @@ def _create_new_parameter_entry(telescope, param, param_data, model_parameters_d
         Name of the parameter.
     param_data: dict
         Dictionary containing the parameter data including version and value.
-    model_parameters_dir: str
-        Path to the model parameters directory.
+    simulation_models_path: Path
+        Path to the simulation models directory.
     """
-    telescope_dir = Path(model_parameters_dir) / telescope
+    telescope_dir = simulation_models_path / "model_parameters" / telescope
     if not telescope_dir.exists():
-        raise FileNotFoundError(
-            f"Directory for telescope '{telescope}' does not exist in '{model_parameters_dir}'."
-        )
+        _logger.info(f"Create directory for array element '{telescope}': '{telescope_dir}'.")
+        telescope_dir.mkdir(parents=True, exist_ok=True)
 
     param_dir = telescope_dir / param
-    if not param_dir.exists():
-        raise FileNotFoundError(
-            f"Directory for parameter '{param}' does not exist in '{telescope}'."
-        )
-
-    latest_file = _get_latest_model_parameter_file(param_dir, param)
-    if not latest_file:
-        raise FileNotFoundError(
-            f"No files found for parameter '{param}' in directory '{param_dir}'."
+    try:
+        latest_file = _get_latest_model_parameter_file(param_dir, param)
+    except FileNotFoundError:
+        latest_file = None
+
+    if latest_file is not None:
+        json_data = ascii_handler.collect_data_from_file(latest_file)
+        param_data["version"] = _check_for_major_version_jump(
+            json_data, param_data, param, telescope
         )
-
-    json_data = ascii_handler.collect_data_from_file(latest_file)
-
-    json_data["parameter_version"] = _update_model_parameter_version(
-        json_data, param_data, param, telescope
+        # important for e.g. nsb_pixel_rate
+        if isinstance(json_data["value"], list) and not isinstance(param_data["value"], list):
+            param_data["value"] = [param_data["value"]] * len(json_data["value"])
+        param_data["meta_parameter"] = json_data.get("meta_parameter", False)
+
+    writer.ModelDataWriter.dump_model_parameter(
+        parameter_name=param,
+        value=param_data["value"],
+        instrument=telescope,
+        parameter_version=param_data["version"],
+        output_file=f"{param}-{param_data['version']}.json",
+        output_path=param_dir,
+        unit=param_data.get("unit"),
+        meta_parameter=param_data.get("meta_parameter", False),
     )
-    # important for e.g. nsb_pixel_rate
-    if isinstance(json_data["value"], list) and not isinstance(param_data["value"], list):
-        json_data["value"] = [param_data["value"]] * len(json_data["value"])
-    else:
-        json_data["value"] = param_data["value"]
-
-    new_file_name = f"{param}-{param_data['version']}.json"
-    new_file_path = param_dir / new_file_name
-
-    ascii_handler.write_data_to_file(json_data, new_file_path, sort_keys=True)
-    _logger.info(f"Created new model parameter JSON file: {new_file_path}")
 
 
 def _get_latest_model_parameter_file(directory, parameter):
@@ -362,21 +393,23 @@ def _get_latest_model_parameter_file(directory, parameter):
             f"No JSON files found for parameter '{parameter}' in directory '{directory}'."
         )
 
-    # Sort files by version number (assumes version is part of the filename)
-    def safe_parse_version(filename):
-        version_str = filename.stem.split("-")[-1]
-        parts = version_str.split(".")
-        return tuple(part.zfill(8) for part in parts)
+    def extract_version(path: Path):
+        # version is part after first '-'
+        return parse_version(path.stem.split("-", 1)[1])
 
-    files.sort(key=safe_parse_version)
-    return str(files[-1])
+    latest_file = max(files, key=extract_version)
+    return str(latest_file)
 
 
-def _update_model_parameter_version(json_data, param_data, param, telescope):
-    """Check for major version jump and print a warning if necessary."""
-    latest_version = int(json_data.get("parameter_version", "0").split(".")[0])
-    new_version = int(param_data["version"].split(".")[0])
-    if new_version > latest_version + 1:
+def _check_for_major_version_jump(json_data, param_data, param, telescope):
+    """
+    Check for major version jump and print a warning if necessary.
+
+    Generally a jump from e.g. '3.1.0' to '5.0.0' should be avoided.
+    """
+    latest_version = parse_version(json_data.get("parameter_version", "0"))
+    new_version = parse_version(param_data["version"])
+    if new_version.major > latest_version.major + 1:
         _logger.warning(
             f"Major version jump from {latest_version} to {new_version} "
             f"for parameter '{param}' in telescope '{telescope}'."

simtools/model/model_utils.py

@@ -3,19 +3,17 @@
 
 import math
 
+from simtools.model.calibration_model import CalibrationModel
 from simtools.model.site_model import SiteModel
 from simtools.model.telescope_model import TelescopeModel
 from simtools.utils import names
 
-__all__ = [
-    "compute_telescope_transmission",
-    "is_two_mirror_telescope",
-]
 
-
-def initialize_simulation_models(label, db_config, site, telescope_name, model_version):
+def initialize_simulation_models(
+    label, db_config, model_version, site, telescope_name, calibration_device_name=None
+):
     """
-    Initialize simulation models for a single telescope and site model.
+    Initialize simulation models for a single telescope, site, and calibration device model.
 
     Parameters
     ----------
@@ -23,17 +21,19 @@ def initialize_simulation_models(label, db_config, site, telescope_name, model_v
         Label for the simulation.
     db_config: dict
         Database configuration.
+    model_version: str
+        Version of the simulation model
     site: str
         Name of the site.
     telescope_name: str
         Name of the telescope.
-    model_version: str
-        Version of the simulation model
+    calibration_device_name: str, optional
+        Name of the calibration device.
 
     Returns
     -------
     Tuple
-        Tuple containing the telescope model and site model.
+        Tuple containing the telescope site, (optional) calibration device model.
     """
     tel_model = TelescopeModel(
         site=site,
@@ -48,9 +48,19 @@ def initialize_simulation_models(label, db_config, site, telescope_name, model_v
         mongo_db_config=db_config,
         label=label,
     )
+    if calibration_device_name is not None:
+        calibration_model = CalibrationModel(
+            site=site,
+            calibration_device_model_name=calibration_device_name,
+            mongo_db_config=db_config,
+            model_version=model_version,
+            label=label,
+        )
+    else:
+        calibration_model = None
     for model in tel_model, site_model:
         model.export_model_files()
-    return tel_model, site_model
+    return tel_model, site_model, calibration_model
 
 
 def compute_telescope_transmission(pars: list[float], off_axis: float) -> float:

simtools/model/site_model.py

@@ -9,8 +9,6 @@ from astropy import units as u
 
 from simtools.model.model_parameter import ModelParameter
 
-__all__ = ["SiteModel"]
-
 
 class SiteModel(ModelParameter):
     """

simtools/model/telescope_model.py

@@ -13,8 +13,6 @@ from simtools.model.mirrors import Mirrors
 from simtools.model.model_parameter import InvalidModelParameterError, ModelParameter
 from simtools.utils import names
 
-__all__ = ["TelescopeModel"]
-
 
 class TelescopeModel(ModelParameter):
     """
@@ -378,3 +376,23 @@ class TelescopeModel(ModelParameter):
         except InvalidModelParameterError as exc:
             self._logger.error(f"Coordinate system {coordinate_system} not found.")
             raise exc
+
+    def get_calibration_device_name(self, device_type):
+        """
+        Get the calibration device name for this telescope.
+
+        Parameters
+        ----------
+        device_type: str
+            Type of the calibration device (e.g., 'flasher', 'illuminator')
+
+        Returns
+        -------
+        str or None
+            Calibration device name or None if not defined.
+        """
+        try:
+            devices = self.get_parameter_value("calibration_devices") or {}
+        except InvalidModelParameterError:
+            return None
+        return devices.get(device_type)

simtools/production_configuration/calculate_statistical_uncertainties_grid_point.py

@@ -6,8 +6,6 @@ import numpy as np
 from astropy import units as u
 from astropy.io import fits
 
-__all__ = ["StatisticalUncertaintyEvaluator"]
-
 
 class StatisticalUncertaintyEvaluator:
     """

simtools/production_configuration/derive_corsika_limits.py

@@ -116,7 +116,7 @@ def write_results(results, args_dict):
     """
     table = _create_results_table(results, args_dict["loss_fraction"])
 
-    output_dir = io_handler.IOHandler().get_output_directory("corsika_limits")
+    output_dir = io_handler.IOHandler().get_output_directory()
     output_file = output_dir / args_dict["output_file"]
 
     table.write(output_file, format="ascii.ecsv", overwrite=True)

simtools/production_configuration/derive_production_statistics.py

@@ -10,8 +10,6 @@ metrics and the evaluator's results.
 import astropy.units as u
 import numpy as np
 
-__all__ = ["ProductionStatisticsDerivator"]
-
 
 class ProductionStatisticsDerivator:
     """

simtools/production_configuration/interpolation_handler.py

@@ -10,8 +10,6 @@ from simtools.production_configuration.derive_production_statistics import (
     ProductionStatisticsDerivator,
 )
 
-__all__ = ["InterpolationHandler"]
-
 
 class InterpolationHandler:
     """

simtools/ray_tracing/incident_angles.py

@@ -80,12 +80,12 @@ class IncidentAnglesCalculator:
             config_data["site"],
             config_data["telescope"],
         )
-        self.telescope_model, self.site_model = initialize_simulation_models(
-            self.label,
-            db_config,
-            config_data["site"],
-            config_data["telescope"],
-            config_data["model_version"],
+        self.telescope_model, self.site_model, _ = initialize_simulation_models(
+            label=self.label,
+            db_config=db_config,
+            site=config_data["site"],
+            telescope_name=config_data["telescope"],
+            model_version=config_data["model_version"],
         )
 
     def _label_suffix(self):
@@ -110,7 +110,7 @@ class IncidentAnglesCalculator:
             Table containing at least the ``angle_incidence_focal`` column
             and, when configured, primary/secondary angles and hit geometry.
         """
-        self.telescope_model.write_sim_telarray_config_file(additional_model=self.site_model)
+        self.telescope_model.write_sim_telarray_config_file(additional_models=self.site_model)
 
         photons_file, stars_file, log_file = self._prepare_psf_io_files()
         run_script = self._write_run_script(photons_file, stars_file, log_file)

simtools/ray_tracing/mirror_panel_psf.py

@@ -70,7 +70,7 @@ class MirrorPanelPSF:
         site_model : SiteModel
             The site model.
         """
-        tel_model, site_model = initialize_simulation_models(
+        tel_model, site_model, _ = initialize_simulation_models(
             label=label,
             db_config=db_config,
             site=self.args_dict["site"],

simtools/ray_tracing/psf_analysis.py

@@ -19,8 +19,6 @@ import numpy as np
 
 from simtools.utils.general import collect_kwargs, set_default_kwargs
 
-__all__ = ["PSFImage"]
-
 
 class PSFImage:
     """