gammasimtools 0.20.0__py3-none-any.whl → 0.22.0__py3-none-any.whl

simtools/model/model_repository.py

@@ -5,11 +5,13 @@ a gitlab repository ('SimulationModels'). This module provides service
 functions to interact with and verify the repository.
 """
 
-import json
 import logging
-import shutil
 from pathlib import Path
 
+from packaging.version import Version
+from packaging.version import parse as parse_version
+
+import simtools.data_model.model_data_writer as writer
 from simtools.io import ascii_handler
 from simtools.utils import names
 
@@ -136,101 +138,186 @@ def _get_model_parameter_file_path(
     )
 
 
-def copy_and_update_production_table(args_dict):
+def generate_new_production(modifications, simulation_models_path):
     """
-    Copy and update simulation model production tables.
+    Generate a new production definition (production tables and model parameters).
+
+    The following steps are performed:
+
+    - copy of production tables from an existing base model version
+    - update production tables with changes defined in a YAML file
+    - generate new model parameter entries for changed parameters
+    - allows for full or patch updates
 
     Parameters
     ----------
-    args_dict: dict
-        Dictionary containing the arguments for copying and updating production tables.
+    modifications: str
+        Path to the YAML file defining the changes to be applied.
+    simulation_models_path: str
+        Path to the simulation models repository.
     """
-    modifications = ascii_handler.collect_data_from_file(args_dict["modifications"])
-    changes = modifications.get("changes", {})
+    modifications = ascii_handler.collect_data_from_file(modifications)
+    model_version_history = modifications.get("model_version_history", [])
+    try:
+        # oldest version is the base version
+        base_model_version = min(set(model_version_history), key=Version)
+    except ValueError as exc:
+        raise ValueError(f"Base model version not found in {modifications}") from exc
     model_version = modifications["model_version"]
+    changes = modifications.get("changes", {})
 
-    simulation_models_path = Path(args_dict["simulation_models_path"])
-    source_prod_table_path = (
-        simulation_models_path / "productions" / args_dict["source_prod_table_dir"]
+    _apply_changes_to_production_tables(
+        changes,
+        base_model_version,
+        model_version,
+        modifications.get("model_update", "full_update"),
+        simulation_models_path,
     )
-    target_prod_table_path = simulation_models_path / "productions" / model_version
-    model_parameters_dir = simulation_models_path / "model_parameters"
 
-    _logger.info(
-        f"Copying production tables from {source_prod_table_path} to {target_prod_table_path}"
-    )
+    _apply_changes_to_model_parameters(changes, simulation_models_path)
 
-    if Path(target_prod_table_path).exists():
-        raise FileExistsError(
-            f"The target production table directory '{target_prod_table_path}' already exists."
-        )
-    shutil.copytree(source_prod_table_path, target_prod_table_path)
 
-    _apply_changes_to_production_tables(target_prod_table_path, changes, model_version)
+def _apply_changes_to_production_tables(
+    changes, base_model_version, model_version, update_type, simulation_models_path
+):
+    """
+    Apply changes to production tables and write them to target directory.
 
-    for telescope, parameters in changes.items():
-        for param, param_data in parameters.items():
-            if param_data.get("value"):
-                _create_new_parameter_entry(telescope, param, param_data, model_parameters_dir)
+    Parameters
+    ----------
+    changes: dict
+        The changes to be applied.
+    base_model_version: str
+        The base model version (source directory for production tables).
+    model_version: str
+        The model version to be set in the JSON data.
+    update_type: str
+        Update mode, either 'full_update' or 'patch_update'.
+    simulation_models_path: Path
+        Path to the simulation models repository.
+    """
+    source = simulation_models_path / "productions" / base_model_version
+    target = simulation_models_path / "productions" / model_version
+    _logger.info(f"Production tables {update_type} from {source} to {target}")
+    target.mkdir(parents=True, exist_ok=True)
+
+    # load existing tables
+    tables = {}
+    for file_path in Path(source).rglob("*.json"):
+        data = ascii_handler.collect_data_from_file(file_path)
+        if not isinstance(data, dict):
+            raise TypeError(f"Unsupported data type {type(data)} in {file_path}")
+        tables[data["production_table_name"]] = data
 
+    # placeholder for new tables
+    for table_name in changes:
+        tables.setdefault(table_name, {})
 
-def _apply_changes_to_production_tables(target_prod_table_path, changes, model_version):
-    """Apply changes to the production tables in the target directory."""
-    for file_path in Path(target_prod_table_path).rglob("*.json"):
-        if file_path.name.startswith("configuration"):
-            continue
-        data = ascii_handler.collect_data_from_file(file_path)
-        _apply_changes_to_production_table(data, changes, model_version)
-        with file_path.open("w", encoding="utf-8") as f:
-            json.dump(data, f, indent=4, sort_keys=True)
-            f.write("\n")
+    for table_name, data in tables.items():
+        if _apply_changes_to_production_table(
+            table_name, data, changes, model_version, update_type == "patch_update"
+        ):
+            _logger.info(f"Writing updated production table '{table_name}'")
+            data["production_table_name"] = table_name
+            ascii_handler.write_data_to_file(data, target / f"{table_name}.json", sort_keys=True)
 
 
-def _apply_changes_to_production_table(data, changes, model_version):
+def _apply_changes_to_production_table(table_name, data, changes, model_version, patch_update):
     """
-    Recursively apply changes to the new production tables.
+    Apply changes to a single production table.
 
     Parameters
     ----------
-    data: dict or list
-        The JSON data to be updated.
+    data: dict
+        The data to be updated.
     changes: dict
         The changes to be applied.
     model_version: str
         The model version to be set in the JSON data.
+    patch_update: bool
+        True if patch update (modify only changed parameters), False for full update.
+
+    Returns
+    -------
+    bool
+        True if data was modified and should be written to disk (patch updates);
+        always True for full updates.
     """
-    if isinstance(data, dict):
-        if "model_version" in data:
-            data["model_version"] = model_version
-        _update_parameters(data.get("parameters", {}), changes)
-
-    elif isinstance(data, list):
-        for item in data:
-            _apply_changes_to_production_table(item, changes, model_version)
-
-
-def _update_parameters(params, changes):
-    """Update parameters in the given dictionary based on changes."""
-    for telescope, updates in changes.items():
-        if telescope not in params:
-            continue
-        for param, param_data in updates.items():
-            if param in params[telescope]:
-                old = params[telescope][param]
-                new = param_data["version"]
-                _logger.info(f"Updating '{telescope} - {param}' from {old} to {new}")
-                params[telescope][param] = new
-            else:
-                _logger.info(
-                    f"Adding new parameter '{telescope} - {param}' "
-                    f"with version {param_data['version']}"
+    data["model_version"] = model_version
+    if table_name in changes:
+        table_parameters = {} if patch_update else data.get("parameters", {}).get(table_name, {})
+        parameters, deprecated = _update_parameters_dict(table_parameters, changes, table_name)
+        data["parameters"] = parameters
+        if deprecated:
+            data["deprecated_parameters"] = deprecated
+    elif patch_update:
+        return False
+
+    return True
+
+
+def _update_parameters_dict(table_parameters, changes, table_name):
+    """
+    Create a new parameters dictionary for the production tables.
+
+    Include only changes relevant to the specific telescope.
+    Do not include parameters if 'deprecated' flag is set to True.
+
+    Parameters
+    ----------
+    table_parameters: dict
+        Parameters for the specific table.
+    changes: dict
+        The changes to be applied, containing table and parameter information.
+    table_name: str
+        The name of the production table to filter parameters for.
+
+    Returns
+    -------
+    dict, list
+        Dictionary containing only the new/changed parameters for the specified table.
+        List of deprecated parameters.
+    """
+    new_params = {table_name: table_parameters}
+    deprecated_params = []
+
+    for param, data in changes[table_name].items():
+        if data.get("deprecated", False):
+            _logger.info(f"Removing model parameter '{table_name} - {param}'")
+            deprecated_params.append(param)
+        else:
+            version = data["version"]
+            _logger.info(f"Setting '{table_name} - {param}' to version {version}")
+            new_params[table_name][param] = version
+
+    return new_params, deprecated_params
+
+
+def _apply_changes_to_model_parameters(changes, simulation_models_path):
+    """
+    Apply changes to model parameters by creating new parameter entries.
+
+    Parameters
+    ----------
+    changes: dict
+        The changes to be applied.
+    simulation_models_path: Path
+        Path to the simulation models directory.
+    """
+    for telescope, parameters in changes.items():
+        for param, param_data in parameters.items():
+            if param_data.get("value") is not None:
+                _create_new_model_parameter_entry(
+                    telescope, param, param_data, simulation_models_path
                 )
-                params[telescope][param] = param_data["version"]
 
 
-def _create_new_parameter_entry(telescope, param, param_data, model_parameters_dir):
+def _create_new_model_parameter_entry(telescope, param, param_data, simulation_models_path):
     """
-    Create new model parameter JSON file by copying the latest version and updating fields.
+    Create new model parameter entry in the model parameters directory.
+
+    If a model parameter files exists, copy latest version and update the fields.
+    Otherwise generate new file using the model parameter schema.
 
     Parameters
     ----------
@@ -240,41 +327,40 @@ def _create_new_parameter_entry(telescope, param, param_data, model_parameters_d
         Name of the parameter.
     param_data: dict
         Dictionary containing the parameter data including version and value.
-    model_parameters_dir: str
-        Path to the model parameters directory.
+    simulation_models_path: Path
+        Path to the simulation models directory.
     """
-    telescope_dir = Path(model_parameters_dir) / telescope
+    telescope_dir = simulation_models_path / "model_parameters" / telescope
     if not telescope_dir.exists():
-        raise FileNotFoundError(
-            f"Directory for telescope '{telescope}' does not exist in '{model_parameters_dir}'."
-        )
+        _logger.info(f"Create directory for array element '{telescope}': '{telescope_dir}'.")
+        telescope_dir.mkdir(parents=True, exist_ok=True)
 
     param_dir = telescope_dir / param
-    if not param_dir.exists():
-        raise FileNotFoundError(
-            f"Directory for parameter '{param}' does not exist in '{telescope}'."
+    try:
+        latest_file = _get_latest_model_parameter_file(param_dir, param)
+    except FileNotFoundError:
+        latest_file = None
+
+    if latest_file is not None:
+        json_data = ascii_handler.collect_data_from_file(latest_file)
+        param_data["version"] = _check_for_major_version_jump(
+            json_data, param_data, param, telescope
         )
-
-    latest_file = _get_latest_model_parameter_file(param_dir, param)
-    if not latest_file:
-        raise FileNotFoundError(
-            f"No files found for parameter '{param}' in directory '{param_dir}'."
-        )
-
-    json_data = ascii_handler.collect_data_from_file(latest_file)
-
-    json_data["parameter_version"] = _update_model_parameter_version(
-        json_data, param_data, param, telescope
+        # important for e.g. nsb_pixel_rate
+        if isinstance(json_data["value"], list) and not isinstance(param_data["value"], list):
+            param_data["value"] = [param_data["value"]] * len(json_data["value"])
+        param_data["meta_parameter"] = json_data.get("meta_parameter", False)
+
+    writer.ModelDataWriter.dump_model_parameter(
+        parameter_name=param,
+        value=param_data["value"],
+        instrument=telescope,
+        parameter_version=param_data["version"],
+        output_file=f"{param}-{param_data['version']}.json",
+        output_path=param_dir,
+        unit=param_data.get("unit"),
+        meta_parameter=param_data.get("meta_parameter", False),
     )
-    json_data["value"] = param_data["value"]
-
-    new_file_name = f"{param}-{param_data['version']}.json"
-    new_file_path = param_dir / new_file_name
-
-    with new_file_path.open("w", encoding="utf-8") as f:
-        json.dump(json_data, f, indent=4)
-        f.write("\n")
-    _logger.info(f"Created new model parameter JSON file: {new_file_path}")
 
 
 def _get_latest_model_parameter_file(directory, parameter):
@@ -307,21 +393,23 @@ def _get_latest_model_parameter_file(directory, parameter):
             f"No JSON files found for parameter '{parameter}' in directory '{directory}'."
         )
 
-    # Sort files by version number (assumes version is part of the filename)
-    def safe_parse_version(filename):
-        version_str = filename.stem.split("-")[-1]
-        parts = version_str.split(".")
-        return tuple(part.zfill(8) for part in parts)
+    def extract_version(path: Path):
+        # version is part after first '-'
+        return parse_version(path.stem.split("-", 1)[1])
+
+    latest_file = max(files, key=extract_version)
+    return str(latest_file)
 
-    files.sort(key=safe_parse_version)
-    return str(files[-1])
 
+def _check_for_major_version_jump(json_data, param_data, param, telescope):
+    """
+    Check for major version jump and print a warning if necessary.
 
-def _update_model_parameter_version(json_data, param_data, param, telescope):
-    """Check for major version jump and print a warning if necessary."""
-    latest_version = int(json_data.get("parameter_version", "0").split(".")[0])
-    new_version = int(param_data["version"].split(".")[0])
-    if new_version > latest_version + 1:
+    Generally a jump from e.g. '3.1.0' to '5.0.0' should be avoided.
+    """
+    latest_version = parse_version(json_data.get("parameter_version", "0"))
+    new_version = parse_version(param_data["version"])
+    if new_version.major > latest_version.major + 1:
         _logger.warning(
             f"Major version jump from {latest_version} to {new_version} "
             f"for parameter '{param}' in telescope '{telescope}'."

simtools/model/model_utils.py

@@ -3,6 +3,7 @@
 
 import math
 
+from simtools.model.calibration_model import CalibrationModel
 from simtools.model.site_model import SiteModel
 from simtools.model.telescope_model import TelescopeModel
 from simtools.utils import names
@@ -13,9 +14,11 @@ __all__ = [
 ]
 
 
-def initialize_simulation_models(label, db_config, site, telescope_name, model_version):
+def initialize_simulation_models(
+    label, db_config, model_version, site, telescope_name, calibration_device_name=None
+):
     """
-    Initialize simulation models for a single telescope and site model.
+    Initialize simulation models for a single telescope, site, and calibration device model.
 
     Parameters
     ----------
@@ -23,17 +26,19 @@ def initialize_simulation_models(label, db_config, site, telescope_name, model_v
         Label for the simulation.
     db_config: dict
         Database configuration.
+    model_version: str
+        Version of the simulation model
     site: str
         Name of the site.
     telescope_name: str
         Name of the telescope.
-    model_version: str
-        Version of the simulation model
+    calibration_device_name: str, optional
+        Name of the calibration device.
 
     Returns
     -------
     Tuple
-        Tuple containing the telescope model and site model.
+        Tuple containing the telescope site, (optional) calibration device model.
     """
     tel_model = TelescopeModel(
         site=site,
@@ -48,9 +53,19 @@ def initialize_simulation_models(label, db_config, site, telescope_name, model_v
         mongo_db_config=db_config,
         label=label,
     )
+    if calibration_device_name is not None:
+        calibration_model = CalibrationModel(
+            site=site,
+            calibration_device_model_name=calibration_device_name,
+            mongo_db_config=db_config,
+            model_version=model_version,
+            label=label,
+        )
+    else:
+        calibration_model = None
     for model in tel_model, site_model:
         model.export_model_files()
-    return tel_model, site_model
+    return tel_model, site_model, calibration_model
 
 
 def compute_telescope_transmission(pars: list[float], off_axis: float) -> float:

simtools/model/telescope_model.py

@@ -378,3 +378,23 @@ class TelescopeModel(ModelParameter):
         except InvalidModelParameterError as exc:
             self._logger.error(f"Coordinate system {coordinate_system} not found.")
             raise exc
+
+    def get_calibration_device_name(self, device_type):
+        """
+        Get the calibration device name for this telescope.
+
+        Parameters
+        ----------
+        device_type: str
+            Type of the calibration device (e.g., 'flasher', 'illuminator')
+
+        Returns
+        -------
+        str or None
+            Calibration device name or None if not defined.
+        """
+        try:
+            devices = self.get_parameter_value("calibration_devices") or {}
+        except InvalidModelParameterError:
+            return None
+        return devices.get(device_type)

simtools/production_configuration/derive_corsika_limits.py

@@ -116,7 +116,7 @@ def write_results(results, args_dict):
     """
     table = _create_results_table(results, args_dict["loss_fraction"])
 
-    output_dir = io_handler.IOHandler().get_output_directory("corsika_limits")
+    output_dir = io_handler.IOHandler().get_output_directory()
     output_file = output_dir / args_dict["output_file"]
 
     table.write(output_file, format="ascii.ecsv", overwrite=True)

simtools/ray_tracing/incident_angles.py

@@ -80,12 +80,12 @@ class IncidentAnglesCalculator:
             config_data["site"],
             config_data["telescope"],
         )
-        self.telescope_model, self.site_model = initialize_simulation_models(
-            self.label,
-            db_config,
-            config_data["site"],
-            config_data["telescope"],
-            config_data["model_version"],
+        self.telescope_model, self.site_model, _ = initialize_simulation_models(
+            label=self.label,
+            db_config=db_config,
+            site=config_data["site"],
+            telescope_name=config_data["telescope"],
+            model_version=config_data["model_version"],
         )
 
     def _label_suffix(self):
@@ -110,7 +110,7 @@ class IncidentAnglesCalculator:
             Table containing at least the ``angle_incidence_focal`` column
             and, when configured, primary/secondary angles and hit geometry.
         """
-        self.telescope_model.write_sim_telarray_config_file(additional_model=self.site_model)
+        self.telescope_model.write_sim_telarray_config_file(additional_models=self.site_model)
 
         photons_file, stars_file, log_file = self._prepare_psf_io_files()
         run_script = self._write_run_script(photons_file, stars_file, log_file)

simtools/ray_tracing/mirror_panel_psf.py

@@ -70,7 +70,7 @@ class MirrorPanelPSF:
         site_model : SiteModel
             The site model.
         """
-        tel_model, site_model = initialize_simulation_models(
+        tel_model, site_model, _ = initialize_simulation_models(
             label=label,
             db_config=db_config,
             site=self.args_dict["site"],