gammasimtools 0.25.0__py3-none-any.whl → 0.26.0__py3-none-any.whl

simtools/layout/array_layout_utils.py

@@ -3,6 +3,9 @@
 import logging
 from pathlib import Path
 
+import astropy.units as u
+from astropy.table import QTable, Table
+
 import simtools.utils.general as gen
 from simtools.data_model import data_reader
 from simtools.data_model.metadata_collector import MetadataCollector
@@ -137,7 +140,7 @@ def merge_array_layouts(layouts_1, layouts_2):
     return merged_layout
 
 
-def write_array_layouts(array_layouts, args_dict, db_config):
+def write_array_layouts(array_layouts, args_dict):
     """
     Write array layouts as model parameter.
 
@@ -147,8 +150,6 @@ def write_array_layouts(array_layouts, args_dict, db_config):
         Command line arguments.
     array_layouts : dict
         Array layouts to be written.
-    db_config : dict
-        Database configuration.
     """
     site = args_dict.get("site") or array_layouts.get("site")
     _logger.info(f"Writing updated array layouts to the database for site {site}.")
@@ -165,7 +166,6 @@ def write_array_layouts(array_layouts, args_dict, db_config):
         instrument=f"OBS-{site}",
         parameter_version=args_dict.get("updated_parameter_version"),
         output_file=output_file,
-        db_config=db_config,
     )
     MetadataCollector.dump(
         args_dict,
@@ -208,9 +208,7 @@ def validate_array_layouts_with_db(production_table, array_layouts):
     return array_layouts
 
 
-def get_array_layouts_from_parameter_file(
-    file_path, model_version, db_config, coordinate_system="ground"
-):
+def get_array_layouts_from_parameter_file(file_path, model_version, coordinate_system="ground"):
     """
     Retrieve array layouts from parameter file.
 
@@ -220,8 +218,6 @@ def get_array_layouts_from_parameter_file(
         Path to the array layout parameter file.
     model_version : str
         Model version to retrieve.
-    db_config : dict
-        Database configuration.
     coordinate_system : str
         Coordinate system to use for the array elements (default is "ground").
 
@@ -237,24 +233,19 @@ def get_array_layouts_from_parameter_file(
         raise ValueError("Missing 'value' key in layout file.") from exc
     site = array_layouts.get("site")
 
-    layouts = []
-    for layout in value:
-        layouts.append(
-            _get_array_layout_dict(
-                db_config,
-                model_version,
-                site,
-                layout.get("elements"),
-                layout["name"],
-                coordinate_system,
-            )
+    return [
+        _get_array_layout_dict(
+            model_version,
+            site,
+            layout.get("elements"),
+            layout["name"],
+            coordinate_system,
         )
-    return layouts
+        for layout in value
+    ]
 
 
-def get_array_layouts_from_db(
-    layout_name, site, model_version, db_config, coordinate_system="ground"
-):
+def get_array_layouts_from_db(layout_name, site, model_version, coordinate_system="ground"):
     """
     Retrieve all array layouts from the database and return as list of astropy tables.
 
@@ -266,8 +257,6 @@ def get_array_layouts_from_db(
         Site identifier.
     model_version : str
         Model version to retrieve.
-    db_config : dict
-        Database configuration.
     coordinate_system : str
         Coordinate system to use for the array elements (default is "ground").
 
@@ -280,23 +269,20 @@ def get_array_layouts_from_db(
     if layout_name:
         layout_names = gen.ensure_iterable(layout_name)
     else:
-        site_model = SiteModel(site=site, model_version=model_version, db_config=db_config)
+        site_model = SiteModel(site=site, model_version=model_version)
         layout_names = site_model.get_list_of_array_layouts()
 
-    layouts = []
-    for _layout_name in layout_names:
-        layouts.append(
-            _get_array_layout_dict(
-                db_config, model_version, site, None, _layout_name, coordinate_system
-            )
-        )
+    layouts = [
+        _get_array_layout_dict(model_version, site, None, _layout_name, coordinate_system)
+        for _layout_name in layout_names
+    ]
     if len(layouts) == 1:
         return layouts[0]
     return layouts
 
 
 def get_array_layouts_using_telescope_lists_from_db(
-    telescope_lists, site, model_version, db_config, coordinate_system="ground"
+    telescope_lists, site, model_version, coordinate_system="ground"
 ):
     """
     Retrieve array layouts from the database using telescope lists.
@@ -309,8 +295,6 @@ def get_array_layouts_using_telescope_lists_from_db(
         Site identifier.
     model_version : str
         Model version to retrieve.
-    db_config : dict
-        Database configuration.
     coordinate_system : str
         Coordinate system to use for the array elements (default is "ground").
 
@@ -333,9 +317,7 @@ def get_array_layouts_using_telescope_lists_from_db(
             _site = sites.pop()
 
         layouts.append(
-            _get_array_layout_dict(
-                db_config, model_version, _site, telescope_list, None, coordinate_system
-            )
+            _get_array_layout_dict(model_version, _site, telescope_list, None, coordinate_system)
         )
     return layouts
 
@@ -357,23 +339,18 @@ def get_array_layouts_from_file(file_path):
     if isinstance(file_path, str | Path):
         file_path = [file_path]
 
-    layouts = []
-    for _file in file_path:
-        layouts.append(
-            {
-                "name": (Path(_file).name).split(".")[0],
-                "array_elements": data_reader.read_table_from_file(file_name=_file),
-            }
-        )
-    return layouts
+    return [
+        {
+            "name": Path(_file).stem,
+            "array_elements": data_reader.read_table_from_file(file_name=_file),
+        }
+        for _file in file_path
+    ]
 
 
-def _get_array_layout_dict(
-    db_config, model_version, site, telescope_list, layout_name, coordinate_system
-):
+def _get_array_layout_dict(model_version, site, telescope_list, layout_name, coordinate_system):
     """Return array layout dictionary for a given telescope list."""
     array_model = ArrayModel(
-        db_config=db_config,
         model_version=model_version,
         site=site,
         array_elements=telescope_list,
@@ -388,7 +365,7 @@ def _get_array_layout_dict(
     }
 
 
-def get_array_elements_from_db_for_layouts(layouts, site, model_version, db_config):
+def get_array_elements_from_db_for_layouts(layouts, site, model_version):
     """
     Get list of array elements from the database for given list of layout names.
 
@@ -408,17 +385,209 @@ def get_array_elements_from_db_for_layouts(layouts, site, model_version, db_conf
         Site name for the array layouts.
     model_version : str
         Model version for the array layouts.
-    db_config : dict
-        Database configuration dictionary.
 
     Returns
     -------
     dict
         Dictionary mapping layout names to telescope IDs.
     """
-    site_model = SiteModel(site=site, model_version=model_version, db_config=db_config)
+    site_model = SiteModel(site=site, model_version=model_version)
     layout_names = site_model.get_list_of_array_layouts() if layouts == ["all"] else layouts
     layout_dict = {}
     for layout_name in layout_names:
         layout_dict[layout_name] = site_model.get_array_elements_for_layout(layout_name)
     return layout_dict
+
+
+def read_layouts(args_dict):
+    """
+    Read array layouts from the database or parameter file.
+
+    Parameters
+    ----------
+    args_dict : dict
+        Dictionary with command line arguments.
+
+    Returns
+    -------
+    tuple
+        A tuple containing:
+            - list: List of array layouts.
+            - list or None: Background layout or None if not provided.
+    """
+    background_layout = None
+    if args_dict.get("array_layout_name_background"):
+        background_layout = get_array_layouts_from_db(
+            args_dict["array_layout_name_background"],
+            args_dict["site"],
+            args_dict["model_version"],
+            args_dict["coordinate_system"],
+        )["array_elements"]
+
+    if args_dict["array_layout_name"] is not None or args_dict["plot_all_layouts"]:
+        _logger.info("Plotting array from DB using layout array name(s).")
+        layouts = get_array_layouts_from_db(
+            args_dict["array_layout_name"],
+            args_dict["site"],
+            args_dict["model_version"],
+            args_dict["coordinate_system"],
+        )
+        if isinstance(layouts, list):
+            return layouts, background_layout
+        return [layouts], background_layout
+
+    if args_dict["array_layout_parameter_file"] is not None:
+        _logger.info("Plotting array from parameter file(s).")
+        return get_array_layouts_from_parameter_file(
+            args_dict["array_layout_parameter_file"],
+            args_dict["model_version"],
+            args_dict["coordinate_system"],
+        ), background_layout
+
+    if args_dict["array_layout_file"] is not None:
+        _logger.info("Plotting array from telescope table file(s).")
+        return get_array_layouts_from_file(args_dict["array_layout_file"]), background_layout
+    if args_dict["array_element_list"] is not None:
+        _logger.info("Plotting array from list of array elements.")
+        return get_array_layouts_using_telescope_lists_from_db(
+            [args_dict["array_element_list"]],
+            args_dict["site"],
+            args_dict["model_version"],
+            args_dict["coordinate_system"],
+        ), background_layout
+
+    return [], background_layout
+
+
+def _get_array_name(array_name):
+    """
+    Return telescope size and number of telescopes from regular array name.
+
+    Finetuned to array names like "4MST", "1LST", etc.
+
+    Parameters
+    ----------
+    array_name : str
+        Name of the regular array (e.g. "4MST").
+
+    Returns
+    -------
+    tel_size : str
+        Telescope size (e.g. "MST").
+    n_tel : int
+        Number of telescopes (e.g. 4).
+    """
+    if len(array_name) < 2 or not array_name[0].isdigit():
+        raise ValueError(f"Invalid array_name: '{array_name}'")
+
+    return array_name[1:], int(array_name[0])
+
+
+def create_regular_array(array_name, site, telescope_distance):
+    """
+    Create a regular array layout table.
+
+    Parameters
+    ----------
+    array_name : str
+        Name of the regular array (e.g. "4MST").
+    site : str
+        Site identifier.
+    telescope_distance : dict
+        Dictionary with telescope distances per telescope type.
+
+    Returns
+    -------
+    astropy.table.Table
+        Table with the regular array layout.
+    """
+    tel_name, pos_x, pos_y, pos_z = [], [], [], []
+    tel_size, n_tel = _get_array_name(array_name)
+    tel_size = array_name[1:4]
+
+    # Single telescope at the center
+    if n_tel == 1:
+        tel_name.append(names.generate_array_element_name_from_type_site_id(tel_size, site, "01"))
+        pos_x.append(0 * u.m)
+        pos_y.append(0 * u.m)
+        pos_z.append(0 * u.m)
+    # 4 telescopes in a regular square grid
+    elif n_tel == 4:
+        for i in range(1, 5):
+            tel_name.append(
+                names.generate_array_element_name_from_type_site_id(tel_size, site, f"0{i}")
+            )
+            pos_x.append(telescope_distance[tel_size] * (-1) ** (i // 2))
+            pos_y.append(telescope_distance[tel_size] * (-1) ** (i % 2))
+            pos_z.append(0 * u.m)
+    else:
+        raise ValueError(f"Unsupported number of telescopes: {n_tel}.")
+
+    table = QTable(meta={"array_name": array_name, "site": site})
+    table["telescope_name"] = tel_name
+    table["position_x"] = pos_x
+    table["position_y"] = pos_y
+    table["position_z"] = pos_z
+    table.sort("telescope_name")
+    _logger.info(f"Regular array layout table:\n{table}")
+
+    return table
+
+
+def write_array_elements_from_file_to_repository(
+    coordinate_system, input_file, repository_path, parameter_version
+):
+    """
+    Read array elements from file and write their positions to model repository.
+
+    Writes one model parameter file per array elements.
+
+    Parameters
+    ----------
+    coordinate_system : str
+        Coordinate system of array element positions (utm or ground).
+    input_file : str or Path
+        Path to input file with array element positions.
+    repository_path : str or Path
+        Path to model repository.
+    parameter_version : str
+        Parameter version to use when writing to repository.
+    """
+    repository_path = Path(repository_path)
+
+    array_elements = Table.read(input_file)
+
+    if coordinate_system == "ground":
+        parameter_name = "array_element_position_ground"
+        x = array_elements["position_x"].quantity.to(u.m).value
+        y = array_elements["position_y"].quantity.to(u.m).value
+        alt = array_elements["position_z"].quantity.to(u.m).value
+    elif coordinate_system == "utm":
+        x = array_elements["utm_east"].quantity.to(u.m).value
+        y = array_elements["utm_north"].quantity.to(u.m).value
+        alt = array_elements["altitude"].quantity.to(u.m).value
+        parameter_name = "array_element_position_utm"
+    else:
+        raise ValueError(
+            f"Unsupported coordinate system: {coordinate_system}. Allowed are 'utm' and 'ground'."
+        )
+
+    for i, row in enumerate(array_elements):
+        instrument = (
+            row["telescope_name"]
+            if "telescope_name" in array_elements.colnames
+            else f"{row['asset_code']}-{row['sequence_number']}"
+        )
+        output_path = repository_path / f"{instrument}"
+        output_path.mkdir(parents=True, exist_ok=True)
+        _logger.info(f"Writing array element positions ({coordinate_system}) to {output_path}")
+
+        ModelDataWriter.dump_model_parameter(
+            parameter_name=parameter_name,
+            instrument=instrument,
+            value=f"{x[i]} {y[i]} {alt[i]}",
+            unit="m",
+            parameter_version=parameter_version,
+            output_path=repository_path / instrument,
+            output_file=f"{parameter_name}.json",
+        )

simtools/model/array_model.py

@@ -1,4 +1,4 @@
-"""Definition of the ArrayModel class."""
+"""Array model represents an observatory consisting of site, telescopes, and further devices."""
 
 import logging
 from pathlib import Path
@@ -21,8 +21,6 @@ class ArrayModel:
 
     Parameters
     ----------
-    db_config: dict
-        Database configuration.
     model_version: str
         Model version.
     label: str, optional
@@ -36,28 +34,23 @@ class ArrayModel:
         the array element positions).
     calibration_device_types: List[str], optional
         List of calibration device types (e.g., 'flat_fielding') attached to each telescope.
-    simtel_path: str, Path, optional
-        Path to the sim_telarray installation directory.
     overwrite_model_parameters: str, optional
         File name to overwrite model parameters from DB with provided values.
     """
 
     def __init__(
         self,
-        db_config,
         model_version,
         label=None,
         site=None,
         layout_name=None,
         array_elements=None,
         calibration_device_types=None,
-        simtel_path=None,
         overwrite_model_parameters=None,
     ):
         """Initialize ArrayModel."""
         self._logger = logging.getLogger(__name__)
         self._logger.debug("Init ArrayModel")
-        self.db_config = db_config
         self.model_version = model_version
         self.label = label
         self.layout_name = (
@@ -78,7 +71,6 @@ class ArrayModel:
         self._telescope_model_files_exported = False
         self._array_model_file_exported = False
         self._sim_telarray_seeds = None
-        self.simtel_path = simtel_path
 
     def _initialize(self, site, array_elements_config, calibration_device_types):
         """
@@ -105,7 +97,6 @@ class ArrayModel:
         self._logger.debug(f"Getting site parameters from DB ({site})")
         site_model = SiteModel(
             site=names.validate_site_name(site),
-            db_config=self.db_config,
             model_version=self.model_version,
             label=self.label,
             overwrite_model_parameters=self.overwrite_model_parameters,
@@ -250,7 +241,6 @@ class ArrayModel:
                 site=site_model.site,
                 telescope_name=element_name,
                 model_version=self.model_version,
-                db_config=self.db_config,
                 label=self.label,
                 overwrite_model_parameters=self.overwrite_model_parameters,
             )
@@ -282,7 +272,6 @@ class ArrayModel:
             calibration_models[device_name] = CalibrationModel(
                 site=site_model.site,
                 calibration_device_model_name=device_name,
-                db_config=self.db_config,
                 model_version=self.model_version,
                 label=self.label,
                 overwrite_model_parameters=self.overwrite_model_parameters,
@@ -322,7 +311,6 @@ class ArrayModel:
             layout_name=self.layout_name,
             model_version=self.model_version,
             label=self.label,
-            simtel_path=self.simtel_path,
         )
         simtel_writer.write_array_config_file(
             config_file_path=self.config_file_path,

simtools/model/calibration_model.py

@@ -17,8 +17,6 @@ class CalibrationModel(ModelParameter):
         Site name (e.g., South or North).
     calibration_device_model_name: str
         Calibration device model name (ex. ILLS-01, ILLN-01, ...).
-    db_config: dict
-        Database configuration.
     model_version: str
         Model version.
     label: str, optional
@@ -31,7 +29,6 @@ class CalibrationModel(ModelParameter):
         self,
         site,
         calibration_device_model_name,
-        db_config,
         model_version,
         label=None,
         overwrite_model_parameters=None,
@@ -41,7 +38,6 @@ class CalibrationModel(ModelParameter):
             site=site,
             array_element_name=calibration_device_model_name,
             collection="calibration_devices",
-            db_config=db_config,
             model_version=model_version,
             label=label,
             overwrite_model_parameters=overwrite_model_parameters,

simtools/model/legacy_model_parameter.py

@@ -0,0 +1,134 @@
+#!/usr/bin/python3
+"""
+Handling of legacy model parameters for backward compatibility.
+
+Collects routines to convert legacy model parameters to more recent
+formats. This is a fine-tuned process and requires to hard wire the
+changes. All code related to legacy model parameters should go into
+this module.
+
+"""
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+UPDATE_HANDLERS = {}
+
+
+def register_update(name):
+    """Register update handler for legacy model parameter."""
+
+    def deco(func):
+        UPDATE_HANDLERS[name] = func
+        return func
+
+    return deco
+
+
+def apply_legacy_updates_to_parameters(parameters, legacy_updates):
+    """Apply legacy updates to model parameters.
+
+    Modifies the parameters dictionary in-place.
+
+    Parameters
+    ----------
+    parameters: dict
+        Dictionary of model parameters (all parameters).
+    legacy_updates: dict
+        Dictionary of legacy updates to apply.
+    """
+    for par_name, legacy_data in legacy_updates.items():
+        if legacy_data is None or par_name not in parameters:
+            continue
+        for key in parameters[par_name].keys():
+            if key in legacy_data:
+                parameters[par_name][key] = legacy_data[key]
+        if legacy_data.get("remove_parameter", False):
+            parameters.pop(par_name)
+
+
+def update_parameter(par_name, parameters, schema_version):
+    """Update legacy model parameters to recent formats.
+
+    Parameters
+    ----------
+    par_name: str
+        Model parameter name.
+    parameters: dict
+        Dictionary of model parameters (all parameters).
+    schema_version: str
+        Target schema version.
+
+    Returns
+    -------
+    dict
+        Updated model parameter.
+    """
+    handler = UPDATE_HANDLERS.get(par_name)
+    if handler is None:
+        raise ValueError(_get_unsupported_update_message(parameters[par_name], schema_version))
+    return handler(parameters, schema_version)
+
+
+@register_update("dsum_threshold")
+def _update_dsum_threshold(parameters, schema_version):
+    """Update legacy dsum_threshold parameter."""
+    para_data = parameters["dsum_threshold"]
+    if para_data["model_parameter_schema_version"] == "0.1.0" and schema_version == "0.2.0":
+        logger.info(
+            "Updating legacy model parameter dsum_threshold from schema version "
+            f"{para_data['model_parameter_schema_version']} to {schema_version}"
+        )
+        return {
+            para_data["parameter"]: {
+                "value": int(para_data["value"]),
+                "model_parameter_schema_version": schema_version,
+            }
+        }
+    raise ValueError(_get_unsupported_update_message(para_data, schema_version))
+
+
+@register_update("corsika_starting_grammage")
+def _update_corsika_starting_grammage(parameters, schema_version):
+    """Update legacy corsika_starting_grammage parameter (dummy function until model is updated)."""
+    logger.debug(f"No fix applied to corsika_starting_grammage to schema version {schema_version}")
+    return {
+        parameters["corsika_starting_grammage"]["parameter"]: None,
+    }
+
+
+@register_update("flasher_pulse_shape")
+def _update_flasher_pulse_shape(parameters, schema_version):
+    """Update legacy flasher_pulse_shape parameter."""
+    para_data = parameters["flasher_pulse_shape"]
+    if para_data["model_parameter_schema_version"] == "0.1.0" and schema_version == "0.2.0":
+        logger.info(
+            f"Updating legacy model parameter flasher_pulse_shape from schema version "
+            f"{para_data['model_parameter_schema_version']} to {schema_version}"
+        )
+        return {
+            para_data["parameter"]: {
+                "value": [
+                    para_data["value"],
+                    parameters.get("flasher_pulse_width", {}).get("value", 0.0),
+                    parameters.get("flasher_pulse_exp_decay", {}).get("value", 0.0),
+                ],
+                "model_parameter_schema_version": schema_version,
+                "unit": [None, "ns", "ns"],
+                "type": ["string", "float64", "float64"],
+            },
+            "flasher_pulse_width": {"remove_parameter": True},
+            "flasher_pulse_exp_decay": {"remove_parameter": True},
+        }
+
+    raise ValueError(_get_unsupported_update_message(para_data, schema_version))
+
+
+def _get_unsupported_update_message(para_data, schema_version):
+    """Get unsupported update message."""
+    return (
+        f"Unsupported update for legacy parameter {para_data['parameter']}: "
+        f"{para_data['model_parameter_schema_version']} to {schema_version}"
+    )