gammasimtools 0.15.0__py3-none-any.whl → 0.17.0__py3-none-any.whl

simtools/layout/array_layout_utils.py

@@ -0,0 +1,385 @@
+"""Retrieve, merge, and write layout dictionaries."""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.data_model import data_reader
+from simtools.data_model.metadata_collector import MetadataCollector
+from simtools.data_model.model_data_writer import ModelDataWriter
+from simtools.io_operations import io_handler
+from simtools.model.array_model import ArrayModel
+from simtools.model.site_model import SiteModel
+from simtools.utils import names
+
+_logger = logging.getLogger(__name__)
+
+
+def retrieve_ctao_array_layouts(site, repository_url, branch_name="main"):
+    """
+    Retrieve array layouts from CTAO common identifiers repository.
+
+    Parameters
+    ----------
+    site : str
+        Site identifier.
+    repository_url : str
+        URL or path to CTAO common identifiers
+    branch_name : str
+        Repository branch to use for CTAO common identifiers.
+
+    Returns
+    -------
+    dict
+        Array layouts for all CTAO sites.
+    """
+    _logger.info(f"Retrieving array layouts from {repository_url} on branch {branch_name}.")
+
+    if gen.is_url(repository_url):
+        array_element_ids = gen.collect_data_from_http(
+            url=f"{repository_url}/{branch_name}/array-element-ids.json"
+        )
+        sub_arrays = gen.collect_data_from_http(
+            url=f"{repository_url}/{branch_name}/subarray-ids.json"
+        )
+    else:
+        array_element_ids = gen.collect_data_from_file(
+            Path(repository_url) / "array-element-ids.json"
+        )
+        sub_arrays = gen.collect_data_from_file(Path(repository_url) / "subarray-ids.json")
+
+    return _get_ctao_layouts_per_site(site, sub_arrays, array_element_ids)
+
+
+def _get_ctao_layouts_per_site(site, sub_arrays, array_element_ids):
+    """
+    Get array layouts for CTAO sites.
+
+    Parameters
+    ----------
+    site : str
+        Site identifier.
+    sub_arrays : dict
+        Sub-array definitions.
+    array_element_ids : dict
+        Array element definitions.
+
+    Returns
+    -------
+    dict
+        Array layouts for CTAO sites.
+    """
+    layouts_per_site = []
+
+    for array in sub_arrays.get("subarrays", []):
+        elements = []
+        for ids in array.get("array_element_ids", []):
+            element_name = _get_ctao_array_element_name(ids, array_element_ids)
+            if names.get_site_from_array_element_name(element_name) != site:
+                break
+            elements.append(element_name)
+        if len(elements) > 0:
+            array_layout = {
+                "name": array.get("name"),
+                "elements": elements,
+            }
+            layouts_per_site.append(array_layout)
+
+    _logger.info(f"CTAO array layout definition: {layouts_per_site}")
+    return layouts_per_site
+
+
+def _get_ctao_array_element_name(ids, array_element_ids):
+    """Return array element name for common identifier."""
+    for element in array_element_ids.get("array_elements", []):
+        if element.get("id") == ids:
+            return element.get("name")
+    return None
+
+
+def merge_array_layouts(layouts_1, layouts_2):
+    """
+    Compare two array layout dictionaries and merge them.
+
+    Parameters
+    ----------
+    layouts_1 : dict
+        Array layout dictionary 1.
+    layouts_2 : dict
+        Array layout dictionary 2.
+
+    Returns
+    -------
+    dict
+        Merged array layout dictionary based on layout_1.
+    """
+    merged_layout = layouts_1
+    for layout_2 in layouts_2:
+        layout_found = False
+        for layout_1 in layouts_1.get("value", {}):
+            if sorted(layout_1["elements"]) == sorted(layout_2["elements"]):
+                print(
+                    f"Equal telescope list: simtools '{layout_1['name']}' "
+                    f"and CTAO '{layout_2['name']}'"
+                )
+                layout_1["name"] = layout_2["name"]
+                layout_found = True
+        if not layout_found:
+            merged_layout["value"].append(
+                {
+                    "name": layout_2["name"],
+                    "elements": layout_2["elements"],
+                }
+            )
+            _logger.info(f"Adding {layout_2['name']} with {layout_2['elements']}")
+    return merged_layout
+
+
+def write_array_layouts(array_layouts, args_dict, db_config):
+    """
+    Write array layouts as model parameter.
+
+    Parameters
+    ----------
+    args_dict : dict
+        Command line arguments.
+    array_layouts : dict
+        Array layouts to be written.
+    db_config : dict
+        Database configuration.
+    """
+    site = args_dict.get("site") or array_layouts.get("site")
+    _logger.info(f"Writing updated array layouts to the database for site {site}.")
+
+    io_handler_instance = io_handler.IOHandler()
+    io_handler_instance.set_paths(
+        output_path=args_dict["output_path"],
+        use_plain_output_path=args_dict["use_plain_output_path"],
+    )
+    output_file = io_handler_instance.get_output_file(
+        f"array-layouts-{args_dict['updated_parameter_version']}.json"
+    )
+
+    ModelDataWriter.dump_model_parameter(
+        parameter_name="array_layouts",
+        value=array_layouts["value"],
+        instrument=site,
+        parameter_version=args_dict.get("updated_parameter_version"),
+        output_file=output_file,
+        use_plain_output_path=args_dict["use_plain_output_path"],
+        db_config=db_config,
+    )
+    MetadataCollector.dump(
+        args_dict,
+        output_file,
+        add_activity_name=True,
+    )
+
+
+def validate_array_layouts_with_db(production_table, array_layouts):
+    """
+    Validate array layouts against the production table in the database.
+
+    Confirm that every telescope defined in the array layouts exist in the
+    production table.
+
+    Parameters
+    ----------
+    production_table : dict
+        Production table from the database.
+    array_layouts : dict
+        Array layouts to be validated.
+
+    Returns
+    -------
+    dict
+        Validated array layouts.
+    """
+    db_elements = set(production_table.get("parameters", {}).keys())
+
+    invalid_array_elements = [
+        e
+        for layout in array_layouts.get("value", [])
+        for e in layout.get("elements", [])
+        if e not in db_elements
+    ]
+
+    if invalid_array_elements:
+        raise ValueError(f"Invalid array elements found: {invalid_array_elements}. ")
+
+    return array_layouts
+
+
+def get_array_layouts_from_parameter_file(
+    file_path, model_version, db_config, coordinate_system="ground"
+):
+    """
+    Retrieve array layouts from parameter file.
+
+    Parameters
+    ----------
+    file_path : str or Path
+        Path to the array layout parameter file.
+    model_version : str
+        Model version to retrieve.
+    db_config : dict
+        Database configuration.
+    coordinate_system : str
+        Coordinate system to use for the array elements (default is "ground").
+
+    Returns
+    -------
+    list
+        List of dictionaries containing array layout names and their elements.
+    """
+    array_layouts = gen.collect_data_from_file(file_path)
+    try:
+        value = array_layouts["value"]
+    except KeyError as exc:
+        raise ValueError("Missing 'value' key in layout file.") from exc
+    site = array_layouts.get("site")
+
+    layouts = []
+    for layout in value:
+        layouts.append(
+            _get_array_layout_dict(
+                db_config, model_version, site, None, layout["name"], coordinate_system
+            )
+        )
+    return layouts
+
+
+def get_array_layouts_from_db(
+    layout_name, site, model_version, db_config, coordinate_system="ground"
+):
+    """
+    Retrieve all array layouts from the database and return as list of astropy tables.
+
+    Parameters
+    ----------
+    layout_name : str
+        Name of the array layout to retrieve (for None, all layouts are retrieved).
+    site : str
+        Site identifier.
+    model_version : str
+        Model version to retrieve.
+    db_config : dict
+        Database configuration.
+    coordinate_system : str
+        Coordinate system to use for the array elements (default is "ground").
+
+    Returns
+    -------
+    list
+        List of dictionaries containing array layout names and their elements.
+    """
+    layout_names = []
+    if layout_name:
+        layout_names.append(layout_name)
+    else:
+        site_model = SiteModel(site=site, model_version=model_version, mongo_db_config=db_config)
+        layout_names = site_model.get_list_of_array_layouts()
+
+    layouts = []
+    for _layout_name in layout_names:
+        layouts.append(
+            _get_array_layout_dict(
+                db_config, model_version, site, None, _layout_name, coordinate_system
+            )
+        )
+    if len(layouts) == 1:
+        return layouts[0]
+    return layouts
+
+
+def get_array_layouts_using_telescope_lists_from_db(
+    telescope_lists, site, model_version, db_config, coordinate_system="ground"
+):
+    """
+    Retrieve array layouts from the database using telescope lists.
+
+    Parameters
+    ----------
+    telescope_lists : list
+        List of telescope lists to retrieve array layouts for.
+    site : str
+        Site identifier.
+    model_version : str
+        Model version to retrieve.
+    db_config : dict
+        Database configuration.
+    coordinate_system : str
+        Coordinate system to use for the array elements (default is "ground").
+
+    Returns
+    -------
+    list
+        List of dictionaries containing array layout names and their elements.
+
+    """
+    layouts = []
+    for telescope_list in telescope_lists:
+        _site = site
+        if _site is None:
+            sites = {names.get_site_from_array_element_name(t) for t in telescope_list}
+            if len(sites) != 1:
+                raise ValueError(
+                    f"Telescope list contains elements from multiple sites: {sites}."
+                    "Please specify a site."
+                )
+            _site = sites.pop()
+
+        layouts.append(
+            _get_array_layout_dict(
+                db_config, model_version, _site, telescope_list, None, coordinate_system
+            )
+        )
+    return layouts
+
+
+def get_array_layouts_from_file(file_path):
+    """
+    Retrieve array layout(s) from astropy table file(s).
+
+    Parameters
+    ----------
+    file_path : str or Path or list of str or list of Path
+        Path(s) to array layout files(s).
+
+    Returns
+    -------
+    list
+        List of dictionaries containing array layout names and their elements.
+    """
+    if isinstance(file_path, str | Path):
+        file_path = [file_path]
+
+    layouts = []
+    for _file in file_path:
+        layouts.append(
+            {
+                "name": (Path(_file).name).split(".")[0],
+                "array_elements": data_reader.read_table_from_file(file_name=_file),
+            }
+        )
+    return layouts
+
+
+def _get_array_layout_dict(
+    db_config, model_version, site, telescope_list, layout_name, coordinate_system
+):
+    """Return array layout dictionary for a given telescope list."""
+    array_model = ArrayModel(
+        mongo_db_config=db_config,
+        model_version=model_version,
+        site=site,
+        array_elements=telescope_list,
+        layout_name=layout_name,
+    )
+    return {
+        "name": layout_name if layout_name else "list",
+        "site": site,
+        "array_elements": array_model.export_array_elements_as_table(
+            coordinate_system=coordinate_system
+        ),
+    }

simtools/model/array_model.py

@@ -36,6 +36,10 @@ class ArrayModel:
     array_elements: Union[str, Path, List[str]], optional
         Array element definitions (list of array element or path to file with
         the array element positions).
+    sim_telarray_seeds : dict, optional
+        Dictionary with configuration for sim_telarray random instrument setup.
+    simtel_path: str, Path, optional
+        Path to the sim_telarray installation directory.
     """
 
     def __init__(
@@ -46,6 +50,8 @@ class ArrayModel:
         site: str | None = None,
         layout_name: str | None = None,
         array_elements: str | Path | list[str] | None = None,
+        sim_telarray_seeds: dict | None = None,
+        simtel_path: str | Path | None = None,
     ):
         """Initialize ArrayModel."""
         self._logger = logging.getLogger(__name__)
@@ -65,6 +71,8 @@ class ArrayModel:
 
         self._telescope_model_files_exported = False
         self._array_model_file_exported = False
+        self.sim_telarray_seeds = sim_telarray_seeds
+        self.simtel_path = simtel_path
 
     def _initialize(self, site: str, array_elements_config: str | Path | list[str]):
         """
@@ -116,6 +124,26 @@ class ArrayModel:
         telescope_model = self._build_telescope_models(site_model, array_elements)
         return array_elements, site_model, telescope_model
 
+    @property
+    def config_file_path(self) -> Path:
+        """
+        Return the path of the array config file for sim_telarray.
+
+        Returns
+        -------
+        Path
+            Path of the exported config file for sim_telarray.
+        """
+        if self._config_file_path is None:
+            config_file_name = names.simtel_config_file_name(
+                array_name=self.layout_name,
+                site=self.site_model.site,
+                model_version=self.model_version,
+                label=self.label,
+            )
+            self._config_file_path = self.get_config_directory().joinpath(config_file_name)
+        return self._config_file_path
+
     @property
     def number_of_telescopes(self) -> int:
         """
@@ -140,6 +168,34 @@ class ArrayModel:
         """
         return self.site_model.site
 
+    @property
+    def model_version(self):
+        """Model version."""
+        return self._model_version
+
+    @model_version.setter
+    def model_version(self, model_version):
+        """
+        Set model version.
+
+        Parameters
+        ----------
+        _model_version: str or list
+            Model version (e.g., "6.0.0").
+            If a list is passed, it must contain exactly one element,
+            and only that element will be used.
+
+        Raises
+        ------
+        ValueError
+            If more than one model version is passed.
+        """
+        if isinstance(model_version, list):
+            raise ValueError(
+                f"Only one model version can be passed to {self.__class__.__name__}, not a list."
+            )
+        self._model_version = model_version
+
     def _build_telescope_models(self, site_model: SiteModel, array_elements: dict) -> dict:
         """
         Build the the telescope models for all telescopes of this array.
@@ -187,7 +243,7 @@ class ArrayModel:
             )
             if name not in exported_models:
                 self._logger.debug(f"Exporting configuration file for telescope {name}")
-                tel_model.export_config_file()
+                tel_model.write_sim_telarray_config_file()
                 exported_models.append(name)
             else:
                 self._logger.debug(
@@ -196,29 +252,23 @@ class ArrayModel:
 
         self._telescope_model_files_exported = True
 
-    def export_simtel_array_config_file(self):
+    def export_sim_telarray_config_file(self):
         """Export sim_telarray configuration file for the array into the model directory."""
-        # Setting file name and the location
-        config_file_name = names.simtel_config_file_name(
-            array_name=self.layout_name,
-            site=self.site_model.site,
-            model_version=self.model_version,
-            label=self.label,
-        )
-        self._config_file_path = self.get_config_directory().joinpath(config_file_name)
+        self.site_model.export_model_files()
 
-        # Writing parameters to the file
-        self._logger.info(f"Writing array configuration file into {self._config_file_path}")
+        self._logger.info(f"Writing array configuration file into {self.config_file_path}")
         simtel_writer = SimtelConfigWriter(
             site=self.site_model.site,
             layout_name=self.layout_name,
             model_version=self.model_version,
             label=self.label,
+            simtel_path=self.simtel_path,
         )
         simtel_writer.write_array_config_file(
-            config_file_path=self._config_file_path,
+            config_file_path=self.config_file_path,
             telescope_model=self.telescope_model,
             site_model=self.site_model,
+            sim_telarray_seeds=self.sim_telarray_seeds,
         )
         self._array_model_file_exported = True
 
@@ -231,21 +281,7 @@ class ArrayModel:
         if not self._telescope_model_files_exported:
             self.export_simtel_telescope_config_files()
         if not self._array_model_file_exported:
-            self.export_simtel_array_config_file()
-
-    def get_config_file(self) -> Path:
-        """
-        Return the path of the array config file for sim_telarray.
-
-        A new config file is produced if the file is not updated.
-
-        Returns
-        -------
-        Path
-            Path of the exported config file for sim_telarray.
-        """
-        self.export_all_simtel_config_files()
-        return self._config_file_path
+            self.export_sim_telarray_config_file()
 
     def get_config_directory(self) -> Path:
         """
@@ -257,7 +293,9 @@ class ArrayModel:
             Path of the config directory path for sim_telarray.
         """
         if self._config_file_directory is None:
-            self._config_file_directory = self.io_handler.get_output_directory(self.label, "model")
+            self._config_file_directory = self.io_handler.get_output_directory(
+                self.label, f"model/{self.model_version}"
+            )
         return self._config_file_directory
 
     def _load_array_element_positions_from_file(
@@ -331,6 +369,7 @@ class ArrayModel:
             "type": "float64",
             "file": False,
             "meta_parameter": False,
+            "model_parameter_schema_version": "0.1.0",
         }
 
     def _get_array_elements_from_list(self, array_elements_list: list[str]) -> dict: