gammasimtools 0.12.0__py3-none-any.whl → 0.14.0__py3-none-any.whl

simtools/data_model/model_data_writer.py

@@ -6,7 +6,6 @@ from pathlib import Path
 
 import astropy.units as u
 import numpy as np
-import yaml
 from astropy.io.registry.base import IORegistryError
 
 import simtools.utils.general as gen
@@ -95,7 +94,7 @@ class ModelDataWriter:
             Dictionary with configuration parameters (including output file name and path).
         output_file: string or Path
             Name of output file (args["output_file"] is used if this parameter is not set).
-        metadata: dict
+        metadata: MetadataCollector object
             Metadata to be written.
         product_data: astropy Table
             Model data to be written
@@ -174,11 +173,11 @@ class ModelDataWriter:
         if metadata_input_dict is not None:
             metadata_input_dict["output_file"] = output_file
             metadata_input_dict["output_file_format"] = Path(output_file).suffix.lstrip(".")
-            metadata = MetadataCollector(args_dict=metadata_input_dict).get_top_level_metadata()
-            writer.write_metadata_to_yml(
-                metadata=metadata, yml_file=output_path / f"{Path(output_file).stem}"
+            metadata = MetadataCollector(args_dict=metadata_input_dict)
+            metadata.write(output_path / Path(output_file))
+            unique_id = (
+                metadata.get_top_level_metadata().get("cta", {}).get("product", {}).get("id")
             )
-            unique_id = metadata.get("cta", {}).get("product", {}).get("id")
 
         _json_dict = writer.get_validated_parameter_dict(
             parameter_name, value, instrument, parameter_version, unique_id
@@ -270,6 +269,7 @@ class ModelDataWriter:
             "unit": unit,
             "type": self._get_parameter_type(),
             "file": self._parameter_is_a_file(),
+            "meta_parameter": False,
         }
         return self.validate_and_transform(
             product_data_dict=data_dict,
@@ -364,7 +364,7 @@ class ModelDataWriter:
         ----------
         product_data: astropy Table
             Model data to be written
-        metadata: dict
+        metadata: MetadataCollector object
             Metadata to be written.
 
         Raises
@@ -377,7 +377,9 @@ class ModelDataWriter:
             return
 
         if metadata is not None:
-            product_data.meta.update(gen.change_dict_keys_case(metadata, False))
+            product_data.meta.update(
+                gen.change_dict_keys_case(metadata.get_top_level_metadata(), False)
+            )
 
         self._logger.info(f"Writing data to {self.product_data_file}")
         if isinstance(product_data, dict) and Path(self.product_data_file).suffix == ".json":
@@ -390,6 +392,8 @@ class ModelDataWriter:
         except IORegistryError:
             self._logger.error(f"Error writing model data to {self.product_data_file}.")
             raise
+        if metadata is not None:
+            metadata.write(self.product_data_file, add_activity_name=True)
 
     def write_dict_to_model_parameter_json(self, file_name, data_dict):
         """
@@ -423,7 +427,7 @@ class ModelDataWriter:
         """
         Prepare data dictionary for writing to json file.
 
-        Ensure sim_telarray style lists as strings.
+        Ensure sim_telarray style lists as strings 'type' and 'unit' entries.
         Replace "None" with "null" for unit field.
 
         Parameters
@@ -438,64 +442,17 @@ class ModelDataWriter:
 
         """
         try:
-            data_dict["value"] = gen.convert_list_to_string(data_dict["value"])
-            data_dict["unit"] = gen.convert_list_to_string(data_dict["unit"], comma_separated=True)
-            data_dict["type"] = gen.convert_list_to_string(
-                data_dict["type"], comma_separated=True, collapse_list=True
-            )
             if isinstance(data_dict["unit"], str):
                 data_dict["unit"] = data_dict["unit"].replace("None", "null")
+            elif isinstance(data_dict["unit"], list):
+                data_dict["unit"] = [
+                    unit.replace("None", "null") if isinstance(unit, str) else unit
+                    for unit in data_dict["unit"]
+                ]
         except KeyError:
             pass
         return data_dict
 
-    def write_metadata_to_yml(self, metadata, yml_file=None, keys_lower_case=False):
-        """
-        Write model metadata file (yaml file format).
-
-        Parameters
-        ----------
-        metadata: dict
-            Metadata to be stored
-        yml_file: str
-            Name of output file.
-        keys_lower_case: bool
-            Write yaml keys in lower case.
-
-        Returns
-        -------
-        str
-            Name of output file
-
-        Raises
-        ------
-        FileNotFoundError
-            If yml_file not found.
-        TypeError
-            If yml_file is not defined.
-        """
-        try:
-            yml_file = names.file_name_with_version(
-                yml_file or self.product_data_file, ".metadata.yml"
-            )
-            with open(yml_file, "w", encoding="UTF-8") as file:
-                yaml.safe_dump(
-                    gen.change_dict_keys_case(metadata, keys_lower_case),
-                    file,
-                    sort_keys=False,
-                )
-            self._logger.info(f"Writing metadata to {yml_file}")
-            return yml_file
-        except FileNotFoundError:
-            self._logger.error(f"Error writing model data to {yml_file}")
-            raise
-        except AttributeError:
-            self._logger.error("No metadata defined for writing")
-            raise
-        except TypeError:
-            self._logger.error("No output file for metadata defined")
-            raise
-
     @staticmethod
     def _astropy_data_format(product_data_format):
         """

simtools/data_model/schema.py

@@ -108,7 +108,7 @@ def validate_dict_using_schema(data, schema_file=None, json_schema=None):
     """
     if json_schema is None and schema_file is None:
         _logger.warning(f"No schema provided for validation of {data}")
-        return
+        return None
     if json_schema is None:
         json_schema = load_schema(
             schema_file,
@@ -131,6 +131,7 @@ def validate_dict_using_schema(data, schema_file=None, json_schema=None):
         raise FileNotFoundError(f"Meta schema URL does not exist: {data['meta_schema_url']}")
 
     _logger.debug(f"Successful validation of data using schema ({json_schema.get('name')})")
+    return data
 
 
 def load_schema(schema_file=None, schema_version=None):

simtools/data_model/validate_data.py

@@ -220,7 +220,7 @@ class DataValidator:
                 json_schema=self._get_data_description(index).get("json_schema"),
             )
         else:
-            self._check_data_type(np.array(value).dtype, index)
+            self._check_data_type(np.array(value).dtype, index, value)
 
         if self.data_dict.get("type") not in ("string", "dict", "file"):
             self._check_for_not_a_number(value, index)
@@ -436,7 +436,7 @@ class DataValidator:
 
         return u.Unit(reference_unit)
 
-    def _check_data_type(self, dtype, column_name):
+    def _check_data_type(self, dtype, column_name, value=None):
         """
         Check column data type.
 
@@ -446,6 +446,8 @@ class DataValidator:
             data type
         column_name: str
             column name
+        value: value
+            value to be tested (optional)
 
         Raises
         ------
@@ -456,7 +458,7 @@ class DataValidator:
         reference_dtype = self._get_data_description(column_name).get("type", None)
         if not gen.validate_data_type(
             reference_dtype=reference_dtype,
-            value=None,
+            value=value,
             dtype=dtype,
             allow_subtypes=(not self.check_exact_data_type),
         ):

simtools/db/db_handler.py

@@ -2,6 +2,7 @@
 
 import logging
 import re
+from collections import defaultdict
 from pathlib import Path
 from threading import Lock
 
@@ -13,7 +14,8 @@ from pymongo import MongoClient
 
 from simtools.data_model import validate_data
 from simtools.io_operations import io_handler
-from simtools.utils import names, value_conversion
+from simtools.simtel import simtel_table_reader
+from simtools.utils import general, names, value_conversion
 
 __all__ = ["DatabaseHandler"]
 
@@ -113,6 +115,7 @@ class DatabaseHandler:
         direct_connection = self.mongo_db_config["db_server"] in (
             "localhost",
             "simtools-mongodb",
+            "mongodb",
         )
         return MongoClient(
             self.mongo_db_config["db_server"],
@@ -177,48 +180,62 @@ class DatabaseHandler:
     def get_model_parameter(
         self,
         parameter,
-        parameter_version,
         site,
         array_element_name,
+        parameter_version=None,
+        model_version=None,
     ):
         """
-        Get a model parameter using the parameter version.
+        Get a single model parameter using model or parameter version.
+
+        Note that this function should not be called in a loop for many parameters,
+        as it each call queries the database.
 
         Parameters
         ----------
         parameter: str
             Name of the parameter.
-        parameter_version: str
-            Version of the parameter.
         site: str
             Site name.
         array_element_name: str
-            Name of the array element model (e.g. MSTN, SSTS).
+            Name of the array element model.
+        parameter_version: str
+            Version of the parameter.
+        model_version: str
+            Version of the model.
 
         Returns
         -------
         dict containing the parameter
 
         """
+        collection_name = names.get_collection_name_from_parameter_name(parameter)
+        if model_version:
+            production_table = self._read_production_table_from_mongo_db(
+                collection_name, model_version
+            )
+            array_element_list = self._get_array_element_list(
+                array_element_name, site, production_table, collection_name
+            )
+            for array_element in reversed(array_element_list):
+                parameter_version = (
+                    production_table["parameters"].get(array_element, {}).get(parameter)
+                )
+                if parameter_version:
+                    array_element_name = array_element
+                    break
+
         query = {
             "parameter_version": parameter_version,
             "parameter": parameter,
         }
-        if array_element_name is not None:
+        if array_element_name:
             query["instrument"] = array_element_name
-        if site is not None:
+        if site:
             query["site"] = site
-        return self._read_mongo_db(
-            query=query, collection_name=names.get_collection_name_from_parameter_name(parameter)
-        )
+        return self._read_mongo_db(query=query, collection_name=collection_name)
 
-    def get_model_parameters(
-        self,
-        site,
-        array_element_name,
-        collection,
-        model_version=None,
-    ):
+    def get_model_parameters(self, site, array_element_name, collection, model_version):
         """
         Get model parameters using the model version.
 
@@ -239,22 +256,48 @@ class DatabaseHandler:
         -------
         dict containing the parameters
         """
-        model_versions = (
-            self.get_model_versions(collection) if model_version is None else [model_version]
-        )
-
         pars = {}
-        for _model_version in model_versions:
-            production_table = self._read_production_table_from_mongo_db(collection, _model_version)
-            array_element_list = self._get_array_element_list(
-                array_element_name, site, production_table, collection
+        production_table = self._read_production_table_from_mongo_db(collection, model_version)
+        array_element_list = self._get_array_element_list(
+            array_element_name, site, production_table, collection
+        )
+        for array_element in array_element_list:
+            pars.update(
+                self._get_parameter_for_model_version(
+                    array_element, model_version, site, collection, production_table
+                )
             )
-            for array_element in array_element_list:
-                pars.update(
-                    self._get_parameter_for_model_version(
-                        array_element, _model_version, site, collection, production_table
-                    )
+        return pars
+
+    def get_model_parameters_for_all_model_versions(self, site, array_element_name, collection):
+        """
+        Get model parameters for all model versions.
+
+        Parameters
+        ----------
+        site: str
+            Site name.
+        array_element_name: str
+            Name of the array element model (e.g. LSTN-01, MSTx-FlashCam, ILLN-01).
+        collection: str
+            Collection of array element (e.g. telescopes, calibration_devices).
+
+        Returns
+        -------
+        dict containing the parameters with model version as first key
+        """
+        pars = defaultdict(dict)
+        for _model_version in self.get_model_versions(collection):
+            try:
+                parameter_data = self.get_model_parameters(
+                    site, array_element_name, collection, _model_version
                 )
+                pars[_model_version].update(parameter_data)
+            except KeyError:
+                self._logger.debug(
+                    f"Skipping model version {_model_version} - {array_element_name} not found"
+                )
+                continue
         return pars
 
     def _get_parameter_for_model_version(
@@ -331,9 +374,54 @@ class DatabaseHandler:
             return [collection for collection in collections if not collection.startswith("fs.")]
         return collections
 
+    def export_model_file(
+        self,
+        parameter,
+        site,
+        array_element_name,
+        model_version=None,
+        parameter_version=None,
+        export_file_as_table=False,
+    ):
+        """
+        Export single model file from the DB identified by the parameter name.
+
+        The parameter can be identified by model or parameter version.
+        Files can be exported as astropy tables (ecsv format).
+
+        Parameters
+        ----------
+        parameter: str
+            Name of the parameter.
+        site: str
+            Site name.
+        array_element_name: str
+            Name of the array element model (e.g. MSTN, SSTS).
+        parameter_version: str
+            Version of the parameter.
+        model_version: str
+            Version of the model.
+        export_file_as_table: bool
+            If True, export the file as an astropy table (ecsv format).
+        """
+        parameters = self.get_model_parameter(
+            parameter,
+            site,
+            array_element_name,
+            parameter_version=parameter_version,
+            model_version=model_version,
+        )
+        self.export_model_files(parameters=parameters, dest=self.io_handler.get_output_directory())
+        if export_file_as_table:
+            return simtel_table_reader.read_simtel_table(
+                parameter,
+                self.io_handler.get_output_directory().joinpath(parameters[parameter]["value"]),
+            )
+        return None
+
     def export_model_files(self, parameters=None, file_names=None, dest=None, db_name=None):
         """
-        Export files from the DB to the model directory.
+        Export models files from the DB to given directory.
 
         The files to be exported can be specified by file_name or retrieved from the database
         using the parameters dictionary.
@@ -711,10 +799,12 @@ class DatabaseHandler:
                     f"corresponding to the {par_dict['parameter']} parameter, must be provided."
                 )
             file_path = Path(file_prefix).joinpath(par_dict["value"])
+            if not general.is_utf8_file(file_path):
+                raise ValueError(f"File is not UTF-8 encoded: {file_path}")
             files_to_add_to_db.add(f"{file_path}")
 
         self._logger.info(
-            f"Adding a new entry to DB {db_name} and collection {db_name}:\n{par_dict}"
+            f"Adding a new entry to DB {db_name} and collection {collection_name}:\n{par_dict}"
         )
         collection.insert_one(par_dict)
 

simtools/dependencies.py

@@ -81,18 +81,53 @@ def get_sim_telarray_version():
 
 def get_corsika_version():
     """
-    Get the version of the corsika package.
+    Get the version of the CORSIKA package.
 
     Returns
     -------
     str
-        Version of the corsika package.
+        Version of the CORSIKA package.
     """
+    version = None
+    sim_telarray_path = os.getenv("SIMTOOLS_SIMTEL_PATH")
+    if sim_telarray_path is None:
+        _logger.warning("Environment variable SIMTOOLS_SIMTEL_PATH is not set.")
+        return None
+    corsika_command = Path(sim_telarray_path) / "corsika-run" / "corsika"
+
+    # Below I do not use the standard context manager because
+    # it makes mocking in the tests significantly more difficult
+    process = subprocess.Popen(  # pylint: disable=consider-using-with
+        corsika_command,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        stdin=subprocess.PIPE,
+        text=True,
+    )
+
+    # Capture output until it waits for input
+    while True:
+        line = process.stdout.readline()
+        if not line:
+            break
+        # Extract the version from the line "NUMBER OF VERSION :  7.7550"
+        if "NUMBER OF VERSION" in line:
+            version = line.split(":")[1].strip()
+            break
+        # Check for a specific prompt or indication that the program is waiting for input
+        if "DATA CARDS FOR RUN STEERING ARE EXPECTED FROM STANDARD INPUT" in line:
+            break
+
+    process.terminate()
+    # Check it's a valid version string
+    if version and re.match(r"\d+\.\d+", version):
+        return version
     try:
         build_opts = get_build_options()
     except (FileNotFoundError, TypeError):
-        _logger.warning("CORSIKA version not implemented yet.")
+        _logger.warning("Could not get CORSIKA version.")
         return None
+    _logger.debug("Getting the CORSIKA version from the build options.")
     return build_opts.get("corsika_version")
 
 

simtools/layout/array_layout.py

@@ -641,6 +641,7 @@ class ArrayLayout:
             "unit": "m",
             "type": "float64",
             "file": False,
+            "meta_parameter": False,
         }
 
     def get_number_of_telescopes(self):

simtools/layout/ctao_array_layouts.py

@@ -0,0 +1,172 @@
+"""Retrieve, merge, and write layouts from CTAO common identifiers repository."""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.data_model.metadata_collector import MetadataCollector
+from simtools.data_model.model_data_writer import ModelDataWriter
+from simtools.io_operations import io_handler
+from simtools.utils import names
+
+_logger = logging.getLogger(__name__)
+
+
+def retrieve_array_layouts(site, repository_url, branch_name="main"):
+    """
+    Retrieve array layouts from CTAO common identifiers repository.
+
+    Parameters
+    ----------
+    site : str
+        Site identifier.
+    repository_url : str
+        URL or path to CTAO common identifiers
+    branch_name : str
+        Repository branch to use for CTAO common identifiers.
+
+    Returns
+    -------
+    dict
+        Array layouts for all CTAO sites.
+    """
+    _logger.info(f"Retrieving array layouts from {repository_url} on branch {branch_name}.")
+
+    if gen.is_url(repository_url):
+        array_element_ids = gen.collect_data_from_http(
+            url=f"{repository_url}/{branch_name}/array-element-ids.json"
+        )
+        sub_arrays = gen.collect_data_from_http(
+            url=f"{repository_url}/{branch_name}/subarray-ids.json"
+        )
+    else:
+        array_element_ids = gen.collect_data_from_file(
+            Path(repository_url) / "array-element-ids.json"
+        )
+        sub_arrays = gen.collect_data_from_file(Path(repository_url) / "subarray-ids.json")
+
+    return _get_layouts_per_site(site, sub_arrays, array_element_ids)
+
+
+def _get_layouts_per_site(site, sub_arrays, array_element_ids):
+    """
+    Get array layouts for CTAO sites.
+
+    Parameters
+    ----------
+    site : str
+        Site identifier.
+    sub_arrays : dict
+        Sub-array definitions.
+    array_element_ids : dict
+        Array element definitions.
+
+    Returns
+    -------
+    dict
+        Array layouts for CTAO sites.
+    """
+    layouts_per_site = []
+
+    for array in sub_arrays.get("subarrays", []):
+        elements = []
+        for ids in array.get("array_element_ids", []):
+            element_name = _get_array_element_name(ids, array_element_ids)
+            if names.get_site_from_array_element_name(element_name) != site:
+                break
+            elements.append(element_name)
+        if len(elements) > 0:
+            array_layout = {
+                "name": array.get("name"),
+                "elements": elements,
+            }
+            layouts_per_site.append(array_layout)
+
+    _logger.info(f"CTAO array layout definition: {layouts_per_site}")
+    return layouts_per_site
+
+
+def _get_array_element_name(ids, array_element_ids):
+    """Return array element name for common identifier."""
+    for element in array_element_ids.get("array_elements", []):
+        if element.get("id") == ids:
+            return element.get("name")
+    return None
+
+
+def merge_array_layouts(layouts_1, layouts_2):
+    """
+    Compare array layout dictionaries and merge them.
+
+    Parameters
+    ----------
+    layouts_1 : dict
+        Array layout dictionary 1.
+    layouts_2 : dict
+        Array layout dictionary 2.
+
+    Returns
+    -------
+    dict
+        Merged array layout dictionary based on layout_1.
+    """
+    merged_layout = layouts_1
+    for layout_2 in layouts_2:
+        layout_found = False
+        for layout_1 in layouts_1.get("value", {}):
+            if sorted(layout_1["elements"]) == sorted(layout_2["elements"]):
+                print(
+                    f"Equal telescope list: simtools '{layout_1['name']}' "
+                    f"and CTAO '{layout_2['name']}'"
+                )
+                layout_1["name"] = layout_2["name"]
+                layout_found = True
+        if not layout_found:
+            merged_layout["value"].append(
+                {
+                    "name": layout_2["name"],
+                    "elements": layout_2["elements"],
+                }
+            )
+            _logger.info(f"Adding {layout_2['name']} with {layout_2['elements']}")
+    return merged_layout
+
+
+def write_array_layouts(array_layouts, args_dict, db_config):
+    """
+    Write array layouts as model parameter.
+
+    Parameters
+    ----------
+    args_dict : dict
+        Command line arguments.
+    array_layouts : dict
+        Array layouts to be written.
+    db_config : dict
+        Database configuration.
+    """
+    _logger.info(f"Writing updated array layouts to the database for site {args_dict['site']}.")
+
+    io_handler_instance = io_handler.IOHandler()
+    io_handler_instance.set_paths(
+        output_path=args_dict["output_path"],
+        use_plain_output_path=args_dict["use_plain_output_path"],
+    )
+    output_file = io_handler_instance.get_output_file(
+        f"array-layouts-{args_dict['updated_parameter_version']}.json"
+    )
+
+    ModelDataWriter.dump_model_parameter(
+        parameter_name="array_layouts",
+        value=array_layouts["value"],
+        instrument=args_dict["site"],
+        parameter_version=args_dict.get("updated_parameter_version"),
+        output_file=output_file,
+        use_plain_output_path=args_dict["use_plain_output_path"],
+        db_config=db_config,
+    )
+    MetadataCollector.dump(
+        args_dict,
+        output_file,
+        add_activity_name=True,
+    )