gammasimtools 0.11.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

simtools/data_model/model_data_writer.py

@@ -6,7 +6,6 @@ from pathlib import Path
 
 import astropy.units as u
 import numpy as np
-import yaml
 from astropy.io.registry.base import IORegistryError
 
 import simtools.utils.general as gen
@@ -95,7 +94,7 @@ class ModelDataWriter:
             Dictionary with configuration parameters (including output file name and path).
         output_file: string or Path
             Name of output file (args["output_file"] is used if this parameter is not set).
-        metadata: dict
+        metadata: MetadataCollector object
             Metadata to be written.
         product_data: astropy Table
             Model data to be written
@@ -174,11 +173,11 @@ class ModelDataWriter:
         if metadata_input_dict is not None:
             metadata_input_dict["output_file"] = output_file
             metadata_input_dict["output_file_format"] = Path(output_file).suffix.lstrip(".")
-            metadata = MetadataCollector(args_dict=metadata_input_dict).get_top_level_metadata()
-            writer.write_metadata_to_yml(
-                metadata=metadata, yml_file=output_path / f"{Path(output_file).stem}"
+            metadata = MetadataCollector(args_dict=metadata_input_dict)
+            metadata.write(output_path / Path(output_file))
+            unique_id = (
+                metadata.get_top_level_metadata().get("cta", {}).get("product", {}).get("id")
             )
-            unique_id = metadata.get("cta", {}).get("product", {}).get("id")
 
         _json_dict = writer.get_validated_parameter_dict(
             parameter_name, value, instrument, parameter_version, unique_id
@@ -364,7 +363,7 @@ class ModelDataWriter:
         ----------
         product_data: astropy Table
             Model data to be written
-        metadata: dict
+        metadata: MetadataCollector object
             Metadata to be written.
 
         Raises
@@ -377,7 +376,9 @@ class ModelDataWriter:
             return
 
         if metadata is not None:
-            product_data.meta.update(gen.change_dict_keys_case(metadata, False))
+            product_data.meta.update(
+                gen.change_dict_keys_case(metadata.get_top_level_metadata(), False)
+            )
 
         self._logger.info(f"Writing data to {self.product_data_file}")
         if isinstance(product_data, dict) and Path(self.product_data_file).suffix == ".json":
@@ -390,6 +391,8 @@ class ModelDataWriter:
         except IORegistryError:
             self._logger.error(f"Error writing model data to {self.product_data_file}.")
             raise
+        if metadata is not None:
+            metadata.write(self.product_data_file, add_activity_name=True)
 
     def write_dict_to_model_parameter_json(self, file_name, data_dict):
         """
@@ -449,53 +452,6 @@ class ModelDataWriter:
             pass
         return data_dict
 
-    def write_metadata_to_yml(self, metadata, yml_file=None, keys_lower_case=False):
-        """
-        Write model metadata file (yaml file format).
-
-        Parameters
-        ----------
-        metadata: dict
-            Metadata to be stored
-        yml_file: str
-            Name of output file.
-        keys_lower_case: bool
-            Write yaml keys in lower case.
-
-        Returns
-        -------
-        str
-            Name of output file
-
-        Raises
-        ------
-        FileNotFoundError
-            If yml_file not found.
-        TypeError
-            If yml_file is not defined.
-        """
-        try:
-            yml_file = names.file_name_with_version(
-                yml_file or self.product_data_file, ".metadata.yml"
-            )
-            with open(yml_file, "w", encoding="UTF-8") as file:
-                yaml.safe_dump(
-                    gen.change_dict_keys_case(metadata, keys_lower_case),
-                    file,
-                    sort_keys=False,
-                )
-            self._logger.info(f"Writing metadata to {yml_file}")
-            return yml_file
-        except FileNotFoundError:
-            self._logger.error(f"Error writing model data to {yml_file}")
-            raise
-        except AttributeError:
-            self._logger.error("No metadata defined for writing")
-            raise
-        except TypeError:
-            self._logger.error("No output file for metadata defined")
-            raise
-
     @staticmethod
     def _astropy_data_format(product_data_format):
         """

simtools/data_model/schema.py

@@ -108,7 +108,7 @@ def validate_dict_using_schema(data, schema_file=None, json_schema=None):
     """
     if json_schema is None and schema_file is None:
         _logger.warning(f"No schema provided for validation of {data}")
-        return
+        return None
     if json_schema is None:
         json_schema = load_schema(
             schema_file,
@@ -131,6 +131,7 @@ def validate_dict_using_schema(data, schema_file=None, json_schema=None):
         raise FileNotFoundError(f"Meta schema URL does not exist: {data['meta_schema_url']}")
 
     _logger.debug(f"Successful validation of data using schema ({json_schema.get('name')})")
+    return data
 
 
 def load_schema(schema_file=None, schema_version=None):

simtools/data_model/validate_data.py

@@ -220,7 +220,7 @@ class DataValidator:
                 json_schema=self._get_data_description(index).get("json_schema"),
             )
         else:
-            self._check_data_type(np.array(value).dtype, index)
+            self._check_data_type(np.array(value).dtype, index, value)
 
         if self.data_dict.get("type") not in ("string", "dict", "file"):
             self._check_for_not_a_number(value, index)
@@ -436,7 +436,7 @@ class DataValidator:
 
         return u.Unit(reference_unit)
 
-    def _check_data_type(self, dtype, column_name):
+    def _check_data_type(self, dtype, column_name, value=None):
         """
         Check column data type.
 
@@ -446,6 +446,8 @@ class DataValidator:
             data type
         column_name: str
             column name
+        value: value
+            value to be tested (optional)
 
         Raises
         ------
@@ -456,7 +458,7 @@ class DataValidator:
         reference_dtype = self._get_data_description(column_name).get("type", None)
         if not gen.validate_data_type(
             reference_dtype=reference_dtype,
-            value=None,
+            value=value,
             dtype=dtype,
             allow_subtypes=(not self.check_exact_data_type),
         ):

simtools/db/db_handler.py

@@ -2,6 +2,7 @@
 
 import logging
 import re
+from collections import defaultdict
 from pathlib import Path
 from threading import Lock
 
@@ -13,6 +14,7 @@ from pymongo import MongoClient
 
 from simtools.data_model import validate_data
 from simtools.io_operations import io_handler
+from simtools.simtel import simtel_table_reader
 from simtools.utils import names, value_conversion
 
 __all__ = ["DatabaseHandler"]
@@ -40,7 +42,7 @@ jsonschema_db_dict = {
         "db_api_user": {"type": "string", "description": "API username"},
         "db_api_pw": {"type": "string", "description": "Password for the API user"},
         "db_api_authentication_database": {
-            "type": "string",
+            "type": ["string", "null"],
             "default": "admin",
             "description": "DB with user info (optional)",
         },
@@ -113,13 +115,16 @@ class DatabaseHandler:
         direct_connection = self.mongo_db_config["db_server"] in (
             "localhost",
             "simtools-mongodb",
+            "mongodb",
         )
         return MongoClient(
             self.mongo_db_config["db_server"],
             port=self.mongo_db_config["db_api_port"],
             username=self.mongo_db_config["db_api_user"],
             password=self.mongo_db_config["db_api_pw"],
-            authSource=self.mongo_db_config.get("db_api_authentication_database", "admin"),
+            authSource=self.mongo_db_config.get("db_api_authentication_database")
+            if self.mongo_db_config.get("db_api_authentication_database")
+            else "admin",
             directConnection=direct_connection,
             ssl=not direct_connection,
             tlsallowinvalidhostnames=True,
@@ -175,48 +180,62 @@ class DatabaseHandler:
     def get_model_parameter(
         self,
         parameter,
-        parameter_version,
         site,
         array_element_name,
+        parameter_version=None,
+        model_version=None,
     ):
         """
-        Get a model parameter using the parameter version.
+        Get a single model parameter using model or parameter version.
+
+        Note that this function should not be called in a loop for many parameters,
+        as it each call queries the database.
 
         Parameters
         ----------
         parameter: str
             Name of the parameter.
-        parameter_version: str
-            Version of the parameter.
         site: str
             Site name.
         array_element_name: str
-            Name of the array element model (e.g. MSTN, SSTS).
+            Name of the array element model.
+        parameter_version: str
+            Version of the parameter.
+        model_version: str
+            Version of the model.
 
         Returns
         -------
         dict containing the parameter
 
         """
+        collection_name = names.get_collection_name_from_parameter_name(parameter)
+        if model_version:
+            production_table = self._read_production_table_from_mongo_db(
+                collection_name, model_version
+            )
+            array_element_list = self._get_array_element_list(
+                array_element_name, site, production_table, collection_name
+            )
+            for array_element in reversed(array_element_list):
+                parameter_version = (
+                    production_table["parameters"].get(array_element, {}).get(parameter)
+                )
+                if parameter_version:
+                    array_element_name = array_element
+                    break
+
         query = {
             "parameter_version": parameter_version,
             "parameter": parameter,
         }
-        if array_element_name is not None:
+        if array_element_name:
             query["instrument"] = array_element_name
-        if site is not None:
+        if site:
             query["site"] = site
-        return self._read_mongo_db(
-            query=query, collection_name=names.get_collection_name_from_parameter_name(parameter)
-        )
+        return self._read_mongo_db(query=query, collection_name=collection_name)
 
-    def get_model_parameters(
-        self,
-        site,
-        array_element_name,
-        collection,
-        model_version=None,
-    ):
+    def get_model_parameters(self, site, array_element_name, collection, model_version):
         """
         Get model parameters using the model version.
 
@@ -237,22 +256,44 @@ class DatabaseHandler:
         -------
         dict containing the parameters
         """
-        model_versions = (
-            self.get_model_versions(collection) if model_version is None else [model_version]
+        pars = {}
+        production_table = self._read_production_table_from_mongo_db(collection, model_version)
+        array_element_list = self._get_array_element_list(
+            array_element_name, site, production_table, collection
         )
+        for array_element in array_element_list:
+            pars.update(
+                self._get_parameter_for_model_version(
+                    array_element, model_version, site, collection, production_table
+                )
+            )
+        return pars
 
-        pars = {}
-        for _model_version in model_versions:
-            production_table = self._read_production_table_from_mongo_db(collection, _model_version)
-            array_element_list = self._get_array_element_list(
-                array_element_name, site, production_table, collection
+    def get_model_parameters_for_all_model_versions(self, site, array_element_name, collection):
+        """
+        Get model parameters for all model versions.
+
+        Queries parameters for design and for the specified array element (if necessary).
+
+        Parameters
+        ----------
+        site: str
+            Site name.
+        array_element_name: str
+            Name of the array element model (e.g. LSTN-01, MSTx-FlashCam, ILLN-01).
+        collection: str
+            Collection of array element (e.g. telescopes, calibration_devices).
+
+        Returns
+        -------
+        dict containing the parameters with model version as first key
+        """
+        pars = defaultdict(dict)
+        for _model_version in self.get_model_versions(collection):
+            parameter_data = self.get_model_parameters(
+                site, array_element_name, collection, _model_version
             )
-            for array_element in array_element_list:
-                pars.update(
-                    self._get_parameter_for_model_version(
-                        array_element, _model_version, site, collection, production_table
-                    )
-                )
+            pars[_model_version].update(parameter_data)
         return pars
 
     def _get_parameter_for_model_version(
@@ -329,9 +370,54 @@ class DatabaseHandler:
             return [collection for collection in collections if not collection.startswith("fs.")]
         return collections
 
+    def export_model_file(
+        self,
+        parameter,
+        site,
+        array_element_name,
+        model_version=None,
+        parameter_version=None,
+        export_file_as_table=False,
+    ):
+        """
+        Export single model file from the DB identified by the parameter name.
+
+        The parameter can be identified by model or parameter version.
+        Files can be exported as astropy tables (ecsv format).
+
+        Parameters
+        ----------
+        parameter: str
+            Name of the parameter.
+        site: str
+            Site name.
+        array_element_name: str
+            Name of the array element model (e.g. MSTN, SSTS).
+        parameter_version: str
+            Version of the parameter.
+        model_version: str
+            Version of the model.
+        export_file_as_table: bool
+            If True, export the file as an astropy table (ecsv format).
+        """
+        parameters = self.get_model_parameter(
+            parameter,
+            site,
+            array_element_name,
+            parameter_version=parameter_version,
+            model_version=model_version,
+        )
+        self.export_model_files(parameters=parameters, dest=self.io_handler.get_output_directory())
+        if export_file_as_table:
+            return simtel_table_reader.read_simtel_table(
+                parameter,
+                self.io_handler.get_output_directory().joinpath(parameters[parameter]["value"]),
+            )
+        return None
+
     def export_model_files(self, parameters=None, file_names=None, dest=None, db_name=None):
         """
-        Export files from the DB to the model directory.
+        Export models files from the DB to given directory.
 
         The files to be exported can be specified by file_name or retrieved from the database
         using the parameters dictionary.

simtools/model/model_parameter.py

@@ -6,12 +6,10 @@ import shutil
 from copy import copy
 
 import astropy.units as u
-from astropy.table import Table
 
 import simtools.utils.general as gen
 from simtools.db import db_handler
 from simtools.io_operations import io_handler
-from simtools.simtel import simtel_table_reader
 from simtools.simtel.simtel_config_writer import SimtelConfigWriter
 from simtools.utils import names
 
@@ -517,35 +515,6 @@ class ModelParameter:
         self.db.export_model_files(parameters=pars_from_db, dest=self.config_file_directory)
         self._is_exported_model_files_up_to_date = True
 
-    def get_model_file_as_table(self, par_name):
-        """
-        Return tabular data from file as astropy table.
-
-        Parameters
-        ----------
-        par_name: str
-            Name of the parameter.
-
-        Returns
-        -------
-        Table
-            Astropy table.
-        """
-        _par_entry = {}
-        try:
-            _par_entry[par_name] = self._parameters[par_name]
-        except KeyError as exc:
-            raise ValueError(f"Parameter {par_name} not found in the model.") from exc
-        self.db.export_model_files(parameters=_par_entry, dest=self.config_file_directory)
-        if _par_entry[par_name]["value"].endswith("ecsv"):
-            return Table.read(
-                self.config_file_directory.joinpath(_par_entry[par_name]["value"]),
-                format="ascii.ecsv",
-            )
-        return simtel_table_reader.read_simtel_table(
-            par_name, self.config_file_directory.joinpath(_par_entry[par_name]["value"])
-        )
-
     def export_config_file(self):
         """Export the config file used by sim_telarray."""
         # Exporting model file