gammasimtools 0.10.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

simtools/corsika/corsika_histograms.py

@@ -944,10 +944,10 @@ class CorsikaHistograms:
 
         hist_2d_values_list, x_position_list, y_position_list = self.get_2d_photon_position_distr()
 
-        for i_hist, _ in enumerate(x_position_list):
+        for i_hist, x_pos in enumerate(x_position_list):
             hist_1d, bin_edges_1d = convert_2d_to_radial_distr(
                 hist_2d_values_list[i_hist],
-                x_position_list[i_hist],  # pylint: disable=unnecessary-list-index-lookup
+                x_pos,
                 y_position_list[i_hist],
                 bins=bins,
                 max_dist=max_dist,
@@ -1248,8 +1248,7 @@ class CorsikaHistograms:
                     meta_data=self._meta_dict,
                 )
                 self._logger.info(
-                    f"Writing 1D histogram with name {hdf5_table_name} to "
-                    f"{self.hdf5_file_name}."
+                    f"Writing 1D histogram with name {hdf5_table_name} to {self.hdf5_file_name}."
                 )
                 # overwrite takes precedence over append
                 if overwrite is True:
@@ -1382,8 +1381,7 @@ class CorsikaHistograms:
                 )
 
                 self._logger.info(
-                    f"Writing 2D histogram with name {hdf5_table_name} to "
-                    f"{self.hdf5_file_name}."
+                    f"Writing 2D histogram with name {hdf5_table_name} to {self.hdf5_file_name}."
                 )
                 # Always appending to table due to the file previously created
                 # by self._export_1d_histograms.

simtools/corsika/corsika_histograms_visualize.py

@@ -93,8 +93,7 @@ def _kernel_plot_2d_photons(histograms_instance, property_name, log_z=False):
         all_figs.append(fig)
         if histograms_instance.individual_telescopes is False:
             ax.set_title(
-                f"{histograms_instance.dict_2d_distributions[property_name]['file name']}"
-                "_all_tels"
+                f"{histograms_instance.dict_2d_distributions[property_name]['file name']}_all_tels"
             )
         else:
             ax.text(
@@ -285,8 +284,7 @@ def _kernel_plot_1d_photons(histograms_instance, property_name, log_y=True):
             ax.set_yscale("log")
         if histograms_instance.individual_telescopes is False:
             ax.set_title(
-                f"{histograms_instance.dict_1d_distributions[property_name]['file name']}"
-                "_all_tels"
+                f"{histograms_instance.dict_1d_distributions[property_name]['file name']}_all_tels"
             )
         else:
             ax.set_title(

simtools/data_model/metadata_collector.py

@@ -6,7 +6,6 @@ implementation of the observatory metadata model.
 
 """
 
-import datetime
 import getpass
 import logging
 import uuid
@@ -15,6 +14,7 @@ from pathlib import Path
 import simtools.constants
 import simtools.utils.general as gen
 import simtools.version
+from simtools.constants import METADATA_JSON_SCHEMA
 from simtools.data_model import metadata_model, schema
 from simtools.io_operations import io_handler
 from simtools.utils import names
@@ -96,9 +96,9 @@ class MetadataCollector:
 
         """
         try:
-            self.top_level_meta[self.observatory]["activity"][
-                "end"
-            ] = datetime.datetime.now().isoformat(timespec="seconds")
+            self.top_level_meta[self.observatory]["activity"]["end"] = (
+                gen.now_date_time_in_isoformat()
+            )
         except KeyError:
             pass
         return self.top_level_meta
@@ -199,8 +199,17 @@ class MetadataCollector:
         contact_dict: dict
             Dictionary for contact metadata fields.
         """
-        if contact_dict.get("name", None) is None:
+        contact_dict["name"] = contact_dict.get("name") or self.args_dict.get("user_name")
+        if contact_dict["name"] is None:
+            self._logger.warning("No user name provided, take user info from system level.")
             contact_dict["name"] = getpass.getuser()
+        meta_dict = {
+            "email": "user_mail",
+            "orcid": "user_orcid",
+            "organization": "user_organization",
+        }
+        for key, value in meta_dict.items():
+            contact_dict[key] = contact_dict.get(key) or self.args_dict.get(value)
 
     def _fill_context_meta(self, context_dict):
         """
@@ -266,7 +275,7 @@ class MetadataCollector:
             self._logger.error("Unknown metadata file format: %s", metadata_file_name)
             raise gen.InvalidConfigDataError
 
-        schema.validate_dict_using_schema(_input_metadata, None)
+        schema.validate_dict_using_schema(_input_metadata, schema_file=METADATA_JSON_SCHEMA)
 
         return gen.change_dict_keys_case(
             self._process_metadata_from_file(_input_metadata),
@@ -326,7 +335,7 @@ class MetadataCollector:
         self.schema_dict = self.get_data_model_schema_dict()
 
         product_dict["id"] = str(uuid.uuid4())
-        product_dict["creation_time"] = datetime.datetime.now().isoformat(timespec="seconds")
+        product_dict["creation_time"] = gen.now_date_time_in_isoformat()
         product_dict["description"] = self.schema_dict.get("description", None)
 
         # DATA:CATEGORY
@@ -366,7 +375,7 @@ class MetadataCollector:
         )
         if instrument_dict["ID"]:
             instrument_dict["class"] = names.get_collection_name_from_array_element_name(
-                instrument_dict["ID"]
+                instrument_dict["ID"], False
             )
 
     def _fill_process_meta(self, process_dict):
@@ -394,7 +403,7 @@ class MetadataCollector:
         activity_dict["name"] = self.args_dict.get("label", None)
         activity_dict["type"] = "software"
         activity_dict["id"] = self.args_dict.get("activity_id", "UNDEFINED_ACTIVITY_ID")
-        activity_dict["start"] = datetime.datetime.now().isoformat(timespec="seconds")
+        activity_dict["start"] = gen.now_date_time_in_isoformat()
         activity_dict["end"] = activity_dict["start"]
         activity_dict["software"]["name"] = "simtools"
         activity_dict["software"]["version"] = simtools.version.__version__

simtools/data_model/model_data_writer.py

@@ -12,6 +12,7 @@ from astropy.io.registry.base import IORegistryError
 import simtools.utils.general as gen
 from simtools.data_model import schema, validate_data
 from simtools.data_model.metadata_collector import MetadataCollector
+from simtools.db import db_handler
 from simtools.io_operations import io_handler
 from simtools.utils import names, value_conversion
 
@@ -126,6 +127,7 @@ class ModelDataWriter:
         output_path=None,
         use_plain_output_path=False,
         metadata_input_dict=None,
+        db_config=None,
     ):
         """
         Generate DB-style model parameter dict and write it to json file.
@@ -148,6 +150,8 @@ class ModelDataWriter:
             Use plain output path.
         metadata_input_dict: dict
             Input to metadata collector.
+        db_config: dict
+            Database configuration. If not None, check if parameter with the same version exists.
 
         Returns
         -------
@@ -161,21 +165,72 @@ class ModelDataWriter:
             output_path=output_path,
             use_plain_output_path=use_plain_output_path,
         )
-        _json_dict = writer.get_validated_parameter_dict(
-            parameter_name, value, instrument, parameter_version
-        )
-        writer.write_dict_to_model_parameter_json(output_file, _json_dict)
+        if db_config is not None:
+            writer.check_db_for_existing_parameter(
+                parameter_name, instrument, parameter_version, db_config
+            )
+
+        unique_id = None
         if metadata_input_dict is not None:
             metadata_input_dict["output_file"] = output_file
             metadata_input_dict["output_file_format"] = Path(output_file).suffix.lstrip(".")
+            metadata = MetadataCollector(args_dict=metadata_input_dict).get_top_level_metadata()
             writer.write_metadata_to_yml(
-                metadata=MetadataCollector(args_dict=metadata_input_dict).get_top_level_metadata(),
-                yml_file=output_path / f"{Path(output_file).stem}",
+                metadata=metadata, yml_file=output_path / f"{Path(output_file).stem}"
             )
+            unique_id = metadata.get("cta", {}).get("product", {}).get("id")
+
+        _json_dict = writer.get_validated_parameter_dict(
+            parameter_name, value, instrument, parameter_version, unique_id
+        )
+        writer.write_dict_to_model_parameter_json(output_file, _json_dict)
         return _json_dict
 
+    def check_db_for_existing_parameter(
+        self, parameter_name, instrument, parameter_version, db_config
+    ):
+        """
+        Check if a parameter with the same version exists in the simulation model database.
+
+        Parameters
+        ----------
+        parameter_name: str
+            Name of the parameter.
+        instrument: str
+            Name of the instrument.
+        parameter_version: str
+            Version of the parameter.
+        db_config: dict
+            Database configuration.
+
+        Raises
+        ------
+        ValueError
+            If parameter with the same version exists in the database.
+        """
+        db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+        try:
+            db.get_model_parameter(
+                parameter=parameter_name,
+                parameter_version=parameter_version,
+                site=names.get_site_from_array_element_name(instrument),
+                array_element_name=instrument,
+            )
+        except ValueError:
+            pass  # parameter does not exist - expected behavior
+        else:
+            raise ValueError(
+                f"Parameter {parameter_name} with version {parameter_version} already exists."
+            )
+
     def get_validated_parameter_dict(
-        self, parameter_name, value, instrument, parameter_version, schema_version=None
+        self,
+        parameter_name,
+        value,
+        instrument,
+        parameter_version,
+        unique_id=None,
+        schema_version=None,
     ):
         """
         Get validated parameter dictionary.
@@ -202,20 +257,15 @@ class ModelDataWriter:
         schema_file = schema.get_model_parameter_schema_file(parameter_name)
         self.schema_dict = gen.collect_data_from_file(schema_file)
 
-        try:  # e.g. instrument is 'North"
-            site = names.validate_site_name(instrument)
-        except ValueError:  # e.g. instrument is 'LSTN-01'
-            site = names.get_site_from_array_element_name(instrument)
-
         value, unit = value_conversion.split_value_and_unit(value)
 
         data_dict = {
             "schema_version": schema.get_model_parameter_schema_version(schema_version),
             "parameter": parameter_name,
             "instrument": instrument,
-            "site": site,
+            "site": names.get_site_from_array_element_name(instrument),
             "parameter_version": parameter_version,
-            "unique_id": None,
+            "unique_id": unique_id,
             "value": value,
             "unit": unit,
             "type": self._get_parameter_type(),
@@ -425,7 +475,9 @@ class ModelDataWriter:
             If yml_file is not defined.
         """
         try:
-            yml_file = Path(yml_file or self.product_data_file).with_suffix(".metadata.yml")
+            yml_file = names.file_name_with_version(
+                yml_file or self.product_data_file, ".metadata.yml"
+            )
             with open(yml_file, "w", encoding="UTF-8") as file:
                 yaml.safe_dump(
                     gen.change_dict_keys_case(metadata, keys_lower_case),

simtools/data_model/schema.py

@@ -112,15 +112,22 @@ def validate_dict_using_schema(data, schema_file=None, json_schema=None):
     if json_schema is None:
         json_schema = load_schema(
             schema_file,
-            data.get("schema_version", "0.1.0"),  # default version to ensure backward compatibility
+            data.get("schema_version")
+            or data.get(
+                "SCHEMA_VERSION", "0.1.0"
+            ),  # default version to ensure backward compatibility
         )
 
     try:
         jsonschema.validate(data, schema=json_schema, format_checker=format_checkers.format_checker)
     except jsonschema.exceptions.ValidationError as exc:
-        _logger.error(f"Validation failed using schema: {json_schema}")
+        _logger.error(f"Validation failed using schema: {json_schema} for data: {data}")
         raise exc
-    if data.get("meta_schema_url") and not gen.url_exists(data["meta_schema_url"]):
+    if (
+        isinstance(data, dict)
+        and data.get("meta_schema_url")
+        and not gen.url_exists(data["meta_schema_url"])
+    ):
         raise FileNotFoundError(f"Meta schema URL does not exist: {data['meta_schema_url']}")
 
     _logger.debug(f"Successful validation of data using schema ({json_schema.get('name')})")

simtools/data_model/validate_data.py

@@ -12,7 +12,7 @@ from astropy.utils.diff import report_diff_values
 
 import simtools.utils.general as gen
 from simtools.data_model import schema
-from simtools.utils import value_conversion
+from simtools.utils import names, value_conversion
 
 __all__ = ["DataValidator"]
 
@@ -79,7 +79,7 @@ class DataValidator:
 
         """
         if self.data_file_name:
-            self.validate_data_file()
+            self.validate_data_file(is_model_parameter)
         if isinstance(self.data_dict, dict):
             return self._validate_data_dict(is_model_parameter, lists_as_strings)
         if isinstance(self.data_table, Table):
@@ -87,11 +87,16 @@ class DataValidator:
         self._logger.error("No data or data table to validate")
         raise TypeError
 
-    def validate_data_file(self):
+    def validate_data_file(self, is_model_parameter=None):
         """
         Open data file and read data from file.
 
         Doing this successfully is understood as file validation.
+
+        Parameters
+        ----------
+        is_model_parameter: bool
+            This is a model parameter file.
         """
         try:
             if Path(self.data_file_name).suffix in (".yml", ".yaml", ".json"):
@@ -102,14 +107,30 @@ class DataValidator:
                 self._logger.info(f"Validating tabled data from: {self.data_file_name}")
         except (AttributeError, TypeError):
             pass
+        if is_model_parameter:
+            self.validate_parameter_and_file_name()
 
     def validate_parameter_and_file_name(self):
-        """Validate that file name and key 'parameter_name' in data dict are the same."""
-        if not str(Path(self.data_file_name).stem).startswith(self.data_dict.get("parameter")):
-            raise ValueError(
-                f"Parameter name in data dict {self.data_dict.get('parameter')} and "
-                f"file name {Path(self.data_file_name).stem} do not match."
-            )
+        """
+        Validate model parameter file name.
+
+        Expect that the following convention is used:
+
+        - file name starts with the parameter name
+        - file name ends with parameter version string
+
+        """
+        file_stem = Path(self.data_file_name).stem
+        param = self.data_dict.get("parameter")
+        param_version = self.data_dict.get("parameter_version")
+        if not file_stem.startswith(param):
+            raise ValueError(f"Mismatch: parameter '{param}' vs. file '{file_stem}'.")
+
+        if param_version and not file_stem.endswith(param_version):
+            raise ValueError(f"Mismatch: version '{param_version}' vs. file '{file_stem}'.")
+
+        if param_version is None:
+            self._logger.warning(f"File '{file_stem}' has no parameter version defined.")
 
     @staticmethod
     def validate_model_parameter(par_dict):
@@ -170,7 +191,17 @@ class DataValidator:
         else:
             self.data_dict["value"], self.data_dict["unit"] = value_as_list, unit_as_list
 
-        self._check_version_string(self.data_dict.get("version"))
+        if self.data_dict.get("instrument"):
+            if self.data_dict["instrument"] == self.data_dict["site"]:  # site parameters
+                names.validate_site_name(self.data_dict["site"])
+            else:
+                names.validate_array_element_name(self.data_dict["instrument"])
+                self._check_site_and_array_element_consistency(
+                    self.data_dict.get("instrument"), self.data_dict.get("site")
+                )
+
+        for version_string in ("version", "parameter_version", "model_version"):
+            self._check_version_string(self.data_dict.get(version_string))
 
         if lists_as_strings:
             self._convert_results_to_model_format()
@@ -766,20 +797,14 @@ class DataValidator:
         Converts strings to numerical values or lists of values, if required.
 
         """
-        value = self.data_dict["value"]
-        if not isinstance(value, str):
-            return
-
-        # assume float value if type is not defined
-        _is_float = self.data_dict.get("type", "float").startswith(("float", "double"))
-
-        if value.isnumeric():
-            self.data_dict["value"] = float(value) if _is_float else int(value)
-        else:
-            self.data_dict["value"] = gen.convert_string_to_list(value, is_float=_is_float)
-
-        if self.data_dict["unit"] is not None:
-            self.data_dict["unit"] = gen.convert_string_to_list(self.data_dict["unit"])
+        self.data_dict["value"], _ = value_conversion.split_value_and_unit(
+            self.data_dict["value"],
+            "int" in self.data_dict.get("type", "float"),
+        )
+        if isinstance(self.data_dict["unit"], str):
+            self.data_dict["unit"] = gen.convert_string_to_list(
+                self.data_dict["unit"], force_comma_separation=True
+            )
 
     def _convert_results_to_model_format(self):
         """
@@ -814,3 +839,24 @@ class DataValidator:
         if not re.match(semver_regex, version):
             raise ValueError(f"Invalid version string '{version}'")
         self._logger.debug(f"Valid version string '{version}'")
+
+    def _check_site_and_array_element_consistency(self, instrument, site):
+        """
+        Check that site and array element names are consistent.
+
+        An example for an inconsistency is 'LSTN' at site 'South'
+        """
+        if not all([instrument, site]) or "OBS" in instrument:
+            return
+
+        def to_sorted_list(value):
+            """Return value as sorted list."""
+            return [value] if isinstance(value, str) else sorted(value)
+
+        instrument_site = to_sorted_list(names.get_site_from_array_element_name(instrument))
+        site = to_sorted_list(site)
+
+        if instrument_site != site:
+            raise ValueError(
+                f"Site '{site}' and instrument site '{instrument_site}' are inconsistent."
+            )

simtools/db/db_handler.py

@@ -40,7 +40,7 @@ jsonschema_db_dict = {
         "db_api_user": {"type": "string", "description": "API username"},
         "db_api_pw": {"type": "string", "description": "Password for the API user"},
         "db_api_authentication_database": {
-            "type": "string",
+            "type": ["string", "null"],
             "default": "admin",
             "description": "DB with user info (optional)",
         },
@@ -119,7 +119,9 @@ class DatabaseHandler:
             port=self.mongo_db_config["db_api_port"],
             username=self.mongo_db_config["db_api_user"],
             password=self.mongo_db_config["db_api_pw"],
-            authSource=self.mongo_db_config.get("db_api_authentication_database", "admin"),
+            authSource=self.mongo_db_config.get("db_api_authentication_database")
+            if self.mongo_db_config.get("db_api_authentication_database")
+            else "admin",
             directConnection=direct_connection,
             ssl=not direct_connection,
             tlsallowinvalidhostnames=True,
@@ -178,7 +180,6 @@ class DatabaseHandler:
         parameter_version,
         site,
         array_element_name,
-        collection,
     ):
         """
         Get a model parameter using the parameter version.
@@ -193,8 +194,6 @@ class DatabaseHandler:
             Site name.
         array_element_name: str
             Name of the array element model (e.g. MSTN, SSTS).
-        collection: str
-            Collection of array element (e.g. telescopes, calibration_devices).
 
         Returns
         -------
@@ -209,7 +208,9 @@ class DatabaseHandler:
             query["instrument"] = array_element_name
         if site is not None:
             query["site"] = site
-        return self._read_mongo_db(query=query, collection_name=collection)
+        return self._read_mongo_db(
+            query=query, collection_name=names.get_collection_name_from_parameter_name(parameter)
+        )
 
     def get_model_parameters(
         self,
@@ -228,7 +229,7 @@ class DatabaseHandler:
         site: str
             Site name.
         array_element_name: str
-            Name of the array element model (e.g. LSTN-01, MSTS-design, ILLN-01).
+            Name of the array element model (e.g. LSTN-01, MSTx-FlashCam, ILLN-01).
         model_version: str, list
             Version(s) of the model.
         collection: str
@@ -476,7 +477,7 @@ class DatabaseHandler:
         """
         collection = self.get_collection(self._get_db_name(), "production_tables")
         return sorted(
-            [post["model_version"] for post in collection.find({"collection": collection_name})]
+            {post["model_version"] for post in collection.find({"collection": collection_name})}
         )
 
     def get_array_elements(self, model_version, collection="telescopes"):
@@ -499,6 +500,32 @@ class DatabaseHandler:
         production_table = self._read_production_table_from_mongo_db(collection, model_version)
         return sorted([entry for entry in production_table["parameters"] if "-design" not in entry])
 
+    def get_design_model(self, model_version, array_element_name, collection="telescopes"):
+        """
+        Get the design model used for a given array element and a given model version.
+
+        Parameters
+        ----------
+        model_version: str
+            Version of the model.
+        array_element_name: str
+            Name of the array element model (e.g. MSTN, SSTS).
+        collection: str
+            Which collection to get the array elements from:
+            i.e. telescopes, calibration_devices.
+
+        Returns
+        -------
+        str
+            Design model for a given array element.
+        """
+        production_table = self._read_production_table_from_mongo_db(collection, model_version)
+        try:
+            return production_table["design_model"][array_element_name]
+        except KeyError:
+            # for eg. array_element_name == 'LSTN-design' returns 'LSTN-design'
+            return array_element_name
+
     def get_array_elements_of_type(self, array_element_type, model_version, collection):
         """
         Get array elements of a certain type (e.g. 'LSTN') for a DB collection.
@@ -837,15 +864,20 @@ class DatabaseHandler:
             return ["xSTx-design"]  # placeholder to ignore 'instrument' field in query.
         if collection == "sites":
             return [f"OBS-{site}"]
-        if "-design" in array_element_name:
+        if names.is_design_type(array_element_name):
             return [array_element_name]
+        if collection == "configuration_sim_telarray":
+            # get design model from 'telescope' or 'calibration_device' production tables
+            production_table = self._read_production_table_from_mongo_db(
+                names.get_collection_name_from_array_element_name(array_element_name),
+                production_table["model_version"],
+            )
         try:
             return [
                 production_table["design_model"][array_element_name],
                 array_element_name,
             ]
-        except KeyError:
-            return [
-                f"{names.get_array_element_type_from_name(array_element_name)}-design",
-                array_element_name,
-            ]
+        except KeyError as exc:
+            raise KeyError(
+                f"Failed generated array element list for db query for {array_element_name}"
+            ) from exc