gammasimtools 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

simtools/data_model/validate_data.py

@@ -5,16 +5,14 @@ import os
 import re
 from pathlib import Path
 
-import jsonschema
 import numpy as np
 from astropy import units as u
 from astropy.table import Column, Table, unique
 from astropy.utils.diff import report_diff_values
 
 import simtools.utils.general as gen
-from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH
-from simtools.data_model import format_checkers
-from simtools.utils import value_conversion
+from simtools.data_model import schema
+from simtools.utils import names, value_conversion
 
 __all__ = ["DataValidator"]
 
@@ -81,7 +79,7 @@ class DataValidator:
 
         """
         if self.data_file_name:
-            self.validate_data_file()
+            self.validate_data_file(is_model_parameter)
         if isinstance(self.data_dict, dict):
             return self._validate_data_dict(is_model_parameter, lists_as_strings)
         if isinstance(self.data_table, Table):
@@ -89,11 +87,16 @@ class DataValidator:
         self._logger.error("No data or data table to validate")
         raise TypeError
 
-    def validate_data_file(self):
+    def validate_data_file(self, is_model_parameter=None):
         """
         Open data file and read data from file.
 
         Doing this successfully is understood as file validation.
+
+        Parameters
+        ----------
+        is_model_parameter: bool
+            This is a model parameter file.
         """
         try:
             if Path(self.data_file_name).suffix in (".yml", ".yaml", ".json"):
@@ -104,14 +107,30 @@ class DataValidator:
                 self._logger.info(f"Validating tabled data from: {self.data_file_name}")
         except (AttributeError, TypeError):
             pass
+        if is_model_parameter:
+            self.validate_parameter_and_file_name()
 
     def validate_parameter_and_file_name(self):
-        """Validate that file name and key 'parameter_name' in data dict are the same."""
-        if not str(Path(self.data_file_name).stem).startswith(self.data_dict.get("parameter")):
-            raise ValueError(
-                f"Parameter name in data dict {self.data_dict.get('parameter')} and "
-                f"file name {Path(self.data_file_name).stem} do not match."
-            )
+        """
+        Validate model parameter file name.
+
+        Expect that the following convention is used:
+
+        - file name starts with the parameter name
+        - file name ends with parameter version string
+
+        """
+        file_stem = Path(self.data_file_name).stem
+        param = self.data_dict.get("parameter")
+        param_version = self.data_dict.get("parameter_version")
+        if not file_stem.startswith(param):
+            raise ValueError(f"Mismatch: parameter '{param}' vs. file '{file_stem}'.")
+
+        if param_version and not file_stem.endswith(param_version):
+            raise ValueError(f"Mismatch: version '{param_version}' vs. file '{file_stem}'.")
+
+        if param_version is None:
+            self._logger.warning(f"File '{file_stem}' has no parameter version defined.")
 
     @staticmethod
     def validate_model_parameter(par_dict):
@@ -129,7 +148,7 @@ class DataValidator:
             Validated data dictionary
         """
         data_validator = DataValidator(
-            schema_file=MODEL_PARAMETER_SCHEMA_PATH / "f{par_dict['parameter']}.schema.yml",
+            schema_file=schema.get_model_parameter_schema_file(f"{par_dict['parameter']}"),
             data_dict=par_dict,
             check_exact_data_type=False,
         )
@@ -158,9 +177,7 @@ class DataValidator:
         if is_model_parameter:
             self._prepare_model_parameter()
 
-        if not (_name := self.data_dict.get("name") or self.data_dict.get("parameter")):
-            raise KeyError("Data dict does not contain a 'name' or 'parameter' key.")
-        self._data_description = self._read_validation_schema(self.schema_file_name, _name)
+        self._data_description = self._read_validation_schema(self.schema_file_name)
 
         value_as_list, unit_as_list = self._get_value_and_units_as_lists()
 
@@ -174,7 +191,17 @@ class DataValidator:
         else:
             self.data_dict["value"], self.data_dict["unit"] = value_as_list, unit_as_list
 
-        self._check_version_string(self.data_dict.get("version"))
+        if self.data_dict.get("instrument"):
+            if self.data_dict["instrument"] == self.data_dict["site"]:  # site parameters
+                names.validate_site_name(self.data_dict["site"])
+            else:
+                names.validate_array_element_name(self.data_dict["instrument"])
+                self._check_site_and_array_element_consistency(
+                    self.data_dict.get("instrument"), self.data_dict.get("site")
+                )
+
+        for version_string in ("version", "parameter_version", "model_version"):
+            self._check_version_string(self.data_dict.get(version_string))
 
         if lists_as_strings:
             self._convert_results_to_model_format()
@@ -188,8 +215,9 @@ class DataValidator:
         Take into account different data types and allow to use json_schema for testing.
         """
         if self._get_data_description(index).get("type", None) == "dict":
-            self._validate_data_dict_using_json_schema(
-                self.data_dict["value"], self._get_data_description(index).get("json_schema")
+            schema.validate_dict_using_schema(
+                data=self.data_dict["value"],
+                json_schema=self._get_data_description(index).get("json_schema"),
             )
         else:
             self._check_data_type(np.array(value).dtype, index)
@@ -235,27 +263,6 @@ class DataValidator:
         except TypeError:
             return [None], target_unit
 
-    def _validate_data_dict_using_json_schema(self, data, json_schema):
-        """
-        Validate a dictionary using a json schema.
-
-        Parameters
-        ----------
-        data: dict
-            Data dictionary
-        json_schema: dict
-            JSON schema
-        """
-        if json_schema is None:
-            self._logger.debug("Skipping validation of dict type")
-            return
-        self._logger.debug("Validation of dict type using JSON schema")
-        try:
-            jsonschema.validate(data, json_schema, format_checker=format_checkers.format_checker)
-        except jsonschema.exceptions.ValidationError as exc:
-            self._logger.error(f"Validation error: {exc}")
-            raise exc
-
     def _validate_data_table(self):
         """Validate tabulated data."""
         try:
@@ -692,7 +699,7 @@ class DataValidator:
 
         return False
 
-    def _read_validation_schema(self, schema_file, parameter=None):
+    def _read_validation_schema(self, schema_file):
         """
         Read validation schema from file.
 
@@ -700,11 +707,6 @@ class DataValidator:
         ----------
         schema_file: Path
             Schema file describing input data.
-            If this is a directory, a filename of
-            '<par>.schema.yml' is assumed.
-        parameter: str
-            Parameter name of required schema
-            (if None, return first schema in file)
 
         Returns
         -------
@@ -715,17 +717,11 @@ class DataValidator:
         ------
         KeyError
             if 'data' can not be read from dict in schema file
-
         """
         try:
-            if Path(schema_file).is_dir():
-                return gen.collect_data_from_file(
-                    file_name=Path(schema_file) / (parameter + ".schema.yml"),
-                )["data"]
             return gen.collect_data_from_file(file_name=schema_file)["data"]
-        except KeyError:
-            self._logger.error(f"Error reading validation schema from {schema_file}")
-            raise
+        except KeyError as exc:
+            raise KeyError(f"Error reading validation schema from {schema_file}") from exc
 
     def _get_data_description(self, column_name=None, status_test=False):
         """
@@ -801,20 +797,14 @@ class DataValidator:
         Converts strings to numerical values or lists of values, if required.
 
         """
-        value = self.data_dict["value"]
-        if not isinstance(value, str):
-            return
-
-        # assume float value if type is not defined
-        _is_float = self.data_dict.get("type", "float").startswith(("float", "double"))
-
-        if value.isnumeric():
-            self.data_dict["value"] = float(value) if _is_float else int(value)
-        else:
-            self.data_dict["value"] = gen.convert_string_to_list(value, is_float=_is_float)
-
-        if self.data_dict["unit"] is not None:
-            self.data_dict["unit"] = gen.convert_string_to_list(self.data_dict["unit"])
+        self.data_dict["value"], _ = value_conversion.split_value_and_unit(
+            self.data_dict["value"],
+            "int" in self.data_dict.get("type", "float"),
+        )
+        if isinstance(self.data_dict["unit"], str):
+            self.data_dict["unit"] = gen.convert_string_to_list(
+                self.data_dict["unit"], force_comma_separation=True
+            )
 
     def _convert_results_to_model_format(self):
         """
@@ -849,3 +839,24 @@ class DataValidator:
         if not re.match(semver_regex, version):
             raise ValueError(f"Invalid version string '{version}'")
         self._logger.debug(f"Valid version string '{version}'")
+
+    def _check_site_and_array_element_consistency(self, instrument, site):
+        """
+        Check that site and array element names are consistent.
+
+        An example for an inconsistency is 'LSTN' at site 'South'
+        """
+        if not all([instrument, site]) or "OBS" in instrument:
+            return
+
+        def to_sorted_list(value):
+            """Return value as sorted list."""
+            return [value] if isinstance(value, str) else sorted(value)
+
+        instrument_site = to_sorted_list(names.get_site_from_array_element_name(instrument))
+        site = to_sorted_list(site)
+
+        if instrument_site != site:
+            raise ValueError(
+                f"Site '{site}' and instrument site '{instrument_site}' are inconsistent."
+            )