gammasimtools 0.8.2__py3-none-any.whl → 0.10.0__py3-none-any.whl

simtools/data_model/model_data_writer.py

@@ -10,8 +10,7 @@ import yaml
 from astropy.io.registry.base import IORegistryError
 
 import simtools.utils.general as gen
-from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH
-from simtools.data_model import validate_data
+from simtools.data_model import schema, validate_data
 from simtools.data_model.metadata_collector import MetadataCollector
 from simtools.io_operations import io_handler
 from simtools.utils import names, value_conversion
@@ -122,7 +121,7 @@ class ModelDataWriter:
         parameter_name,
         value,
         instrument,
-        model_version,
+        parameter_version,
         output_file,
         output_path=None,
         use_plain_output_path=False,
@@ -139,8 +138,8 @@ class ModelDataWriter:
             Value of the parameter.
         instrument: str
             Name of the instrument.
-        model_version: str
-            Version of the model.
+        parameter_version: str
+            Version of the parameter.
         output_file: str
             Name of output file.
         output_path: str or Path
@@ -163,19 +162,21 @@ class ModelDataWriter:
             use_plain_output_path=use_plain_output_path,
         )
         _json_dict = writer.get_validated_parameter_dict(
-            parameter_name, value, instrument, model_version
+            parameter_name, value, instrument, parameter_version
         )
         writer.write_dict_to_model_parameter_json(output_file, _json_dict)
         if metadata_input_dict is not None:
             metadata_input_dict["output_file"] = output_file
             metadata_input_dict["output_file_format"] = Path(output_file).suffix.lstrip(".")
             writer.write_metadata_to_yml(
-                metadata=MetadataCollector(args_dict=metadata_input_dict).top_level_meta,
+                metadata=MetadataCollector(args_dict=metadata_input_dict).get_top_level_metadata(),
                 yml_file=output_path / f"{Path(output_file).stem}",
             )
         return _json_dict
 
-    def get_validated_parameter_dict(self, parameter_name, value, instrument, model_version):
+    def get_validated_parameter_dict(
+        self, parameter_name, value, instrument, parameter_version, schema_version=None
+    ):
         """
         Get validated parameter dictionary.
 
@@ -187,8 +188,10 @@ class ModelDataWriter:
             Value of the parameter.
         instrument: str
             Name of the instrument.
-        model_version: str
-            Version of the model.
+        parameter_version: str
+            Version of the parameter.
+        schema_version: str
+            Version of the schema.
 
         Returns
         -------
@@ -196,29 +199,26 @@ class ModelDataWriter:
             Validated parameter dictionary.
         """
         self._logger.debug(f"Getting validated parameter dictionary for {instrument}")
-        schema_file = self._read_model_parameter_schema(parameter_name)
+        schema_file = schema.get_model_parameter_schema_file(parameter_name)
+        self.schema_dict = gen.collect_data_from_file(schema_file)
 
         try:  # e.g. instrument is 'North"
             site = names.validate_site_name(instrument)
         except ValueError:  # e.g. instrument is 'LSTN-01'
             site = names.get_site_from_array_element_name(instrument)
 
-        try:
-            applicable = self._get_parameter_applicability(instrument)
-        except ValueError:
-            applicable = True  # Default to True (expect that this field goes in future)
-
         value, unit = value_conversion.split_value_and_unit(value)
 
         data_dict = {
+            "schema_version": schema.get_model_parameter_schema_version(schema_version),
             "parameter": parameter_name,
             "instrument": instrument,
             "site": site,
-            "version": model_version,
+            "parameter_version": parameter_version,
+            "unique_id": None,
             "value": value,
             "unit": unit,
             "type": self._get_parameter_type(),
-            "applicable": applicable,
             "file": self._parameter_is_a_file(),
         }
         return self.validate_and_transform(
@@ -227,22 +227,6 @@ class ModelDataWriter:
             is_model_parameter=True,
         )
 
-    def _read_model_parameter_schema(self, parameter_name):
-        """
-        Read model parameter schema.
-
-        Parameters
-        ----------
-        parameter_name: str
-            Name of the parameter.
-        """
-        schema_file = MODEL_PARAMETER_SCHEMA_PATH / f"{parameter_name}.schema.yml"
-        try:
-            self.schema_dict = gen.collect_data_from_file(file_name=schema_file)
-        except FileNotFoundError as exc:
-            raise FileNotFoundError(f"Schema file not found: {schema_file}") from exc
-        return schema_file
-
     def _get_parameter_type(self):
         """
         Return parameter type from schema.
@@ -273,36 +257,6 @@ class ModelDataWriter:
             pass
         return False
 
-    def _get_parameter_applicability(self, telescope_name):
-        """
-        Check if a parameter is applicable for a given telescope using schema files.
-
-        First check for exact telescope name (e.g., LSTN-01), if not listed in the schema
-        use telescope type (LSTN).
-
-        Parameters
-        ----------
-        telescope_name: str
-            Telescope name (e.g., LSTN-01)
-
-        Returns
-        -------
-        bool
-            True if parameter is applicable to telescope.
-
-        """
-        try:
-            if telescope_name in self.schema_dict["instrument"]["type"]:
-                return True
-        except KeyError as exc:
-            self._logger.error("Schema file does not contain 'instrument:type' key.")
-            raise exc
-
-        return (
-            names.get_array_element_type_from_name(telescope_name)
-            in self.schema_dict["instrument"]["type"]
-        )
-
     def _get_unit_from_schema(self):
         """
         Return unit(s) from schema dict.

simtools/data_model/schema.py

@@ -0,0 +1,213 @@
+"""Module providing functionality to read and validate dictionaries using schema."""
+
+import logging
+from pathlib import Path
+
+import jsonschema
+
+import simtools.utils.general as gen
+from simtools.constants import (
+    METADATA_JSON_SCHEMA,
+    MODEL_PARAMETER_METASCHEMA,
+    MODEL_PARAMETER_SCHEMA_PATH,
+    SCHEMA_PATH,
+)
+from simtools.data_model import format_checkers
+from simtools.utils import names
+
+_logger = logging.getLogger(__name__)
+
+
+def get_get_model_parameter_schema_files(schema_directory=MODEL_PARAMETER_SCHEMA_PATH):
+    """
+    Return list of parameters and schema files located in schema file directory.
+
+    Returns
+    -------
+    list
+        List of parameters found in schema file directory.
+    list
+        List of schema files found in schema file directory.
+
+    """
+    schema_files = sorted(Path(schema_directory).rglob("*.schema.yml"))
+    if not schema_files:
+        raise FileNotFoundError(f"No schema files found in {schema_directory}")
+    parameters = []
+    for schema_file in schema_files:
+        schema_dict = gen.collect_data_from_file(file_name=schema_file)
+        parameters.append(schema_dict.get("name"))
+    return parameters, schema_files
+
+
+def get_model_parameter_schema_file(parameter):
+    """
+    Return schema file path for a given model parameter.
+
+    Parameters
+    ----------
+    parameter: str
+        Model parameter name.
+
+    Returns
+    -------
+    Path
+        Schema file path.
+
+    """
+    schema_file = MODEL_PARAMETER_SCHEMA_PATH / f"{parameter}.schema.yml"
+    if not schema_file.exists():
+        raise FileNotFoundError(f"Schema file not found: {schema_file}")
+    return schema_file
+
+
+def get_model_parameter_schema_version(schema_version=None):
+    """
+    Validate  and return schema versions.
+
+    If no schema_version is given, the most recent version is provided.
+
+    Parameters
+    ----------
+    schema_version: str
+        Schema version.
+
+    Returns
+    -------
+    str
+        Schema version.
+
+    """
+    schemas = gen.collect_data_from_file(MODEL_PARAMETER_METASCHEMA)
+
+    if schema_version is None and schemas:
+        return schemas[0].get("version")
+
+    if any(schema.get("version") == schema_version for schema in schemas):
+        return schema_version
+
+    raise ValueError(f"Schema version {schema_version} not found in {MODEL_PARAMETER_METASCHEMA}.")
+
+
+def validate_dict_using_schema(data, schema_file=None, json_schema=None):
+    """
+    Validate a data dictionary against a schema.
+
+    Parameters
+    ----------
+    data
+        dictionary to be validated
+    schema_file (dict)
+        schema used for validation
+
+    Raises
+    ------
+    jsonschema.exceptions.ValidationError
+        if validation fails
+
+    """
+    if json_schema is None and schema_file is None:
+        _logger.warning(f"No schema provided for validation of {data}")
+        return
+    if json_schema is None:
+        json_schema = load_schema(
+            schema_file,
+            data.get("schema_version", "0.1.0"),  # default version to ensure backward compatibility
+        )
+
+    try:
+        jsonschema.validate(data, schema=json_schema, format_checker=format_checkers.format_checker)
+    except jsonschema.exceptions.ValidationError as exc:
+        _logger.error(f"Validation failed using schema: {json_schema}")
+        raise exc
+    if data.get("meta_schema_url") and not gen.url_exists(data["meta_schema_url"]):
+        raise FileNotFoundError(f"Meta schema URL does not exist: {data['meta_schema_url']}")
+
+    _logger.debug(f"Successful validation of data using schema ({json_schema.get('name')})")
+
+
+def load_schema(schema_file=None, schema_version=None):
+    """
+    Load parameter schema from file.
+
+    Parameters
+    ----------
+    schema_file: str
+        Path to schema file.
+    schema_version: str
+        Schema version.
+
+    Returns
+    -------
+    schema: dict
+        Schema dictionary.
+
+    Raises
+    ------
+    FileNotFoundError
+        if schema file is not found
+
+    """
+    schema_file = schema_file or METADATA_JSON_SCHEMA
+
+    for path in (schema_file, SCHEMA_PATH / schema_file):
+        try:
+            schema = gen.collect_data_from_file(file_name=path)
+            break
+        except FileNotFoundError:
+            continue
+    else:
+        raise FileNotFoundError(f"Schema file not found: {schema_file}")
+
+    if isinstance(schema, list):  # schema file with several schemas defined
+        if schema_version is None:
+            raise ValueError(f"Schema version not given in {schema_file}.")
+        schema = next((doc for doc in schema if doc.get("version") == schema_version), None)
+        if schema is None:
+            raise ValueError(f"Schema version {schema_version} not found in {schema_file}.")
+    elif schema_version is not None and schema_version != schema.get("version"):
+        _logger.warning(f"Schema version {schema_version} does not match {schema.get('version')}")
+
+    _logger.debug(f"Loading schema from {schema_file}")
+    _add_array_elements("InstrumentTypeElement", schema)
+
+    return schema
+
+
+def _add_array_elements(key, schema):
+    """
+    Add list of array elements to schema.
+
+    Avoids having to list all array elements in multiple schema.
+    Assumes an element [key]['enum'] is a list of elements.
+
+    Parameters
+    ----------
+    key: str
+        Key in schema dictionary
+    schema: dict
+        Schema dictionary
+
+    Returns
+    -------
+    dict
+        Schema dictionary with added array elements.
+
+    """
+    _list_of_array_elements = sorted(names.array_elements().keys())
+
+    def recursive_search(sub_schema, key):
+        if key in sub_schema:
+            if "enum" in sub_schema[key] and isinstance(sub_schema[key]["enum"], list):
+                sub_schema[key]["enum"] = list(
+                    set(sub_schema[key]["enum"] + _list_of_array_elements)
+                )
+            else:
+                sub_schema[key]["enum"] = _list_of_array_elements
+        else:
+            for _, v in sub_schema.items():
+                if isinstance(v, dict):
+                    recursive_search(v, key)
+
+    recursive_search(schema, key)
+    return schema

simtools/data_model/validate_data.py

@@ -5,14 +5,13 @@ import os
 import re
 from pathlib import Path
 
-import jsonschema
 import numpy as np
 from astropy import units as u
 from astropy.table import Column, Table, unique
 from astropy.utils.diff import report_diff_values
 
 import simtools.utils.general as gen
-from simtools.data_model import format_checkers
+from simtools.data_model import schema
 from simtools.utils import value_conversion
 
 __all__ = ["DataValidator"]
@@ -57,7 +56,7 @@ class DataValidator:
         self.data_table = data_table
         self.check_exact_data_type = check_exact_data_type
 
-    def validate_and_transform(self, is_model_parameter=False):
+    def validate_and_transform(self, is_model_parameter=False, lists_as_strings=False):
         """
         Validate data and data file.
 
@@ -65,6 +64,8 @@ class DataValidator:
         ----------
         is_model_parameter: bool
             This is a model parameter (add some data preparation)
+        lists_as_strings: bool
+            Convert lists to strings (as needed for model parameters)
 
         Returns
         -------
@@ -80,13 +81,9 @@ class DataValidator:
         if self.data_file_name:
             self.validate_data_file()
         if isinstance(self.data_dict, dict):
-            if is_model_parameter:
-                self._prepare_model_parameter()
-            self._validate_data_dict()
-            return self.data_dict
+            return self._validate_data_dict(is_model_parameter, lists_as_strings)
         if isinstance(self.data_table, Table):
-            self._validate_data_table()
-            return self.data_table
+            return self._validate_data_table()
         self._logger.error("No data or data table to validate")
         raise TypeError
 
@@ -108,28 +105,58 @@ class DataValidator:
 
     def validate_parameter_and_file_name(self):
         """Validate that file name and key 'parameter_name' in data dict are the same."""
-        if self.data_dict.get("parameter") != Path(self.data_file_name).stem:
+        if not str(Path(self.data_file_name).stem).startswith(self.data_dict.get("parameter")):
             raise ValueError(
                 f"Parameter name in data dict {self.data_dict.get('parameter')} and "
                 f"file name {Path(self.data_file_name).stem} do not match."
             )
 
-    def _validate_data_dict(self):
+    @staticmethod
+    def validate_model_parameter(par_dict):
+        """
+        Validate a simulation model parameter (static method).
+
+        Parameters
+        ----------
+        par_dict: dict
+            Data dictionary
+
+        Returns
+        -------
+        dict
+            Validated data dictionary
+        """
+        data_validator = DataValidator(
+            schema_file=schema.get_model_parameter_schema_file(f"{par_dict['parameter']}"),
+            data_dict=par_dict,
+            check_exact_data_type=False,
+        )
+        return data_validator.validate_and_transform(is_model_parameter=True)
+
+    def _validate_data_dict(self, is_model_parameter=False, lists_as_strings=False):
         """
         Validate values in a dictionary.
 
         Handles different types of naming in data dicts (using 'name' or 'parameter'
         keys for name fields).
 
+        Parameters
+        ----------
+        is_model_parameter: bool
+            This is a model parameter (add some data preparation)
+        lists_as_strings: bool
+            Convert lists to strings (as needed for model parameters)
+
         Raises
         ------
         KeyError
             if data dict does not contain a 'name' or 'parameter' key.
 
         """
-        if not (_name := self.data_dict.get("name") or self.data_dict.get("parameter")):
-            raise KeyError("Data dict does not contain a 'name' or 'parameter' key.")
-        self._data_description = self._read_validation_schema(self.schema_file_name, _name)
+        if is_model_parameter:
+            self._prepare_model_parameter()
+
+        self._data_description = self._read_validation_schema(self.schema_file_name)
 
         value_as_list, unit_as_list = self._get_value_and_units_as_lists()
 
@@ -145,6 +172,11 @@ class DataValidator:
 
         self._check_version_string(self.data_dict.get("version"))
 
+        if lists_as_strings:
+            self._convert_results_to_model_format()
+
+        return self.data_dict
+
     def _validate_value_and_unit(self, value, unit, index):
         """
         Validate value, unit, and perform type checking and conversions.
@@ -152,8 +184,9 @@ class DataValidator:
         Take into account different data types and allow to use json_schema for testing.
         """
         if self._get_data_description(index).get("type", None) == "dict":
-            self._validate_data_dict_using_json_schema(
-                self.data_dict["value"], self._get_data_description(index).get("json_schema")
+            schema.validate_dict_using_schema(
+                data=self.data_dict["value"],
+                json_schema=self._get_data_description(index).get("json_schema"),
             )
         else:
             self._check_data_type(np.array(value).dtype, index)
@@ -191,28 +224,13 @@ class DataValidator:
         conversion_factor = [
             1 if v is None else u.Unit(v).to(u.Unit(t)) for v, t in zip(unit, target_unit)
         ]
-        return [v * c for v, c in zip(value, conversion_factor)], target_unit
-
-    def _validate_data_dict_using_json_schema(self, data, json_schema):
-        """
-        Validate a dictionary using a json schema.
-
-        Parameters
-        ----------
-        data: dict
-            Data dictionary
-        json_schema: dict
-            JSON schema
-        """
-        if json_schema is None:
-            self._logger.debug("Skipping validation of dict type")
-            return
-        self._logger.debug("Validation of dict type using JSON schema")
         try:
-            jsonschema.validate(data, json_schema, format_checker=format_checkers.format_checker)
-        except jsonschema.exceptions.ValidationError as exc:
-            self._logger.error(f"Validation error: {exc}")
-            raise exc
+            return [
+                v * c if not isinstance(v, bool) and not isinstance(v, dict) else v
+                for v, c in zip(value, conversion_factor)
+            ], target_unit
+        except TypeError:
+            return [None], target_unit
 
     def _validate_data_table(self):
         """Validate tabulated data."""
@@ -228,6 +246,7 @@ class DataValidator:
             self._validate_data_columns()
             self._check_data_for_duplicates()
             self._sort_data()
+        return self.data_table
 
     def _validate_data_columns(self):
         """
@@ -649,7 +668,7 @@ class DataValidator:
 
         return False
 
-    def _read_validation_schema(self, schema_file, parameter=None):
+    def _read_validation_schema(self, schema_file):
         """
         Read validation schema from file.
 
@@ -657,11 +676,6 @@ class DataValidator:
         ----------
         schema_file: Path
             Schema file describing input data.
-            If this is a directory, a filename of
-            '<par>.schema.yml' is assumed.
-        parameter: str
-            Parameter name of required schema
-            (if None, return first schema in file)
 
         Returns
         -------
@@ -672,17 +686,11 @@ class DataValidator:
         ------
         KeyError
             if 'data' can not be read from dict in schema file
-
         """
         try:
-            if Path(schema_file).is_dir():
-                return gen.collect_data_from_file(
-                    file_name=Path(schema_file) / (parameter + ".schema.yml"),
-                )["data"]
             return gen.collect_data_from_file(file_name=schema_file)["data"]
-        except KeyError:
-            self._logger.error(f"Error reading validation schema from {schema_file}")
-            raise
+        except KeyError as exc:
+            raise KeyError(f"Error reading validation schema from {schema_file}") from exc
 
     def _get_data_description(self, column_name=None, status_test=False):
         """
@@ -726,6 +734,8 @@ class DataValidator:
                 )
             )
         except IndexError as exc:
+            if len(self._data_description) == 1:  # all columns are described by the same schema
+                return self._data_description[0]
             self._logger.error(
                 f"Data column '{column_name}' not found in reference column definition"
             )
@@ -771,6 +781,18 @@ class DataValidator:
         if self.data_dict["unit"] is not None:
             self.data_dict["unit"] = gen.convert_string_to_list(self.data_dict["unit"])
 
+    def _convert_results_to_model_format(self):
+        """
+        Convert results to model format.
+
+        Convert lists to strings (as needed for model parameters).
+        """
+        value = self.data_dict["value"]
+        if isinstance(value, list):
+            self.data_dict["value"] = gen.convert_list_to_string(value)
+        if isinstance(self.data_dict["unit"], list):
+            self.data_dict["unit"] = gen.convert_list_to_string(self.data_dict["unit"])
+
     def _check_version_string(self, version):
         """
         Check that version string follows semantic versioning.