gammasimtools 0.6.1__py3-none-any.whl → 0.8.1__py3-none-any.whl

simtools/data_model/metadata_model.py

@@ -1,5 +1,6 @@
 """
 Definition of metadata model for input to and output of simtools.
+
 Follows CTAO top-level data model definition.
 
 * data products submitted to SimPipe ('input')
@@ -14,6 +15,8 @@ import jsonschema
 
 import simtools.constants
 import simtools.utils.general as gen
+from simtools.data_model import format_checkers
+from simtools.utils import names
 
 _logger = logging.getLogger(__name__)
 
@@ -35,11 +38,10 @@ def validate_schema(data, schema_file):
         if validation fails
 
     """
-
     schema, schema_file = _load_schema(schema_file)
 
     try:
-        jsonschema.validate(data, schema=schema)
+        jsonschema.validate(data, schema=schema, format_checker=format_checkers.format_checker)
     except jsonschema.exceptions.ValidationError:
         _logger.error(f"Failed using {schema}")
         raise
@@ -48,7 +50,8 @@ def validate_schema(data, schema_file):
 
 def get_default_metadata_dict(schema_file=None, observatory="CTA"):
     """
-    Returns metadata schema with default values.
+    Return metadata schema with default values.
+
     Follows the CTA Top-Level Data Model.
 
     Parameters
@@ -65,7 +68,6 @@ def get_default_metadata_dict(schema_file=None, observatory="CTA"):
 
 
     """
-
     schema, _ = _load_schema(schema_file)
     return _fill_defaults(schema["definitions"], observatory)
 
@@ -88,16 +90,58 @@ def _load_schema(schema_file=None):
         if schema file is not found
 
     """
-
     if schema_file is None:
         schema_file = files("simtools").joinpath(simtools.constants.METADATA_JSON_SCHEMA)
 
-    schema = gen.collect_data_from_file_or_dict(file_name=schema_file, in_dict=None)
+    try:
+        schema = gen.collect_data_from_file(file_name=schema_file)
+    except FileNotFoundError:
+        schema_file = files("simtools").joinpath("schemas") / schema_file
+        schema = gen.collect_data_from_file(file_name=schema_file)
     _logger.debug(f"Loading schema from {schema_file}")
+    _add_array_elements("InstrumentTypeElement", schema)
 
     return schema, schema_file
 
 
+def _add_array_elements(key, schema):
+    """
+    Add list of array elements to schema.
+
+    This assumes an element [key]['enum'] is a list of elements.
+
+    Parameters
+    ----------
+    key: str
+        Key in schema dictionary
+    schema: dict
+        Schema dictionary
+
+    Returns
+    -------
+    dict
+        Schema dictionary with added array elements.
+
+    """
+    _list_of_array_elements = sorted(names.array_elements().keys())
+
+    def recursive_search(sub_schema, key):
+        if key in sub_schema:
+            if "enum" in sub_schema[key] and isinstance(sub_schema[key]["enum"], list):
+                sub_schema[key]["enum"] = list(
+                    set(sub_schema[key]["enum"] + _list_of_array_elements)
+                )
+            else:
+                sub_schema[key]["enum"] = _list_of_array_elements
+        else:
+            for _, v in sub_schema.items():
+                if isinstance(v, dict):
+                    recursive_search(v, key)
+
+    recursive_search(schema, key)
+    return schema
+
+
 def _resolve_references(yaml_data, observatory="CTA"):
     """
     Resolve references in yaml data and expand the received dictionary accordingly.
@@ -126,17 +170,23 @@ def _resolve_references(yaml_data, observatory="CTA"):
             ref_data = ref_data.get(part, {})
         return ref_data
 
+    def resolve_dict(data):
+        if "$ref" in data:
+            ref = data["$ref"]
+            resolved_data = expand_ref(ref)
+            if isinstance(resolved_data, dict) and len(resolved_data) > 1:
+                return _resolve_references_recursive(resolved_data)
+            return resolved_data
+        return {k: _resolve_references_recursive(v) for k, v in data.items()}
+
+    def resolve_list(data):
+        return [_resolve_references_recursive(item) for item in data]
+
     def _resolve_references_recursive(data):
         if isinstance(data, dict):
-            if "$ref" in data:
-                ref = data["$ref"]
-                resolved_data = expand_ref(ref)
-                if isinstance(resolved_data, dict) and len(resolved_data) > 1:
-                    return _resolve_references_recursive(resolved_data)
-                return resolved_data
-            return {k: _resolve_references_recursive(v) for k, v in data.items()}
+            return resolve_dict(data)
         if isinstance(data, list):
-            return [_resolve_references_recursive(item) for item in data]
+            return resolve_list(data)
         return data
 
     return _resolve_references_recursive(yaml_data)
@@ -157,30 +207,58 @@ def _fill_defaults(schema, observatory="CTA"):
     -------
     dict
         Dictionary with default values.
+    """
+    defaults = {observatory: {}}
+    resolved_schema = _resolve_references(schema[observatory])
+    _fill_defaults_recursive(resolved_schema, defaults[observatory])
+    return defaults
+
 
+def _fill_defaults_recursive(subschema, current_dict):
     """
+    Recursively fill default values from the subschema into the current dictionary.
 
-    defaults = {observatory: {}}
+    Parameters
+    ----------
+    subschema: dict
+        Subschema describing part of the input data.
+    current_dict: dict
+        Current dictionary to fill with default values.
+    """
+    if "properties" not in subschema:
+        _raise_missing_properties_error()
 
-    schema = _resolve_references(schema[observatory])
-
-    def _fill_defaults_recursive(subschema, current_dict):
-        try:
-            for prop, prop_schema in subschema["properties"].items():
-                if "default" in prop_schema:
-                    current_dict[prop] = prop_schema["default"]
-                elif "type" in prop_schema:
-                    if prop_schema["type"] == "object":
-                        current_dict[prop] = {}
-                        _fill_defaults_recursive(prop_schema, current_dict[prop])
-                    elif prop_schema["type"] == "array":
-                        current_dict[prop] = [{}]
-                        if "items" in prop_schema and isinstance(prop_schema["items"], dict):
-                            _fill_defaults_recursive(prop_schema["items"], current_dict[prop][0])
-        except KeyError:
-            msg = "Missing 'properties' key in schema."
-            _logger.error(msg)
-            raise
-
-    _fill_defaults_recursive(schema, defaults[observatory])
-    return defaults
+    for prop, prop_schema in subschema["properties"].items():
+        _process_property(prop, prop_schema, current_dict)
+
+
+def _process_property(prop, prop_schema, current_dict):
+    """
+    Process each property and fill the default values accordingly.
+
+    Parameters
+    ----------
+    prop: str
+        Property name.
+    prop_schema: dict
+        Schema of the property.
+    current_dict: dict
+        Current dictionary to fill with default values.
+    """
+    if "default" in prop_schema:
+        current_dict[prop] = prop_schema["default"]
+    elif "type" in prop_schema:
+        if prop_schema["type"] == "object":
+            current_dict[prop] = {}
+            _fill_defaults_recursive(prop_schema, current_dict[prop])
+        elif prop_schema["type"] == "array":
+            current_dict[prop] = [{}]
+            if "items" in prop_schema and isinstance(prop_schema["items"], dict):
+                _fill_defaults_recursive(prop_schema["items"], current_dict[prop][0])
+
+
+def _raise_missing_properties_error():
+    """Raise an error when the 'properties' key is missing in the schema."""
+    msg = "Missing 'properties' key in schema."
+    _logger.error(msg)
+    raise KeyError(msg)

simtools/data_model/model_data_writer.py

@@ -1,16 +1,41 @@
+"""Model data writer module."""
+
+import json
 import logging
 from pathlib import Path
 
-import astropy
+import astropy.units as u
+import numpy as np
 import yaml
+from astropy.io.registry.base import IORegistryError
 
 import simtools.utils.general as gen
+from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH
 from simtools.data_model import validate_data
+from simtools.data_model.metadata_collector import MetadataCollector
 from simtools.io_operations import io_handler
+from simtools.utils import names, value_conversion
 
 __all__ = ["ModelDataWriter"]
 
 
+class JsonNumpyEncoder(json.JSONEncoder):
+    """Convert numpy to python types as accepted by json.dump."""
+
+    def default(self, o):
+        if isinstance(o, np.floating):
+            return float(o)
+        if isinstance(o, np.integer):
+            return int(o)
+        if isinstance(o, np.ndarray):
+            return o.tolist()
+        if isinstance(o, u.core.CompositeUnit | u.core.IrreducibleUnit | u.core.Unit):
+            return str(o) if o != u.dimensionless_unscaled else None
+        if np.issubdtype(type(o), np.bool_):
+            return bool(o)
+        return super().default(o)
+
+
 class ModelDataWriter:
     """
     Writer for simulation model data and metadata.
@@ -23,24 +48,36 @@ class ModelDataWriter:
         Format of output file.
     args_dict: Dictionary
         Dictionary with configuration parameters.
-    """
+    output_path: str or Path
+        Path to output file.
+    use_plain_output_path: bool
+        Use plain output path.
+    args_dict: dict
+        Dictionary with configuration parameters.
 
-    def __init__(self, product_data_file=None, product_data_format=None, args_dict=None):
-        """
-        Initialize model data writer.
-        """
+    """
 
+    def __init__(
+        self,
+        product_data_file=None,
+        product_data_format=None,
+        output_path=None,
+        use_plain_output_path=True,
+        args_dict=None,
+    ):
+        """Initialize model data writer."""
         self._logger = logging.getLogger(__name__)
         self.io_handler = io_handler.IOHandler()
+        self.schema_dict = {}
         if args_dict is not None:
+            output_path = args_dict.get("output_path", output_path)
+            use_plain_output_path = args_dict.get("use_plain_output_path", use_plain_output_path)
+        if output_path is not None:
             self.io_handler.set_paths(
-                output_path=args_dict.get("output_path", None),
-                use_plain_output_path=args_dict.get("use_plain_output_path", False),
+                output_path=output_path, use_plain_output_path=use_plain_output_path
             )
         try:
-            self.product_data_file = self.io_handler.get_output_file(
-                file_name=product_data_file, dir_type="simtools-result"
-            )
+            self.product_data_file = self.io_handler.get_output_file(file_name=product_data_file)
         except TypeError:
             self.product_data_file = None
         self.product_data_format = self._astropy_data_format(product_data_format)
@@ -66,7 +103,6 @@ class ModelDataWriter:
             Schema file used in validation of output data.
 
         """
-
         writer = ModelDataWriter(
             product_data_file=(
                 args_dict.get("output_file", None) if output_file is None else output_file
@@ -76,34 +112,249 @@ class ModelDataWriter:
         )
         if validate_schema_file is not None and not args_dict.get("skip_output_validation", True):
             product_data = writer.validate_and_transform(
-                product_data=product_data,
+                product_data_table=product_data,
                 validate_schema_file=validate_schema_file,
             )
         writer.write(metadata=metadata, product_data=product_data)
 
-    def validate_and_transform(self, product_data=None, validate_schema_file=None):
+    @staticmethod
+    def dump_model_parameter(
+        parameter_name,
+        value,
+        instrument,
+        model_version,
+        output_file,
+        output_path=None,
+        use_plain_output_path=False,
+        metadata_input_dict=None,
+    ):
+        """
+        Generate DB-style model parameter dict and write it to json file.
+
+        Parameters
+        ----------
+        parameter_name: str
+            Name of the parameter.
+        value: any
+            Value of the parameter.
+        instrument: str
+            Name of the instrument.
+        model_version: str
+            Version of the model.
+        output_file: str
+            Name of output file.
+        output_path: str or Path
+            Path to output file.
+        use_plain_output_path: bool
+            Use plain output path.
+        metadata_input_dict: dict
+            Input to metadata collector.
+
+        Returns
+        -------
+        dict
+            Validated parameter dictionary.
+        """
+        writer = ModelDataWriter(
+            product_data_file=output_file,
+            product_data_format="json",
+            args_dict=None,
+            output_path=output_path,
+            use_plain_output_path=use_plain_output_path,
+        )
+        _json_dict = writer.get_validated_parameter_dict(
+            parameter_name, value, instrument, model_version
+        )
+        writer.write_dict_to_model_parameter_json(output_file, _json_dict)
+        if metadata_input_dict is not None:
+            metadata_input_dict["output_file"] = output_file
+            metadata_input_dict["output_file_format"] = Path(output_file).suffix.lstrip(".")
+            writer.write_metadata_to_yml(
+                metadata=MetadataCollector(args_dict=metadata_input_dict).top_level_meta,
+                yml_file=output_path / f"{Path(output_file).stem}",
+            )
+        return _json_dict
+
+    def get_validated_parameter_dict(self, parameter_name, value, instrument, model_version):
+        """
+        Get validated parameter dictionary.
+
+        Parameters
+        ----------
+        parameter_name: str
+            Name of the parameter.
+        value: any
+            Value of the parameter.
+        instrument: str
+            Name of the instrument.
+        model_version: str
+            Version of the model.
+
+        Returns
+        -------
+        dict
+            Validated parameter dictionary.
+        """
+        self._logger.debug(f"Getting validated parameter dictionary for {instrument}")
+        schema_file = self._read_model_parameter_schema(parameter_name)
+
+        try:  # e.g. instrument is 'North"
+            site = names.validate_site_name(instrument)
+        except ValueError:  # e.g. instrument is 'LSTN-01'
+            site = names.get_site_from_array_element_name(instrument)
+
+        try:
+            applicable = self._get_parameter_applicability(instrument)
+        except ValueError:
+            applicable = True  # Default to True (expect that this field goes in future)
+
+        value, unit = value_conversion.split_value_and_unit(value)
+
+        data_dict = {
+            "parameter": parameter_name,
+            "instrument": instrument,
+            "site": site,
+            "version": model_version,
+            "value": value,
+            "unit": unit,
+            "type": self._get_parameter_type(),
+            "applicable": applicable,
+            "file": self._parameter_is_a_file(),
+        }
+        return self.validate_and_transform(
+            product_data_dict=data_dict,
+            validate_schema_file=schema_file,
+            is_model_parameter=True,
+        )
+
+    def _read_model_parameter_schema(self, parameter_name):
+        """
+        Read model parameter schema.
+
+        Parameters
+        ----------
+        parameter_name: str
+            Name of the parameter.
+        """
+        schema_file = MODEL_PARAMETER_SCHEMA_PATH / f"{parameter_name}.schema.yml"
+        try:
+            self.schema_dict = gen.collect_data_from_file(file_name=schema_file)
+        except FileNotFoundError as exc:
+            raise FileNotFoundError(f"Schema file not found: {schema_file}") from exc
+        return schema_file
+
+    def _get_parameter_type(self):
+        """
+        Return parameter type from schema.
+
+        Returns
+        -------
+        str
+            Parameter type
+        """
+        _parameter_type = []
+        for data in self.schema_dict["data"]:
+            _parameter_type.append(data["type"])
+        return _parameter_type if len(_parameter_type) > 1 else _parameter_type[0]
+
+    def _parameter_is_a_file(self):
+        """
+        Check if parameter is a file.
+
+        Returns
+        -------
+        bool
+            True if parameter is a file.
+
+        """
+        try:
+            return self.schema_dict["data"][0]["type"] == "file"
+        except (KeyError, IndexError):
+            pass
+        return False
+
+    def _get_parameter_applicability(self, telescope_name):
+        """
+        Check if a parameter is applicable for a given telescope using schema files.
+
+        First check for exact telescope name (e.g., LSTN-01), if not listed in the schema
+        use telescope type (LSTN).
+
+        Parameters
+        ----------
+        telescope_name: str
+            Telescope name (e.g., LSTN-01)
+
+        Returns
+        -------
+        bool
+            True if parameter is applicable to telescope.
+
+        """
+        try:
+            if telescope_name in self.schema_dict["instrument"]["type"]:
+                return True
+        except KeyError as exc:
+            self._logger.error("Schema file does not contain 'instrument:type' key.")
+            raise exc
+
+        return (
+            names.get_array_element_type_from_name(telescope_name)
+            in self.schema_dict["instrument"]["type"]
+        )
+
+    def _get_unit_from_schema(self):
+        """
+        Return unit(s) from schema dict.
+
+        Returns
+        -------
+        str or list
+            Parameter unit(s)
+        """
+        try:
+            unit_list = []
+            for data in self.schema_dict["data"]:
+                unit_list.append(data["unit"] if data["unit"] != "dimensionless" else None)
+            return unit_list if len(unit_list) > 1 else unit_list[0]
+        except (KeyError, IndexError):
+            pass
+        return None
+
+    def validate_and_transform(
+        self,
+        product_data_table=None,
+        product_data_dict=None,
+        validate_schema_file=None,
+        is_model_parameter=False,
+    ):
         """
         Validate product data using jsonschema given in metadata.
+
         If necessary, transform product data to match schema.
 
         Parameters
         ----------
-        product_data: astropy Table
-            Model data to be validated
+        product_data_table: astropy Table
+            Model data to be validated.
+        product_data_dict: dict
+            Model data to be validated.
         validate_schema_file: str
             Schema file used in validation of output data.
-
+        is_model_parameter: bool
+            True if data describes a model parameter.
         """
-
         _validator = validate_data.DataValidator(
             schema_file=validate_schema_file,
-            data_table=product_data,
+            data_table=product_data_table,
+            data_dict=product_data_dict,
+            check_exact_data_type=False,
         )
-        return _validator.validate_and_transform()
+        return _validator.validate_and_transform(is_model_parameter)
 
     def write(self, product_data=None, metadata=None):
         """
-        Write model data and metadata
+        Write model data and metadata.
 
         Parameters
         ----------
@@ -118,22 +369,82 @@ class ModelDataWriter:
             if data writing was not successful.
 
         """
-
         if product_data is None:
             return
 
         if metadata is not None:
             product_data.meta.update(gen.change_dict_keys_case(metadata, False))
 
+        self._logger.info(f"Writing data to {self.product_data_file}")
+        if isinstance(product_data, dict) and Path(self.product_data_file).suffix == ".json":
+            self.write_dict_to_model_parameter_json(self.product_data_file, product_data)
+            return
         try:
-            self._logger.info(f"Writing data to {self.product_data_file}")
             product_data.write(
                 self.product_data_file, format=self.product_data_format, overwrite=True
             )
-        except astropy.io.registry.base.IORegistryError:
+        except IORegistryError:
             self._logger.error(f"Error writing model data to {self.product_data_file}.")
             raise
 
+    def write_dict_to_model_parameter_json(self, file_name, data_dict):
+        """
+        Write dictionary to model-parameter-style json file.
+
+        Parameters
+        ----------
+        file_name : str
+            Name of output file.
+        data_dict : dict
+            Data dictionary.
+
+        Raises
+        ------
+        FileNotFoundError
+            if data writing was not successful.
+        """
+        data_dict = ModelDataWriter.prepare_data_dict_for_writing(data_dict)
+        try:
+            self._logger.info(f"Writing data to {self.io_handler.get_output_file(file_name)}")
+            with open(self.io_handler.get_output_file(file_name), "w", encoding="UTF-8") as file:
+                json.dump(data_dict, file, indent=4, sort_keys=False, cls=JsonNumpyEncoder)
+                file.write("\n")
+        except FileNotFoundError as exc:
+            raise FileNotFoundError(
+                f"Error writing model data to {self.io_handler.get_output_file(file_name)}"
+            ) from exc
+
+    @staticmethod
+    def prepare_data_dict_for_writing(data_dict):
+        """
+        Prepare data dictionary for writing to json file.
+
+        Ensure sim_telarray style lists as strings.
+        Replace "None" with "null" for unit field.
+
+        Parameters
+        ----------
+        data_dict: dict
+            Dictionary with lists.
+
+        Returns
+        -------
+        dict
+            Dictionary with lists converted to strings.
+
+        """
+        try:
+            data_dict["value"] = gen.convert_list_to_string(data_dict["value"])
+            data_dict["unit"] = gen.convert_list_to_string(data_dict["unit"], comma_separated=True)
+            data_dict["type"] = gen.convert_list_to_string(
+                data_dict["type"], comma_separated=True, collapse_list=True
+            )
+            if isinstance(data_dict["unit"], str):
+                data_dict["unit"] = data_dict["unit"].replace("None", "null")
+        except KeyError:
+            pass
+        return data_dict
+
     def write_metadata_to_yml(self, metadata, yml_file=None, keys_lower_case=False):
         """
         Write model metadata file (yaml file format).
@@ -159,7 +470,6 @@ class ModelDataWriter:
         TypeError
             If yml_file is not defined.
         """
-
         try:
             yml_file = Path(yml_file or self.product_data_file).with_suffix(".metadata.yml")
             with open(yml_file, "w", encoding="UTF-8") as file:
@@ -191,7 +501,6 @@ class ModelDataWriter:
             format identifier
 
         """
-
         if product_data_format == "ecsv":
             product_data_format = "ascii.ecsv"
         return product_data_format