gammasimtools 0.17.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

simtools/data_model/metadata_collector.py

@@ -11,13 +11,11 @@ import logging
 import uuid
 from pathlib import Path
 
-import yaml
-
 import simtools.utils.general as gen
 import simtools.version
 from simtools.constants import METADATA_JSON_SCHEMA
 from simtools.data_model import metadata_model, schema
-from simtools.io_operations import io_handler
+from simtools.io import ascii_handler, io_handler
 from simtools.utils import names
 
 __all__ = ["MetadataCollector"]
@@ -41,33 +39,36 @@ class MetadataCollector:
         Command line parameters
     metadata_file_name: str
         Name of metadata file (only required when args_dict is None)
-    data_model_name: str
-        Name of data model parameter
+    model_parameter_name: str
+        Name of model parameter
     observatory: str
         Name of observatory (default: "cta")
     clean_meta: bool
         Clean metadata from None values and empty lists (default: True)
+    schema_version: str
+        Version of the metadata schema to use (default: 'latest')
     """
 
     def __init__(
         self,
-        args_dict,
+        args_dict=None,
         metadata_file_name=None,
-        data_model_name=None,
+        model_parameter_name=None,
         observatory="cta",
         clean_meta=True,
+        schema_version="latest",
     ):
         """Initialize metadata collector."""
         self._logger = logging.getLogger(__name__)
         self.observatory = observatory
         self.io_handler = io_handler.IOHandler()
 
-        self.args_dict = args_dict if args_dict else {}
-        self.data_model_name = data_model_name
+        self.args_dict = args_dict or {}
+        self.model_parameter_name = model_parameter_name
         self.schema_file = None
         self.schema_dict = None
-        self.top_level_meta = gen.change_dict_keys_case(
-            data_dict=metadata_model.get_default_metadata_dict(), lower_case=True
+        self.top_level_meta = metadata_model.get_default_metadata_dict(
+            schema_version=schema_version
         )
         self.input_metadata = self._read_input_metadata_from_file(metadata_file_name)
         self.collect_meta_data()
@@ -119,7 +120,7 @@ class MetadataCollector:
         collector = MetadataCollector(args_dict)
         collector.write(output_file, add_activity_name=add_activity_name)
 
-    def write(self, yml_file=None, keys_lower_case=False, add_activity_name=False):
+    def write(self, yml_file=None, keys_lower_case=True, add_activity_name=False):
         """
         Write toplevel metadata to file (yaml file format).
 
@@ -155,16 +156,13 @@ class MetadataCollector:
 
         try:
             yml_file = names.file_name_with_version(yml_file, suffix)
-            with open(yml_file, "w", encoding="UTF-8") as file:
-                yaml.safe_dump(
-                    gen.change_dict_keys_case(
-                        gen.remove_substring_recursively_from_dict(metadata, substring="\n"),
-                        keys_lower_case,
-                    ),
-                    file,
-                    sort_keys=False,
-                    explicit_start=True,
-                )
+            ascii_handler.write_data_to_file(
+                data=gen.change_dict_keys_case(
+                    gen.remove_substring_recursively_from_dict(metadata, substring="\n"),
+                    keys_lower_case,
+                ),
+                output_file=yml_file,
+            )
             self._logger.info(f"Writing metadata to {yml_file}")
             return yml_file
         except FileNotFoundError as exc:
@@ -198,10 +196,10 @@ class MetadataCollector:
         except KeyError:
             pass
 
-        # from data model name
-        if self.data_model_name:
-            self._logger.debug(f"Schema file from data model name: {self.data_model_name}")
-            return str(schema.get_model_parameter_schema_file(self.data_model_name))
+        # from model parameter name
+        if self.model_parameter_name:
+            self._logger.debug(f"Schema file from data model name: {self.model_parameter_name}")
+            return str(schema.get_model_parameter_schema_file(self.model_parameter_name))
 
         # from first entry in input metadata (least preferred)
         try:
@@ -225,7 +223,7 @@ class MetadataCollector:
 
         """
         try:
-            return gen.collect_data_from_file(file_name=self.schema_file)
+            return ascii_handler.collect_data_from_file(file_name=self.schema_file)
         except TypeError:
             self._logger.debug(f"No valid schema file provided ({self.schema_file}).")
         return {}
@@ -334,7 +332,7 @@ class MetadataCollector:
 
         Raises
         ------
-        gen.InvalidConfigDataError, FileNotFoundError
+        ValueError:
             if metadata cannot be read from file.
         KeyError:
             if metadata does not exist
@@ -365,7 +363,7 @@ class MetadataCollector:
                 )
                 continue
             else:
-                raise gen.InvalidConfigDataError(f"Unknown metadata file format: {metadata_file}")
+                raise ValueError(f"Unknown metadata file format: {metadata_file}")
 
             schema.validate_dict_using_schema(_input_metadata, schema_file=METADATA_JSON_SCHEMA)
             metadata.append(gen.change_dict_keys_case(_input_metadata, lower_case=True))
@@ -391,16 +389,14 @@ class MetadataCollector:
     def _read_input_metadata_from_yml_or_json(self, metadata_file_name):
         """Read input metadata from yml or json file."""
         try:
-            _input_metadata = gen.collect_data_from_file(file_name=metadata_file_name)
+            _input_metadata = ascii_handler.collect_data_from_file(file_name=metadata_file_name)
             _json_type_metadata = {"Metadata", "metadata", "METADATA"}.intersection(_input_metadata)
             if len(_json_type_metadata) == 1:
                 _input_metadata = _input_metadata[_json_type_metadata.pop()]
             if len(_json_type_metadata) > 1:
-                self._logger.error("More than one metadata entry found in %s", metadata_file_name)
-                raise gen.InvalidConfigDataError
-        except (gen.InvalidConfigDataError, FileNotFoundError) as exc:
-            self._logger.error("Failed reading metadata from %s", metadata_file_name)
-            raise exc
+                raise ValueError(f"More than one metadata entry found in {metadata_file_name}")
+        except FileNotFoundError as exc:
+            raise FileNotFoundError(f"Failed reading metadata from {metadata_file_name}") from exc
         return _input_metadata
 
     def _fill_product_meta(self, product_dict):
@@ -443,7 +439,7 @@ class MetadataCollector:
             or self.args_dict.get("metadata_product_data_name")
             or "undefined_model_name"
         )
-        product_dict["data"]["model"]["version"] = self.schema_dict.get("version", "0.0.0")
+        product_dict["data"]["model"]["version"] = self.schema_dict.get("schema_version", "0.0.0")
         product_dict["data"]["model"]["type"] = self.schema_dict.get("meta_schema", None)
         product_dict["data"]["model"]["url"] = self.schema_file or self.args_dict.get(
             "metadata_product_data_url"

simtools/data_model/metadata_model.py

@@ -11,11 +11,14 @@ Follows CTAO top-level data model definition.
 import logging
 
 import simtools.data_model.schema
+import simtools.utils.general as gen
 
 _logger = logging.getLogger(__name__)
 
 
-def get_default_metadata_dict(schema_file=None, observatory="CTA"):
+def get_default_metadata_dict(
+    schema_file=None, observatory="CTA", schema_version="latest", lower_case=True
+):
     """
     Return metadata schema with default values.
 
@@ -27,6 +30,10 @@ def get_default_metadata_dict(schema_file=None, observatory="CTA"):
         Schema file (jsonschema format) used for validation
     observatory: str
         Observatory name
+    schema_version: str, optional
+        Version of the schema to use. If not provided, the latest version is used.
+    lower_case: bool, optional
+        If True, all keys in the returned dictionary will be converted to lower case.
 
     Returns
     -------
@@ -35,8 +42,11 @@ def get_default_metadata_dict(schema_file=None, observatory="CTA"):
 
 
     """
-    schema = simtools.data_model.schema.load_schema(schema_file)
-    return _fill_defaults(schema["definitions"], observatory)
+    schema = simtools.data_model.schema.load_schema(schema_file, schema_version=schema_version)
+    return gen.change_dict_keys_case(
+        data_dict=_fill_defaults(schema["definitions"], observatory.lower()),
+        lower_case=lower_case,
+    )
 
 
 def _resolve_references(yaml_data, observatory="CTA"):
@@ -62,7 +72,7 @@ def _resolve_references(yaml_data, observatory="CTA"):
         parts = ref_path.split("/")
         ref_data = yaml_data
         for part in parts:
-            if part in ("definitions", observatory):
+            if part in ("definitions", observatory.lower()):
                 continue
             ref_data = ref_data.get(part, {})
         return ref_data
@@ -123,7 +133,7 @@ def _fill_defaults_recursive(sub_schema, current_dict):
         Current dictionary to fill with default values.
     """
     if "properties" not in sub_schema:
-        _raise_missing_properties_error()
+        raise KeyError("Missing 'properties' key in schema.")
 
     for prop, prop_schema in sub_schema["properties"].items():
         _process_property(prop, prop_schema, current_dict)
@@ -152,10 +162,3 @@ def _process_property(prop, prop_schema, current_dict):
             current_dict[prop] = [{}]
             if "items" in prop_schema and isinstance(prop_schema["items"], dict):
                 _fill_defaults_recursive(prop_schema["items"], current_dict[prop][0])
-
-
-def _raise_missing_properties_error():
-    """Raise an error when the 'properties' key is missing in the schema."""
-    msg = "Missing 'properties' key in schema."
-    _logger.error(msg)
-    raise KeyError(msg)

simtools/data_model/model_data_writer.py

@@ -1,40 +1,20 @@
 """Model data writer module."""
 
-import json
 import logging
 from pathlib import Path
 
-import astropy.units as u
-import numpy as np
 from astropy.io.registry.base import IORegistryError
 
 import simtools.utils.general as gen
 from simtools.data_model import schema, validate_data
 from simtools.data_model.metadata_collector import MetadataCollector
 from simtools.db import db_handler
-from simtools.io_operations import io_handler
+from simtools.io import ascii_handler, io_handler
 from simtools.utils import names, value_conversion
 
 __all__ = ["ModelDataWriter"]
 
 
-class JsonNumpyEncoder(json.JSONEncoder):
-    """Convert numpy to python types as accepted by json.dump."""
-
-    def default(self, o):
-        if isinstance(o, np.floating):
-            return float(o)
-        if isinstance(o, np.integer):
-            return int(o)
-        if isinstance(o, np.ndarray):
-            return o.tolist()
-        if isinstance(o, u.core.CompositeUnit | u.core.IrreducibleUnit | u.core.Unit):
-            return str(o) if o != u.dimensionless_unscaled else None
-        if np.issubdtype(type(o), np.bool_):
-            return bool(o)
-        return super().default(o)
-
-
 class ModelDataWriter:
     """
     Writer for simulation model data and metadata.
@@ -254,7 +234,7 @@ class ModelDataWriter:
         """
         self._logger.debug(f"Getting validated parameter dictionary for {instrument}")
         schema_file = schema.get_model_parameter_schema_file(parameter_name)
-        self.schema_dict = gen.collect_data_from_file(schema_file)
+        self.schema_dict = ascii_handler.collect_data_from_file(schema_file)
 
         value, unit = value_conversion.split_value_and_unit(value)
 
@@ -381,7 +361,7 @@ class ModelDataWriter:
 
         if metadata is not None:
             product_data.meta.update(
-                gen.change_dict_keys_case(metadata.get_top_level_metadata(), False)
+                gen.change_dict_keys_case(metadata.get_top_level_metadata(), True)
             )
 
         self._logger.info(f"Writing data to {self.product_data_file}")
@@ -415,15 +395,13 @@ class ModelDataWriter:
             if data writing was not successful.
         """
         data_dict = ModelDataWriter.prepare_data_dict_for_writing(data_dict)
-        try:
-            self._logger.info(f"Writing data to {self.io_handler.get_output_file(file_name)}")
-            with open(self.io_handler.get_output_file(file_name), "w", encoding="UTF-8") as file:
-                json.dump(data_dict, file, indent=4, sort_keys=False, cls=JsonNumpyEncoder)
-                file.write("\n")
-        except FileNotFoundError as exc:
-            raise FileNotFoundError(
-                f"Error writing model data to {self.io_handler.get_output_file(file_name)}"
-            ) from exc
+        self._logger.info(f"Writing data to {self.io_handler.get_output_file(file_name)}")
+        ascii_handler.write_data_to_file(
+            data=data_dict,
+            output_file=self.io_handler.get_output_file(file_name),
+            sort_keys=False,
+            numpy_types=True,
+        )
 
     @staticmethod
     def prepare_data_dict_for_writing(data_dict):
@@ -432,6 +410,7 @@ class ModelDataWriter:
 
         Ensure sim_telarray style lists as strings 'type' and 'unit' entries.
         Replace "None" with "null" for unit field.
+        Replace list of equal units with single unit string.
 
         Parameters
         ----------
@@ -452,6 +431,8 @@ class ModelDataWriter:
                     unit.replace("None", "null") if isinstance(unit, str) else unit
                     for unit in data_dict["unit"]
                 ]
+                if all(u == data_dict["unit"][0] for u in data_dict["unit"]):
+                    data_dict["unit"] = data_dict["unit"][0]
         except KeyError:
             pass
         return data_dict

simtools/data_model/schema.py

@@ -4,6 +4,7 @@ import logging
 from pathlib import Path
 
 import jsonschema
+from referencing import Registry, Resource
 
 import simtools.utils.general as gen
 from simtools.constants import (
@@ -13,6 +14,7 @@ from simtools.constants import (
     SCHEMA_PATH,
 )
 from simtools.data_model import format_checkers
+from simtools.io import ascii_handler
 from simtools.utils import names
 
 _logger = logging.getLogger(__name__)
@@ -36,7 +38,7 @@ def get_model_parameter_schema_files(schema_directory=MODEL_PARAMETER_SCHEMA_PAT
     parameters = []
     for schema_file in schema_files:
         # reading parameter 'name' only - first document in schema file should be ok
-        schema_dict = gen.collect_data_from_file(file_name=schema_file, yaml_document=0)
+        schema_dict = ascii_handler.collect_data_from_file(file_name=schema_file, yaml_document=0)
         parameters.append(schema_dict.get("name"))
     return parameters, schema_files
 
@@ -79,12 +81,12 @@ def get_model_parameter_schema_version(schema_version=None):
         Schema version.
 
     """
-    schemas = gen.collect_data_from_file(MODEL_PARAMETER_METASCHEMA)
+    schemas = ascii_handler.collect_data_from_file(MODEL_PARAMETER_METASCHEMA)
 
     if schema_version is None and schemas:
-        return schemas[0].get("version")
+        return schemas[0].get("schema_version")
 
-    if any(schema.get("version") == schema_version for schema in schemas):
+    if any(schema.get("schema_version") == schema_version for schema in schemas):
         return schema_version
 
     raise ValueError(f"Schema version {schema_version} not found in {MODEL_PARAMETER_METASCHEMA}.")
@@ -111,16 +113,16 @@ def validate_dict_using_schema(data, schema_file=None, json_schema=None):
         _logger.warning(f"No schema provided for validation of {data}")
         return None
     if json_schema is None:
-        json_schema = load_schema(
-            schema_file,
-            data.get("schema_version")
-            or data.get(
-                "SCHEMA_VERSION", "0.1.0"
-            ),  # default version to ensure backward compatibility
-        )
+        json_schema = load_schema(schema_file, get_schema_version_from_data(data))
+
+    validator = jsonschema.Draft6Validator(
+        schema=json_schema,
+        format_checker=format_checkers.format_checker,
+        registry=Registry(retrieve=_retrieve_yaml_schema_from_uri),
+    )
 
     try:
-        jsonschema.validate(data, schema=json_schema, format_checker=format_checkers.format_checker)
+        validator.validate(instance=data)
     except jsonschema.exceptions.ValidationError as exc:
         _logger.error(f"Validation failed using schema: {json_schema} for data: {data}")
         raise exc
@@ -135,7 +137,39 @@ def validate_dict_using_schema(data, schema_file=None, json_schema=None):
     return data
 
 
-def load_schema(schema_file=None, schema_version=None):
+def _retrieve_yaml_schema_from_uri(uri):
+    """Load schema from a file URI."""
+    path = SCHEMA_PATH / Path(uri.removeprefix("file:/"))
+    contents = ascii_handler.collect_data_from_file(file_name=path)
+    return Resource.from_contents(contents)
+
+
+def get_schema_version_from_data(data, observatory="cta"):
+    """
+    Get schema version from data dictionary.
+
+    Parameters
+    ----------
+    data: dict
+        data dictionary.
+
+    Returns
+    -------
+    str
+        Schema version. If not found, returns 'latest'.
+    """
+    schema_version = data.get("schema_version") or data.get("SCHEMA_VERSION")
+    if schema_version:
+        return schema_version
+    reference_version = data.get(observatory.upper(), {}).get("REFERENCE", {}).get(
+        "VERSION"
+    ) or data.get(observatory.lower(), {}).get("reference", {}).get("version")
+    if reference_version:
+        return reference_version
+    return "latest"
+
+
+def load_schema(schema_file=None, schema_version="latest"):
     """
     Load parameter schema from file.
 
@@ -161,28 +195,58 @@ def load_schema(schema_file=None, schema_version=None):
 
     for path in (schema_file, SCHEMA_PATH / schema_file):
         try:
-            schema = gen.collect_data_from_file(file_name=path)
+            schema = ascii_handler.collect_data_from_file(file_name=path)
             break
         except FileNotFoundError:
             continue
     else:
         raise FileNotFoundError(f"Schema file not found: {schema_file}")
 
-    if isinstance(schema, list):  # schema file with several schemas defined
-        if schema_version is None:
-            raise ValueError(f"Schema version not given in {schema_file}.")
-        schema = next((doc for doc in schema if doc.get("version") == schema_version), None)
-        if schema is None:
-            raise ValueError(f"Schema version {schema_version} not found in {schema_file}.")
-    elif schema_version is not None and schema_version != schema.get("version"):
-        _logger.warning(f"Schema version {schema_version} does not match {schema.get('version')}")
-
-    _logger.debug(f"Loading schema from {schema_file}")
+    _logger.debug(f"Loading schema from {schema_file} for schema version {schema_version}")
+    schema = _get_schema_for_version(schema, schema_file, schema_version)
     _add_array_elements("InstrumentTypeElement", schema)
 
     return schema
 
 
+def _get_schema_for_version(schema, schema_file, schema_version):
+    """
+    Get schema for a specific version.
+
+    Allow for 'latest' version to return the most recent schema.
+
+    Parameters
+    ----------
+    schema: dict or list
+        Schema dictionary or list of dictionaries.
+    schema_file: str
+        Path to schema file.
+    schema_version: str or None
+        Schema version to retrieve. If 'latest', the most recent version is returned.
+
+    Returns
+    -------
+    dict
+        Schema dictionary for the specified version.
+    """
+    if schema_version is None:
+        raise ValueError(f"Schema version not given in {schema_file}.")
+
+    if isinstance(schema, list):  # schema file with several schemas defined
+        if len(schema) == 0:
+            raise ValueError(f"No schemas found in {schema_file}.")
+        if schema_version == "latest":
+            schema_version = schema[0].get("schema_version")
+        schema = next((doc for doc in schema if doc.get("schema_version") == schema_version), None)
+    if schema is None:
+        raise ValueError(f"Schema version {schema_version} not found in {schema_file}.")
+    if schema_version not in (None, "latest") and schema_version != schema.get("schema_version"):
+        _logger.warning(
+            f"Schema version {schema_version} does not match {schema.get('schema_version')}"
+        )
+    return schema
+
+
 def _get_array_element_list():
     """Build complete list of array elements including design types."""
     elements = set(names.array_elements().keys())

simtools/data_model/validate_data.py

@@ -12,6 +12,7 @@ from astropy.utils.diff import report_diff_values
 
 import simtools.utils.general as gen
 from simtools.data_model import schema
+from simtools.io import ascii_handler
 from simtools.utils import names, value_conversion
 
 __all__ = ["DataValidator"]
@@ -100,7 +101,7 @@ class DataValidator:
         """
         try:
             if Path(self.data_file_name).suffix in (".yml", ".yaml", ".json"):
-                self.data_dict = gen.collect_data_from_file(self.data_file_name)
+                self.data_dict = ascii_handler.collect_data_from_file(self.data_file_name)
                 self._logger.info(f"Validating data from: {self.data_file_name}")
             else:
                 self.data_table = Table.read(self.data_file_name, guess=True, delimiter=r"\s")
@@ -563,7 +564,7 @@ class DataValidator:
             If unit conversions fails
 
         """
-        self._logger.debug(f"Checking data column '{col_name}'")
+        self._rate_limited_logger(col_name, f"Checking data column '{col_name}'")
 
         reference_unit = self._get_reference_unit(col_name)
         try:
@@ -574,9 +575,10 @@ class DataValidator:
         if self._is_dimensionless(column_unit) and self._is_dimensionless(reference_unit):
             return data, u.dimensionless_unscaled
 
-        self._logger.debug(
+        self._rate_limited_logger(
+            col_name,
             f"Data column '{col_name}' with reference unit "
-            f"'{reference_unit}' and data unit '{column_unit}'"
+            f"'{reference_unit}' and data unit '{column_unit}'",
         )
         try:
             if isinstance(data, u.Quantity | Column):
@@ -652,7 +654,9 @@ class DataValidator:
             range columns
 
         """
-        self._logger.debug(f"Checking data in column '{col_name}' for '{range_type}' ")
+        self._rate_limited_logger(
+            col_name, f"Checking data in column '{col_name}' for '{range_type}'"
+        )
 
         if range_type not in ("allowed_range", "required_range"):
             raise KeyError("Allowed range types are 'allowed_range', 'required_range'")
@@ -673,6 +677,26 @@ class DataValidator:
                 f"{_entry[range_type].get('max', np.inf)}])"
             )
 
+    def _rate_limited_logger(self, col_name, message, max_logs=10):
+        """
+        Log debug messages at a limited rate defined by a numerical column name.
+
+        Parameters
+        ----------
+        col_name: str or int
+            Column name or index to limit the rate of logging.
+        message: str
+            Message to log.
+        max_logs: int
+            Maximum number of logging messages to be printed.
+        """
+        try:
+            col_index = int(col_name)
+            if col_index < max_logs:
+                self._logger.debug(message)
+        except (ValueError, TypeError):
+            self._logger.debug(message)
+
     @staticmethod
     def _interval_check(data, axis_range, range_type):
         """
@@ -732,11 +756,11 @@ class DataValidator:
         ValueError
             if schema version is not found in schema file
         """
-        schema_data = gen.collect_data_from_file(file_name=schema_file)
+        schema_data = ascii_handler.collect_data_from_file(file_name=schema_file)
         entries = schema_data if isinstance(schema_data, list) else [schema_data]
 
         for entry in entries:
-            if not schema_version or entry.get("version") == schema_version:
+            if not schema_version or entry.get("schema_version") == schema_version:
                 try:
                     return entry["data"]
                 except KeyError as exc:
@@ -774,8 +798,9 @@ class DataValidator:
             If data column is not found.
 
         """
-        self._logger.debug(
-            f"Getting reference data column {column_name} from schema {self._data_description}"
+        self._rate_limited_logger(
+            column_name,
+            f"Getting reference data column {column_name} from schema {self._data_description}",
         )
         try:
             return (