gammasimtools 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

simtools/data_model/metadata_model.py

@@ -9,48 +9,12 @@ Follows CTAO top-level data model definition.
 """
 
 import logging
-from importlib.resources import files
 
-import jsonschema
-
-import simtools.constants
-import simtools.utils.general as gen
-from simtools.data_model import format_checkers
-from simtools.utils import names
+import simtools.data_model.schema
 
 _logger = logging.getLogger(__name__)
 
 
-def validate_schema(data, schema_file):
-    """
-    Validate dictionary against schema.
-
-    Parameters
-    ----------
-    data
-        dictionary to be validated
-    schema_file (dict)
-        schema used for validation
-
-    Raises
-    ------
-    jsonschema.exceptions.ValidationError
-        if validation fails
-
-    """
-    schema, schema_file = _load_schema(
-        schema_file,
-        data.get("schema_version", "0.1.0"),  # default version to ensure backward compatibility
-    )
-
-    try:
-        jsonschema.validate(data, schema=schema, format_checker=format_checkers.format_checker)
-    except jsonschema.exceptions.ValidationError:
-        _logger.error(f"Failed using {schema}")
-        raise
-    _logger.debug(f"Successful validation of data using schema from {schema_file}")
-
-
 def get_default_metadata_dict(schema_file=None, observatory="CTA"):
     """
     Return metadata schema with default values.
@@ -71,90 +35,10 @@ def get_default_metadata_dict(schema_file=None, observatory="CTA"):
 
 
     """
-    schema, _ = _load_schema(schema_file)
+    schema = simtools.data_model.schema.load_schema(schema_file)
     return _fill_defaults(schema["definitions"], observatory)
 
 
-def _load_schema(schema_file=None, schema_version=None):
-    """
-    Load parameter schema from file from simpipe metadata schema.
-
-    Returns
-    -------
-    schema_file: str
-        File name schema is loaded from. If schema_file is not given,
-        the default schema file name is returned.
-    schema_version: str
-        Schema version.
-
-    Raises
-    ------
-    FileNotFoundError
-        if schema file is not found
-
-    """
-    if schema_file is None:
-        schema_file = files("simtools").joinpath(simtools.constants.METADATA_JSON_SCHEMA)
-
-    try:
-        schema = gen.collect_data_from_file(file_name=schema_file)
-    except FileNotFoundError:
-        schema_file = files("simtools").joinpath("schemas") / schema_file
-        schema = gen.collect_data_from_file(file_name=schema_file)
-
-    if isinstance(schema, list):  # schema file with several schemas defined
-        if schema_version is None:
-            raise ValueError(f"Schema version not given in {schema_file}.")
-        schema = next((doc for doc in schema if doc.get("version") == schema_version), None)
-        if schema is None:
-            raise ValueError(f"Schema version {schema_version} not found in {schema_file}.")
-    elif schema_version is not None and schema_version != schema.get("version"):
-        _logger.warning(f"Schema version {schema_version} does not match {schema.get('version')}")
-
-    _logger.debug(f"Loading schema from {schema_file}")
-    _add_array_elements("InstrumentTypeElement", schema)
-
-    return schema, schema_file
-
-
-def _add_array_elements(key, schema):
-    """
-    Add list of array elements to schema.
-
-    This assumes an element [key]['enum'] is a list of elements.
-
-    Parameters
-    ----------
-    key: str
-        Key in schema dictionary
-    schema: dict
-        Schema dictionary
-
-    Returns
-    -------
-    dict
-        Schema dictionary with added array elements.
-
-    """
-    _list_of_array_elements = sorted(names.array_elements().keys())
-
-    def recursive_search(sub_schema, key):
-        if key in sub_schema:
-            if "enum" in sub_schema[key] and isinstance(sub_schema[key]["enum"], list):
-                sub_schema[key]["enum"] = list(
-                    set(sub_schema[key]["enum"] + _list_of_array_elements)
-                )
-            else:
-                sub_schema[key]["enum"] = _list_of_array_elements
-        else:
-            for _, v in sub_schema.items():
-                if isinstance(v, dict):
-                    recursive_search(v, key)
-
-    recursive_search(schema, key)
-    return schema
-
-
 def _resolve_references(yaml_data, observatory="CTA"):
     """
     Resolve references in yaml data and expand the received dictionary accordingly.
@@ -227,21 +111,21 @@ def _fill_defaults(schema, observatory="CTA"):
     return defaults
 
 
-def _fill_defaults_recursive(subschema, current_dict):
+def _fill_defaults_recursive(sub_schema, current_dict):
     """
-    Recursively fill default values from the subschema into the current dictionary.
+    Recursively fill default values from the sub_schema into the current dictionary.
 
     Parameters
     ----------
-    subschema: dict
-        Subschema describing part of the input data.
+    sub_schema: dict
+        Sub schema describing part of the input data.
     current_dict: dict
         Current dictionary to fill with default values.
     """
-    if "properties" not in subschema:
+    if "properties" not in sub_schema:
         _raise_missing_properties_error()
 
-    for prop, prop_schema in subschema["properties"].items():
+    for prop, prop_schema in sub_schema["properties"].items():
         _process_property(prop, prop_schema, current_dict)
 
 

simtools/data_model/model_data_writer.py

@@ -10,9 +10,9 @@ import yaml
 from astropy.io.registry.base import IORegistryError
 
 import simtools.utils.general as gen
-from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH
-from simtools.data_model import validate_data
+from simtools.data_model import schema, validate_data
 from simtools.data_model.metadata_collector import MetadataCollector
+from simtools.db import db_handler
 from simtools.io_operations import io_handler
 from simtools.utils import names, value_conversion
 
@@ -122,11 +122,12 @@ class ModelDataWriter:
         parameter_name,
         value,
         instrument,
-        model_version,
+        parameter_version,
         output_file,
         output_path=None,
         use_plain_output_path=False,
         metadata_input_dict=None,
+        db_config=None,
     ):
         """
         Generate DB-style model parameter dict and write it to json file.
@@ -139,8 +140,8 @@ class ModelDataWriter:
             Value of the parameter.
         instrument: str
             Name of the instrument.
-        model_version: str
-            Version of the model.
+        parameter_version: str
+            Version of the parameter.
         output_file: str
             Name of output file.
         output_path: str or Path
@@ -149,6 +150,8 @@ class ModelDataWriter:
             Use plain output path.
         metadata_input_dict: dict
             Input to metadata collector.
+        db_config: dict
+            Database configuration. If not None, check if parameter with the same version exists.
 
         Returns
         -------
@@ -162,20 +165,73 @@ class ModelDataWriter:
             output_path=output_path,
             use_plain_output_path=use_plain_output_path,
         )
-        _json_dict = writer.get_validated_parameter_dict(
-            parameter_name, value, instrument, model_version
-        )
-        writer.write_dict_to_model_parameter_json(output_file, _json_dict)
+        if db_config is not None:
+            writer.check_db_for_existing_parameter(
+                parameter_name, instrument, parameter_version, db_config
+            )
+
+        unique_id = None
         if metadata_input_dict is not None:
             metadata_input_dict["output_file"] = output_file
             metadata_input_dict["output_file_format"] = Path(output_file).suffix.lstrip(".")
+            metadata = MetadataCollector(args_dict=metadata_input_dict).get_top_level_metadata()
             writer.write_metadata_to_yml(
-                metadata=MetadataCollector(args_dict=metadata_input_dict).get_top_level_metadata(),
-                yml_file=output_path / f"{Path(output_file).stem}",
+                metadata=metadata, yml_file=output_path / f"{Path(output_file).stem}"
             )
+            unique_id = metadata.get("cta", {}).get("product", {}).get("id")
+
+        _json_dict = writer.get_validated_parameter_dict(
+            parameter_name, value, instrument, parameter_version, unique_id
+        )
+        writer.write_dict_to_model_parameter_json(output_file, _json_dict)
         return _json_dict
 
-    def get_validated_parameter_dict(self, parameter_name, value, instrument, model_version):
+    def check_db_for_existing_parameter(
+        self, parameter_name, instrument, parameter_version, db_config
+    ):
+        """
+        Check if a parameter with the same version exists in the simulation model database.
+
+        Parameters
+        ----------
+        parameter_name: str
+            Name of the parameter.
+        instrument: str
+            Name of the instrument.
+        parameter_version: str
+            Version of the parameter.
+        db_config: dict
+            Database configuration.
+
+        Raises
+        ------
+        ValueError
+            If parameter with the same version exists in the database.
+        """
+        db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+        try:
+            db.get_model_parameter(
+                parameter=parameter_name,
+                parameter_version=parameter_version,
+                site=names.get_site_from_array_element_name(instrument),
+                array_element_name=instrument,
+            )
+        except ValueError:
+            pass  # parameter does not exist - expected behavior
+        else:
+            raise ValueError(
+                f"Parameter {parameter_name} with version {parameter_version} already exists."
+            )
+
+    def get_validated_parameter_dict(
+        self,
+        parameter_name,
+        value,
+        instrument,
+        parameter_version,
+        unique_id=None,
+        schema_version=None,
+    ):
         """
         Get validated parameter dictionary.
 
@@ -187,8 +243,10 @@ class ModelDataWriter:
             Value of the parameter.
         instrument: str
             Name of the instrument.
-        model_version: str
-            Version of the model.
+        parameter_version: str
+            Version of the parameter.
+        schema_version: str
+            Version of the schema.
 
         Returns
         -------
@@ -196,29 +254,21 @@ class ModelDataWriter:
             Validated parameter dictionary.
         """
         self._logger.debug(f"Getting validated parameter dictionary for {instrument}")
-        schema_file = self._read_model_parameter_schema(parameter_name)
-
-        try:  # e.g. instrument is 'North"
-            site = names.validate_site_name(instrument)
-        except ValueError:  # e.g. instrument is 'LSTN-01'
-            site = names.get_site_from_array_element_name(instrument)
-
-        try:
-            applicable = self._get_parameter_applicability(instrument)
-        except ValueError:
-            applicable = True  # Default to True (expect that this field goes in future)
+        schema_file = schema.get_model_parameter_schema_file(parameter_name)
+        self.schema_dict = gen.collect_data_from_file(schema_file)
 
         value, unit = value_conversion.split_value_and_unit(value)
 
         data_dict = {
+            "schema_version": schema.get_model_parameter_schema_version(schema_version),
             "parameter": parameter_name,
             "instrument": instrument,
-            "site": site,
-            "version": model_version,
+            "site": names.get_site_from_array_element_name(instrument),
+            "parameter_version": parameter_version,
+            "unique_id": unique_id,
             "value": value,
             "unit": unit,
             "type": self._get_parameter_type(),
-            "applicable": applicable,
             "file": self._parameter_is_a_file(),
         }
         return self.validate_and_transform(
@@ -227,22 +277,6 @@ class ModelDataWriter:
             is_model_parameter=True,
         )
 
-    def _read_model_parameter_schema(self, parameter_name):
-        """
-        Read model parameter schema.
-
-        Parameters
-        ----------
-        parameter_name: str
-            Name of the parameter.
-        """
-        schema_file = MODEL_PARAMETER_SCHEMA_PATH / f"{parameter_name}.schema.yml"
-        try:
-            self.schema_dict = gen.collect_data_from_file(file_name=schema_file)
-        except FileNotFoundError as exc:
-            raise FileNotFoundError(f"Schema file not found: {schema_file}") from exc
-        return schema_file
-
     def _get_parameter_type(self):
         """
         Return parameter type from schema.
@@ -273,36 +307,6 @@ class ModelDataWriter:
             pass
         return False
 
-    def _get_parameter_applicability(self, telescope_name):
-        """
-        Check if a parameter is applicable for a given telescope using schema files.
-
-        First check for exact telescope name (e.g., LSTN-01), if not listed in the schema
-        use telescope type (LSTN).
-
-        Parameters
-        ----------
-        telescope_name: str
-            Telescope name (e.g., LSTN-01)
-
-        Returns
-        -------
-        bool
-            True if parameter is applicable to telescope.
-
-        """
-        try:
-            if telescope_name in self.schema_dict["instrument"]["type"]:
-                return True
-        except KeyError as exc:
-            self._logger.error("Schema file does not contain 'instrument:type' key.")
-            raise exc
-
-        return (
-            names.get_array_element_type_from_name(telescope_name)
-            in self.schema_dict["instrument"]["type"]
-        )
-
     def _get_unit_from_schema(self):
         """
         Return unit(s) from schema dict.
@@ -471,7 +475,9 @@ class ModelDataWriter:
             If yml_file is not defined.
         """
         try:
-            yml_file = Path(yml_file or self.product_data_file).with_suffix(".metadata.yml")
+            yml_file = names.file_name_with_version(
+                yml_file or self.product_data_file, ".metadata.yml"
+            )
             with open(yml_file, "w", encoding="UTF-8") as file:
                 yaml.safe_dump(
                     gen.change_dict_keys_case(metadata, keys_lower_case),

simtools/data_model/schema.py

@@ -0,0 +1,220 @@
+"""Module providing functionality to read and validate dictionaries using schema."""
+
+import logging
+from pathlib import Path
+
+import jsonschema
+
+import simtools.utils.general as gen
+from simtools.constants import (
+    METADATA_JSON_SCHEMA,
+    MODEL_PARAMETER_METASCHEMA,
+    MODEL_PARAMETER_SCHEMA_PATH,
+    SCHEMA_PATH,
+)
+from simtools.data_model import format_checkers
+from simtools.utils import names
+
+_logger = logging.getLogger(__name__)
+
+
+def get_get_model_parameter_schema_files(schema_directory=MODEL_PARAMETER_SCHEMA_PATH):
+    """
+    Return list of parameters and schema files located in schema file directory.
+
+    Returns
+    -------
+    list
+        List of parameters found in schema file directory.
+    list
+        List of schema files found in schema file directory.
+
+    """
+    schema_files = sorted(Path(schema_directory).rglob("*.schema.yml"))
+    if not schema_files:
+        raise FileNotFoundError(f"No schema files found in {schema_directory}")
+    parameters = []
+    for schema_file in schema_files:
+        schema_dict = gen.collect_data_from_file(file_name=schema_file)
+        parameters.append(schema_dict.get("name"))
+    return parameters, schema_files
+
+
+def get_model_parameter_schema_file(parameter):
+    """
+    Return schema file path for a given model parameter.
+
+    Parameters
+    ----------
+    parameter: str
+        Model parameter name.
+
+    Returns
+    -------
+    Path
+        Schema file path.
+
+    """
+    schema_file = MODEL_PARAMETER_SCHEMA_PATH / f"{parameter}.schema.yml"
+    if not schema_file.exists():
+        raise FileNotFoundError(f"Schema file not found: {schema_file}")
+    return schema_file
+
+
+def get_model_parameter_schema_version(schema_version=None):
+    """
+    Validate  and return schema versions.
+
+    If no schema_version is given, the most recent version is provided.
+
+    Parameters
+    ----------
+    schema_version: str
+        Schema version.
+
+    Returns
+    -------
+    str
+        Schema version.
+
+    """
+    schemas = gen.collect_data_from_file(MODEL_PARAMETER_METASCHEMA)
+
+    if schema_version is None and schemas:
+        return schemas[0].get("version")
+
+    if any(schema.get("version") == schema_version for schema in schemas):
+        return schema_version
+
+    raise ValueError(f"Schema version {schema_version} not found in {MODEL_PARAMETER_METASCHEMA}.")
+
+
+def validate_dict_using_schema(data, schema_file=None, json_schema=None):
+    """
+    Validate a data dictionary against a schema.
+
+    Parameters
+    ----------
+    data
+        dictionary to be validated
+    schema_file (dict)
+        schema used for validation
+
+    Raises
+    ------
+    jsonschema.exceptions.ValidationError
+        if validation fails
+
+    """
+    if json_schema is None and schema_file is None:
+        _logger.warning(f"No schema provided for validation of {data}")
+        return
+    if json_schema is None:
+        json_schema = load_schema(
+            schema_file,
+            data.get("schema_version")
+            or data.get(
+                "SCHEMA_VERSION", "0.1.0"
+            ),  # default version to ensure backward compatibility
+        )
+
+    try:
+        jsonschema.validate(data, schema=json_schema, format_checker=format_checkers.format_checker)
+    except jsonschema.exceptions.ValidationError as exc:
+        _logger.error(f"Validation failed using schema: {json_schema} for data: {data}")
+        raise exc
+    if (
+        isinstance(data, dict)
+        and data.get("meta_schema_url")
+        and not gen.url_exists(data["meta_schema_url"])
+    ):
+        raise FileNotFoundError(f"Meta schema URL does not exist: {data['meta_schema_url']}")
+
+    _logger.debug(f"Successful validation of data using schema ({json_schema.get('name')})")
+
+
+def load_schema(schema_file=None, schema_version=None):
+    """
+    Load parameter schema from file.
+
+    Parameters
+    ----------
+    schema_file: str
+        Path to schema file.
+    schema_version: str
+        Schema version.
+
+    Returns
+    -------
+    schema: dict
+        Schema dictionary.
+
+    Raises
+    ------
+    FileNotFoundError
+        if schema file is not found
+
+    """
+    schema_file = schema_file or METADATA_JSON_SCHEMA
+
+    for path in (schema_file, SCHEMA_PATH / schema_file):
+        try:
+            schema = gen.collect_data_from_file(file_name=path)
+            break
+        except FileNotFoundError:
+            continue
+    else:
+        raise FileNotFoundError(f"Schema file not found: {schema_file}")
+
+    if isinstance(schema, list):  # schema file with several schemas defined
+        if schema_version is None:
+            raise ValueError(f"Schema version not given in {schema_file}.")
+        schema = next((doc for doc in schema if doc.get("version") == schema_version), None)
+        if schema is None:
+            raise ValueError(f"Schema version {schema_version} not found in {schema_file}.")
+    elif schema_version is not None and schema_version != schema.get("version"):
+        _logger.warning(f"Schema version {schema_version} does not match {schema.get('version')}")
+
+    _logger.debug(f"Loading schema from {schema_file}")
+    _add_array_elements("InstrumentTypeElement", schema)
+
+    return schema
+
+
+def _add_array_elements(key, schema):
+    """
+    Add list of array elements to schema.
+
+    Avoids having to list all array elements in multiple schema.
+    Assumes an element [key]['enum'] is a list of elements.
+
+    Parameters
+    ----------
+    key: str
+        Key in schema dictionary
+    schema: dict
+        Schema dictionary
+
+    Returns
+    -------
+    dict
+        Schema dictionary with added array elements.
+
+    """
+    _list_of_array_elements = sorted(names.array_elements().keys())
+
+    def recursive_search(sub_schema, key):
+        if key in sub_schema:
+            if "enum" in sub_schema[key] and isinstance(sub_schema[key]["enum"], list):
+                sub_schema[key]["enum"] = list(
+                    set(sub_schema[key]["enum"] + _list_of_array_elements)
+                )
+            else:
+                sub_schema[key]["enum"] = _list_of_array_elements
+        else:
+            for _, v in sub_schema.items():
+                if isinstance(v, dict):
+                    recursive_search(v, key)
+
+    recursive_search(schema, key)
+    return schema