gammasimtools 0.5.1__py3-none-any.whl → 0.6.1__py3-none-any.whl

simtools/data_model/metadata_model.py

@@ -1,281 +1,186 @@
 """
 Definition of metadata model for input to and output of simtools.
-Follows CTA top-level data model definition.
-Metadata is broken into hierarchical categories in this module.
-
-Metadata definitions require the three following fields to be defined:
-
-* type: data type
-* required: boolean if this field is required as input(!)
-* default: default value
-
-These definitions are used for:
+Follows CTAO top-level data model definition.
 
 * data products submitted to SimPipe ('input')
 * data products generated by SimPipe ('output')
 
 """
 
+import logging
+from importlib.resources import files
 
-class InvalidSchemaList(Exception):
-    """
-    Exception raised for requests of unknown schema lists
-    """
-
-
-def _category_reference():
-    """
-    Metadata field REFERENCE.
-
-    Refers to the CTA Top-Level Data Model (document CTA-SPE-OSO-000000-0001.)
-
-    """
-    return {"VERSION": {"type": "str", "required": True, "default": "1.0.0"}}
-
-
-def _category_contact():
-    """
-    Metadata field CONTACT.
+import jsonschema
 
-    Organisation and person submitting or processing these data.
+import simtools.constants
+import simtools.utils.general as gen
 
-    """
-    return {
-        "ORGANIZATION": {"type": "str", "required": True, "default": None},
-        "NAME": {"type": "str", "required": True, "default": None},
-        "EMAIL": {"type": "email", "required": True, "default": None},
-    }
+_logger = logging.getLogger(__name__)
 
 
-def _category_product():
+def validate_schema(data, schema_file):
     """
-    Metadata field PRODUCT.
-
-    Describes the data product, especially its validity and associations.
-
-    """
-    return {
-        "DESCRIPTION": {"type": "str", "required": True, "default": None},
-        "CREATION_TIME": {"type": "datetime", "required": True, "default": None},
-        "ID": {"type": "str", "required": False, "default": None},
-        "DATA": {
-            "CATEGORY": {"type": "str", "required": False, "default": None},
-            "LEVEL": {"type": "str", "required": False, "default": None},
-            "TYPE": {"type": "str", "required": False, "default": None},
-            "MODEL": {
-                "NAME": {"type": "str", "required": False, "default": None},
-                "VERSION": {"type": "str", "required": False, "default": None},
-                "URL": {"type": "str", "required": False, "default": None},
-                "TYPE": {"type": "str", "required": False, "default": None},
-                "SUBTYPE": {"type": "str", "required": False, "default": None},
-            },
-        },
-        "FORMAT": {"type": "str", "required": False, "default": "ecsv"},
-        "VALID": {
-            "START": {"type": "datetime", "required": False, "default": None},
-            "END": {"type": "datetime", "required": False, "default": None},
-        },
-        "ASSOCIATION": {"type": "instrumentlist", "required": True, "default": []},
-    }
-
-
-def _category_process():
-    """
-    Metadata field PROCESS.
-
-    Description of the process which generated these data.
+    Validate dictionary against schema.
 
-    """
-    return {
-        "TYPE": {"type": "str", "required": True, "default": None},
-        "SUBTYPE": {"type": "str", "required": False, "default": None},
-        "ID": {"type": "str", "required": True, "default": None},
-    }
+    Parameters
+    ----------
+    data
+        dictionary to be validated
+    schema_file (dict)
+        schema used for validation
 
+    Raises
+    ------
+    jsonschema.exceptions.ValidationError
+        if validation fails
 
-def _category_context():
     """
-    Metadata field CONTEXT.
 
-    Describes list of context documents.
-
-    """
-    return {"DOCUMENT": {"type": "documentlist", "required": False, "default": None}}
+    schema, schema_file = _load_schema(schema_file)
 
+    try:
+        jsonschema.validate(data, schema=schema)
+    except jsonschema.exceptions.ValidationError:
+        _logger.error(f"Failed using {schema}")
+        raise
+    _logger.debug(f"Successful validation of data using schema from {schema_file}")
 
-def _category_document():
-    """
-    Metadata field DOCUMENT.
 
+def get_default_metadata_dict(schema_file=None, observatory="CTA"):
     """
+    Returns metadata schema with default values.
+    Follows the CTA Top-Level Data Model.
 
-    return {
-        "TYPE": {"type": "str", "required": False, "default": None},
-        "ID": {"type": "str", "required": False, "default": None},
-        "LINK": {"type": "str", "required": False, "default": None},
-        "TITLE": {"type": "str", "required": False, "default": None},
-    }
-
+    Parameters
+    ----------
+    schema_file: str
+        Schema file (jsonschema format) used for validation
+    observatory: str
+        Observatory name
 
-def _category_instrument():
-    """
-    Metadata field INSTRUMENT.
+    Returns
+    -------
+    dict
+        Reference schema dictionary.
 
-    Describes the instrument used to obtain these data or the instrument these data is applied for.
 
     """
-    return {
-        "SITE": {"type": "str", "required": True, "default": None},
-        "CLASS": {"type": "str", "required": True, "default": None},
-        "TYPE": {"type": "str", "required": True, "default": None},
-        "SUBTYPE": {"type": "str", "required": False, "default": None},
-        "ID": {"type": "str", "required": True, "default": None},
-    }
 
+    schema, _ = _load_schema(schema_file)
+    return _fill_defaults(schema["definitions"], observatory)
 
-def _category_activity():
-    """
-    Metadata field ACTIVITY.
-
-    Describes the software used to process these data.
 
+def _load_schema(schema_file=None):
     """
-    return {
-        "NAME": {"type": "str", "required": True, "default": None},
-        "TYPE": {"type": "str", "required": True, "default": "software"},
-        "ID": {"type": "str", "required": True, "default": None},
-        "START": {"type": "datetime", "required": False, "default": None},
-        "SOFTWARE": {
-            "NAME": {"type": "str", "required": True, "default": None},
-            "VERSION": {"type": "str", "required": True, "default": None},
-        },
-    }
-
-
-def top_level_reference_schema():
-    """
-    Reference schema following the CTA Top-Level Data Model.
-
-    This metadata schema is used for simtools data products.
+    Load parameter schema from file from simpipe metadata schema.
 
     Returns
     -------
-    dict with reference schema
+    schema_file dict
+        Schema used for validation.
+    schema_file str
+        File name schema is loaded from. If schema_file is not given,
+        the default schema file name is returned.
 
+    Raises
+    ------
+    FileNotFoundError
+        if schema file is not found
 
     """
 
-    _ref_schema = {
-        "CTA": {
-            "REFERENCE": _category_reference(),
-            "CONTACT": _category_contact(),
-            "PRODUCT": _category_product(),
-            "INSTRUMENT": _category_instrument(),
-            "PROCESS": _category_process(),
-            "ACTIVITY": _category_activity(),
-            "CONTEXT": {
-                "SIM": {
-                    "ASSOCIATION": [
-                        _category_instrument(),
-                    ],
-                    "DOCUMENT": [
-                        _category_document(),
-                    ],
-                },
-            },
-        }
-    }
-    _ref_schema = _metadata_dict_with_defaults(_ref_schema)
-    return _remove_empty_lists(_ref_schema)
-
-
-def metadata_input_reference_schema():
-    """
-    Reference data model scheme for input metadata.
-    Describes metadata provided for input to simtools applications.
+    if schema_file is None:
+        schema_file = files("simtools").joinpath(simtools.constants.METADATA_JSON_SCHEMA)
 
-    Returns
-    -------
-    dict with input reference schema
-
-    """
+    schema = gen.collect_data_from_file_or_dict(file_name=schema_file, in_dict=None)
+    _logger.debug(f"Loading schema from {schema_file}")
 
-    return {
-        "REFERENCE": _category_reference(),
-        "CONTACT": _category_contact(),
-        "PRODUCT": _category_product(),
-        "INSTRUMENT": _category_instrument(),
-        "PROCESS": _category_process(),
-        "CONTEXT": _category_context(),
-    }
+    return schema, schema_file
 
 
-def metadata_input_reference_document_list(schema_list):
+def _resolve_references(yaml_data, observatory="CTA"):
     """
-    Reference model data for input metata data of type documentlist or instrumentlist
+    Resolve references in yaml data and expand the received dictionary accordingly.
 
     Parameters
     ----------
-    schema_list: str
-        List type to be returned (e.g., instrumentlist or documentlist)
+    yaml_data: dict
+        Dictionary with yaml data.
+    observatory: str
+        Observatory name
 
     Returns
     -------
-    dict with input reference schema for documentlist or instrumentlist
+    dict
+        Dictionary with resolved references.
 
     """
-    if schema_list.lower() == "instrumentlist":
-        return _category_instrument()
-    if schema_list.lower() == "documentlist":
-        return _category_document()
 
-    msg = f"Invalid schema list: {schema_list}"
-    raise InvalidSchemaList(msg)
+    def expand_ref(ref):
+        ref_path = ref.lstrip("#/")
+        parts = ref_path.split("/")
+        ref_data = yaml_data
+        for part in parts:
+            if part in ("definitions", observatory):
+                continue
+            ref_data = ref_data.get(part, {})
+        return ref_data
+
+    def _resolve_references_recursive(data):
+        if isinstance(data, dict):
+            if "$ref" in data:
+                ref = data["$ref"]
+                resolved_data = expand_ref(ref)
+                if isinstance(resolved_data, dict) and len(resolved_data) > 1:
+                    return _resolve_references_recursive(resolved_data)
+                return resolved_data
+            return {k: _resolve_references_recursive(v) for k, v in data.items()}
+        if isinstance(data, list):
+            return [_resolve_references_recursive(item) for item in data]
+        return data
+
+    return _resolve_references_recursive(yaml_data)
 
 
-def _metadata_dict_with_defaults(meta_dict):
+def _fill_defaults(schema, observatory="CTA"):
     """
-    Prepare dictionary with default values filled and removal of all type/required/default dicts.
+    Fill default values from json schema.
 
     Parameters
     ----------
-    meta_dict
-        Metadata dictionary
+    schema: dict
+        Schema describing the input data.
+    observatory: str
+        Observatory name
 
     Returns
     -------
     dict
-        Metadata dictionary with default values filled
-
-    """
-
-    for key, value in meta_dict.items():
-        if isinstance(value, dict):
-            if value.keys() >= {"type", "required", "default"}:
-                meta_dict[key] = value["default"]
-            else:
-                _metadata_dict_with_defaults(value)
-        elif isinstance(value, list):
-            for list_entry in value:
-                _metadata_dict_with_defaults(list_entry)
-        else:
-            msg = f"Invalid schema list with missing type, required, or default fields: {key}"
-            raise InvalidSchemaList(msg)
-    return meta_dict
-
-
-def _remove_empty_lists(meta_dict):
-    """
-    Remove entries of type list with length zero, as those are not used for output.
-
-    """
-    _entries_to_pop = []
-    for key, value in meta_dict.items():
-        if isinstance(value, dict):
-            _remove_empty_lists(value)
-        elif isinstance(value, list) and len(value) == 0:
-            _entries_to_pop.append(key)
-    for key in _entries_to_pop:
-        meta_dict.pop(key)
-
-    return meta_dict
+        Dictionary with default values.
+
+    """
+
+    defaults = {observatory: {}}
+
+    schema = _resolve_references(schema[observatory])
+
+    def _fill_defaults_recursive(subschema, current_dict):
+        try:
+            for prop, prop_schema in subschema["properties"].items():
+                if "default" in prop_schema:
+                    current_dict[prop] = prop_schema["default"]
+                elif "type" in prop_schema:
+                    if prop_schema["type"] == "object":
+                        current_dict[prop] = {}
+                        _fill_defaults_recursive(prop_schema, current_dict[prop])
+                    elif prop_schema["type"] == "array":
+                        current_dict[prop] = [{}]
+                        if "items" in prop_schema and isinstance(prop_schema["items"], dict):
+                            _fill_defaults_recursive(prop_schema["items"], current_dict[prop][0])
+        except KeyError:
+            msg = "Missing 'properties' key in schema."
+            _logger.error(msg)
+            raise
+
+    _fill_defaults_recursive(schema, defaults[observatory])
+    return defaults

simtools/data_model/model_data_writer.py

@@ -5,7 +5,8 @@ import astropy
 import yaml
 
 import simtools.utils.general as gen
-from simtools import io_handler
+from simtools.data_model import validate_data
+from simtools.io_operations import io_handler
 
 __all__ = ["ModelDataWriter"]
 
@@ -16,66 +17,124 @@ class ModelDataWriter:
 
     Parameters
     ----------
+    product_data_file: str
+        Name of output file.
+    product_data_format: str
+        Format of output file.
     args_dict: Dictionary
         Dictionary with configuration parameters.
     """
 
-    def __init__(self, product_data_file=None, product_data_format=None):
+    def __init__(self, product_data_file=None, product_data_format=None, args_dict=None):
         """
         Initialize model data writer.
         """
 
         self._logger = logging.getLogger(__name__)
         self.io_handler = io_handler.IOHandler()
+        if args_dict is not None:
+            self.io_handler.set_paths(
+                output_path=args_dict.get("output_path", None),
+                use_plain_output_path=args_dict.get("use_plain_output_path", False),
+            )
         try:
-            self.product_data_file = self.io_handler.get_output_file(file_name=product_data_file)
+            self.product_data_file = self.io_handler.get_output_file(
+                file_name=product_data_file, dir_type="simtools-result"
+            )
         except TypeError:
             self.product_data_file = None
         self.product_data_format = self._astropy_data_format(product_data_format)
 
-    def write(self, metadata=None, product_data=None):
+    @staticmethod
+    def dump(
+        args_dict, output_file=None, metadata=None, product_data=None, validate_schema_file=None
+    ):
         """
-        Write model data and metadata
+        Write model data and metadata (as static method).
 
-        Parameters:
-        -----------
+        Parameters
+        ----------
+        args_dict: dict
+            Dictionary with configuration parameters (including output file name and path).
+        output_file: string or Path
+            Name of output file (args["output_file"] is used if this parameter is not set).
         metadata: dict
             Metadata to be written.
         product_data: astropy Table
             Model data to be written
+        validate_schema_file: str
+            Schema file used in validation of output data.
 
         """
 
-        self.write_metadata(metadata=metadata)
-        self.write_data(product_data=product_data)
+        writer = ModelDataWriter(
+            product_data_file=(
+                args_dict.get("output_file", None) if output_file is None else output_file
+            ),
+            product_data_format=args_dict.get("output_file_format", "ascii.ecsv"),
+            args_dict=args_dict,
+        )
+        if validate_schema_file is not None and not args_dict.get("skip_output_validation", True):
+            product_data = writer.validate_and_transform(
+                product_data=product_data,
+                validate_schema_file=validate_schema_file,
+            )
+        writer.write(metadata=metadata, product_data=product_data)
+
+    def validate_and_transform(self, product_data=None, validate_schema_file=None):
+        """
+        Validate product data using jsonschema given in metadata.
+        If necessary, transform product data to match schema.
+
+        Parameters
+        ----------
+        product_data: astropy Table
+            Model data to be validated
+        validate_schema_file: str
+            Schema file used in validation of output data.
 
-    def write_data(self, product_data):
         """
-        Write model data.
+
+        _validator = validate_data.DataValidator(
+            schema_file=validate_schema_file,
+            data_table=product_data,
+        )
+        return _validator.validate_and_transform()
+
+    def write(self, product_data=None, metadata=None):
+        """
+        Write model data and metadata
 
         Parameters
         ----------
         product_data: astropy Table
-            Model data to be written.
+            Model data to be written
+        metadata: dict
+            Metadata to be written.
 
         Raises
         ------
         FileNotFoundError
-            if data writing was not sucessfull.
+            if data writing was not successful.
 
         """
 
+        if product_data is None:
+            return
+
+        if metadata is not None:
+            product_data.meta.update(gen.change_dict_keys_case(metadata, False))
+
         try:
-            if product_data is not None:
-                self._logger.info(f"Writing data to {self.product_data_file}")
-                product_data.write(
-                    self.product_data_file, format=self.product_data_format, overwrite=True
-                )
+            self._logger.info(f"Writing data to {self.product_data_file}")
+            product_data.write(
+                self.product_data_file, format=self.product_data_format, overwrite=True
+            )
         except astropy.io.registry.base.IORegistryError:
             self._logger.error(f"Error writing model data to {self.product_data_file}.")
             raise
 
-    def write_metadata(self, metadata, yml_file=None, keys_lower_case=False):
+    def write_metadata_to_yml(self, metadata, yml_file=None, keys_lower_case=False):
         """
         Write model metadata file (yaml file format).
 
@@ -97,21 +156,19 @@ class ModelDataWriter:
         ------
         FileNotFoundError
             If yml_file not found.
-        AttributeError
-            If no metadata defined for writing.
         TypeError
             If yml_file is not defined.
         """
 
         try:
             yml_file = Path(yml_file or self.product_data_file).with_suffix(".metadata.yml")
-            self._logger.info(f"Writing metadata to {yml_file}")
             with open(yml_file, "w", encoding="UTF-8") as file:
                 yaml.safe_dump(
                     gen.change_dict_keys_case(metadata, keys_lower_case),
                     file,
                     sort_keys=False,
                 )
+            self._logger.info(f"Writing metadata to {yml_file}")
             return yml_file
         except FileNotFoundError:
             self._logger.error(f"Error writing model data to {yml_file}")