gammasimtools 0.5.1__py3-none-any.whl → 0.6.1__py3-none-any.whl

simtools/data_model/metadata_collector.py

@@ -1,30 +1,46 @@
+"""
+Metadata collector for simtools.
+
+This should be the only module in simtools with knowledge on the
+implementation of the metadata model.
+
+"""
 import datetime
+import getpass
 import logging
 from pathlib import Path
 
+from astropy.table import Table
+
+import simtools.constants
 import simtools.utils.general as gen
 import simtools.version
-from simtools import io_handler
-from simtools.data_model import metadata_model, validate_schema
-from simtools.utils import names
+from simtools.data_model import metadata_model
+from simtools.io_operations import io_handler
 
 __all__ = ["MetadataCollector"]
 
 
 class MetadataCollector:
     """
-    Collects and combines metadata associated with the current activity
-    (e.g., the execution of an application).
-    Follows CTAO top-level metadata definition.
+    Collects and combines metadata associated to describe the current
+    simtools activity and its data products. Collect as much metadata
+    as possible from command line configuration, input data, environment,
+    schema descriptions.
+    Depends on the CTAO top-level metadata definition.
 
     Parameters
     ----------
-    args: argparse.Namespace
+    args_dict: dict
         Command line parameters
+    metadata_file_name: str
+        Name of metadata file (only required when args_dict is None)
+    data_model_name: str
+        Name of data model parameter
 
     """
 
-    def __init__(self, args_dict):
+    def __init__(self, args_dict, metadata_file_name=None, data_model_name=None):
         """
         Initialize metadata collector.
 
@@ -33,38 +49,128 @@ class MetadataCollector:
         self._logger = logging.getLogger(__name__)
         self.io_handler = io_handler.IOHandler()
 
-        self.args_dict = args_dict
+        self.args_dict = args_dict if args_dict else {}
+        self.data_model_name = data_model_name
+        self.schema_file = None
+        self.schema_dict = None
         self.top_level_meta = gen.change_dict_keys_case(
-            data_dict=metadata_model.top_level_reference_schema(), lower_case=True
+            data_dict=metadata_model.get_default_metadata_dict(), lower_case=True
         )
-        self.collect_product_meta_data()
+        self.input_metadata = self._read_input_metadata_from_file(
+            metadata_file_name=metadata_file_name
+        )
+        self.collect_meta_data()
 
-    def collect_product_meta_data(self):
+    def collect_meta_data(self):
         """
         Collect and verify product metadata from different sources.
 
         """
 
-        self._fill_association_meta_from_args(
-            self.top_level_meta["cta"]["context"]["sim"]["association"]
+        self._fill_contact_meta(self.top_level_meta["cta"]["contact"])
+        self._fill_product_meta(self.top_level_meta["cta"]["product"])
+        self._fill_activity_meta(self.top_level_meta["cta"]["activity"])
+        self._fill_process_meta(self.top_level_meta["cta"]["process"])
+        self._fill_context_from_input_meta(self.top_level_meta["cta"]["context"])
+        self._fill_associated_elements_from_args(
+            self.top_level_meta["cta"]["context"]["associated_elements"]
         )
 
-        self._fill_product_meta(self.top_level_meta["cta"]["product"])
+    def get_data_model_schema_file_name(self):
+        """
+        Return data model schema file name.
+        The schema file name is taken (in this order) from the command line,
+        from the metadata file, from the data model name, or from the input
+        metadata file.
+
+        Returns
+        -------
+        str
+            Name of schema file.
 
-        self._fill_top_level_meta_from_file(self.top_level_meta["cta"])
+        """
+
+        # from command line
+        try:
+            if self.args_dict["schema"]:
+                self._logger.debug(f"Schema file from command line: {self.args_dict['schema']}")
+                return self.args_dict["schema"]
+        except KeyError:
+            pass
 
-        self._fill_association_id(self.top_level_meta["cta"]["context"]["sim"]["association"])
+        # from metadata
+        try:
+            if self.top_level_meta["cta"]["product"]["data"]["model"]["url"]:
+                self._logger.debug(
+                    "Schema file from product metadata: "
+                    f"{self.top_level_meta['cta']['product']['data']['model']['url']}"
+                )
+                return self.top_level_meta["cta"]["product"]["data"]["model"]["url"]
+        except KeyError:
+            pass
 
-        self._fill_activity_meta(self.top_level_meta["cta"]["activity"])
+        # from data model name
+        if self.data_model_name:
+            self._logger.debug(f"Schema file from data model name: {self.data_model_name}")
+            return f"{simtools.constants.SCHEMA_URL}{self.data_model_name}.schema.yml"
+
+        # from input metadata
+        try:
+            self._logger.debug(
+                "Schema file from input metadata: "
+                f"{self.input_metadata['cta']['product']['data']['model']['url']}"
+            )
+            return self.input_metadata["cta"]["product"]["data"]["model"]["url"]
+        except KeyError:
+            pass
+
+        self._logger.warning("No schema file found.")
+        return None
+
+    def get_data_model_schema_dict(self):
+        """
+        Return data model schema dictionary.
+
+        Returns
+        -------
+        dict
+            Data model schema dictionary.
+
+        """
+
+        try:
+            return gen.collect_data_from_file_or_dict(file_name=self.schema_file, in_dict=None)
+        except gen.InvalidConfigData:
+            self._logger.debug(f"No valid schema file provided ({self.schema_file}).")
+        return {}
+
+    def _fill_contact_meta(self, contact_dict):
+        """
+        Fill contact metadata fields.
+
+        Parameters
+        ----------
+        contact_dict: dict
+            Dictionary for contact metadata fields.
 
-    def _fill_association_meta_from_args(self, association_dict):
+        """
+
+        if contact_dict.get("name", None) is None:
+            contact_dict["name"] = getpass.getuser()
+
+    def _fill_associated_elements_from_args(self, associated_elements_dict):
         """
         Append association metadata set through configurator.
 
+        Note
+        ----
+        This function might go in future, as instrument
+        information will not be given via command line.
+
         Parameters
         ----------
-        association_dict: dict
-            Dictionary for assocation metadata field.
+        associated_elements_dict: dict
+            Dictionary for associated elements field.
 
         Raises
         ------
@@ -90,16 +196,16 @@ class MetadataCollector:
             self._logger.error("Error reading association metadata from args")
             raise
 
-        self._fill_context_sim_list(association_dict, _association)
+        self._fill_context_sim_list(associated_elements_dict, _association)
 
-    def _fill_top_level_meta_from_file(self, top_level_dict):
+    def _fill_context_from_input_meta(self, context_dict):
         """
-        Read and validate metadata from file. Fill metadata into top-level template.
+        Read and validate input metadata from file and fill CONTEXT metadata fields.
 
         Parameters
         ----------
-        top_level_dict: dict
-            Dictionary for top level metadata.
+        context_dict: dict
+            Dictionary with context level metadata.
 
         Raises
         ------
@@ -108,26 +214,99 @@ class MetadataCollector:
 
         """
 
-        if self.args_dict.get("input_meta", None) is None:
-            self._logger.debug("Skipping metadata reading; no metadata file defined.")
-            return
-
-        _schema_validator = validate_schema.SchemaValidator()
-        _input_meta = _schema_validator.validate_and_transform(
-            meta_file_name=self.args_dict["input_meta"],
-        )
+        try:
+            self._merge_config_dicts(context_dict, self.input_metadata["cta"]["context"])
+            for key in ("document", "associated_elements", "associated_data"):
+                self._copy_list_type_metadata(context_dict, self.input_metadata["cta"], key)
+        except KeyError:
+            self._logger.debug("No context metadata defined in input metadata file.")
 
-        self._merge_config_dicts(top_level_dict, _input_meta)
-        # list entry copies
-        for association in _input_meta["product"]["association"]:
+        try:
             self._fill_context_sim_list(
-                top_level_dict["context"]["sim"]["association"], association
+                context_dict["associated_data"], self.input_metadata["cta"]["product"]
             )
+        except (KeyError, TypeError):
+            self._logger.debug("No input product metadata appended to associated data.")
+
+    def _read_input_metadata_from_file(self, metadata_file_name=None, observatory="CTA"):
+        """
+        Read and validate input metadata from file. In case of an ecsv file including a
+        table, the metadata is read from the table meta data. Returns empty dict in case
+        no file is given.
+
+        Parameter
+        ---------
+        metadata_file_name: str or Path
+            Name of metadata file.
+        observatory: str
+            Observatory name.
+
+        Returns
+        -------
+        dict
+            Metadata dictionary.
+
+        Raises
+        ------
+        gen.InvalidConfigData, FileNotFoundError
+            if metadata cannot be read from file.
+        KeyError:
+            if metadata does not exist for the given observatory.
+
+        """
+
         try:
-            for document in _input_meta["product"]["document"]:
-                self._fill_context_sim_list(top_level_dict["context"]["sim"]["document"], document)
-        except KeyError:
-            top_level_dict["context"]["sim"].pop("document")
+            metadata_file_name = (
+                self.args_dict.get("input_meta", None)
+                if metadata_file_name is None
+                else metadata_file_name
+            )
+        except TypeError:
+            pass
+
+        if metadata_file_name is None:
+            self._logger.debug("No input metadata file defined.")
+            return {}
+
+        # metadata from yml or json file
+        if Path(metadata_file_name).suffix in (".yaml", ".yml", ".json"):
+            try:
+                self._logger.debug("Reading meta data from %s", metadata_file_name)
+                _input_metadata = gen.collect_data_from_file_or_dict(
+                    file_name=metadata_file_name, in_dict=None
+                )
+                _json_type_metadata = {"Metadata", "metadata", "METADATA"}.intersection(
+                    _input_metadata
+                )
+                if len(_json_type_metadata) == 1:
+                    _input_metadata = _input_metadata[_json_type_metadata.pop()]
+                elif len(_json_type_metadata) > 1:
+                    self._logger.error(
+                        "More than one metadata entry found in %s", metadata_file_name
+                    )
+                    raise gen.InvalidConfigData
+            except (gen.InvalidConfigData, FileNotFoundError):
+                self._logger.error("Failed reading metadata from %s", metadata_file_name)
+                raise
+        # metadata from table meta in ecsv file
+        elif Path(metadata_file_name).suffix == ".ecsv":
+            try:
+                _input_metadata = {observatory: Table.read(metadata_file_name).meta[observatory]}
+            except (FileNotFoundError, KeyError):
+                self._logger.error(
+                    "Failed reading metadata for %s from %s", observatory, metadata_file_name
+                )
+                raise
+        else:
+            self._logger.error("Unknown metadata file format: %s", metadata_file_name)
+            raise gen.InvalidConfigData
+
+        metadata_model.validate_schema(_input_metadata, None)
+
+        return gen.change_dict_keys_case(
+            self._process_metadata_from_file(_input_metadata),
+            lower_case=True,
+        )
 
     def _fill_product_meta(self, product_dict):
         """
@@ -146,69 +325,43 @@ class MetadataCollector:
 
         """
 
+        self.schema_file = self.get_data_model_schema_file_name()
+        self.schema_dict = self.get_data_model_schema_dict()
+
         product_dict["id"] = self.args_dict.get("activity_id", "UNDEFINED_ACTIVITY_ID")
-        self._logger.debug(f"Reading activitiy UUID {product_dict['id']}")
+        product_dict["creation_time"] = datetime.datetime.now().isoformat(timespec="seconds")
+        product_dict["description"] = self.schema_dict.get("description", None)
 
+        # DATA:CATEGORY
         product_dict["data"]["category"] = "SIM"
         product_dict["data"]["level"] = "R1"
-        product_dict["data"]["type"] = "service"
+        product_dict["data"]["type"] = "Service"
+        try:
+            product_dict["data"]["association"] = self.schema_dict["instrument"]["class"]
+        except KeyError:
+            pass
+
+        # DATA:MODEL
+        helper_dict = {"name": "name", "version": "version", "type": "base_schema"}
+        for key, value in helper_dict.items():
+            product_dict["data"]["model"][key] = self.schema_dict.get(value, None)
+        product_dict["data"]["model"]["url"] = self.schema_file
 
-        _schema_dict = self._collect_schema_dict()
-        product_dict["data"]["model"]["name"] = _schema_dict.get("name", "simpipe-schema")
-        product_dict["data"]["model"]["version"] = _schema_dict.get("version", "0.0.0")
         product_dict["format"] = self.args_dict.get("output_file_format", None)
         product_dict["filename"] = str(self.args_dict.get("output_file", None))
 
-    def _collect_schema_dict(self):
+    def _fill_process_meta(self, process_dict):
         """
-        Read schema from file.
-
-        The schema configuration parameter points to a directory or a file.
-        For the case of a directory, the schema file is assumed to be named
-        <parameter_name>.schema.yml.
-
-        Returns
-        -------
-        dict
-            Dictionary containing schema metadata.
-
-        """
-
-        _schema = self.args_dict.get("schema", "")
-        if Path(_schema).is_dir():
-            try:
-                _data_dict = gen.collect_data_from_yaml_or_dict(
-                    in_yaml=self.args_dict.get("input", None), in_dict=None, allow_empty=True
-                )
-                return gen.collect_dict_from_file(
-                    file_path=_schema,
-                    file_name=f"{_data_dict['name']}.schema.yml",
-                )
-            except (TypeError, KeyError):
-                return {}
-        return gen.collect_dict_from_file(_schema)
-
-    @staticmethod
-    def _fill_association_id(association_dict):
-        """
-        Fill association id from site and telescope class, type, subtype.
+        Fill process fields in metadata.
 
         Parameters
         ----------
-        association_dict: dict
-            Association dictionary.
+        process_dict: dict
+            Dictionary for process metadata fields.
 
         """
-        for association in association_dict:
-            try:
-                association["id"] = names.simtools_instrument_name(
-                    site=association["site"],
-                    telescope_class_name=association["class"],
-                    sub_system_name=association["type"],
-                    telescope_id_name=association.get("subtype", "D"),
-                )
-            except ValueError:
-                association["id"] = None
+
+        process_dict["type"] = "simulation"
 
     def _fill_activity_meta(self, activity_dict):
         """
@@ -217,11 +370,13 @@ class MetadataCollector:
         Parameters
         ----------
         activity_dict: dict
-            Dictionary for top-level activitiy metadata.
+            Dictionary for top-level activity metadata.
 
         """
 
         activity_dict["name"] = self.args_dict.get("label", None)
+        activity_dict["type"] = "software"
+        activity_dict["id"] = self.args_dict.get("activity_id", "UNDEFINED_ACTIVITY_ID")
         activity_dict["start"] = datetime.datetime.now().isoformat(timespec="seconds")
         activity_dict["end"] = activity_dict["start"]
         activity_dict["software"]["name"] = "simtools"
@@ -265,47 +420,120 @@ class MetadataCollector:
             self._logger.error("Error merging dictionaries")
             raise
 
-    def input_data_file_name(self):
+    def _fill_context_sim_list(self, meta_list, new_entry_dict):
         """
-        Return input data file (full path).
+        Fill list-type entries into metadata. Take into account the first list entry is the default
+        value filled with Nones.
+
+        Parameters
+        ----------
+        meta_list: list
+            List of metadata entries.
+        new_entry_dict: dict
+            New metadata entry to be added to meta_list.
 
         Returns
         -------
-        str
-            Input data file (full path).
+        list
+            Updated meta list.
 
-        Raises
-        ------
-        KeyError
-            if missing description of INPUT_DATA
         """
 
+        if len(new_entry_dict) == 0:
+            return []
         try:
-            return self.args_dict["input_data"]
-        except KeyError:
-            self._logger.error("Missing description of INPUT_DATA")
-            raise
+            if self._all_values_none(meta_list[0]):
+                meta_list[0] = new_entry_dict
+            else:
+                meta_list.append(new_entry_dict)
+        except (TypeError, IndexError):
+            meta_list = [new_entry_dict]
+        return meta_list
+
+    def _process_metadata_from_file(self, meta_dict):
+        """
+        Process metadata from file to ensure compatibility with metadata model.
+        Changes keys to lower case and removes line feeds from description fields.
+
+        Parameters
+        ----------
+        meta_dict: dict
+            Input metadata dictionary.
+
+        Returns
+        -------
+        dict
+            Metadata dictionary.
+
+        """
+
+        meta_dict = gen.change_dict_keys_case(meta_dict, True)
+        try:
+            meta_dict["cta"]["product"]["description"] = self._remove_line_feed(
+                meta_dict["cta"]["product"]["description"]
+            )
+        except (KeyError, AttributeError):
+            pass
+
+        return meta_dict
 
     @staticmethod
-    def _fill_context_sim_list(product_list, new_entry_dict):
+    def _remove_line_feed(string):
         """
-        Fill list-type entries into metadata. Take into account the first list entry is the default
-        value filled with Nones.
+        Remove all line feeds from a string
+
+        Parameters
+        ----------
+        str
+            input string
 
         Returns
         -------
-        list
-            Updated product list.
+        str
+            with line feeds removed
+        """
+
+        return string.replace("\n", " ").replace("\r", "").replace("  ", " ")
+
+    def _copy_list_type_metadata(self, context_dict, _input_metadata, key):
+        """
+        Copy list-type metadata from file.
+        Very fine tuned.
+
+        Parameters
+        ----------
+        context_dict: dict
+            Dictionary for top level metadata (context level)
+        _input_metadata: dict
+            Dictionary for metadata from file.
+        key: str
+            Key for metadata entry.
 
         """
 
-        if len(new_entry_dict) == 0:
-            return []
         try:
-            if any(v is not None for v in product_list[0].values()):
-                product_list.append(new_entry_dict)
-            else:
-                product_list[0] = new_entry_dict
-        except (TypeError, IndexError):
-            product_list = [new_entry_dict]
-        return product_list
+            for document in _input_metadata["context"][key]:
+                self._fill_context_sim_list(context_dict[key], document)
+        except KeyError:
+            pass
+
+    def _all_values_none(self, input_dict):
+        """
+        Check recursively if all values in a dictionary are None.
+
+        Parameters
+        ----------
+        input_dict: dict
+            Input dictionary.
+
+        Returns
+        -------
+        bool
+            True if all entries are None, False otherwise.
+
+        """
+
+        if not isinstance(input_dict, dict):
+            return input_dict is None
+
+        return all(self._all_values_none(value) for value in input_dict.values())