gammasimtools 0.6.1__py3-none-any.whl → 0.8.1__py3-none-any.whl

simtools/data_model/metadata_collector.py

@@ -2,32 +2,38 @@
 Metadata collector for simtools.
 
 This should be the only module in simtools with knowledge on the
-implementation of the metadata model.
+implementation of the observatory metadata model.
 
 """
+
 import datetime
 import getpass
 import logging
+import uuid
+from importlib.resources import files
 from pathlib import Path
 
-from astropy.table import Table
-
 import simtools.constants
 import simtools.utils.general as gen
 import simtools.version
 from simtools.data_model import metadata_model
 from simtools.io_operations import io_handler
+from simtools.utils import names
 
 __all__ = ["MetadataCollector"]
 
 
 class MetadataCollector:
     """
-    Collects and combines metadata associated to describe the current
-    simtools activity and its data products. Collect as much metadata
-    as possible from command line configuration, input data, environment,
-    schema descriptions.
-    Depends on the CTAO top-level metadata definition.
+    Collects metadata to describe the current simtools activity and its data products.
+
+    Collect metadata from command line configuration, input data, environment,
+    and schema descriptions. Depends on the CTAO top-level metadata definition.
+
+    Two dictionaries store two different types of metadata:
+
+    - top_level_meta: metadata for the current activity
+    - input_metadata: metadata from input data
 
     Parameters
     ----------
@@ -37,16 +43,23 @@ class MetadataCollector:
         Name of metadata file (only required when args_dict is None)
     data_model_name: str
         Name of data model parameter
-
+    observatory: str
+        Name of observatory (default: "cta")
+    clean_meta: bool
+        Clean metadata from None values and empty lists (default: True)
     """
 
-    def __init__(self, args_dict, metadata_file_name=None, data_model_name=None):
-        """
-        Initialize metadata collector.
-
-        """
-
+    def __init__(
+        self,
+        args_dict,
+        metadata_file_name=None,
+        data_model_name=None,
+        observatory="cta",
+        clean_meta=True,
+    ):
+        """Initialize metadata collector."""
         self._logger = logging.getLogger(__name__)
+        self.observatory = observatory
         self.io_handler = io_handler.IOHandler()
 
         self.args_dict = args_dict if args_dict else {}
@@ -60,25 +73,23 @@ class MetadataCollector:
             metadata_file_name=metadata_file_name
         )
         self.collect_meta_data()
+        if clean_meta:
+            self.top_level_meta = self.clean_meta_data(self.top_level_meta)
 
     def collect_meta_data(self):
-        """
-        Collect and verify product metadata from different sources.
-
-        """
-
-        self._fill_contact_meta(self.top_level_meta["cta"]["contact"])
-        self._fill_product_meta(self.top_level_meta["cta"]["product"])
-        self._fill_activity_meta(self.top_level_meta["cta"]["activity"])
-        self._fill_process_meta(self.top_level_meta["cta"]["process"])
-        self._fill_context_from_input_meta(self.top_level_meta["cta"]["context"])
-        self._fill_associated_elements_from_args(
-            self.top_level_meta["cta"]["context"]["associated_elements"]
-        )
+        """Collect and verify product metadata for each main-level metadata type."""
+        meta_types = self.top_level_meta[self.observatory].keys()
+        for meta_type in meta_types:
+            try:
+                fill_method = getattr(self, f"_fill_{meta_type}_meta")
+                fill_method(self.top_level_meta[self.observatory][meta_type])
+            except AttributeError:
+                self._logger.debug(f"Method _fill_{meta_type}_meta not implemented")
 
     def get_data_model_schema_file_name(self):
         """
         Return data model schema file name.
+
         The schema file name is taken (in this order) from the command line,
         from the metadata file, from the data model name, or from the input
         metadata file.
@@ -89,38 +100,30 @@ class MetadataCollector:
             Name of schema file.
 
         """
-
         # from command line
-        try:
-            if self.args_dict["schema"]:
-                self._logger.debug(f"Schema file from command line: {self.args_dict['schema']}")
-                return self.args_dict["schema"]
-        except KeyError:
-            pass
+        if self.args_dict.get("schema"):
+            self._logger.debug(f"Schema file from command line: {self.args_dict['schema']}")
+            return self.args_dict["schema"]
 
         # from metadata
         try:
-            if self.top_level_meta["cta"]["product"]["data"]["model"]["url"]:
-                self._logger.debug(
-                    "Schema file from product metadata: "
-                    f"{self.top_level_meta['cta']['product']['data']['model']['url']}"
-                )
-                return self.top_level_meta["cta"]["product"]["data"]["model"]["url"]
+            url = self.top_level_meta[self.observatory]["product"]["data"]["model"]["url"]
+            if url:
+                self._logger.debug(f"Schema file from product metadata: {url}")
+                return url
         except KeyError:
             pass
 
         # from data model name
         if self.data_model_name:
             self._logger.debug(f"Schema file from data model name: {self.data_model_name}")
-            return f"{simtools.constants.SCHEMA_URL}{self.data_model_name}.schema.yml"
+            return f"{files('simtools')}/schemas/model_parameters/{self.data_model_name}.schema.yml"
 
         # from input metadata
         try:
-            self._logger.debug(
-                "Schema file from input metadata: "
-                f"{self.input_metadata['cta']['product']['data']['model']['url']}"
-            )
-            return self.input_metadata["cta"]["product"]["data"]["model"]["url"]
+            url = self.input_metadata[self.observatory]["product"]["data"]["model"]["url"]
+            self._logger.debug(f"Schema file from input metadata: {url}")
+            return url
         except KeyError:
             pass
 
@@ -137,109 +140,82 @@ class MetadataCollector:
             Data model schema dictionary.
 
         """
-
         try:
-            return gen.collect_data_from_file_or_dict(file_name=self.schema_file, in_dict=None)
-        except gen.InvalidConfigData:
+            return gen.collect_data_from_file(file_name=self.schema_file)
+        except TypeError:
             self._logger.debug(f"No valid schema file provided ({self.schema_file}).")
         return {}
 
-    def _fill_contact_meta(self, contact_dict):
+    def get_site(self, from_input_meta=False):
         """
-        Fill contact metadata fields.
+        Get site entry from metadata. Allow to get from collected or from input metadata.
 
         Parameters
         ----------
-        contact_dict: dict
-            Dictionary for contact metadata fields.
-
-        """
+        from_input_meta: bool
+            Get site from input metadata (default: False)
 
-        if contact_dict.get("name", None) is None:
-            contact_dict["name"] = getpass.getuser()
+        Returns
+        -------
+        str
+            Site name
 
-    def _fill_associated_elements_from_args(self, associated_elements_dict):
         """
-        Append association metadata set through configurator.
+        try:
+            _site = (
+                self.top_level_meta[self.observatory]["instrument"]["site"]
+                if not from_input_meta
+                else self.input_metadata[self.observatory]["instrument"]["site"]
+            )
+            if _site is not None:
+                return names.validate_site_name(_site)
+        except KeyError:
+            pass
+        return None
 
-        Note
-        ----
-        This function might go in future, as instrument
-        information will not be given via command line.
+    def _fill_contact_meta(self, contact_dict):
+        """
+        Fill contact metadata fields. Get user name from system level if not given.
 
         Parameters
         ----------
-        associated_elements_dict: dict
-            Dictionary for associated elements field.
-
-        Raises
-        ------
-        TypeError, KeyError
-            if error reading association metadata from args.
-        KeyError
-            if metadata description cannot be filled.
-
+        contact_dict: dict
+            Dictionary for contact metadata fields.
         """
-        self._logger.debug(f"Fill metadata from args: {self.args_dict}")
-
-        _association = {}
-
-        try:
-            if "site" in self.args_dict:
-                _association["site"] = self.args_dict["site"]
-            if "telescope" in self.args_dict:
-                _split_telescope_name = self.args_dict["telescope"].split("-")
-                _association["class"] = _split_telescope_name[0]
-                _association["type"] = _split_telescope_name[1]
-                _association["subtype"] = _split_telescope_name[2]
-        except (TypeError, KeyError):
-            self._logger.error("Error reading association metadata from args")
-            raise
-
-        self._fill_context_sim_list(associated_elements_dict, _association)
+        if contact_dict.get("name", None) is None:
+            contact_dict["name"] = getpass.getuser()
 
-    def _fill_context_from_input_meta(self, context_dict):
+    def _fill_context_meta(self, context_dict):
         """
-        Read and validate input metadata from file and fill CONTEXT metadata fields.
+        Fill context metadata fields with product metadata from input data.
 
         Parameters
         ----------
         context_dict: dict
-            Dictionary with context level metadata.
-
-        Raises
-        ------
-        KeyError
-            if corresponding fields cannot by accessed in the top-level or metadata dictionaries.
+            Dictionary for context metadata fields.
 
         """
-
-        try:
-            self._merge_config_dicts(context_dict, self.input_metadata["cta"]["context"])
-            for key in ("document", "associated_elements", "associated_data"):
-                self._copy_list_type_metadata(context_dict, self.input_metadata["cta"], key)
-        except KeyError:
-            self._logger.debug("No context metadata defined in input metadata file.")
-
-        try:
-            self._fill_context_sim_list(
-                context_dict["associated_data"], self.input_metadata["cta"]["product"]
-            )
+        try:  # wide try..except as for some cases we expect that there is no product metadata
+            reduced_product_meta = {
+                key: value
+                for key, value in self.input_metadata[self.observatory]["product"].items()
+                if key in {"description", "id", "creation_time", "valid", "format", "filename"}
+            }
+            self._fill_context_sim_list(context_dict["associated_data"], reduced_product_meta)
         except (KeyError, TypeError):
             self._logger.debug("No input product metadata appended to associated data.")
 
-    def _read_input_metadata_from_file(self, metadata_file_name=None, observatory="CTA"):
+    def _read_input_metadata_from_file(self, metadata_file_name=None):
         """
-        Read and validate input metadata from file. In case of an ecsv file including a
-        table, the metadata is read from the table meta data. Returns empty dict in case
-        no file is given.
+        Read and validate input metadata from file.
+
+        In case of an ecsv file including a table, the metadata is read from the table meta data.
+        Returns empty dict in case no file is given.
 
         Parameter
         ---------
         metadata_file_name: str or Path
             Name of metadata file.
-        observatory: str
-            Observatory name.
 
         Returns
         -------
@@ -248,58 +224,30 @@ class MetadataCollector:
 
         Raises
         ------
-        gen.InvalidConfigData, FileNotFoundError
+        gen.InvalidConfigDataError, FileNotFoundError
             if metadata cannot be read from file.
         KeyError:
-            if metadata does not exist for the given observatory.
+            if metadata does not exist
 
         """
-
-        try:
-            metadata_file_name = (
-                self.args_dict.get("input_meta", None)
-                if metadata_file_name is None
-                else metadata_file_name
-            )
-        except TypeError:
-            pass
+        metadata_file_name = (
+            self.args_dict.get("input_meta", None) or self.args_dict.get("input", None)
+            if metadata_file_name is None
+            else metadata_file_name
+        )
 
         if metadata_file_name is None:
             self._logger.debug("No input metadata file defined.")
             return {}
 
-        # metadata from yml or json file
+        self._logger.debug("Reading meta data from %s", metadata_file_name)
         if Path(metadata_file_name).suffix in (".yaml", ".yml", ".json"):
-            try:
-                self._logger.debug("Reading meta data from %s", metadata_file_name)
-                _input_metadata = gen.collect_data_from_file_or_dict(
-                    file_name=metadata_file_name, in_dict=None
-                )
-                _json_type_metadata = {"Metadata", "metadata", "METADATA"}.intersection(
-                    _input_metadata
-                )
-                if len(_json_type_metadata) == 1:
-                    _input_metadata = _input_metadata[_json_type_metadata.pop()]
-                elif len(_json_type_metadata) > 1:
-                    self._logger.error(
-                        "More than one metadata entry found in %s", metadata_file_name
-                    )
-                    raise gen.InvalidConfigData
-            except (gen.InvalidConfigData, FileNotFoundError):
-                self._logger.error("Failed reading metadata from %s", metadata_file_name)
-                raise
-        # metadata from table meta in ecsv file
+            _input_metadata = self._read_input_metadata_from_yml_or_json(metadata_file_name)
         elif Path(metadata_file_name).suffix == ".ecsv":
-            try:
-                _input_metadata = {observatory: Table.read(metadata_file_name).meta[observatory]}
-            except (FileNotFoundError, KeyError):
-                self._logger.error(
-                    "Failed reading metadata for %s from %s", observatory, metadata_file_name
-                )
-                raise
+            _input_metadata = self._read_input_metadata_from_ecsv(metadata_file_name)
         else:
             self._logger.error("Unknown metadata file format: %s", metadata_file_name)
-            raise gen.InvalidConfigData
+            raise gen.InvalidConfigDataError
 
         metadata_model.validate_schema(_input_metadata, None)
 
@@ -308,10 +256,43 @@ class MetadataCollector:
             lower_case=True,
         )
 
+    def _read_input_metadata_from_ecsv(self, metadata_file_name):
+        """Read input metadata from ecsv file."""
+        from astropy.table import Table  # pylint: disable=C0415
+
+        try:
+            return {
+                self.observatory.upper(): Table.read(metadata_file_name).meta[
+                    self.observatory.upper()
+                ]
+            }
+        except (FileNotFoundError, KeyError, AttributeError) as exc:
+            self._logger.error(
+                "Failed reading metadata for %s from %s", self.observatory, metadata_file_name
+            )
+            raise exc
+
+    def _read_input_metadata_from_yml_or_json(self, metadata_file_name):
+        """Read input metadata from yml or json file."""
+        try:
+            _input_metadata = gen.collect_data_from_file(file_name=metadata_file_name)
+            _json_type_metadata = {"Metadata", "metadata", "METADATA"}.intersection(_input_metadata)
+            if len(_json_type_metadata) == 1:
+                _input_metadata = _input_metadata[_json_type_metadata.pop()]
+            if len(_json_type_metadata) > 1:
+                self._logger.error("More than one metadata entry found in %s", metadata_file_name)
+                raise gen.InvalidConfigDataError
+        except (gen.InvalidConfigDataError, FileNotFoundError) as exc:
+            self._logger.error("Failed reading metadata from %s", metadata_file_name)
+            raise exc
+        return _input_metadata
+
     def _fill_product_meta(self, product_dict):
         """
-        Fill metadata for data products fields. If a schema file is given for the data products,
-        try and read product:data:model metadata from there.
+        Fill metadata for data products fields.
+
+        If a schema file is given for the data products, try and read product:data:model metadata
+        from there.
 
         Parameters
         ----------
@@ -324,11 +305,10 @@ class MetadataCollector:
             if relevant fields are not defined in top level metadata dictionary.
 
         """
-
         self.schema_file = self.get_data_model_schema_file_name()
         self.schema_dict = self.get_data_model_schema_dict()
 
-        product_dict["id"] = self.args_dict.get("activity_id", "UNDEFINED_ACTIVITY_ID")
+        product_dict["id"] = str(uuid.uuid4())
         product_dict["creation_time"] = datetime.datetime.now().isoformat(timespec="seconds")
         product_dict["description"] = self.schema_dict.get("description", None)
 
@@ -342,7 +322,7 @@ class MetadataCollector:
             pass
 
         # DATA:MODEL
-        helper_dict = {"name": "name", "version": "version", "type": "base_schema"}
+        helper_dict = {"name": "name", "version": "version", "type": "meta_schema"}
         for key, value in helper_dict.items():
             product_dict["data"]["model"][key] = self.schema_dict.get(value, None)
         product_dict["data"]["model"]["url"] = self.schema_file
@@ -350,6 +330,28 @@ class MetadataCollector:
         product_dict["format"] = self.args_dict.get("output_file_format", None)
         product_dict["filename"] = str(self.args_dict.get("output_file", None))
 
+    def _fill_instrument_meta(self, instrument_dict):
+        """
+        Fill instrument metadata fields.
+
+        Note inconsistency in command line arguments for 'ID',
+        which is either 'instrument' or 'telescope'.
+
+        Parameters
+        ----------
+        instrument_dict: dict
+            Dictionary for instrument metadata fields.
+
+        """
+        instrument_dict["site"] = self.args_dict.get("site", None)
+        instrument_dict["ID"] = self.args_dict.get("instrument") or self.args_dict.get(
+            "telescope", None
+        )
+        if instrument_dict["ID"]:
+            instrument_dict["class"] = names.get_collection_name_from_array_element_name(
+                instrument_dict["ID"]
+            )
+
     def _fill_process_meta(self, process_dict):
         """
         Fill process fields in metadata.
@@ -360,12 +362,11 @@ class MetadataCollector:
             Dictionary for process metadata fields.
 
         """
-
         process_dict["type"] = "simulation"
 
     def _fill_activity_meta(self, activity_dict):
         """
-        Fill activity (software) related metadata
+        Fill activity (software) related metadata.
 
         Parameters
         ----------
@@ -373,7 +374,6 @@ class MetadataCollector:
             Dictionary for top-level activity metadata.
 
         """
-
         activity_dict["name"] = self.args_dict.get("label", None)
         activity_dict["type"] = "software"
         activity_dict["id"] = self.args_dict.get("activity_id", "UNDEFINED_ACTIVITY_ID")
@@ -384,8 +384,9 @@ class MetadataCollector:
 
     def _merge_config_dicts(self, dict_high, dict_low, add_new_fields=False):
         """
-        Merge two config dicts and replace values in dict_high which are Nonetype. Priority to \
-         dict_high in case of conflicting entries.
+        Merge two config dicts and replace values in dict_high which are Nonetype.
+
+        Priority to dict_high in case of conflicting entries.
 
         Parameters
         ----------
@@ -397,11 +398,6 @@ class MetadataCollector:
             If true: add fields from dict_low to dict_high, if they don't exist in dict_high
 
         """
-
-        if dict_high is None and dict_low:
-            dict_high = dict_low
-            return
-
         try:
             for k in dict_low:
                 if k in dict_high:
@@ -416,14 +412,14 @@ class MetadataCollector:
                         )
                 elif add_new_fields:
                     dict_high[k] = dict_low[k]
-        except (KeyError, TypeError):
-            self._logger.error("Error merging dictionaries")
-            raise
+        except TypeError as exc:
+            raise TypeError("Error merging dictionaries") from exc
 
     def _fill_context_sim_list(self, meta_list, new_entry_dict):
         """
-        Fill list-type entries into metadata. Take into account the first list entry is the default
-        value filled with Nones.
+        Fill list-type entries into metadata.
+
+        Take into account the first list entry is the default value filled with Nones.
 
         Parameters
         ----------
@@ -438,7 +434,6 @@ class MetadataCollector:
             Updated meta list.
 
         """
-
         if len(new_entry_dict) == 0:
             return []
         try:
@@ -453,6 +448,7 @@ class MetadataCollector:
     def _process_metadata_from_file(self, meta_dict):
         """
         Process metadata from file to ensure compatibility with metadata model.
+
         Changes keys to lower case and removes line feeds from description fields.
 
         Parameters
@@ -466,11 +462,10 @@ class MetadataCollector:
             Metadata dictionary.
 
         """
-
         meta_dict = gen.change_dict_keys_case(meta_dict, True)
         try:
-            meta_dict["cta"]["product"]["description"] = self._remove_line_feed(
-                meta_dict["cta"]["product"]["description"]
+            meta_dict[self.observatory]["product"]["description"] = self._remove_line_feed(
+                meta_dict[self.observatory]["product"]["description"]
             )
         except (KeyError, AttributeError):
             pass
@@ -480,7 +475,7 @@ class MetadataCollector:
     @staticmethod
     def _remove_line_feed(string):
         """
-        Remove all line feeds from a string
+        Remove all line feeds from a string.
 
         Parameters
         ----------
@@ -492,12 +487,12 @@ class MetadataCollector:
         str
             with line feeds removed
         """
-
         return string.replace("\n", " ").replace("\r", "").replace("  ", " ")
 
     def _copy_list_type_metadata(self, context_dict, _input_metadata, key):
         """
         Copy list-type metadata from file.
+
         Very fine tuned.
 
         Parameters
@@ -510,7 +505,6 @@ class MetadataCollector:
             Key for metadata entry.
 
         """
-
         try:
             for document in _input_metadata["context"][key]:
                 self._fill_context_sim_list(context_dict[key], document)
@@ -532,8 +526,40 @@ class MetadataCollector:
             True if all entries are None, False otherwise.
 
         """
-
         if not isinstance(input_dict, dict):
             return input_dict is None
 
         return all(self._all_values_none(value) for value in input_dict.values())
+
+    def clean_meta_data(self, meta_dict):
+        """
+        Clean metadata dictionary from None values and empty lists.
+
+        Parameters
+        ----------
+        meta_dict: dict
+            Metadata dictionary.
+
+        """
+
+        def clean_list(value):
+            nested_list = [
+                self.clean_meta_data(item) if isinstance(item, dict) else item for item in value
+            ]
+            return [item for item in nested_list if item not in (None, "", [], {})]
+
+        cleaned = {}
+        for key, value in meta_dict.items():
+            if value in (None, []):
+                continue
+            if isinstance(value, dict):
+                nested = self.clean_meta_data(value)
+                if nested:  # Only add if not empty
+                    cleaned[key] = nested
+            elif isinstance(value, list):
+                nested_list = clean_list(value)
+                if nested_list:  # Only add if not empty
+                    cleaned[key] = nested_list
+            else:
+                cleaned[key] = value
+        return cleaned