imap-processing 0.6.0__py3-none-any.whl

imap_processing/cdf/cdf_attribute_manager.py

@@ -0,0 +1,322 @@
+"""
+Class for creating and managing CDF attrs.
+
+Developed based of HermesDataSchema from HERMES-SOC/hermes_core.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+import yaml
+
+DEFAULT_GLOBAL_CDF_ATTRS_FILE = "imap_default_global_cdf_attrs.yaml"
+DEFAULT_GLOBAL_CDF_ATTRS_SCHEMA_FILE = "default_global_cdf_attrs_schema.yaml"
+DEFAULT_VARIABLE_CDF_ATTRS_SCHEMA_FILE = "default_variable_cdf_attrs_schema.yaml"
+
+
+class CdfAttributeManager:
+    """
+    Class for creating and managing CDF attributes based out of yaml files.
+
+    This class is based on the HERMES SWxSOC project for managing CDF attributes, but
+    is intended to be a flexible and very lightweight way of managing CDF attribute
+    creation and validation.
+
+    To use, you can load one or many global and variable attribute files:
+
+    .. code::
+
+        cdf_attr_manager = CdfAttributeManager(data_dir)
+        cdf_attr_manager.load_global_attributes("global_attrs.yaml")
+        cdf_attr_manager.load_global_attributes("instrument_global_attrs.yaml")
+        cdf_attr_manager.load_variable_attributes("variable_attrs.yaml")
+
+    Later files will overwrite earlier files if the same attribute is defined.
+
+    You can then get the global and variable attributes:
+
+    If you provide an instrument_id, it will also add the attributes defined under
+    instrument_id. If this is not included, then only the attributes defined in the top
+    level of the file are used.
+
+    .. code::
+
+        # Instrument ID is optional for refining the attributes used from the file
+        global_attrs = cdf_attr_manager.get_global_attributes(instrument_id)
+        variable_attrs = cdf_attr_manager.get_variable_attributes(variable_name)
+
+    The variable and global attributes are validated against the schemas upon calling
+    ``get_global_attributes`` and ``get_variable_attributes``.
+
+    Parameters
+    ----------
+    data_dir : pathlib.Path
+        The directory containing the schema and variable files (nominally config/).
+
+    Attributes
+    ----------
+    source_dir : pathlib.Path
+        The directory containing the schema and variable files - nominally config/
+    """
+
+    def __init__(self, data_dir: Path):
+        """Initialize the CdfAttributeManager and read schemas from data_dir."""
+        # TODO: Split up schema source and data source?
+        self.source_dir = data_dir
+
+        # TODO: copied from hermes_core. Currently we can use default schema, but
+        # We should add some way of extending the schema and remove all the HERMES
+        # specific stuff
+        # Data Validation, Complaiance,
+        self.global_attribute_schema = self._load_default_global_attr_schema()
+
+        # Data Validation and Compliance for Variable Data
+        self.variable_attribute_schema = self._load_default_variable_attr_schema()
+
+        # Load Default IMAP Global Attributes
+        self._global_attributes = CdfAttributeManager._load_yaml_data(
+            self.source_dir / DEFAULT_GLOBAL_CDF_ATTRS_FILE
+        )
+        self._variable_attributes: dict = {}
+
+    def _load_default_global_attr_schema(self) -> yaml:
+        """
+        Load the default global schema from the source directory.
+
+        Returns
+        -------
+        dict
+            The dict representing the global schema.
+        """
+        default_schema_path = (
+            self.source_dir / "shared" / DEFAULT_GLOBAL_CDF_ATTRS_SCHEMA_FILE
+        )
+        # Load the Schema
+        return CdfAttributeManager._load_yaml_data(default_schema_path)
+
+    def _load_default_variable_attr_schema(self) -> yaml:
+        """
+        Load the default variable schema from the source directory.
+
+        Returns
+        -------
+        dict
+            The dict representing the variable schema.
+        """
+        default_schema_path = (
+            self.source_dir / "shared" / DEFAULT_VARIABLE_CDF_ATTRS_SCHEMA_FILE
+        )
+        # Load the Schema
+        return CdfAttributeManager._load_yaml_data(default_schema_path)
+
+    # TODO Change Returning Any from function declared to return "dict[Any, Any]"
+
+    def load_global_attributes(self, file_path: str) -> None:
+        """
+        Update the global attributes property with the attributes from the file.
+
+        Calling this method multiple times on different files will add all the
+        attributes from the files, overwriting existing attributes if they are
+        duplicated.
+
+        Parameters
+        ----------
+        file_path : str
+            File path to load, under self.source_dir.
+        """
+        self._global_attributes.update(
+            CdfAttributeManager._load_yaml_data(self.source_dir / file_path)
+        )
+
+    def add_global_attribute(self, attribute_name: str, attribute_value: str) -> None:
+        """
+        Add a single global attribute to the global attributes.
+
+        This is intended only for dynamic global attributes which change per-file, such
+        as Data_version. It is not intended to be used for static attributes, which
+        should all be included in the YAML files.
+
+        This will overwrite any existing value in attribute_name if it exists. The
+        attribute must be in the global schema, or it will not be included as output.
+
+        Parameters
+        ----------
+        attribute_name : str
+            The name of the attribute to add.
+        attribute_value : str
+            The value of the attribute to add.
+        """
+        self._global_attributes[attribute_name] = attribute_value
+
+    @staticmethod
+    def _load_yaml_data(file_path: str | Path) -> yaml:
+        """
+        Load a yaml file from the provided path.
+
+        Parameters
+        ----------
+        file_path : str | Path
+            Path to the yaml file to load.
+
+        Returns
+        -------
+        yaml
+            Loaded yaml.
+        """
+        with open(file_path) as file:
+            return yaml.safe_load(file)
+
+    def get_global_attributes(self, instrument_id: str | None = None) -> dict:
+        """
+        Generate a dictionary global attributes based off the loaded schema and attrs.
+
+        Validates against the global schema to ensure all required variables are
+        present. It can also include instrument specific global attributes if
+        instrumet_id is set.
+
+        If an instrument_id is provided, the level and instrument specific
+        attributes that were previously loaded using add_instrument_global_attrs will
+        be included.
+
+        Parameters
+        ----------
+        instrument_id : str
+            The id of the CDF file, used to retrieve instrument and level
+            specific global attributes. Suggested value is the logical_source_id.
+
+        Returns
+        -------
+        output : dict
+            The global attribute values created from the input global attribute files
+            and schemas.
+        """
+        output = dict()
+        for attr_name, attr_schema in self.global_attribute_schema.items():
+            if attr_name in self._global_attributes:
+                output[attr_name] = self._global_attributes[attr_name]
+            # Retrieve instrument specific global attributes from the variable file
+            elif (
+                instrument_id is not None
+                and attr_name in self._global_attributes[instrument_id]
+            ):
+                output[attr_name] = self._global_attributes[instrument_id][attr_name]
+            elif attr_schema["required"] and attr_name not in self._global_attributes:
+                # TODO throw an error
+                output[attr_name] = None
+
+        return output
+
+    def load_variable_attributes(self, file_name: str) -> None:
+        """
+        Add variable attributes for a given filename.
+
+        Parameters
+        ----------
+        file_name : str
+            The name of the file to load from self.source_dir.
+        """
+        # Add variable attributes from file_name. Each variable name should have the
+        # Required subfields as defined in the variable schema.
+        raw_var_attrs = CdfAttributeManager._load_yaml_data(self.source_dir / file_name)
+        var_attrs = raw_var_attrs.copy()
+
+        self._variable_attributes.update(var_attrs)
+
+    def get_variable_attributes(
+        self, variable_name: str, check_schema: bool = True
+    ) -> dict:
+        """
+        Get the attributes for a given variable name.
+
+        It retrieves the variable from previously loaded variable definition files and
+        validates against the defined variable schemas.
+
+        Parameters
+        ----------
+        variable_name : str
+            The name of the variable to retrieve attributes for.
+
+        check_schema : bool
+            Flag to bypass schema validation.
+
+        Returns
+        -------
+        dict
+            Information containing specific variable attributes
+            associated with "variable_name".
+        """
+        # Case to handle attributes not in schema
+        if check_schema is False:
+            if variable_name in self._variable_attributes:
+                return_dict: dict = self._variable_attributes[variable_name]
+                return return_dict
+            # TODO: throw an error?
+            return {}
+
+        output = dict()
+        for attr_name in self.variable_attribute_schema["attribute_key"]:
+            # Standard case
+            if attr_name in self._variable_attributes[variable_name]:
+                output[attr_name] = self._variable_attributes[variable_name][attr_name]
+            # Case to handle DEPEND_i schema issues
+            elif attr_name == "DEPEND_i":
+                # DEFAULT_0 is not required, UNLESS we are dealing with
+                # variable_name = epoch
+                # Find all the attributes of variable_name that contain "DEPEND"
+                variable_depend_attrs = [
+                    key
+                    for key in self._variable_attributes[variable_name].keys()
+                    if "DEPEND" in key
+                ]
+                # Confirm that each DEPEND_i attribute is unique
+                if len(set(variable_depend_attrs)) != len(variable_depend_attrs):
+                    logging.warning(
+                        f"Found duplicate DEPEND_i attribute in variable "
+                        f"{variable_name}: {variable_depend_attrs}"
+                    )
+                for variable_depend_attr in variable_depend_attrs:
+                    output[variable_depend_attr] = self._variable_attributes[
+                        variable_name
+                    ][variable_depend_attr]
+                # TODO: Add more DEPEND_0 variable checks!
+            # Case to handle LABL_PTR_i schema issues
+            elif attr_name == "LABL_PTR_i":
+                # Find all the attributes of variable_name that contain "LABL_PTR"
+                variable_labl_attrs = [
+                    key
+                    for key in self._variable_attributes[variable_name].keys()
+                    if "LABL_PTR" in key
+                ]
+                for variable_labl_attr in variable_labl_attrs:
+                    output[variable_labl_attr] = self._variable_attributes[
+                        variable_name
+                    ][variable_labl_attr]
+            # Case to handle REPRESENTATION_i schema issues
+            elif attr_name == "REPRESENTATION_i":
+                # Find all the attributes of variable_name that contain
+                # "REPRESENTATION_i"
+                variable_rep_attrs = [
+                    key
+                    for key in self._variable_attributes[variable_name].keys()
+                    if "REPRESENTATION" in key
+                ]
+                for variable_rep_attr in variable_rep_attrs:
+                    output[variable_rep_attr] = self._variable_attributes[
+                        variable_name
+                    ][variable_rep_attr]
+            # Validating required schema
+            elif (
+                self.variable_attribute_schema["attribute_key"][attr_name]["required"]
+                and attr_name not in self._variable_attributes[variable_name]
+            ):
+                logging.warning(
+                    "Required schema '"
+                    + attr_name
+                    + "' attribute not present for "
+                    + variable_name
+                )
+                output[attr_name] = ""
+
+        return output

imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml

@@ -0,0 +1,212 @@
+instrument_base: &instrument_base
+  Descriptor: &desc
+    CoDICE>Compact Dual Ion Composition Experiment
+  TEXT: &txt >
+    The Compact Dual Ion Composition Experiment (CoDICE) will measure the distributions and composition of interstellar
+    pickup ions (PUIs), particles that make it through the heliosheath into the heliosphere. CoDICE also collects and
+    characterizes solar wind ions including the mass and composition of highly energized particles (called suprathermal)
+    from the Sun. CoDICE combines an electrostatic analyzer (ESA) with a Time-Of-Flight versus Energy (TOF / E)
+    subsystem to simultaneously measure the velocity, arrival direction, ionic charge state, and mass of specific
+    species of ions in the LISM. CoDICE also has a path for higher energy particles to skip the ESA but still get
+    measured by the common TOF / E system. These measurements are critical in determining the Local Interstellar Medium
+    (LISM) composition and flow properties, the origin of the enigmatic suprathermal tails on the solar wind
+    distributions and advance understanding of the acceleration of particles in the heliosphere.
+  Instrument_type: Particles (space)
+
+# L1a
+imap_codice_l1a_hskp:
+    <<: *instrument_base
+    Data_level: 1A
+    Data_type: L1A_hskp->Level-1A Housekeeping Data
+    Logical_source: imap_codice_l1a_hskp
+    Logical_source_description: IMAP Mission CoDICE Instrument Level-1A Housekeeping Data.
+
+imap_codice_l1a_hi-counters-aggregated:
+    <<: *instrument_base
+    Data_level: 1A
+    Data_type: L1A_hi-counters-aggregated->Level-1A Hi Aggregated Instrument Counts Data
+    Logical_source: imap_codice_l1a_hi-counters-aggregated
+    Logical_source_description: IMAP Mission CoDICE Hi Level-1A Aggregated Instrument Counts Data.
+
+imap_codice_l1a_hi-counters-singles:
+    <<: *instrument_base
+    Data_level: 1A
+    Data_type: L1A_hi-counters-singles->Level-1A Hi Single Instrument Counts Data
+    Logical_source: imap_codice_l1a_hi-counters-singles
+    Logical_source_description: IMAP Mission CoDICE Hi Level-1A Single Instrument Counts Data.
+
+imap_codice_l1a_hi-omni:
+    <<: *instrument_base
+    Data_level: 1A
+    Data_type: L1A_hi-omni->Level-1A Hi Omnidirectional Data
+    Logical_source: imap_codice_l1a_hi-omni
+    Logical_source_description: IMAP Mission CoDICE Hi Level-1A Omnidirectional Data.
+
+imap_codice_l1a_hi-sectored:
+    <<: *instrument_base
+    Data_level: 1A
+    Data_type: L1A_hi-sectored->Level-1A Hi Sectored Data
+    Logical_source: imap_codice_l1a_hi-sectored
+    Logical_source_description: IMAP Mission CoDICE Hi Level-1A Sectored Data.
+
+imap_codice_l1a_hi-pha:
+    <<: *instrument_base
+    Data_level: 1A
+    Data_type: L1A_hi-pha->Level-1A Hi Event Data
+    Logical_source: imap_codice_l1a_hi-pha
+    Logical_source_description: IMAP Mission CoDICE Hi Level-1A Event Data.
+
+imap_codice_l1a_lo-counters-aggregated:
+    <<: *instrument_base
+    Data_level: 1A
+    Data_type: L1A_lo-counters-aggregated->Level-1A Lo Aggregated Instrument Counts Data
+    Logical_source: imap_codice_l1a_lo-counters-aggregated
+    Logical_source_description: IMAP Mission CoDICE Lo Level-1A Aggregated Instrument Counts Data.
+
+imap_codice_l1a_lo-counters-singles:
+    <<: *instrument_base
+    Data_level: 1A
+    Data_type: L1A_lo-counters-aggregated->Level-1A Lo Single Instrument Counts Data
+    Logical_source: imap_codice_l1a_lo-counters-singles
+    Logical_source_description: IMAP Mission CoDICE Lo Level-1A Single Instrument Counts Data.
+
+imap_codice_l1a_lo-sw-angular:
+    <<: *instrument_base
+    Data_level: 1A
+    Data_type: L1A_lo-sw-angular->Level-1A Lo Sunward Angular Counts Data
+    Logical_source: imap_codice_l1a_lo-sw-angular
+    Logical_source_description: IMAP Mission CoDICE Lo Level-1A Sunward Angular Counts Data.
+
+imap_codice_l1a_lo-nsw-angular:
+    <<: *instrument_base
+    Data_level: 1A
+    Data_type: L1A_lo-nsw-angular->Level-1A Lo Non-Sunward Angular Counts Data
+    Logical_source: imap_codice_l1a_lo-nsw-angular
+    Logical_source_description: IMAP Mission CoDICE Lo Level-1A Non-Sunward Angular Counts Data.
+
+imap_codice_l1a_lo-sw-priority:
+    <<: *instrument_base
+    Data_level: 1A
+    Data_type: L1A_lo-sw-priority->Level-1A Lo Sunward Priority Counts Data
+    Logical_source: imap_codice_l1a_lo-sw-priority
+    Logical_source_description: IMAP Mission CoDICE Lo Level-1A Sunward Priority Counts Data.
+
+imap_codice_l1a_lo-nsw-priority:
+    <<: *instrument_base
+    Data_level: 1A
+    Data_type: L1A_lo-nsw-priority->Level-1A Lo Non-Sunward Priority Counts Data
+    Logical_source: imap_codice_l1a_lo-nsw-priority
+    Logical_source_description: IMAP Mission CoDICE Lo Level-1A Non-Sunward Priority Counts Data.
+
+imap_codice_l1a_lo-sw-species:
+    <<: *instrument_base
+    Data_level: 1A
+    Data_type: L1A_lo-sw-species->Level-1A Lo Sunward Species Counts Data
+    Logical_source: imap_codice_l1a_lo-sw-species
+    Logical_source_description: IMAP Mission CoDICE Lo Level-1A Sunward Species Counts Data.
+
+imap_codice_l1a_lo-nsw-species:
+    <<: *instrument_base
+    Data_level: 1A
+    Data_type: L1A_lo-nsw-species->Level-1A Lo Non-Sunward Species Counts Data
+    Logical_source: imap_codice_l1a_lo-nsw-species
+    Logical_source_description: IMAP Mission CoDICE Lo Level-1A Non-Sunward Species Counts Data.
+
+imap_codice_l1a_lo-pha:
+    <<: *instrument_base
+    Data_level: 1A
+    Data_type: L1A_lo-pha->Level-1A Lo Event Data
+    Logical_source: imap_codice_l1a_lo-pha
+    Logical_source_description: IMAP Mission CoDICE Lo Level-1A Event Data.
+
+# L1b
+imap_codice_l1b_hskp:
+    <<: *instrument_base
+    Data_level: 1B
+    Data_type: L1B_hskp->Level-1B Housekeeping Data
+    Logical_source: imap_codice_l1b_hskp
+    Logical_source_description: IMAP Mission CoDICE Instrument Level-1B Housekeeping Data.
+
+imap_codice_l1b_hi_counters_aggregated:
+    <<: *instrument_base
+    Data_level: 1B
+    Data_type: L1B_hi-counters-aggregated->Level-1B Hi Aggregated Instrument Rates Data
+    Logical_source: imap_codice_l1b_hi-counters-aggregated
+    Logical_source_description: IMAP Mission CoDICE Hi Level-1B Aggregated Instrument Rates Data.
+
+imap_codice_l1b_hi_counters_singles:
+    <<: *instrument_base
+    Data_level: 1B
+    Data_type: L1B_hi-counters-singles->Level-1B Hi Single Instrument Rates Data
+    Logical_source: imap_codice_l1b_hi-counters-singles
+    Logical_source_description: IMAP Mission CoDICE Hi Level-1B Single Instrument Rates Data.
+
+imap_codice_l1b_hi_omni:
+    <<: *instrument_base
+    Data_level: 1B
+    Data_type: L1B_hi-omni->Level-1B Hi Omnidirectional Data
+    Logical_source: imap_codice_l1b_hi-omni
+    Logical_source_description: IMAP Mission CoDICE Hi Level-1B Omnidirectional Data.
+
+imap_codice_l1b_hi_sectored:
+    <<: *instrument_base
+    Data_level: 1B
+    Data_type: L1B_hi-sectored->Level-1B Hi Sectored Data
+    Logical_source: imap_codice_l1b_hi-sectored
+    Logical_source_description: IMAP Mission CoDICE Hi Level-1B Sectored Data.
+
+imap_codice_l1b_lo_counters_aggregated:
+    <<: *instrument_base
+    Data_level: 1B
+    Data_type: L1B_lo-counters-aggregated->Level-1B Lo Aggregated Instrument Rates Data
+    Logical_source: imap_codice_l1b_lo-counters-aggregated
+    Logical_source_description: IMAP Mission CoDICE Lo Level-1B Aggregated Instrument Rates Data.
+
+imap_codice_l1b_lo_counters_singles:
+    <<: *instrument_base
+    Data_level: 1B
+    Data_type: L1B_lo-counters-aggregated->Level-1B Lo Single Instrument Rates Data
+    Logical_source: imap_codice_l1b_lo-counters-singles
+    Logical_source_description: IMAP Mission CoDICE Lo Level-1B Single Instrument Rates Data.
+
+imap_codice_l1b_lo_sw_angular:
+    <<: *instrument_base
+    Data_level: 1B
+    Data_type: L1B_lo-sw-angular->Level-1B Lo Sunward Angular Rates Data
+    Logical_source: imap_codice_l1b_lo-sw-angular
+    Logical_source_description: IMAP Mission CoDICE Lo Level-1B Sunward Angular Rates Data.
+
+imap_codice_l1b_lo_nsw_angular:
+    <<: *instrument_base
+    Data_level: 1B
+    Data_type: L1B_lo-nsw-angular->Level-1B Lo Non-Sunward Angular Rates Data
+    Logical_source: imap_codice_l1b_lo-nsw-angular
+    Logical_source_description: IMAP Mission CoDICE Lo Level-1B Non-Sunward Angular Rates Data.
+
+imap_codice_l1b_lo_sw_priority:
+    <<: *instrument_base
+    Data_level: 1B
+    Data_type: L1B_lo-sw-priority->Level-1B Lo Sunward Priority Rates Data
+    Logical_source: imap_codice_l1b_lo-sw-priority
+    Logical_source_description: IMAP Mission CoDICE Lo Level-1B Sunward Priority Rates Data.
+
+imap_codice_l1b_lo_nsw_priority:
+    <<: *instrument_base
+    Data_level: 1B
+    Data_type: L1B_lo-nsw-priority->Level-1B Lo Non-Sunward Priority Rates Data
+    Logical_source: imap_codice_l1b_lo-nsw-priority
+    Logical_source_description: IMAP Mission CoDICE Lo Level-1B Non-Sunward Priority Rates Data.
+
+imap_codice_l1b_lo_sw_species:
+    <<: *instrument_base
+    Data_level: 1B
+    Data_type: L1B_lo-sw-species->Level-1B Lo Sunward Species Rates Data
+    Logical_source: imap_codice_l1b_lo-sw-species
+    Logical_source_description: IMAP Mission CoDICE Lo Level-1B Sunward Species Rates Data.
+
+imap_codice_l1b_lo_nsw_species:
+    <<: *instrument_base
+    Data_level: 1B
+    Data_type: L1B_lo-nsw-species->Level-1B Lo Non-Sunward Species Rates Data
+    Logical_source: imap_codice_l1b_lo-nsw-species
+    Logical_source_description: IMAP Mission CoDICE Lo Level-1B Non-Sunward Species Rates Data.