imap-processing 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

imap_processing/cdf/cdf_attribute_manager.py

@@ -1,322 +0,0 @@
-"""
-Class for creating and managing CDF attrs.
-
-Developed based of HermesDataSchema from HERMES-SOC/hermes_core.
-"""
-
-from __future__ import annotations
-
-import logging
-from pathlib import Path
-
-import yaml
-
-DEFAULT_GLOBAL_CDF_ATTRS_FILE = "imap_default_global_cdf_attrs.yaml"
-DEFAULT_GLOBAL_CDF_ATTRS_SCHEMA_FILE = "default_global_cdf_attrs_schema.yaml"
-DEFAULT_VARIABLE_CDF_ATTRS_SCHEMA_FILE = "default_variable_cdf_attrs_schema.yaml"
-
-
-class CdfAttributeManager:
-    """
-    Class for creating and managing CDF attributes based out of yaml files.
-
-    This class is based on the HERMES SWxSOC project for managing CDF attributes, but
-    is intended to be a flexible and very lightweight way of managing CDF attribute
-    creation and validation.
-
-    To use, you can load one or many global and variable attribute files:
-
-    .. code::
-
-        cdf_attr_manager = CdfAttributeManager(data_dir)
-        cdf_attr_manager.load_global_attributes("global_attrs.yaml")
-        cdf_attr_manager.load_global_attributes("instrument_global_attrs.yaml")
-        cdf_attr_manager.load_variable_attributes("variable_attrs.yaml")
-
-    Later files will overwrite earlier files if the same attribute is defined.
-
-    You can then get the global and variable attributes:
-
-    If you provide an instrument_id, it will also add the attributes defined under
-    instrument_id. If this is not included, then only the attributes defined in the top
-    level of the file are used.
-
-    .. code::
-
-        # Instrument ID is optional for refining the attributes used from the file
-        global_attrs = cdf_attr_manager.get_global_attributes(instrument_id)
-        variable_attrs = cdf_attr_manager.get_variable_attributes(variable_name)
-
-    The variable and global attributes are validated against the schemas upon calling
-    ``get_global_attributes`` and ``get_variable_attributes``.
-
-    Parameters
-    ----------
-    data_dir : pathlib.Path
-        The directory containing the schema and variable files (nominally config/).
-
-    Attributes
-    ----------
-    source_dir : pathlib.Path
-        The directory containing the schema and variable files - nominally config/
-    """
-
-    def __init__(self, data_dir: Path):
-        """Initialize the CdfAttributeManager and read schemas from data_dir."""
-        # TODO: Split up schema source and data source?
-        self.source_dir = data_dir
-
-        # TODO: copied from hermes_core. Currently we can use default schema, but
-        # We should add some way of extending the schema and remove all the HERMES
-        # specific stuff
-        # Data Validation, Complaiance,
-        self.global_attribute_schema = self._load_default_global_attr_schema()
-
-        # Data Validation and Compliance for Variable Data
-        self.variable_attribute_schema = self._load_default_variable_attr_schema()
-
-        # Load Default IMAP Global Attributes
-        self._global_attributes = CdfAttributeManager._load_yaml_data(
-            self.source_dir / DEFAULT_GLOBAL_CDF_ATTRS_FILE
-        )
-        self._variable_attributes: dict = {}
-
-    def _load_default_global_attr_schema(self) -> yaml:
-        """
-        Load the default global schema from the source directory.
-
-        Returns
-        -------
-        dict
-            The dict representing the global schema.
-        """
-        default_schema_path = (
-            self.source_dir / "shared" / DEFAULT_GLOBAL_CDF_ATTRS_SCHEMA_FILE
-        )
-        # Load the Schema
-        return CdfAttributeManager._load_yaml_data(default_schema_path)
-
-    def _load_default_variable_attr_schema(self) -> yaml:
-        """
-        Load the default variable schema from the source directory.
-
-        Returns
-        -------
-        dict
-            The dict representing the variable schema.
-        """
-        default_schema_path = (
-            self.source_dir / "shared" / DEFAULT_VARIABLE_CDF_ATTRS_SCHEMA_FILE
-        )
-        # Load the Schema
-        return CdfAttributeManager._load_yaml_data(default_schema_path)
-
-    # TODO Change Returning Any from function declared to return "dict[Any, Any]"
-
-    def load_global_attributes(self, file_path: str) -> None:
-        """
-        Update the global attributes property with the attributes from the file.
-
-        Calling this method multiple times on different files will add all the
-        attributes from the files, overwriting existing attributes if they are
-        duplicated.
-
-        Parameters
-        ----------
-        file_path : str
-            File path to load, under self.source_dir.
-        """
-        self._global_attributes.update(
-            CdfAttributeManager._load_yaml_data(self.source_dir / file_path)
-        )
-
-    def add_global_attribute(self, attribute_name: str, attribute_value: str) -> None:
-        """
-        Add a single global attribute to the global attributes.
-
-        This is intended only for dynamic global attributes which change per-file, such
-        as Data_version. It is not intended to be used for static attributes, which
-        should all be included in the YAML files.
-
-        This will overwrite any existing value in attribute_name if it exists. The
-        attribute must be in the global schema, or it will not be included as output.
-
-        Parameters
-        ----------
-        attribute_name : str
-            The name of the attribute to add.
-        attribute_value : str
-            The value of the attribute to add.
-        """
-        self._global_attributes[attribute_name] = attribute_value
-
-    @staticmethod
-    def _load_yaml_data(file_path: str | Path) -> yaml:
-        """
-        Load a yaml file from the provided path.
-
-        Parameters
-        ----------
-        file_path : str | Path
-            Path to the yaml file to load.
-
-        Returns
-        -------
-        yaml
-            Loaded yaml.
-        """
-        with open(file_path) as file:
-            return yaml.safe_load(file)
-
-    def get_global_attributes(self, instrument_id: str | None = None) -> dict:
-        """
-        Generate a dictionary global attributes based off the loaded schema and attrs.
-
-        Validates against the global schema to ensure all required variables are
-        present. It can also include instrument specific global attributes if
-        instrumet_id is set.
-
-        If an instrument_id is provided, the level and instrument specific
-        attributes that were previously loaded using add_instrument_global_attrs will
-        be included.
-
-        Parameters
-        ----------
-        instrument_id : str
-            The id of the CDF file, used to retrieve instrument and level
-            specific global attributes. Suggested value is the logical_source_id.
-
-        Returns
-        -------
-        output : dict
-            The global attribute values created from the input global attribute files
-            and schemas.
-        """
-        output = dict()
-        for attr_name, attr_schema in self.global_attribute_schema.items():
-            if attr_name in self._global_attributes:
-                output[attr_name] = self._global_attributes[attr_name]
-            # Retrieve instrument specific global attributes from the variable file
-            elif (
-                instrument_id is not None
-                and attr_name in self._global_attributes[instrument_id]
-            ):
-                output[attr_name] = self._global_attributes[instrument_id][attr_name]
-            elif attr_schema["required"] and attr_name not in self._global_attributes:
-                # TODO throw an error
-                output[attr_name] = None
-
-        return output
-
-    def load_variable_attributes(self, file_name: str) -> None:
-        """
-        Add variable attributes for a given filename.
-
-        Parameters
-        ----------
-        file_name : str
-            The name of the file to load from self.source_dir.
-        """
-        # Add variable attributes from file_name. Each variable name should have the
-        # Required subfields as defined in the variable schema.
-        raw_var_attrs = CdfAttributeManager._load_yaml_data(self.source_dir / file_name)
-        var_attrs = raw_var_attrs.copy()
-
-        self._variable_attributes.update(var_attrs)
-
-    def get_variable_attributes(
-        self, variable_name: str, check_schema: bool = True
-    ) -> dict:
-        """
-        Get the attributes for a given variable name.
-
-        It retrieves the variable from previously loaded variable definition files and
-        validates against the defined variable schemas.
-
-        Parameters
-        ----------
-        variable_name : str
-            The name of the variable to retrieve attributes for.
-
-        check_schema : bool
-            Flag to bypass schema validation.
-
-        Returns
-        -------
-        dict
-            Information containing specific variable attributes
-            associated with "variable_name".
-        """
-        # Case to handle attributes not in schema
-        if check_schema is False:
-            if variable_name in self._variable_attributes:
-                return_dict: dict = self._variable_attributes[variable_name]
-                return return_dict
-            # TODO: throw an error?
-            return {}
-
-        output = dict()
-        for attr_name in self.variable_attribute_schema["attribute_key"]:
-            # Standard case
-            if attr_name in self._variable_attributes[variable_name]:
-                output[attr_name] = self._variable_attributes[variable_name][attr_name]
-            # Case to handle DEPEND_i schema issues
-            elif attr_name == "DEPEND_i":
-                # DEFAULT_0 is not required, UNLESS we are dealing with
-                # variable_name = epoch
-                # Find all the attributes of variable_name that contain "DEPEND"
-                variable_depend_attrs = [
-                    key
-                    for key in self._variable_attributes[variable_name].keys()
-                    if "DEPEND" in key
-                ]
-                # Confirm that each DEPEND_i attribute is unique
-                if len(set(variable_depend_attrs)) != len(variable_depend_attrs):
-                    logging.warning(
-                        f"Found duplicate DEPEND_i attribute in variable "
-                        f"{variable_name}: {variable_depend_attrs}"
-                    )
-                for variable_depend_attr in variable_depend_attrs:
-                    output[variable_depend_attr] = self._variable_attributes[
-                        variable_name
-                    ][variable_depend_attr]
-                # TODO: Add more DEPEND_0 variable checks!
-            # Case to handle LABL_PTR_i schema issues
-            elif attr_name == "LABL_PTR_i":
-                # Find all the attributes of variable_name that contain "LABL_PTR"
-                variable_labl_attrs = [
-                    key
-                    for key in self._variable_attributes[variable_name].keys()
-                    if "LABL_PTR" in key
-                ]
-                for variable_labl_attr in variable_labl_attrs:
-                    output[variable_labl_attr] = self._variable_attributes[
-                        variable_name
-                    ][variable_labl_attr]
-            # Case to handle REPRESENTATION_i schema issues
-            elif attr_name == "REPRESENTATION_i":
-                # Find all the attributes of variable_name that contain
-                # "REPRESENTATION_i"
-                variable_rep_attrs = [
-                    key
-                    for key in self._variable_attributes[variable_name].keys()
-                    if "REPRESENTATION" in key
-                ]
-                for variable_rep_attr in variable_rep_attrs:
-                    output[variable_rep_attr] = self._variable_attributes[
-                        variable_name
-                    ][variable_rep_attr]
-            # Validating required schema
-            elif (
-                self.variable_attribute_schema["attribute_key"][attr_name]["required"]
-                and attr_name not in self._variable_attributes[variable_name]
-            ):
-                logging.warning(
-                    "Required schema '"
-                    + attr_name
-                    + "' attribute not present for "
-                    + variable_name
-                )
-                output[attr_name] = ""
-
-        return output

imap_processing/cdf/config/shared/default_global_cdf_attrs_schema.yaml

@@ -1,246 +0,0 @@
-DOI:
-  description: >
-    DOI is a persistent Unique Digital Identifier with the form
-    https://doi.org/<PREFIX>/<SUFFIX> with the <PREFIX> identifying the DOI
-    registration authority and the <SUFFIX> identifying the dataset. The DOI should point to
-    a landing page for additional information about the dataset. DOIs are typically created by
-    the SPASE naming authority or archive.
-  default: null
-  required: false # NOT Required in ISTP Guide (Recommended)
-  validate: true # include in validation
-  overwrite: false
-Data_level:
-  description: >
-    This attribute is used in file name creation and records the level of processsing done
-    on the dataset. For HERMES the following are valid values:
-    - l0>Level 0
-    - l1>Level 1
-    - l2>Level 2
-    - l3>Level 3
-    - l4>Level 4
-    - ql>Quicklook
-  default: null
-  required: true # NOT Required in ISTP Guide (Derived)
-  validate: false
-  overwrite: true
-Data_product_descriptor:
-  description: >
-    This is an optional field that may not be needed for all products. Where it is used, identifier
-    should be short (e.q. 3-8 characters) descriptors that are helpful to end- users. If a
-    descriptor contains multiple components, underscores are used to separate those components.
-  default: null
-  required: false # NOT Required in ISTP Guide (Derived)
-  validate: false
-  overwrite: true
-Data_type:
-  description: >
-    This attribute is used by CDF file writing software to create a filename. It is a
-    combination of the following filename components: mode, data level, and optional data
-    product descriptor.
-  default: null
-  required: false # NOT Required in ISTP Guide (Derived)
-  validate: false
-  overwrite: true
-Data_version:
-  description: >
-    This attribute identifies the version of a particular CDF data file.
-  default: null
-  required: true
-  validate: true
-  overwrite: false
-Descriptor:
-  description: >
-    This attribute identifies the name of the instrument or sensor that collected the data. Both
-    a long name and a short name are given. For any data file, only a single value is allowed.
-    For HERMES, the following are valid values:
-    - EEA>Electron Electrostatic Analyzer
-    - MERIT>Miniaturized Electron pRoton Telescope
-    - NEMISIS> Noise Eliminating Magnetometer In a Small Integrated System
-    - SPAN-I>Solar Probe Analyzer for Ions
-  default: null
-  required: true
-  validate: true
-  overwrite: false
-Discipline:
-  description: >
-    This attribute describes both the science discipline and sub discipline. For HERMES,
-    this value should always be "Space Physics>Magnetospheric Science."
-  default: Space Physics>Magnetospheric Science
-  required: true
-  validate: true
-  overwrite: false
-File_naming_convention:
-  description: >
-    If File_naming_convention was not set, it uses default setting:
-      source_datatype_descriptor_yyyyMMdd
-  default: source_datatype_descriptor_yyyyMMdd
-  required: false
-  validate: false
-  overwrite: true
-Generation_date:
-  description: >
-    Date stamps the creation of the file using the syntax yyyymmdd, e.g., "
-  default: null
-  required: false # NOT Required in ISTP Guide (Recommended)
-  validate: true
-  overwrite: true
-HTTP_LINK:
-  description: >
-    The 'HTTP_LINK', 'LINK_TEXT', and 'LINK_TITLE' attributes store the URL with a
-    description of this dataset at the HERMES SDC. The use of HTTP_LINK attribute requires
-    the existence and equal number of corresponding LINK_TEXT and LINK_TITLE attributes.
-    If text is not needed for these attributes, use an empty string "".
-  default: null
-  required: false # NOT Required in ISTP Guide (Recommended)
-  validate: true
-  overwrite: false
-Instrument_mode:
-  description: >
-    TBS
-  default: null
-  required: false # NOT Required in ISTP Guide (Derived)
-  validate: false
-  overwrite: false
-Instrument_type:
-  description: >
-    This attribute is used to facilitate making choices of instrument type. More than one entry
-    is allowed. Acceptable values for HERMES include:
-    - Magnetic Fields (space)
-    - Particles (space)
-    - Plasma and Solar Wind
-    - Ephemeris -> Ephemeris/Attitude/Ancillary
-  default: null
-  required: true
-  validate: true
-  overwrite: false
-LINK_TEXT:
-  description: >
-    The 'HTTP_LINK', 'LINK_TEXT', and 'LINK_TITLE' attributes store the URL with a
-    description of this dataset at the HERMES SDC. The use of HTTP_LINK attribute requires
-    the existence and equal number of corresponding LINK_TEXT and LINK_TITLE attributes.
-    If text is not needed for these attributes, use an empty string "".
-  default: null
-  required: false # NOT Required in ISTP Guide (Recommended)
-  validate: true
-  overwrite: false
-LINK_TITLE:
-  description: >
-    The 'HTTP_LINK', 'LINK_TEXT', and 'LINK_TITLE' attributes store the URL with a
-    description of this dataset at the HERMES SDC. The use of HTTP_LINK attribute requires
-    the existence and equal number of corresponding LINK_TEXT and LINK_TITLE attributes.
-    If text is not needed for these attributes, use an empty string "".
-  default: null
-  required: false # NOT Required in ISTP Guide (Recommended)
-  validate: true
-  overwrite: false
-Logical_file_id:
-  description: >
-    This attribute stores the name of the CDF file but without the
-    file extension (e.g. ".cdf"). This attribute is required to avoid loss of the original source
-    in the case of accidental (or intentional) renaming.
-  default: null
-  required: true
-  validate: true
-  overwrite: true
-Logical_source:
-  description: >
-    This attribute determines the file naming convention in the SKT Editor and is used by
-    CDA Web. It is composed of the following values:
-    - source_name - (e.g. spacecraft identifier)
-    - descriptor - (e.g. instrument identifier - see Section Error! Reference source not
-    found.)
-    - data_type - (e.g. mode, data level, and optional data product descriptor - value
-    come from 'Data_type' attribute)
-  default: null
-  required: true
-  validate: true
-  overwrite: true
-Logical_source_description:
-  description: >
-    This attribute writes out the full words associated with the encrypted Logical_source
-    above, e.g., "Level 1 Dual Electron Spectrometer Survey Data". Users on CDAWeb see
-    this value on their website.
-  default: null
-  required: true
-  validate: true
-  overwrite: true
-MODS:
-  description: >
-    This attribute is an SPDF standard global attribute, which is used to denote the history of
-    modifications made to the CDF data set. The MODS attribute should contain a
-    description of all significant changes to the data set, essentially capturing a log of high-
-    level release notes. This attribute can have as many entries as necessary and should be
-    updated if the Interface Number ("X") of the version number changes.
-  default: null
-  required: false # NOT Required in ISTP Guide (Recommended)
-  validate: true
-  overwrite: false
-Mission_group:
-  description: >
-    This attribute has a single value and is used to facilitate making choices of source through
-    CDAWeb. This value should be "HERMES."
-  default: HERMES
-  required: true
-  validate: true
-  overwrite: false
-PI_affiliation:
-  description: >
-    This attribute value should include the HERMES mission PI affiliation followed by a
-    comma-separated list of any Co-I affiliations that are responsible for this particular
-    dataset. The following are valid HERMES values, of which the abbreviations should be
-    used exclusively within this attribute value, and the full text of the affiliation included in
-    the general 'text' attribute as it is used solely in plot labels.
-    - GSFC - Goddard Space Flight Center
-    - UCB - University of California, Berkeley
-    - SSL - Space Sciences Laboratory, UCB
-    - UM - University of Michigan
-  default: null
-  required: true
-  validate: true
-  overwrite: false
-PI_name:
-  description: >
-    This attribute value should include first initial and last name of the HERMES mission PI
-    followed by a comma-separated list of any Co-Is that are responsible for this particular
-    dataset.
-  default: null
-  required: true
-  validate: true
-  overwrite: false
-Project:
-  description: >
-    This attribute identifies the name of the project and indicates ownership. For HERMES,
-    this value should be "STP>Solar-Terrestrial Physics".
-  default: STP>Solar-Terrestrial Physics
-  required: true
-  validate: true
-  overwrite: false
-Source_name:
-  description: >
-    This attribute identifies the observatory where the data originated. The following are
-    valid values for HERMES:
-    - HERMES>Heliophysics Environmental and Radiation Measurement Experiment Suite
-  default: HERMES>Heliophysics Environmental and Radiation Measurement Experiment Suite
-  required: true
-  validate: true
-  overwrite: false
-Start_time:
-  description: >
-    The start time of the contained data given in YYYYMMDD_hhmmss
-  default: null
-  required: false # NOT Required in ISTP Guide (Derived)
-  validate: false
-  overwrite: true
-TEXT:
-  description: >
-    This attribute is an SPDF standard global attribute, which is a text description of the
-    experiment whose data is included in the CDF. A reference to a journal article(s) or to a
-    World Wide Web page describing the experiment is essential and constitutes the
-    minimum requirement. A written description of the data set is also desirable. This
-    attribute can have as many entries as necessary to contain the desired information.
-    Typically, this attribute is about a paragraph in length and is not shown on CDAWeb.
-    CDAWeb is the web portal for access to SPDF data, available at https://cdaweb.gsfc.nasa.gov.
-  default: null
-  required: true
-  validate: true
-  overwrite: false