imap-processing 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

imap_processing/idex/idex_l1b.py

@@ -16,19 +16,47 @@ Examples
 
 import logging
 from enum import Enum
+from typing import Union
 
 import pandas as pd
 import xarray as xr
 
 from imap_processing import imap_module_directory
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.spice.geometry import (
+    SpiceBody,
+    SpiceFrame,
+    cartesian_to_spherical,
+    imap_state,
+    instrument_pointing,
+    solar_longitude,
+)
+from imap_processing.spice.spin import get_spacecraft_spin_phase, get_spin_angle
+from imap_processing.spice.time import ttj2000ns_to_et
 from imap_processing.utils import convert_raw_to_eu
 
 logger = logging.getLogger(__name__)
 
 
 class ConversionFactors(float, Enum):
-    """Enum class for conversion factor values."""
+    """
+    Enum class for conversion factor values.
+
+    Attributes
+    ----------
+    TOF_High : float
+        Time of flight high conversion factor.
+    TOF_Low : float
+        Time of flight low conversion factor.
+    TOF_Mid : float
+        Time of flight mid conversion factor.
+    Target_Low : float
+        Target Low conversion factor.
+    Target_High : float
+        Target High conversion factor.
+    Ion_Grid : float
+        Ion Grid conversion factor.
+    """
 
     TOF_High = 2.89e-4
     TOF_Low = 5.14e-4
@@ -38,6 +66,44 @@ class ConversionFactors(float, Enum):
     Ion_Grid = 7.46e-4
 
 
+class TriggerMode(Enum):
+    """
+    Enum class for data collection trigger Modes.
+
+    Attributes
+    ----------
+    Threshold : int
+        Mode 1 - Triggers when signal reaches the threshold value.
+    SinglePulse : int
+        Mode 2 - Triggers when a single pulse is detected.
+    DoublePulse : int
+        Mode 3 - Triggers when two pulses are detected.
+    """
+
+    Threshold = 1
+    SinglePulse = 2
+    DoublePulse = 3
+
+    @staticmethod
+    def get_mode_label(mode: int, channel: str) -> str:
+        """
+        Return trigger mode label.
+
+        Parameters
+        ----------
+        mode : int
+            Raw mode value.
+        channel : str
+            Channel gain level.
+
+        Returns
+        -------
+        str
+            Mode label.
+        """
+        return f"{channel.upper()}{TriggerMode(mode).name}"
+
+
 def idex_l1b(l1a_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
     """
     Will process IDEX l1a data to create l1b data products.
@@ -82,10 +148,23 @@ def idex_l1b(l1a_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
         dims=["epoch"],
         attrs=idex_attrs.get_variable_attributes("epoch"),
     )
+    # Get spice data and save them as xr.DataArrays in the output. Spice data is not
+    # used for calculations yet but are saved in the CDF for reference.
+    spice_data = get_spice_data(l1a_dataset, idex_attrs)
+
+    trigger_settings = get_trigger_mode_and_level(l1a_dataset)
+    if trigger_settings:
+        trigger_settings["triggerlevel"].attrs = idex_attrs.get_variable_attributes(
+            "trigger_level"
+        )
+        trigger_settings["triggermode"].attrs = idex_attrs.get_variable_attributes(
+            "trigger_mode"
+        )
+
     # Create l1b Dataset
     l1b_dataset = xr.Dataset(
         coords={"epoch": epoch_da},
-        data_vars=processed_vars | waveforms_converted,
+        data_vars=processed_vars | waveforms_converted | trigger_settings | spice_data,
         attrs=idex_attrs.get_global_attributes("imap_idex_l1b_sci"),
     )
     # Convert variables
@@ -94,19 +173,16 @@ def idex_l1b(l1a_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
         conversion_table_path=var_information_path,
         packet_name="IDEX_SCI",
     )
+    prefixes = ["shcoarse", "shfine", "time_high_sample", "time_low_sample"]
     vars_to_copy = [
-        "shcoarse",
-        "shfine",
-        "time_high_sr",
-        "time_low_sr",
-        "time_high_sr_label",
-        "time_low_sr_label",
+        var
+        for var in l1a_dataset.variables
+        if any(prefix in var for prefix in prefixes)
     ]
     # Copy arrays from the l1a_dataset that do not need l1b processing
     for var in vars_to_copy:
         l1b_dataset[var] = l1a_dataset[var].copy()
 
-    # TODO: Add TriggerMode and TriggerLevel attr
     # TODO: Spice data?
 
     logger.info("IDEX L1B science data processing completed.")
@@ -126,7 +202,7 @@ def unpack_instrument_settings(
     ----------
     l1a_dataset : xarray.Dataset
         IDEX L1a dataset containing the 6 waveform arrays.
-    var_information_df : pd.DataFrame
+    var_information_df : pandas.DataFrame
         Pandas data frame that contains information about each variable
         (e.g., bit-size, starting bit, and padding). This is used to unpack raw
         telemetry data from the input dataset (`l1a_dataset`).
@@ -190,3 +266,161 @@ def convert_waveforms(
         )
 
     return waveforms_pc
+
+
+def get_trigger_mode_and_level(
+    l1a_dataset: xr.Dataset,
+) -> Union[dict[str, xr.DataArray], dict]:
+    """
+    Determine the trigger mode and threshold level for each event.
+
+    Parameters
+    ----------
+    l1a_dataset : xarray.Dataset
+        IDEX L1a dataset containing the six waveform arrays and instrument settings.
+
+    Returns
+    -------
+    dict
+        A dictionary containing the trigger mode and level values.
+    """
+    # low, mid, and high gain channels
+    channels = ["lg", "mg", "hg"]
+    # 10 bit mask
+    mask = 0b1111111111
+    trigger_modes = []
+    trigger_levels = []
+
+    def compute_trigger_values(
+        trigger_mode: int, trigger_controls: int, gain_channel: str
+    ) -> Union[tuple[str, Union[int, float]], tuple[None, None]]:
+        """
+        Compute the trigger mode label and threshold level.
+
+        Parameters
+        ----------
+        trigger_mode : float
+            Raw trigger mode value.
+        trigger_controls : int
+            Raw trigger control values.
+        gain_channel : float
+            Gain channel (low, mid, or high).
+
+        Returns
+        -------
+        tuple
+            Mode label and threshold level.
+        """
+        # If the trigger mode is zero, then the channel did not trigger the event and
+        # therefore there is no threshold level
+        if trigger_mode == 0:
+            return None, None
+
+        mode_label = TriggerMode.get_mode_label(mode=trigger_mode, channel=gain_channel)
+        # The trigger control variable is 32 bits with the first 10 bits representing
+        # the Threshold level.
+        # Bit-shift right 22 places and use a 10-bit mask to extract the level value.
+        threshold_level = float((trigger_controls >> 22) & mask)
+
+        # If it is the high gain channel multiply the level by the conversion factor.
+        # TODO: determine why the idex team is only doing this for the high gain channel
+        if gain_channel == "hg":
+            threshold_level *= ConversionFactors["TOF_High"]
+        return mode_label, threshold_level
+
+    for channel in channels:
+        # Get all the modes and controls for each event for the current channel
+        modes = l1a_dataset[f"idx__txhdr{channel}trigmode"].copy()
+        controls = l1a_dataset[f"idx__txhdr{channel}trigctrl1"].copy()
+
+        # Apply the function across the arrays
+        mode_array, level_array = xr.apply_ufunc(
+            compute_trigger_values,
+            modes,
+            controls,
+            channel,
+            output_core_dims=([], []),
+            vectorize=True,
+            output_dtypes=[object, float],
+        )
+        trigger_modes.append(mode_array.rename("trigger_mode"))
+        trigger_levels.append(level_array.rename("trigger_level"))
+
+    try:
+        # There should be an array of modes and threshold levels for each channel.
+        # At each index (event) only one of the three arrays should have a value that is
+        # not 'None' because each event can only be triggered by one channel.
+        # By merging the three arrays, we get value for each event.
+        merged_modes = xr.merge([trigger_modes[0], xr.merge(trigger_modes[1:])])
+        merged_levels = xr.merge([trigger_levels[0], xr.merge(trigger_levels[1:])])
+
+        return {
+            "triggermode": merged_modes.trigger_mode,
+            "triggerlevel": merged_levels.trigger_level,
+        }
+
+    except xr.MergeError as e:
+        raise ValueError(
+            f"Only one channel can trigger a dust event. Please make sure "
+            f"there is only one valid trigger value per event. This "
+            f"caused Merge Error: {e}"
+        ) from e
+
+
+def get_spice_data(
+    l1a_dataset: xr.Dataset, idex_attrs: ImapCdfAttributes
+) -> dict[str, xr.DataArray]:
+    """
+    Use spice to query ephemeris, attitude, celestial coordinates for each dust event.
+
+    Parameters
+    ----------
+    l1a_dataset : xarray.Dataset
+        IDEX L1a dataset containing the six waveform arrays and instrument settings.
+    idex_attrs : ImapCdfAttributes
+        CDF attribute manager object.
+
+    Returns
+    -------
+    dict
+        Spice array names and xr.DataArrays.
+    """
+    # convert 'epoch' from nanoseconds to seconds since j2000
+    et = ttj2000ns_to_et(l1a_dataset["epoch"].data)
+    # Get 'shcoarse' (Mission Elapsed Time)
+    met = l1a_dataset["shcoarse"].data
+    # Get spacecraft spin phase in degrees
+    spin_phase = get_spacecraft_spin_phase(query_met_times=met)
+    imap_spin_phase = get_spin_angle(spin_phase, degrees=True)
+    # Get position and velocity of IMAP in ecliptic frame
+    ephemeris = imap_state(et, observer=SpiceBody.SUN)
+    # Get Idex pointing in the j2000 equatorial frame
+    idex_pointing = instrument_pointing(
+        et, SpiceFrame.IMAP_IDEX, SpiceFrame.J2000, cartesian=True
+    )
+    solar_lon = solar_longitude(et, degrees=True)
+
+    range_ra_and_dec = cartesian_to_spherical(idex_pointing)
+
+    spice_data = {
+        "ephemeris_position_x": ephemeris[:, 0],
+        "ephemeris_position_y": ephemeris[:, 1],
+        "ephemeris_position_z": ephemeris[:, 2],
+        "ephemeris_velocity_x": ephemeris[:, 3],
+        "ephemeris_velocity_y": ephemeris[:, 4],
+        "ephemeris_velocity_z": ephemeris[:, 5],
+        "right_ascension": range_ra_and_dec[:, 1],
+        "declination": range_ra_and_dec[:, 2],
+        "spin_phase": imap_spin_phase,
+        "solar_longitude": solar_lon,
+    }
+
+    for name, array in spice_data.items():
+        spice_data[name] = xr.DataArray(
+            name=name,
+            data=array,
+            dims="epoch",
+            attrs=idex_attrs.get_variable_attributes(name),
+        )
+
+    return spice_data

imap_processing/lo/l0/lo_science.py

@@ -20,6 +20,7 @@ from imap_processing.lo.l0.utils.bit_decompression import (
     Decompress,
     decompress_int,
 )
+from imap_processing.spice.time import met_to_ttj2000ns
 from imap_processing.utils import convert_to_binary_string
 
 logger = logging.getLogger(__name__)
@@ -439,3 +440,63 @@ def find_valid_groups(
     ]
     valid_groups = [is_sequential(seq_ctrs) for seq_ctrs in grouped_seq_ctrs]
     return valid_groups
+
+
+def organize_spin_data(dataset: xr.Dataset, attr_mgr: ImapCdfAttributes) -> xr.Dataset:
+    """
+    Organize the spin data for Lo.
+
+    The spin data is spread across 28 fields. This function
+    combines each of those fields into 2D arrays for each
+    epoch and spin.
+
+    Parameters
+    ----------
+    dataset : xr.Dataset
+        Lo spin data from packets_to_dataset function.
+    attr_mgr : ImapCdfAttributes
+        CDF attribute manager for Lo L1A.
+
+    Returns
+    -------
+    dataset : xr.Dataset
+        Updated dataset with the spin data organized.
+    """
+    # Get the spin data fields
+    spin_fields = [
+        "start_sec_spin",
+        "start_subsec_spin",
+        "esa_neg_dac_spin",
+        "esa_pos_dac_spin",
+        "valid_period_spin",
+        "valid_phase_spin",
+        "period_source_spin",
+    ]
+
+    # Set epoch to the acq_start time
+    # acq_start_sec is in units of seconds
+    # acq_start_subsec is in units of microseconds
+    acq_start = dataset.acq_start_sec + (1e-6 * dataset.acq_start_subsec)
+    epoch = met_to_ttj2000ns(acq_start)
+    dataset = dataset.assign_coords(epoch=("epoch", epoch))
+    for spin_field in spin_fields:
+        # Get the field attributes
+        field_attrs = attr_mgr.get_variable_attributes(spin_field, check_schema=False)
+        dtype = field_attrs.pop("dtype")
+
+        packet_fields = [f"{spin_field}_{i}" for i in range(1, 29)]
+        # Combine the spin data fields along a new dimension
+        combined_spin_data = xr.concat(
+            [dataset[field].astype(dtype) for field in packet_fields], dim="spin"
+        )
+
+        # Assign the combined data back to the dataset
+        dataset[spin_field] = xr.DataArray(
+            combined_spin_data.transpose(),
+            dims=["epoch", "spin"],
+            attrs=field_attrs,
+        )
+        # Drop the individual spin data fields
+        dataset = dataset.drop_vars(packet_fields)
+
+    return dataset

imap_processing/lo/l1a/lo_l1a.py

@@ -11,6 +11,7 @@ from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.lo.l0.lo_apid import LoAPID
 from imap_processing.lo.l0.lo_science import (
     combine_segmented_packets,
+    organize_spin_data,
     parse_events,
     parse_histogram,
 )
@@ -52,6 +53,19 @@ def lo_l1a(dependency: Path, data_version: str) -> list[xr.Dataset]:
     attr_mgr.add_instrument_variable_attrs(instrument="lo", level="l1a")
     attr_mgr.add_global_attribute("Data_version", data_version)
 
+    if LoAPID.ILO_SPIN in datasets_by_apid:
+        logger.info(
+            f"\nProcessing {LoAPID(LoAPID.ILO_SPIN).name} "
+            f"packet (APID: {LoAPID.ILO_SPIN.value})"
+        )
+        logical_source = "imap_lo_l1a_spin"
+        datasets_by_apid[LoAPID.ILO_SPIN] = organize_spin_data(
+            datasets_by_apid[LoAPID.ILO_SPIN], attr_mgr
+        )
+
+        datasets_by_apid[LoAPID.ILO_SPIN] = add_dataset_attrs(
+            datasets_by_apid[LoAPID.ILO_SPIN], attr_mgr, logical_source
+        )
     if LoAPID.ILO_SCI_CNT in datasets_by_apid:
         logger.info(
             f"\nProcessing {LoAPID(LoAPID.ILO_SCI_CNT).name} "
@@ -90,7 +104,7 @@ def lo_l1a(dependency: Path, data_version: str) -> list[xr.Dataset]:
             datasets_by_apid[LoAPID.ILO_SCI_DE], attr_mgr, logical_source
         )
 
-    good_apids = [LoAPID.ILO_SCI_CNT, LoAPID.ILO_SCI_DE]
+    good_apids = [LoAPID.ILO_SPIN, LoAPID.ILO_SCI_CNT, LoAPID.ILO_SCI_DE]
     logger.info(f"\nReturning datasets: {[LoAPID(apid) for apid in good_apids]}")
     return [datasets_by_apid[good_apid] for good_apid in good_apids]
 
@@ -116,8 +130,63 @@ def add_dataset_attrs(
         Data with attributes added.
     """
     # TODO: may want up split up these if statements into their
-    # own functions
-    if logical_source == "imap_lo_l1a_histogram":
+    #  own functions
+    # Get global attributes
+    dataset.attrs.update(attr_mgr.get_global_attributes(logical_source))
+    # Get attributes for shcoarse and epoch
+    dataset.shcoarse.attrs.update(attr_mgr.get_variable_attributes("shcoarse"))
+    dataset.epoch.attrs.update(attr_mgr.get_variable_attributes("epoch"))
+
+    if logical_source == "imap_lo_l1a_spin":
+        spin = xr.DataArray(
+            data=np.arange(0, 28, dtype=np.uint8),
+            name="spin",
+            dims=["spin"],
+            attrs=attr_mgr.get_variable_attributes("spin"),
+        )
+        spin_label = xr.DataArray(
+            data=spin.values.astype(str),
+            name="spin_label",
+            dims=["spin_label"],
+            attrs=attr_mgr.get_variable_attributes("spin_label"),
+        )
+
+        dataset = dataset.assign_coords(spin=spin, spin_label=spin_label)
+        dataset.num_completed.attrs.update(
+            attr_mgr.get_variable_attributes("num_completed")
+        )
+        dataset.acq_start_sec.attrs.update(
+            attr_mgr.get_variable_attributes("acq_start_sec")
+        )
+        dataset.acq_start_subsec.attrs.update(
+            attr_mgr.get_variable_attributes("acq_start_subsec")
+        )
+        dataset.acq_end_sec.attrs.update(
+            attr_mgr.get_variable_attributes("acq_end_sec")
+        )
+        dataset.acq_end_subsec.attrs.update(
+            attr_mgr.get_variable_attributes("acq_end_subsec")
+        )
+
+        dataset = dataset.drop_vars(
+            [
+                "version",
+                "type",
+                "sec_hdr_flg",
+                "pkt_apid",
+                "seq_flgs",
+                "src_seq_ctr",
+                "pkt_len",
+                "chksum",
+            ]
+        )
+        # An empty DEPEND_0 is being added to support_data
+        # variables that should only have DEPEND_1
+        # Removing Depend_0 here.
+        # TODO: Should look for a fix to this issue
+        del dataset["spin"].attrs["DEPEND_0"]
+
+    elif logical_source == "imap_lo_l1a_histogram":
         # Create coordinates for the dataset
         azimuth_60 = xr.DataArray(
             data=np.arange(0, 6, dtype=np.uint8),
@@ -157,10 +226,6 @@ def add_dataset_attrs(
             attrs=attr_mgr.get_variable_attributes("esa_step_label"),
         )
 
-        # Get attributes for shcoarse and epoch
-        dataset.shcoarse.attrs.update(attr_mgr.get_variable_attributes("shcoarse"))
-        dataset.epoch.attrs.update(attr_mgr.get_variable_attributes("epoch"))
-
         dataset = dataset.assign_coords(
             azimuth_60=azimuth_60,
             azimuth_60_label=azimuth_60_label,
@@ -169,7 +234,6 @@ def add_dataset_attrs(
             esa_step=esa_step,
             esa_step_label=esa_step_label,
         )
-        dataset.attrs.update(attr_mgr.get_global_attributes(logical_source))
         # remove the binary field and CCSDS header from the dataset
         dataset = dataset.drop_vars(
             [
@@ -184,6 +248,14 @@ def add_dataset_attrs(
                 "pkt_len",
             ]
         )
+        # An empty DEPEND_0 is being added to support_data
+        # variables that should only have DEPEND_1
+        # Removing Depend_0 here.
+        # TODO: Should look for a fix to this issue
+        del dataset["azimuth_60"].attrs["DEPEND_0"]
+        del dataset["azimuth_6"].attrs["DEPEND_0"]
+        del dataset["esa_step"].attrs["DEPEND_0"]
+
     elif logical_source == "imap_lo_l1a_de":
         # Create the coordinates for the dataset
         direct_events = xr.DataArray(
@@ -205,8 +277,6 @@ def add_dataset_attrs(
             direct_events_label=direct_events_label,
         )
         # add the epoch and global attributes
-        dataset.epoch.attrs.update(attr_mgr.get_variable_attributes("epoch"))
-        dataset.attrs.update(attr_mgr.get_global_attributes(logical_source))
         dataset = dataset.drop_vars(
             [
                 "version",
@@ -221,5 +291,23 @@ def add_dataset_attrs(
                 "events",
             ]
         )
+        # An empty DEPEND_0 is being added to support_data
+        # variables that should only have DEPEND_1
+        # Removing Depend_0 here.
+        # TODO: Should look for a fix to this issue
+        for var in [
+            "direct_events",
+            "coincidence_type",
+            "de_time",
+            "mode",
+            "esa_step",
+            "tof0",
+            "tof1",
+            "tof2",
+            "tof3",
+            "pos",
+            "cksm",
+        ]:
+            dataset[var].attrs.pop("DEPEND_0")
 
     return dataset

imap_processing/lo/l1b/lo_l1b.py

@@ -8,7 +8,7 @@ import numpy as np
 import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
-from imap_processing.spice.time import met_to_j2000ns
+from imap_processing.spice.time import met_to_ttj2000ns
 
 
 def lo_l1b(dependencies: dict, data_version: str) -> list[Path]:
@@ -87,7 +87,7 @@ def create_datasets(
     #  and relative L1A DE time to calculate the absolute DE time,
     #  this epoch conversion will go away and the time in the DE dataclass
     #  can be used direction
-    epoch_converted_time = met_to_j2000ns([0, 1, 2])
+    epoch_converted_time = met_to_ttj2000ns([0, 1, 2])
 
     # Create a data array for the epoch time
     # TODO: might need to update the attrs to use new YAML file

imap_processing/lo/l1c/lo_l1c.py

@@ -8,7 +8,7 @@ import numpy as np
 import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
-from imap_processing.spice.time import met_to_j2000ns
+from imap_processing.spice.time import met_to_ttj2000ns
 
 
 def lo_l1c(dependencies: dict, data_version: str) -> list[Path]:
@@ -86,7 +86,7 @@ def create_datasets(
     #  and relative L1A DE time to calculate the absolute DE time,
     #  this epoch conversion will go away and the time in the DE dataclass
     #  can be used direction
-    epoch_converted_time = [met_to_j2000ns(1)]
+    epoch_converted_time = [met_to_ttj2000ns(1)]
 
     # Create a data array for the epoch time
     # TODO: might need to update the attrs to use new YAML file