imap-processing 0.6.0__py3-none-any.whl

imap_processing/hi/l1a/science_direct_event.py

@@ -0,0 +1,341 @@
+"""IMAP-Hi direct event processing."""
+
+import numpy as np
+import xarray as xr
+
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.spice.time import met_to_j2000ns
+
+# TODO: read LOOKED_UP_DURATION_OF_TICK from
+# instrument status summary later. This value
+# is rarely change but want to be able to change
+# it if needed. It stores information about how
+# fast the time was ticking. It is in microseconds.
+LOOKED_UP_DURATION_OF_TICK = 3999
+
+SECOND_TO_NS = 1e9
+MILLISECOND_TO_NS = 1e6
+MICROSECOND_TO_NS = 1e3
+
+
+def parse_direct_event(event_data: str) -> dict:
+    """
+    Parse event data.
+
+    IMAP-Hi direct event data information is stored in
+    48-bits as follow:
+
+    |    Read first two bits (start_bitmask_data) to find
+    |    out which type of event it is. start_bitmask_data value mapping:
+    |
+    |        1 - A
+    |        2 - B
+    |        3 - C
+    |        0 - META
+    |    If it's a metaevent:
+    |
+    |        Read 48-bits into 2, 4, 10, 32 bits. Each of these breaks
+    |        down as:
+    |
+    |            start_bitmask_data - 2 bits (tA=1, tB=2, tC1=3, META=0)
+    |            ESA step - 4 bits
+    |            integer millisecond of MET(subseconds) - 10 bits
+    |            integer MET(seconds) - 32 bits
+    |
+    |    If it's not a metaevent:
+    |        Read 48-bits into 2, 10, 10, 10, 16 bits. Each of these breaks
+    |        down as:
+    |
+    |            start_bitmask_data - 2 bits (tA=1, tB=2, tC1=3, META=0)
+    |            tof_1 - 10 bit counter
+    |            tof_2 - 10 bit counter
+    |            tof_3 - 10 bit counter
+    |            de_tag - 16 bits
+
+    There are at most total of 665 of 48-bits in each data packet.
+    This data packet is of variable length. If there is one event, then
+    DE_TOF will contain 48-bits. If there are 665 events, then
+    DE_TOF will contain 665 x 48-bits. If there is no event, then
+    DE_TOF will contain 0-bits.
+
+    Per ESA, there should be two data packets. First one will begin with
+    metaevent followed by direct events data. Second one will begin with
+    direct event data only. If there is no event record for certain ESA step,
+    then as mentioned above, first packet will contain metaevent in DE_TOF
+    information and second packet will contain 0-bits in DE_TOF. In general,
+    every two packets will look like this.
+
+    |    first packet = [
+    |        (start_bitmask_data, ESA step, int millisecond of MET, int MET),
+    |        (start_bitmask_data, tof_1, tof_2, tof_3, de_tag),
+    |        .....
+    |    ]
+    |    second packet = [
+    |        (start_bitmask_data, tof_1, tof_2, tof_3, de_tag),
+    |       .....
+    |    ]
+
+    In direct event data, if no hit is registered, the tof_x field in
+    the DE to a value of negative one. However, since the field is described as a
+    "10-bit unsigned counter," it cannot actually store negative numbers.
+    Instead, the value negative is represented by the maximum value that can
+    be stored in a 10-bit unsigned integer, which is 0x3FF (in hexadecimal)
+    or 1023 in decimal. This value is used as a special marker to
+    indicate that no hit was registered. Ideally, the system should
+    not be able to register a hit with a value of 1023 for all
+    tof_1, tof_2, tof_3, because this is in case of an error. But,
+    IMAP-Hi like to process it still to investigate the data.
+    Example of what it will look like if no hit was registered.
+
+    |        (start_bitmask_data, 1023, 1023, 1023, de_tag)
+    |        start_bitmask_data will be 1 or 2 or 3.
+
+    Parameters
+    ----------
+    event_data : str
+        48-bits Event data.
+
+    Returns
+    -------
+    dict
+        Parsed event data.
+    """
+    event_type = int(event_data[:2], 2)
+    metaevent = 0
+    if event_type == metaevent:
+        # parse metaevent
+        esa_step = event_data[2:6]
+        subseconds = event_data[6:16]
+        seconds = event_data[16:]
+
+        # return parsed metaevent data
+        return {
+            "start_bitmask_data": event_type,
+            "esa_step": int(esa_step, 2),
+            "subseconds": int(subseconds, 2),
+            "seconds": int(seconds, 2),
+        }
+
+    # parse direct event
+    trigger_id = event_data[:2]
+    tof_1 = event_data[2:12]
+    tof_2 = event_data[12:22]
+    tof_3 = event_data[22:32]
+    de_tag = event_data[32:]
+
+    # return parsed direct event data
+    return {
+        "start_bitmask_data": int(trigger_id, 2),
+        "tof_1": int(tof_1, 2),
+        "tof_2": int(tof_2, 2),
+        "tof_3": int(tof_3, 2),
+        "de_tag": int(de_tag, 2),
+    }
+
+
+def break_into_bits_size(binary_data: str) -> list:
+    """
+    Break binary stream data into 48-bits.
+
+    Parameters
+    ----------
+    binary_data : str
+        Binary data.
+
+    Returns
+    -------
+    list
+        List of 48-bits.
+    """
+    # TODO: ask Paul what to do if the length of
+    # binary_data is not a multiple of 48
+    field_bit_length = 48
+    return [
+        binary_data[i : i + field_bit_length]
+        for i in range(0, len(binary_data), field_bit_length)
+    ]
+
+
+def create_dataset(de_data_list: list, packet_met_time: list) -> xr.Dataset:
+    """
+    Create xarray dataset.
+
+    Parameters
+    ----------
+    de_data_list : list
+        Parsed direct event data list.
+    packet_met_time : list
+        List of packet MET time.
+
+    Returns
+    -------
+    dataset : xarray.Dataset
+        Xarray dataset.
+    """
+    # These are the variables that we will store in the dataset
+    data_dict: dict = {
+        "epoch": list(),
+        "event_met": list(),
+        "ccsds_met": list(),
+        "meta_event_met": list(),
+        "esa_stepping_num": list(),
+        "trigger_id": list(),
+        "tof_1": list(),
+        "tof_2": list(),
+        "tof_3": list(),
+        "de_tag": list(),
+    }
+
+    # How to handle if first event is not metaevent? This
+    # means that current data file started with direct event.
+    # Per Paul, log a warning and discard all direct events
+    # until we see next metaevent because it could mean
+    # that the instrument was turned off during repoint.
+
+    # Find the index of the first occurrence of the metaevent
+    first_metaevent_index = next(
+        (i for i, d in enumerate(de_data_list) if d.get("start_bitmask_data") == 0),
+        None,
+    )
+
+    if first_metaevent_index is None:
+        return None
+    elif first_metaevent_index != 0:
+        # Discard all direct events until we see next metaevent
+        # TODO: log a warning
+        de_data_list = de_data_list[first_metaevent_index:]
+        packet_met_time = packet_met_time[first_metaevent_index:]
+
+    for index, event in enumerate(de_data_list):
+        if event["start_bitmask_data"] == 0:
+            # metaevent is a way to store information
+            # about bigger portion of time information. Eg.
+            # metaevent stores information about, let's say
+            # "20240319T09:30:01.000". Then direct event time
+            # tag stores information of time ticks since
+            # that time. Then we use those two to combine and
+            # get exact time information of each event.
+
+            # set time and esa step values to
+            # be used for direct event followed by
+            # this metaevent
+            int_subseconds = event["subseconds"]
+            int_seconds = event["seconds"]
+            current_esa_step = event["esa_step"]
+
+            metaevent_time_in_ns = (
+                int_seconds * SECOND_TO_NS + int_subseconds * MILLISECOND_TO_NS
+            )
+
+            # Add half a tick once per algorithm document(
+            # section 2.2.5 and second last bullet point)
+            # and Paul Janzen.
+            half_tick = LOOKED_UP_DURATION_OF_TICK / 2
+            # convert microseconds to nanosecond to
+            # match other time format
+            half_tick_ns = half_tick * MICROSECOND_TO_NS
+            metaevent_time_in_ns += half_tick_ns
+            continue
+
+        data_dict["meta_event_met"].append(metaevent_time_in_ns)
+        # calculate direct event time using time information from metaevent
+        # and de_tag. epoch in this dataset uses this time of the event
+        de_met_in_ns = (
+            metaevent_time_in_ns
+            + event["de_tag"] * LOOKED_UP_DURATION_OF_TICK * MICROSECOND_TO_NS
+        )
+        data_dict["event_met"].append(de_met_in_ns)
+        data_dict["epoch"].append(met_to_j2000ns(de_met_in_ns / 1e9))
+        data_dict["esa_stepping_num"].append(current_esa_step)
+        # start_bitmask_data is 1, 2, 3 for detector A, B, C
+        # respectively. This is used to identify which detector
+        # was hit first for this current direct event.
+        data_dict["trigger_id"].append(event["start_bitmask_data"])
+        data_dict["tof_1"].append(event["tof_1"])
+        data_dict["tof_2"].append(event["tof_2"])
+        data_dict["tof_3"].append(event["tof_3"])
+        # IMAP-Hi like to keep de_tag value for diagnostic purposes
+        data_dict["de_tag"].append(event["de_tag"])
+        # add packet time to ccsds_met list
+        data_dict["ccsds_met"].append(packet_met_time[index])
+
+    # Load the CDF attributes
+    attr_mgr = ImapCdfAttributes()
+    attr_mgr.add_instrument_global_attrs("hi")
+    attr_mgr.load_variable_attributes("imap_hi_variable_attrs.yaml")
+    # uncomment this once Maxine's PR is merged
+    # attr_mgr.add_global_attribute("Data_version", data_version)
+
+    epoch_attrs = attr_mgr.get_variable_attributes("epoch")
+    epoch_attrs["CATDESC"] = (
+        "Direct Event time, number of nanoseconds since J2000 with leap "
+        "seconds included"
+    )
+    epoch_time = xr.DataArray(
+        data_dict.pop("epoch"),
+        name="epoch",
+        dims=["epoch"],
+        attrs=epoch_attrs,
+    )
+
+    de_global_attrs = attr_mgr.get_global_attributes("imap_hi_l1a_de_attrs")
+    dataset = xr.Dataset(
+        coords={"epoch": epoch_time},
+        attrs=de_global_attrs,
+    )
+
+    for var_name, data in data_dict.items():
+        attrs = attr_mgr.get_variable_attributes(
+            f"hi_de_{var_name}", check_schema=False
+        ).copy()
+        dtype = attrs.pop("dtype")
+        dataset[var_name] = xr.DataArray(
+            np.array(data, dtype=np.dtype(dtype)),
+            dims="epoch",
+            attrs=attrs,
+        )
+
+    # TODO: figure out how to store information about
+    # input data(one or more) it used to produce this dataset
+    return dataset
+
+
+def science_direct_event(packets_data: xr.Dataset) -> xr.Dataset:
+    """
+    Unpack IMAP-Hi direct event data.
+
+    Processing step:
+
+    |    1. Break binary stream data into unit of 48-bits
+    |    2. Parse direct event data
+    |    5. Save the data into xarray dataset.
+
+    Parameters
+    ----------
+    packets_data : xarray.Dataset
+        Packets extracted into a dataset.
+
+    Returns
+    -------
+    dataset : xarray.Dataset
+        Xarray dataset.
+    """
+    de_data_list = []
+    packet_met_time = []
+
+    # Because DE_TOF is a variable length data,
+    # I am using extend to add another list to the
+    # end of the list. This way, I don't need to flatten
+    # the list later.
+    for i, data in enumerate(packets_data["de_tof"].data):
+        # break binary stream data into unit of 48-bits
+        event_48bits_list = break_into_bits_size(data)
+        # parse 48-bits into meaningful data such as metaevent or direct event
+        de_data_list.extend([parse_direct_event(event) for event in event_48bits_list])
+        # add packet time to packet_met_time
+        packet_met_time.extend(
+            [packets_data["ccsds_met"].data[i]] * len(event_48bits_list)
+        )
+
+    # create dataset
+    return create_dataset(de_data_list, packet_met_time)

imap_processing/hi/l1b/hi_eng_unit_convert_table.csv

@@ -0,0 +1,154 @@
+# Exported from the following files: TLM_H45_20240513.xlsx, TLM_H90_20240513.xlsx
+packetName,mnemonic,convertAs,segNumber,lowValue,highValue,c0,c1,c2,c3,c4,c5,c6,c7,unit
+H45_BOOT_HK,MDM25P_14_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0,DN
+H45_BOOT_HK,MDM25P_15_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0,DN
+H45_BOOT_HK,MDM25P_16_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0,DN
+H45_BOOT_HK,LVPS_12V_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.144461908,0,0,0,0,0,0,DN
+H45_BOOT_HK,LVPS_5V_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.144461908,0,0,0,0,0,0,DN
+H45_BOOT_HK,LVPS_3P3V_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.144461908,0,0,0,0,0,0,DN
+H45_BOOT_HK,LVPS_3P3V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H45_BOOT_HK,LVPS_5V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H45_BOOT_HK,LVPS_N5V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H45_BOOT_HK,LVPS_12V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H45_BOOT_HK,LVPS_N12V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H45_BOOT_HK,LVPS_3P3V_I,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H45_BOOT_HK,LVPS_5V_I,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H45_BOOT_HK,LVPS_N5V_I,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H45_BOOT_HK,LVPS_12V_I,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H45_BOOT_HK,LVPS_N12V_I,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H45_BOOT_HK,CDH_1P5V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H45_BOOT_HK,CDH_1P8V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H45_BOOT_HK,CDH_3P3V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H45_BOOT_HK,CDH_12V,UNSEGMENTED_POLY,1,0,0,0,0.005897117,0,0,0,0,0,0,DN
+H45_BOOT_HK,CDH_N12V,UNSEGMENTED_POLY,1,0,0,0,-0.005840685,0,0,0,0,0,0,DN
+H45_BOOT_HK,CDH_5V,UNSEGMENTED_POLY,1,0,0,0,0.002412457,0,0,0,0,0,0,DN
+H45_BOOT_HK,CDH_5V_ADC,UNSEGMENTED_POLY,1,0,0,0,0.002412457,0,0,0,0,0,0,DN
+H45_BOOT_HK,CDH_PROCESSOR_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0,DN
+H45_BOOT_HK,CDH_1P8V_LDO_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0,DN
+H45_BOOT_HK,CDH_1P5V_LDO_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0,DN
+H45_BOOT_HK,CDH_SDRAM_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0,DN
+H45_DIAG_RTS,SHCOARSE,UNSEGMENTED_POLY,1,0,0,0,1,0,0,0,0,0,0,SEC
+H45_APP_NHK,HV1_BULK1_MON,UNSEGMENTED_POLY,1,0,0,0,2.564102564,0,0,0,0,0,0,V
+H45_APP_NHK,INNER_ESA_HI,UNSEGMENTED_POLY,1,0,0,0,-2.884615385,0,0,0,0,0,0,V
+H45_APP_NHK,INNER_ESA_LO,UNSEGMENTED_POLY,1,0,0,0,-0.288461538,0,0,0,0,0,0,V
+H45_APP_NHK,CEM_BK_A,UNSEGMENTED_POLY,1,0,0,0,-1.282051282,0,0,0,0,0,0,V
+H45_APP_NHK,HV2_12V_RTN_MON,UNSEGMENTED_POLY,1,0,0,0,0.952380952,0,0,0,0,0,0,V
+H45_APP_NHK,HV2_BULK2_MON,UNSEGMENTED_POLY,1,0,0,0,-3.205128205,0,0,0,0,0,0,V
+H45_APP_NHK,OUTER_ESA,UNSEGMENTED_POLY,1,0,0,0,1.602564103,0,0,0,0,0,0,V
+H45_APP_NHK,MCP_B,UNSEGMENTED_POLY,1,0,0,0,-0.641025641,0,0,0,0,0,0,V
+H45_APP_NHK,MCP_F,UNSEGMENTED_POLY,1,0,0,0,-1.282051282,0,0,0,0,0,0,V
+H45_APP_NHK,TOF,UNSEGMENTED_POLY,1,0,0,0,-2.564102564,0,0,0,0,0,0,V
+H45_APP_NHK,CEM_BK_B,UNSEGMENTED_POLY,1,0,0,0,-1.282051282,0,0,0,0,0,0,V
+H45_APP_NHK,POS_DEFL,UNSEGMENTED_POLY,1,0,0,0,2.243589744,0,0,0,0,0,0,V
+H45_APP_NHK,NEG_DEFL,UNSEGMENTED_POLY,1,0,0,0,-2.243589744,0,0,0,0,0,0,V
+H45_APP_NHK,CEM_F,UNSEGMENTED_POLY,1,0,0,0,-1.602564103,0,0,0,0,0,0,V
+H45_APP_NHK,HV2_TEMP_IN,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H45_APP_NHK,HV2_TEMP_REF,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H45_APP_NHK,CEM_A_IMON,UNSEGMENTED_POLY,1,0,0,0,1.60236E-08,0,0,0,0,0,0,A
+H45_APP_NHK,CEM_B_IMON,UNSEGMENTED_POLY,1,0,0,0,1.60236E-08,0,0,0,0,0,0,A
+H45_APP_NHK,MCP_IMON,UNSEGMENTED_POLY,1,0,0,0,3.84567E-08,0,0,0,0,0,0,A
+H45_APP_NHK,CDH_TEMP_MON1,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H45_APP_NHK,CDH_TEMP_MON2,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H45_APP_NHK,CDH_TEMP_MON3,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H45_APP_NHK,CDH_TEMP_MON4,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H45_APP_NHK,LO_TEMP_MON,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H45_APP_NHK,HVPS_TEMP_MON,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H45_APP_NHK,LVPS_TEMP1,UNSEGMENTED_POLY,1,0,0,-274.41,0.1448,0,0,0,0,0,0,C
+H45_APP_NHK,LVPS_TEMP2,UNSEGMENTED_POLY,1,0,0,-274.41,0.1448,0,0,0,0,0,0,C
+H45_APP_NHK,LVPS_TEMP3,UNSEGMENTED_POLY,1,0,0,-274.41,0.1448,0,0,0,0,0,0,C
+H45_APP_NHK,FEE_TEMP1,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H45_APP_NHK,SNS_TEMP2,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H45_APP_NHK,SNS_TEMP3,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H45_APP_NHK,LVPS_3_3V_VMON,UNSEGMENTED_POLY,1,0,0,0,0.001611,0,0,0,0,0,0,V
+H45_APP_NHK,LVPS_5V_P_VMON,UNSEGMENTED_POLY,1,0,0,0,0.002441,0,0,0,0,0,0,V
+H45_APP_NHK,LVPS_5V_N_VMON,UNSEGMENTED_POLY,1,0,0,0,0.002441,0,0,0,0,0,0,V
+H45_APP_NHK,LVPS_12V_P_VMON,UNSEGMENTED_POLY,1,0,0,0,0.005859,0,0,0,0,0,0,V
+H45_APP_NHK,LVPS_12V_N_VMON,UNSEGMENTED_POLY,1,0,0,0,0.005859,0,0,0,0,0,0,V
+H45_APP_NHK,LVPS_3_3V_IMON,UNSEGMENTED_POLY,1,0,0,0,0.000916,0,0,0,0,0,0,A
+H45_APP_NHK,LVPS_5V_P_IMON,UNSEGMENTED_POLY,1,0,0,0,0.000461,0,0,0,0,0,0,A
+H45_APP_NHK,LVPS_5V_N_IMON,UNSEGMENTED_POLY,1,0,0,0,0.000461,0,0,0,0,0,0,A
+H45_APP_NHK,LVPS_12V_P_IMON,UNSEGMENTED_POLY,1,0,0,0,0.000704,0,0,0,0,0,0,A
+H45_APP_NHK,LVPS_12V_N_IMON,UNSEGMENTED_POLY,1,0,0,0,-0.000704,0,0,0,0,0,0,A
+H45_APP_NHK,CDH_P1_5V,UNSEGMENTED_POLY,1,0,0,0,0.001221,0,0,0,0,0,0,V
+H45_APP_NHK,CDH_P1_8V,UNSEGMENTED_POLY,1,0,0,0,0.001221,0,0,0,0,0,0,V
+H45_APP_NHK,CDH_P3_3V,UNSEGMENTED_POLY,1,0,0,0,0.001221,0,0,0,0,0,0,V
+H45_APP_NHK,CDH_P12V,UNSEGMENTED_POLY,1,0,0,0,0.005897,0,0,0,0,0,0,V
+H45_APP_NHK,CDH_N12V,UNSEGMENTED_POLY,1,0,0,0,0.005841,0,0,0,0,0,0,V
+H45_APP_NHK,CDH_P5V,UNSEGMENTED_POLY,1,0,0,0,0.002441,0,0,0,0,0,0,V
+H45_APP_NHK,CDH_ANA_REF,UNSEGMENTED_POLY,1,0,0,0,0.002441,0,0,0,0,0,0,V
+H90_BOOT_HK,MDM25P_14_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0,DN
+H90_BOOT_HK,MDM25P_15_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0,DN
+H90_BOOT_HK,MDM25P_16_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0,DN
+H90_BOOT_HK,LVPS_12V_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.144461908,0,0,0,0,0,0,DN
+H90_BOOT_HK,LVPS_5V_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.144461908,0,0,0,0,0,0,DN
+H90_BOOT_HK,LVPS_3P3V_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.144461908,0,0,0,0,0,0,DN
+H90_BOOT_HK,LVPS_3P3V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H90_BOOT_HK,LVPS_5V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H90_BOOT_HK,LVPS_N5V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H90_BOOT_HK,LVPS_12V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H90_BOOT_HK,LVPS_N12V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H90_BOOT_HK,LVPS_3P3V_I,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H90_BOOT_HK,LVPS_5V_I,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H90_BOOT_HK,LVPS_N5V_I,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H90_BOOT_HK,LVPS_12V_I,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H90_BOOT_HK,LVPS_N12V_I,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H90_BOOT_HK,CDH_1P5V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H90_BOOT_HK,CDH_1P8V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H90_BOOT_HK,CDH_3P3V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0,DN
+H90_BOOT_HK,CDH_12V,UNSEGMENTED_POLY,1,0,0,0,0.005897117,0,0,0,0,0,0,DN
+H90_BOOT_HK,CDH_N12V,UNSEGMENTED_POLY,1,0,0,0,-0.005840685,0,0,0,0,0,0,DN
+H90_BOOT_HK,CDH_5V,UNSEGMENTED_POLY,1,0,0,0,0.002412457,0,0,0,0,0,0,DN
+H90_BOOT_HK,CDH_5V_ADC,UNSEGMENTED_POLY,1,0,0,0,0.002412457,0,0,0,0,0,0,DN
+H90_BOOT_HK,CDH_PROCESSOR_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0,DN
+H90_BOOT_HK,CDH_1P8V_LDO_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0,DN
+H90_BOOT_HK,CDH_1P5V_LDO_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0,DN
+H90_BOOT_HK,CDH_SDRAM_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0,DN
+H90_DIAG_RTS,SHCOARSE,UNSEGMENTED_POLY,1,0,0,0,1,0,0,0,0,0,0,SEC
+H90_APP_NHK,HV1_BULK1_MON,UNSEGMENTED_POLY,1,0,0,0,2.564102564,0,0,0,0,0,0,V
+H90_APP_NHK,INNER_ESA_HI,UNSEGMENTED_POLY,1,0,0,0,-2.884615385,0,0,0,0,0,0,V
+H90_APP_NHK,INNER_ESA_LO,UNSEGMENTED_POLY,1,0,0,0,-0.288461538,0,0,0,0,0,0,V
+H90_APP_NHK,CEM_BK_A,UNSEGMENTED_POLY,1,0,0,0,-1.282051282,0,0,0,0,0,0,V
+H90_APP_NHK,HV2_12V_RTN_MON,UNSEGMENTED_POLY,1,0,0,0,0.952380952,0,0,0,0,0,0,V
+H90_APP_NHK,HV2_BULK2_MON,UNSEGMENTED_POLY,1,0,0,0,-3.205128205,0,0,0,0,0,0,V
+H90_APP_NHK,OUTER_ESA,UNSEGMENTED_POLY,1,0,0,0,1.602564103,0,0,0,0,0,0,V
+H90_APP_NHK,MCP_B,UNSEGMENTED_POLY,1,0,0,0,-0.641025641,0,0,0,0,0,0,V
+H90_APP_NHK,MCP_F,UNSEGMENTED_POLY,1,0,0,0,-1.282051282,0,0,0,0,0,0,V
+H90_APP_NHK,TOF,UNSEGMENTED_POLY,1,0,0,0,-2.564102564,0,0,0,0,0,0,V
+H90_APP_NHK,CEM_BK_B,UNSEGMENTED_POLY,1,0,0,0,-1.282051282,0,0,0,0,0,0,V
+H90_APP_NHK,POS_DEFL,UNSEGMENTED_POLY,1,0,0,0,2.243589744,0,0,0,0,0,0,V
+H90_APP_NHK,NEG_DEFL,UNSEGMENTED_POLY,1,0,0,0,-2.243589744,0,0,0,0,0,0,V
+H90_APP_NHK,CEM_F,UNSEGMENTED_POLY,1,0,0,0,-1.602564103,0,0,0,0,0,0,V
+H90_APP_NHK,HV2_TEMP_IN,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H90_APP_NHK,HV2_TEMP_REF,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H90_APP_NHK,CEM_A_IMON,UNSEGMENTED_POLY,1,0,0,0,1.60236E-08,0,0,0,0,0,0,A
+H90_APP_NHK,CEM_B_IMON,UNSEGMENTED_POLY,1,0,0,0,1.60236E-08,0,0,0,0,0,0,A
+H90_APP_NHK,MCP_IMON,UNSEGMENTED_POLY,1,0,0,0,3.84567E-08,0,0,0,0,0,0,A
+H90_APP_NHK,CDH_TEMP_MON1,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H90_APP_NHK,CDH_TEMP_MON2,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H90_APP_NHK,CDH_TEMP_MON3,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H90_APP_NHK,CDH_TEMP_MON4,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H90_APP_NHK,LO_TEMP_MON,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H90_APP_NHK,HVPS_TEMP_MON,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H90_APP_NHK,LVPS_TEMP1,UNSEGMENTED_POLY,1,0,0,-274.41,0.1448,0,0,0,0,0,0,C
+H90_APP_NHK,LVPS_TEMP2,UNSEGMENTED_POLY,1,0,0,-274.41,0.1448,0,0,0,0,0,0,C
+H90_APP_NHK,LVPS_TEMP3,UNSEGMENTED_POLY,1,0,0,-274.41,0.1448,0,0,0,0,0,0,C
+H90_APP_NHK,FEE_TEMP1,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H90_APP_NHK,SNS_TEMP2,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H90_APP_NHK,SNS_TEMP3,UNSEGMENTED_POLY,1,0,0,-58.397,0.10143,-0.000026929,2.8828E-09,0,0,0,0,C
+H90_APP_NHK,LVPS_3_3V_VMON,UNSEGMENTED_POLY,1,0,0,0,0.001611,0,0,0,0,0,0,V
+H90_APP_NHK,LVPS_5V_P_VMON,UNSEGMENTED_POLY,1,0,0,0,0.002441,0,0,0,0,0,0,V
+H90_APP_NHK,LVPS_5V_N_VMON,UNSEGMENTED_POLY,1,0,0,0,0.002441,0,0,0,0,0,0,V
+H90_APP_NHK,LVPS_12V_P_VMON,UNSEGMENTED_POLY,1,0,0,0,0.005859,0,0,0,0,0,0,V
+H90_APP_NHK,LVPS_12V_N_VMON,UNSEGMENTED_POLY,1,0,0,0,0.005859,0,0,0,0,0,0,V
+H90_APP_NHK,LVPS_3_3V_IMON,UNSEGMENTED_POLY,1,0,0,0,0.000916,0,0,0,0,0,0,A
+H90_APP_NHK,LVPS_5V_P_IMON,UNSEGMENTED_POLY,1,0,0,0,0.000461,0,0,0,0,0,0,A
+H90_APP_NHK,LVPS_5V_N_IMON,UNSEGMENTED_POLY,1,0,0,0,0.000461,0,0,0,0,0,0,A
+H90_APP_NHK,LVPS_12V_P_IMON,UNSEGMENTED_POLY,1,0,0,0,0.000704,0,0,0,0,0,0,A
+H90_APP_NHK,LVPS_12V_N_IMON,UNSEGMENTED_POLY,1,0,0,0,-0.000704,0,0,0,0,0,0,A
+H90_APP_NHK,CDH_P1_5V,UNSEGMENTED_POLY,1,0,0,0,0.001221,0,0,0,0,0,0,V
+H90_APP_NHK,CDH_P1_8V,UNSEGMENTED_POLY,1,0,0,0,0.001221,0,0,0,0,0,0,V
+H90_APP_NHK,CDH_P3_3V,UNSEGMENTED_POLY,1,0,0,0,0.001221,0,0,0,0,0,0,V
+H90_APP_NHK,CDH_P12V,UNSEGMENTED_POLY,1,0,0,0,0.005897,0,0,0,0,0,0,V
+H90_APP_NHK,CDH_N12V,UNSEGMENTED_POLY,1,0,0,0,0.005841,0,0,0,0,0,0,V
+H90_APP_NHK,CDH_P5V,UNSEGMENTED_POLY,1,0,0,0,0.002441,0,0,0,0,0,0,V
+H90_APP_NHK,CDH_ANA_REF,UNSEGMENTED_POLY,1,0,0,0,0.002441,0,0,0,0,0,0,V

imap_processing/hi/l1b/hi_l1b.py

@@ -0,0 +1,127 @@
+"""IMAP-HI L1B processing module."""
+
+import logging
+
+import numpy as np
+import xarray as xr
+
+from imap_processing import imap_module_directory
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.hi.utils import HIAPID
+from imap_processing.utils import convert_raw_to_eu
+
+logger = logging.getLogger(__name__)
+ATTR_MGR = ImapCdfAttributes()
+ATTR_MGR.add_instrument_global_attrs("hi")
+ATTR_MGR.load_variable_attributes("imap_hi_variable_attrs.yaml")
+
+
+def hi_l1b(l1a_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
+    """
+    High level IMAP-HI L1B processing function.
+
+    Parameters
+    ----------
+    l1a_dataset : xarray.Dataset
+        L1A dataset to process.
+    data_version : str
+        Version of the data product being created.
+
+    Returns
+    -------
+    l1b_dataset : xarray.Dataset
+        Processed xarray dataset.
+    """
+    logger.info(
+        f"Running Hi L1B processing on dataset: {l1a_dataset.attrs['Logical_source']}"
+    )
+    logical_source_parts = l1a_dataset.attrs["Logical_source"].split("_")
+    # TODO: apid is not currently stored in all L1A data but should be.
+    #    Use apid to determine what L1B processing function to call
+
+    # Housekeeping processing
+    if logical_source_parts[-1].endswith("hk"):
+        # if packet_enum in (HIAPID.H45_APP_NHK, HIAPID.H90_APP_NHK):
+        packet_enum = HIAPID(l1a_dataset["pkt_apid"].data[0])
+        conversion_table_path = str(
+            imap_module_directory / "hi" / "l1b" / "hi_eng_unit_convert_table.csv"
+        )
+        l1b_dataset = convert_raw_to_eu(
+            l1a_dataset,
+            conversion_table_path=conversion_table_path,
+            packet_name=packet_enum.name,
+            comment="#",  # type: ignore[arg-type]
+            # Todo error, Argument "comment" to "convert_raw_to_eu" has incompatible
+            # type "str"; expected "dict[Any, Any]"
+            converters={"mnemonic": str.lower},
+        )
+
+        l1b_dataset.attrs.update(ATTR_MGR.get_global_attributes("imap_hi_l1b_hk_attrs"))
+    elif logical_source_parts[-1].endswith("de"):
+        l1b_dataset = annotate_direct_events(l1a_dataset)
+    else:
+        raise NotImplementedError(
+            f"No Hi L1B processing defined for file type: "
+            f"{l1a_dataset.attrs['Logical_source']}"
+        )
+    # Update global attributes
+    # TODO: write a function that extracts the sensor from Logical_source
+    #    some functionality can be found in imap_data_access.file_validation but
+    #    only works on full file names
+    sensor_str = logical_source_parts[-1].split("-")[0]
+    l1b_dataset.attrs["Logical_source"] = l1b_dataset.attrs["Logical_source"].format(
+        sensor=sensor_str
+    )
+    # TODO: revisit this
+    l1b_dataset.attrs["Data_version"] = data_version
+    return l1b_dataset
+
+
+def annotate_direct_events(l1a_dataset: xr.Dataset) -> xr.Dataset:
+    """
+    Perform Hi L1B processing on direct event data.
+
+    Parameters
+    ----------
+    l1a_dataset : xarray.Dataset
+        L1A direct event data.
+
+    Returns
+    -------
+    l1b_dataset : xarray.Dataset
+        L1B direct event data.
+    """
+    n_epoch = l1a_dataset["epoch"].size
+    new_data_vars = dict()
+    for var in [
+        "coincidence_type",
+        "esa_step",
+        "delta_t_ab",
+        "delta_t_ac1",
+        "delta_t_bc1",
+        "delta_t_c1c2",
+        "spin_phase",
+        "hae_latitude",
+        "hae_longitude",
+        "quality_flag",
+        "nominal_bin",
+    ]:
+        attrs = ATTR_MGR.get_variable_attributes(
+            f"hi_de_{var}", check_schema=False
+        ).copy()
+        dtype = attrs.pop("dtype")
+        if attrs["FILLVAL"] == "NaN":
+            attrs["FILLVAL"] = np.nan
+        new_data_vars[var] = xr.DataArray(
+            data=np.full(n_epoch, attrs["FILLVAL"], dtype=np.dtype(dtype)),
+            dims=["epoch"],
+            attrs=attrs,
+        )
+    l1b_dataset = l1a_dataset.assign(new_data_vars)
+    l1b_dataset = l1b_dataset.drop_vars(
+        ["tof_1", "tof_2", "tof_3", "de_tag", "ccsds_met", "meta_event_met"]
+    )
+
+    de_global_attrs = ATTR_MGR.get_global_attributes("imap_hi_l1b_de_attrs")
+    l1b_dataset.attrs.update(**de_global_attrs)
+    return l1b_dataset