imap-processing 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

imap_processing/hi/l1a/science_direct_event.py

@@ -1,279 +1,135 @@
 """IMAP-Hi direct event processing."""
 
+from collections import defaultdict
+
 import numpy as np
+import numpy._typing as npt
 import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.spice.time import met_to_j2000ns
-from imap_processing.utils import convert_to_binary_string
 
 # TODO: read LOOKED_UP_DURATION_OF_TICK from
 # instrument status summary later. This value
 # is rarely change but want to be able to change
 # it if needed. It stores information about how
 # fast the time was ticking. It is in microseconds.
-LOOKED_UP_DURATION_OF_TICK = 3999
+LOOKED_UP_DURATION_OF_TICK = 1999
 
 SECOND_TO_NS = 1e9
 MILLISECOND_TO_NS = 1e6
 MICROSECOND_TO_NS = 1e3
 
 
-def parse_direct_event(event_data: str) -> dict:
+def parse_direct_events(de_data: bytes) -> dict[str, npt.ArrayLike]:
     """
-    Parse event data.
+    Parse event data from a binary blob.
 
     IMAP-Hi direct event data information is stored in
-    48-bits as follow:
+    48-bits as follows:
 
-    |    Read first two bits (start_bitmask_data) to find
-    |    out which type of event it is. start_bitmask_data value mapping:
-    |
-    |        1 - A
-    |        2 - B
-    |        3 - C
-    |        0 - META
-    |    If it's a metaevent:
-    |
-    |        Read 48-bits into 2, 4, 10, 32 bits. Each of these breaks
-    |        down as:
-    |
-    |            start_bitmask_data - 2 bits (tA=1, tB=2, tC1=3, META=0)
-    |            ESA step - 4 bits
-    |            integer millisecond of MET(subseconds) - 10 bits
-    |            integer MET(seconds) - 32 bits
-    |
-    |    If it's not a metaevent:
-    |        Read 48-bits into 2, 10, 10, 10, 16 bits. Each of these breaks
+    |        Read 48-bits into 2, 16, 10, 10, 10, bits. Each of these breaks
     |        down as:
     |
     |            start_bitmask_data - 2 bits (tA=1, tB=2, tC1=3, META=0)
+    |            de_tag - 16 bits
     |            tof_1 - 10 bit counter
     |            tof_2 - 10 bit counter
     |            tof_3 - 10 bit counter
-    |            de_tag - 16 bits
 
-    There are at most total of 665 of 48-bits in each data packet.
+    There are at most total of 664 of 48-bits in each data packet.
     This data packet is of variable length. If there is one event, then
-    DE_TOF will contain 48-bits. If there are 665 events, then
-    DE_TOF will contain 665 x 48-bits. If there is no event, then
+    DE_TOF will contain 48-bits. If there are 664 events, then
+    DE_TOF will contain 664 x 48-bits. If there is no event, then
     DE_TOF will contain 0-bits.
 
-    Per ESA, there should be two data packets. First one will begin with
-    metaevent followed by direct events data. Second one will begin with
-    direct event data only. If there is no event record for certain ESA step,
-    then as mentioned above, first packet will contain metaevent in DE_TOF
-    information and second packet will contain 0-bits in DE_TOF. In general,
-    every two packets will look like this.
-
-    |    first packet = [
-    |        (start_bitmask_data, ESA step, int millisecond of MET, int MET),
-    |        (start_bitmask_data, tof_1, tof_2, tof_3, de_tag),
-    |        .....
-    |    ]
-    |    second packet = [
-    |        (start_bitmask_data, tof_1, tof_2, tof_3, de_tag),
-    |       .....
-    |    ]
-
-    In direct event data, if no hit is registered, the tof_x field in
-    the DE to a value of negative one. However, since the field is described as a
-    "10-bit unsigned counter," it cannot actually store negative numbers.
-    Instead, the value negative is represented by the maximum value that can
-    be stored in a 10-bit unsigned integer, which is 0x3FF (in hexadecimal)
-    or 1023 in decimal. This value is used as a special marker to
-    indicate that no hit was registered. Ideally, the system should
-    not be able to register a hit with a value of 1023 for all
-    tof_1, tof_2, tof_3, because this is in case of an error. But,
-    IMAP-Hi like to process it still to investigate the data.
-    Example of what it will look like if no hit was registered.
-
-    |        (start_bitmask_data, 1023, 1023, 1023, de_tag)
-    |        start_bitmask_data will be 1 or 2 or 3.
+    There should be two data packets per ESA. Each packet contains meta-event
+    data that is identical between the two packets for a common ESA.
+    If there is no event record for certain ESA step, then both packets will
+    contain 0-bits in DE_TOF.
 
     Parameters
     ----------
-    event_data : str
-        48-bits Event data.
+    de_data : bytes
+        Binary blob from de_tag field of SCI_DE packet. Must be an integer
+        multiple of 48-bits of data.
 
     Returns
     -------
-    dict
+    Dict[str, list]
         Parsed event data.
     """
-    event_type = int(event_data[:2], 2)
-    metaevent = 0
-    if event_type == metaevent:
-        # parse metaevent
-        esa_step = event_data[2:6]
-        subseconds = event_data[6:16]
-        seconds = event_data[16:]
-
-        # return parsed metaevent data
-        return {
-            "start_bitmask_data": event_type,
-            "esa_step": int(esa_step, 2),
-            "subseconds": int(subseconds, 2),
-            "seconds": int(seconds, 2),
-        }
-
-    # parse direct event
-    trigger_id = event_data[:2]
-    tof_1 = event_data[2:12]
-    tof_2 = event_data[12:22]
-    tof_3 = event_data[22:32]
-    de_tag = event_data[32:]
-
-    # return parsed direct event data
-    return {
-        "start_bitmask_data": int(trigger_id, 2),
-        "tof_1": int(tof_1, 2),
-        "tof_2": int(tof_2, 2),
-        "tof_3": int(tof_3, 2),
-        "de_tag": int(de_tag, 2),
-    }
-
-
-def break_into_bits_size(binary_data: str) -> list:
-    """
-    Break binary stream data into 48-bits.
-
-    Parameters
-    ----------
-    binary_data : str
-        Binary data.
+    # The de_data is a binary blob with Nx6 bytes of data where N = number of
+    # direct events encoded into the binary blob. Interpreting the data as
+    # big-endian uint16 data and reshaping into a (3, -1) ndarray results
+    # in an array with shape (3, N). Indexing the first axis of that array
+    # (e.g. data_uint16[i]) gives the ith 2-bytes of data for each of the N
+    # direct events.
+    # Considering the 6-bytes of data for each DE as 3 2-byte words,
+    # each word contains the following:
+    # word_0: 2-bits of Trigger ID, upper 14-bits of de_tag
+    # word_1: lower 2-bits of de_tag, 10-bits tof_1, upper 4-bits of tof_2
+    # word_2: lower 6-bits of tof_2, 10-bits of tof_3
+    data_uint16 = np.reshape(
+        np.frombuffer(de_data, dtype=">u2"), (3, -1), order="F"
+    ).astype(np.uint16)
+
+    de_dict = dict()
+    de_dict["trigger_id"] = (data_uint16[0] >> 14).astype(np.uint8)
+    de_dict["de_tag"] = (data_uint16[0] << 2) + (data_uint16[1] >> 14)
+    de_dict["tof_1"] = (data_uint16[1] & int(b"00111111_11110000", 2)) >> 4
+    de_dict["tof_2"] = ((data_uint16[1] & int(b"00000000_00001111", 2)) << 6) + (
+        data_uint16[2] >> 10
+    )
+    de_dict["tof_3"] = data_uint16[2] & int(b"00000011_11111111", 2)
 
-    Returns
-    -------
-    list
-        List of 48-bits.
-    """
-    # TODO: ask Paul what to do if the length of
-    # binary_data is not a multiple of 48
-    field_bit_length = 48
-    return [
-        binary_data[i : i + field_bit_length]
-        for i in range(0, len(binary_data), field_bit_length)
-    ]
+    return de_dict
 
 
-def create_dataset(de_data_list: list, packet_met_time: list) -> xr.Dataset:
+def create_dataset(de_data_dict: dict[str, npt.ArrayLike]) -> xr.Dataset:
     """
     Create xarray dataset.
 
     Parameters
     ----------
-    de_data_list : list
-        Parsed direct event data list.
-    packet_met_time : list
-        List of packet MET time.
+    de_data_dict : Dict[list]
+        Dictionary of packet telemetry and direct event data lists.
 
     Returns
     -------
     dataset : xarray.Dataset
         Xarray dataset.
     """
-    # These are the variables that we will store in the dataset
-    data_dict: dict = {
-        "epoch": list(),
-        "event_met": list(),
-        "ccsds_met": list(),
-        "meta_event_met": list(),
-        "esa_step": list(),
-        "trigger_id": list(),
-        "tof_1": list(),
-        "tof_2": list(),
-        "tof_3": list(),
-        "de_tag": list(),
-    }
-
-    # How to handle if first event is not metaevent? This
-    # means that current data file started with direct event.
-    # Per Paul, log a warning and discard all direct events
-    # until we see next metaevent because it could mean
-    # that the instrument was turned off during repoint.
-
-    # Find the index of the first occurrence of the metaevent
-    first_metaevent_index = next(
-        (i for i, d in enumerate(de_data_list) if d.get("start_bitmask_data") == 0),
-        None,
+    # Compute the meta-event MET in nanoseconds
+    de_data_dict["meta_event_met"] = (
+        np.array(de_data_dict.pop("meta_seconds")) * SECOND_TO_NS
+        + np.array(de_data_dict.pop("meta_subseconds")) * MILLISECOND_TO_NS
+    )
+    # Compute the MET of each event in nanoseconds
+    # event MET = meta_event_met + de_clock + 1/2 de_clock_tick
+    # See Hi Algorithm Document section 2.2.5
+    half_tick_ns = LOOKED_UP_DURATION_OF_TICK / 2 * MICROSECOND_TO_NS
+    de_data_dict["event_met"] = (
+        de_data_dict["meta_event_met"]
+        + np.array(de_data_dict["de_tag"])
+        * LOOKED_UP_DURATION_OF_TICK
+        * MICROSECOND_TO_NS
+        + half_tick_ns
     )
-
-    if first_metaevent_index is None:
-        return None
-    elif first_metaevent_index != 0:
-        # Discard all direct events until we see next metaevent
-        # TODO: log a warning
-        de_data_list = de_data_list[first_metaevent_index:]
-        packet_met_time = packet_met_time[first_metaevent_index:]
-
-    for index, event in enumerate(de_data_list):
-        if event["start_bitmask_data"] == 0:
-            # metaevent is a way to store information
-            # about bigger portion of time information. Eg.
-            # metaevent stores information about, let's say
-            # "20240319T09:30:01.000". Then direct event time
-            # tag stores information of time ticks since
-            # that time. Then we use those two to combine and
-            # get exact time information of each event.
-
-            # set time and esa step values to
-            # be used for direct event followed by
-            # this metaevent
-            int_subseconds = event["subseconds"]
-            int_seconds = event["seconds"]
-            current_esa_step = event["esa_step"]
-
-            metaevent_time_in_ns = (
-                int_seconds * SECOND_TO_NS + int_subseconds * MILLISECOND_TO_NS
-            )
-
-            # Add half a tick once per algorithm document(
-            # section 2.2.5 and second last bullet point)
-            # and Paul Janzen.
-            half_tick = LOOKED_UP_DURATION_OF_TICK / 2
-            # convert microseconds to nanosecond to
-            # match other time format
-            half_tick_ns = half_tick * MICROSECOND_TO_NS
-            metaevent_time_in_ns += half_tick_ns
-            continue
-
-        data_dict["meta_event_met"].append(metaevent_time_in_ns)
-        # calculate direct event time using time information from metaevent
-        # and de_tag. epoch in this dataset uses this time of the event
-        de_met_in_ns = (
-            metaevent_time_in_ns
-            + event["de_tag"] * LOOKED_UP_DURATION_OF_TICK * MICROSECOND_TO_NS
-        )
-        data_dict["event_met"].append(de_met_in_ns)
-        data_dict["epoch"].append(met_to_j2000ns(de_met_in_ns / 1e9))
-        data_dict["esa_step"].append(current_esa_step)
-        # start_bitmask_data is 1, 2, 3 for detector A, B, C
-        # respectively. This is used to identify which detector
-        # was hit first for this current direct event.
-        data_dict["trigger_id"].append(event["start_bitmask_data"])
-        data_dict["tof_1"].append(event["tof_1"])
-        data_dict["tof_2"].append(event["tof_2"])
-        data_dict["tof_3"].append(event["tof_3"])
-        # IMAP-Hi like to keep de_tag value for diagnostic purposes
-        data_dict["de_tag"].append(event["de_tag"])
-        # add packet time to ccsds_met list
-        data_dict["ccsds_met"].append(packet_met_time[index])
 
     # Load the CDF attributes
     attr_mgr = ImapCdfAttributes()
     attr_mgr.add_instrument_global_attrs("hi")
-    attr_mgr.load_variable_attributes("imap_hi_variable_attrs.yaml")
-    # uncomment this once Maxine's PR is merged
-    # attr_mgr.add_global_attribute("Data_version", data_version)
+    attr_mgr.add_instrument_variable_attrs(instrument="hi", level=None)
 
-    epoch_attrs = attr_mgr.get_variable_attributes("epoch")
+    # check_schema=False keeps DEPEND_0 = '' from being auto added
+    epoch_attrs = attr_mgr.get_variable_attributes("epoch", check_schema=False)
     epoch_attrs["CATDESC"] = (
         "Direct Event time, number of nanoseconds since J2000 with leap "
         "seconds included"
     )
-    epoch_time = xr.DataArray(
-        data_dict.pop("epoch"),
+    epoch = xr.DataArray(
+        met_to_j2000ns(de_data_dict["event_met"] / 1e9),
         name="epoch",
         dims=["epoch"],
         attrs=epoch_attrs,
@@ -281,11 +137,11 @@ def create_dataset(de_data_list: list, packet_met_time: list) -> xr.Dataset:
 
     de_global_attrs = attr_mgr.get_global_attributes("imap_hi_l1a_de_attrs")
     dataset = xr.Dataset(
-        coords={"epoch": epoch_time},
+        coords={"epoch": epoch},
         attrs=de_global_attrs,
     )
 
-    for var_name, data in data_dict.items():
+    for var_name, data in de_data_dict.items():
         attrs = attr_mgr.get_variable_attributes(
             f"hi_de_{var_name}", check_schema=False
         ).copy()
@@ -321,23 +177,34 @@ def science_direct_event(packets_data: xr.Dataset) -> xr.Dataset:
     dataset : xarray.Dataset
         Xarray dataset.
     """
-    de_data_list = []
-    packet_met_time = []
+    de_data_dict: dict[str, list] = defaultdict(list)
 
     # Because DE_TOF is a variable length data,
     # I am using extend to add another list to the
     # end of the list. This way, I don't need to flatten
     # the list later.
     for i, data in enumerate(packets_data["de_tof"].data):
-        binary_str_val = convert_to_binary_string(data)
-        # break binary stream data into unit of 48-bits
-        event_48bits_list = break_into_bits_size(binary_str_val)
-        # parse 48-bits into meaningful data such as metaevent or direct event
-        de_data_list.extend([parse_direct_event(event) for event in event_48bits_list])
-        # add packet time to packet_met_time
-        packet_met_time.extend(
-            [packets_data["ccsds_met"].data[i]] * len(event_48bits_list)
-        )
+        parsed_de_data = parse_direct_events(data)
+        for key, new_data in parsed_de_data.items():
+            de_data_dict[key].extend(new_data)
+
+        # add packet data to keep_packet_data dictionary, repeating values
+        # for each direct event encoded in the current packet
+        for from_key, to_key in {
+            "shcoarse": "ccsds_met",
+            "src_seq_ctr": "src_seq_ctr",
+            "pkt_len": "pkt_len",
+            "last_spin_num": "last_spin_num",
+            "spin_invalids": "spin_invalids",
+            "esa_step_num": "esa_step",
+            "meta_seconds": "meta_seconds",
+            "meta_subseconds": "meta_subseconds",
+        }.items():
+            # Repeat the ith packet from_key value N times, where N is the
+            # number of events in the ith packet.
+            de_data_dict[to_key].extend(
+                [packets_data[from_key].data[i]] * len(parsed_de_data["de_tag"])
+            )
 
     # create dataset
-    return create_dataset(de_data_list, packet_met_time)
+    return create_dataset(de_data_dict)

imap_processing/hi/l1b/hi_l1b.py

@@ -15,7 +15,12 @@ from imap_processing.hi.utils import (
     create_dataset_variables,
     parse_sensor_number,
 )
-from imap_processing.spice.geometry import SpiceFrame, instrument_pointing
+from imap_processing.spice.geometry import (
+    SpiceFrame,
+    get_instrument_spin_phase,
+    get_spacecraft_spin_phase,
+    instrument_pointing,
+)
 from imap_processing.spice.time import j2000ns_to_j2000s
 from imap_processing.utils import convert_raw_to_eu
 
@@ -40,7 +45,7 @@ class CoincidenceBitmap(IntEnum):
 logger = logging.getLogger(__name__)
 ATTR_MGR = ImapCdfAttributes()
 ATTR_MGR.add_instrument_global_attrs("hi")
-ATTR_MGR.load_variable_attributes("imap_hi_variable_attrs.yaml")
+ATTR_MGR.add_instrument_variable_attrs(instrument="hi", level=None)
 
 
 def hi_l1b(l1a_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
@@ -116,17 +121,16 @@ def annotate_direct_events(l1a_dataset: xr.Dataset) -> xr.Dataset:
     """
     l1b_dataset = l1a_dataset.copy()
     l1b_dataset.update(compute_coincidence_type_and_time_deltas(l1b_dataset))
+    l1b_dataset.update(de_nominal_bin_and_spin_phase(l1b_dataset))
     l1b_dataset.update(compute_hae_coordinates(l1b_dataset))
-    l1b_de_var_names = [
-        "esa_energy_step",
-        "spin_phase",
-        "quality_flag",
-        "nominal_bin",
-    ]
-    new_data_vars = create_dataset_variables(
-        l1b_de_var_names, l1b_dataset["epoch"].size, att_manager_lookup_str="hi_de_{0}"
+    l1b_dataset.update(de_esa_energy_step(l1b_dataset))
+    l1b_dataset.update(
+        create_dataset_variables(
+            ["quality_flag"],
+            l1b_dataset["epoch"].size,
+            att_manager_lookup_str="hi_de_{0}",
+        )
     )
-    l1b_dataset.update(new_data_vars)
     l1b_dataset = l1b_dataset.drop_vars(
         ["tof_1", "tof_2", "tof_3", "de_tag", "ccsds_met", "meta_event_met"]
     )
@@ -258,6 +262,47 @@ def compute_coincidence_type_and_time_deltas(
     return new_vars
 
 
+def de_nominal_bin_and_spin_phase(dataset: xr.Dataset) -> dict[str, xr.DataArray]:
+    """
+    Compute nominal bin and instrument spin-phase for each direct event.
+
+    Parameters
+    ----------
+    dataset : xarray.Dataset
+        Direct event data to compute instrument spin-phase for.
+
+    Returns
+    -------
+    new_vars : dict[str, xarray.DataArray]
+        Dictionary containing new "spin_phase" variable.
+    """
+    new_vars = create_dataset_variables(
+        [
+            "spin_phase",
+            "nominal_bin",
+        ],
+        len(dataset.epoch),
+        att_manager_lookup_str="hi_de_{0}",
+    )
+
+    # nominal_bin is the index number of the 90 4-degree bins that each DE would
+    # be binned into in the histogram packet. The Hi histogram data is binned by
+    # spacecraft spin-phase, not instrument spin-phase, so the same is done here.
+    met_query_times = j2000ns_to_j2000s(dataset.event_met.values)
+    imap_spin_phase = get_spacecraft_spin_phase(met_query_times)
+    new_vars["nominal_bin"].values = np.asarray(imap_spin_phase * 360 / 4).astype(
+        np.uint8
+    )
+
+    sensor_number = parse_sensor_number(dataset.attrs["Logical_source"])
+    new_vars["spin_phase"].values = np.asarray(
+        get_instrument_spin_phase(
+            met_query_times, SpiceFrame[f"IMAP_HI_{sensor_number}"]
+        )
+    ).astype(np.float32)
+    return new_vars
+
+
 def compute_hae_coordinates(dataset: xr.Dataset) -> dict[str, xr.DataArray]:
     """
     Compute HAE latitude and longitude.
@@ -296,3 +341,32 @@ def compute_hae_coordinates(dataset: xr.Dataset) -> dict[str, xr.DataArray]:
     new_vars["hae_longitude"].values = pointing_coordinates[:, 1]
 
     return new_vars
+
+
+def de_esa_energy_step(dataset: xr.Dataset) -> dict[str, xr.DataArray]:
+    """
+    Compute esa_energy_step for each direct event.
+
+    TODO: For now this function just returns the esa_step from the input dataset.
+        Eventually, it will take L1B housekeeping data and determine the esa
+        energy steps from that data.
+
+    Parameters
+    ----------
+    dataset : xarray.Dataset
+        The partial L1B dataset.
+
+    Returns
+    -------
+    new_vars : dict[str, xarray.DataArray]
+        Keys are variable names and values are `xarray.DataArray`.
+    """
+    new_vars = create_dataset_variables(
+        ["esa_energy_step"],
+        len(dataset.epoch),
+        att_manager_lookup_str="hi_de_{0}",
+    )
+    # TODO: Implement this algorithm
+    new_vars["esa_energy_step"].values = dataset.esa_step.values
+
+    return new_vars

imap_processing/hi/l1c/hi_l1c.py

@@ -94,7 +94,7 @@ def allocate_pset_dataset(n_esa_steps: int, sensor_str: str) -> xr.Dataset:
     """
     attr_mgr = ImapCdfAttributes()
     attr_mgr.add_instrument_global_attrs("hi")
-    attr_mgr.load_variable_attributes("imap_hi_variable_attrs.yaml")
+    attr_mgr.add_instrument_variable_attrs(instrument="hi", level=None)
 
     # preallocate coordinates xr.DataArrays
     coords = dict()
@@ -115,6 +115,20 @@ def allocate_pset_dataset(n_esa_steps: int, sensor_str: str) -> xr.Dataset:
         dims=["esa_energy_step"],
         attrs=attrs,
     )
+    # TODO: define calibration product number to coincidence type mapping and
+    #     use the number of calibration products here. I believe it will be 5
+    #     0 for any, 1-4, for the number of detector hits.
+    n_calibration_prod = 5
+    attrs = attr_mgr.get_variable_attributes(
+        "hi_pset_calibration_prod", check_schema=False
+    ).copy()
+    dtype = attrs.pop("dtype")
+    coords["calibration_prod"] = xr.DataArray(
+        np.arange(n_calibration_prod, dtype=dtype),
+        name="calibration_prod",
+        dims=["calibration_prod"],
+        attrs=attrs,
+    )
     # spin angle bins are 0.1 degree bins for full 360 degree spin
     attrs = attr_mgr.get_variable_attributes(
         "hi_pset_spin_angle_bin", check_schema=False
@@ -157,6 +171,14 @@ def allocate_pset_dataset(n_esa_steps: int, sensor_str: str) -> xr.Dataset:
             "hi_pset_esa_energy_step_label", check_schema=False
         ),
     )
+    data_vars["calibration_prod_label"] = xr.DataArray(
+        coords["calibration_prod"].values.astype(str),
+        name="calibration_prod_label",
+        dims=["calibration_prod"],
+        attrs=attr_mgr.get_variable_attributes(
+            "hi_pset_calibration_prod_label", check_schema=False
+        ),
+    )
     data_vars["spin_bin_label"] = xr.DataArray(
         coords["spin_angle_bin"].values.astype(str),
         name="spin_bin_label",