imap-processing 0.8.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

imap_processing/lo/l0/decompression_tables/decompression_tables.py

@@ -13,6 +13,16 @@ DE_BIT_SHIFT = {
     "pos": 0,
 }
 
+# Named tuple for all the packet fields in the
+# direct event packet that need to be parsed
+# but are not part of the compressed DE data.
+PacketFields = namedtuple(
+    "PacketFields",
+    [
+        "passes",
+    ],
+)
+
 # Named tuple for all the fixed fields in the
 # direct event. These fields will always be transmitted.
 FixedFields = namedtuple(
@@ -39,6 +49,11 @@ VariableFields = namedtuple(
         "pos",
     ],
 )
+
+# number of bits for each packet field
+# passes: 32 bits
+PACKET_FIELD_BITS = PacketFields(32)
+
 # number of bits for each fixed field
 # coincidence_type: 4 bits
 # de_time: 12 bits
@@ -55,6 +70,7 @@ FIXED_FIELD_BITS = FixedFields(4, 12, 3, 1)
 # pos: 2 bits
 VARIABLE_FIELD_BITS = VariableFields(10, 9, 9, 6, 4, 2)
 
+
 # Variable fields that are transmitted for each case and mode.
 # (case, mode): tof0, tof1, tof2, tof3, cksm, pos
 CASE_DECODER = {

imap_processing/lo/l0/lo_science.py

@@ -1,5 +1,6 @@
 """Processing function for Lo Science Data."""
 
+import logging
 from collections import namedtuple
 
 import numpy as np
@@ -11,6 +12,7 @@ from imap_processing.lo.l0.decompression_tables.decompression_tables import (
     CASE_DECODER,
     DE_BIT_SHIFT,
     FIXED_FIELD_BITS,
+    PACKET_FIELD_BITS,
     VARIABLE_FIELD_BITS,
 )
 from imap_processing.lo.l0.utils.bit_decompression import (
@@ -20,6 +22,9 @@ from imap_processing.lo.l0.utils.bit_decompression import (
 )
 from imap_processing.utils import convert_to_binary_string
 
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
 HistPacking = namedtuple(
     "HistPacking",
     [
@@ -174,30 +179,55 @@ def parse_events(dataset: xr.Dataset, attr_mgr: ImapCdfAttributes) -> xr.Dataset
     dataset : xr.Dataset
         Parsed and decompressed direct event data.
     """
-    # TODO: Add logging. Want to wait until I have a better understanding of how the
-    #  DEs spread across multiple packets will work first
-
+    logger.info("\n Parsing Lo L1A Direct Events")
     # Sum each count to get the total number of direct events for the pointing
-    num_de: int = np.sum(dataset["count"].values)
+    # parse the count and passes fields. These fields only occur once
+    # at the beginning of each packet group and are not part of the
+    # compressed direct event data
+    dataset["de_count"] = xr.DataArray(
+        [int(pkt[0:16], 2) for pkt in dataset["events"].values],
+        dims="epoch",
+        attrs=attr_mgr.get_variable_attributes("de_count"),
+    )
+    num_de: int = np.sum(dataset["de_count"].values)
+
+    logger.info(f"Total number of direct events in this ASC: {num_de}")
 
-    de_fields = list(FIXED_FIELD_BITS._asdict().keys()) + list(
-        VARIABLE_FIELD_BITS._asdict().keys()
+    de_fields = (
+        list(PACKET_FIELD_BITS._asdict().keys())
+        + list(FIXED_FIELD_BITS._asdict().keys())
+        + list(VARIABLE_FIELD_BITS._asdict().keys())
     )
     # Initialize all Direct Event fields with their fill value
     # L1A Direct event data will not be tied to an epoch
-    # data will use a direct event index for the pointing as its coordinate/dimension
+    # data will use a direct event index for the
+    # pointing as its coordinate/dimension
     for field in de_fields:
         dataset[field] = xr.DataArray(
             np.full(num_de, attr_mgr.get_variable_attributes(field)["FILLVAL"]),
             dims="direct_events",
+            attrs=attr_mgr.get_variable_attributes(field),
         )
+    dataset["passes"] = xr.DataArray(
+        np.full(
+            len(dataset["events"].values),
+            attr_mgr.get_variable_attributes("passes")["FILLVAL"],
+        ),
+        dims="epoch",
+        attrs=attr_mgr.get_variable_attributes("passes"),
+    )
 
     # The DE index for the entire pointing
     pointing_de = 0
     # for each direct event packet in the pointing
-    for pkt_idx, de_count in enumerate(dataset["count"].values):
+    for pkt_idx, de_count in enumerate(dataset["de_count"].values):
         # initialize the bit position for the packet
-        dataset.attrs["bit_pos"] = 0
+        # after the counts field
+        dataset.attrs["bit_pos"] = 16
+        # Parse the passes field for the packet
+        dataset["passes"].values[pkt_idx] = parse_de_bin(dataset, pkt_idx, 32)
+        dataset.attrs["bit_pos"] = dataset.attrs["bit_pos"] + 32
+
         # for each direct event in the packet
         for _ in range(de_count):
             # Parse the fixed fields for the direct event
@@ -209,6 +239,8 @@ def parse_events(dataset: xr.Dataset, attr_mgr: ImapCdfAttributes) -> xr.Dataset
 
             pointing_de += 1
 
+    del dataset.attrs["bit_pos"]
+    logger.info("\n Returning Lo L1A Direct Events Dataset")
     return dataset
 
 
@@ -314,9 +346,10 @@ def parse_de_bin(
         Parsed integer for the direct event field.
     """
     bit_pos = dataset.attrs["bit_pos"]
+
     parsed_int = (
         int(
-            dataset["data"].values[pkt_idx][bit_pos : bit_pos + bit_length],
+            dataset["events"].values[pkt_idx][bit_pos : bit_pos + bit_length],
             2,
         )
         << bit_shift
@@ -364,12 +397,11 @@ def combine_segmented_packets(dataset: xr.Dataset) -> xr.Dataset:
     valid_groups = find_valid_groups(seq_ctrs, seg_starts, seg_ends)
 
     # Combine the segmented packets into a single binary string
-    # Mark the segment splits with comma to identify padding bits
-    # when parsing the binary
     dataset["events"] = [
         "".join(dataset["data"].values[start : end + 1])
         for start, end in zip(seg_starts, seg_ends)
     ]
+
     # drop any group of segmented packets that aren't sequential
     dataset["events"] = dataset["events"].values[valid_groups]
 

imap_processing/lo/l1a/lo_l1a.py

@@ -9,8 +9,12 @@ import xarray as xr
 from imap_processing import imap_module_directory
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.lo.l0.lo_apid import LoAPID
-from imap_processing.lo.l0.lo_science import parse_histogram
-from imap_processing.utils import packet_file_to_datasets
+from imap_processing.lo.l0.lo_science import (
+    combine_segmented_packets,
+    parse_events,
+    parse_histogram,
+)
+from imap_processing.utils import convert_to_binary_string, packet_file_to_datasets
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -48,11 +52,11 @@ def lo_l1a(dependency: Path, data_version: str) -> list[xr.Dataset]:
     attr_mgr.add_instrument_variable_attrs(instrument="lo", level="l1a")
     attr_mgr.add_global_attribute("Data_version", data_version)
 
-    logger.info(
-        f"\nProcessing {LoAPID(LoAPID.ILO_SCI_CNT).name} "
-        f"packet (APID: {LoAPID.ILO_SCI_CNT.value})"
-    )
     if LoAPID.ILO_SCI_CNT in datasets_by_apid:
+        logger.info(
+            f"\nProcessing {LoAPID(LoAPID.ILO_SCI_CNT).name} "
+            f"packet (APID: {LoAPID.ILO_SCI_CNT.value})"
+        )
         logical_source = "imap_lo_l1a_histogram"
         datasets_by_apid[LoAPID.ILO_SCI_CNT] = parse_histogram(
             datasets_by_apid[LoAPID.ILO_SCI_CNT], attr_mgr
@@ -60,8 +64,33 @@ def lo_l1a(dependency: Path, data_version: str) -> list[xr.Dataset]:
         datasets_by_apid[LoAPID.ILO_SCI_CNT] = add_dataset_attrs(
             datasets_by_apid[LoAPID.ILO_SCI_CNT], attr_mgr, logical_source
         )
+    if LoAPID.ILO_SCI_DE in datasets_by_apid:
+        logger.info(
+            f"\nProcessing {LoAPID(LoAPID.ILO_SCI_DE).name} "
+            f"packet (APID: {LoAPID.ILO_SCI_DE.value})"
+        )
+        logical_source = "imap_lo_l1a_de"
+        datasets_by_apid[LoAPID.ILO_SCI_DE]["data"] = xr.DataArray(
+            [
+                convert_to_binary_string(data)
+                for data in datasets_by_apid[LoAPID.ILO_SCI_DE]["data"].values
+            ],
+            dims=datasets_by_apid[LoAPID.ILO_SCI_DE]["data"].dims,
+            attrs=datasets_by_apid[LoAPID.ILO_SCI_DE]["data"].attrs,
+        )
+
+        datasets_by_apid[LoAPID.ILO_SCI_DE] = combine_segmented_packets(
+            datasets_by_apid[LoAPID.ILO_SCI_DE]
+        )
 
-    good_apids = [LoAPID.ILO_SCI_CNT]
+        datasets_by_apid[LoAPID.ILO_SCI_DE] = parse_events(
+            datasets_by_apid[LoAPID.ILO_SCI_DE], attr_mgr
+        )
+        datasets_by_apid[LoAPID.ILO_SCI_DE] = add_dataset_attrs(
+            datasets_by_apid[LoAPID.ILO_SCI_DE], attr_mgr, logical_source
+        )
+
+    good_apids = [LoAPID.ILO_SCI_CNT, LoAPID.ILO_SCI_DE]
     logger.info(f"\nReturning datasets: {[LoAPID(apid) for apid in good_apids]}")
     return [datasets_by_apid[good_apid] for good_apid in good_apids]
 
@@ -89,6 +118,7 @@ def add_dataset_attrs(
     # TODO: may want up split up these if statements into their
     # own functions
     if logical_source == "imap_lo_l1a_histogram":
+        # Create coordinates for the dataset
         azimuth_60 = xr.DataArray(
             data=np.arange(0, 6, dtype=np.uint8),
             name="azimuth_60",
@@ -118,7 +148,7 @@ def add_dataset_attrs(
             data=np.arange(1, 8, dtype=np.uint8),
             name="esa_step",
             dims=["esa_step"],
-            attrs=attr_mgr.get_variable_attributes("esa_step"),
+            attrs=attr_mgr.get_variable_attributes("esa_step_coord"),
         )
         esa_step_label = xr.DataArray(
             esa_step.values.astype(str),
@@ -127,6 +157,7 @@ def add_dataset_attrs(
             attrs=attr_mgr.get_variable_attributes("esa_step_label"),
         )
 
+        # Get attributes for shcoarse and epoch
         dataset.shcoarse.attrs.update(attr_mgr.get_variable_attributes("shcoarse"))
         dataset.epoch.attrs.update(attr_mgr.get_variable_attributes("epoch"))
 
@@ -153,5 +184,42 @@ def add_dataset_attrs(
                 "pkt_len",
             ]
         )
+    elif logical_source == "imap_lo_l1a_de":
+        # Create the coordinates for the dataset
+        direct_events = xr.DataArray(
+            data=np.arange(sum(dataset["de_count"].values), dtype=np.uint16),
+            name="direct_events",
+            dims=["direct_events"],
+            attrs=attr_mgr.get_variable_attributes("direct_events"),
+        )
+
+        direct_events_label = xr.DataArray(
+            direct_events.values.astype(str),
+            name="direct_events_label",
+            dims=["direct_events_label"],
+            attrs=attr_mgr.get_variable_attributes("direct_events_label"),
+        )
+
+        dataset = dataset.assign_coords(
+            direct_events=direct_events,
+            direct_events_label=direct_events_label,
+        )
+        # add the epoch and global attributes
+        dataset.epoch.attrs.update(attr_mgr.get_variable_attributes("epoch"))
+        dataset.attrs.update(attr_mgr.get_global_attributes(logical_source))
+        dataset = dataset.drop_vars(
+            [
+                "version",
+                "type",
+                "sec_hdr_flg",
+                "pkt_apid",
+                "seq_flgs",
+                "src_seq_ctr",
+                "pkt_len",
+                "shcoarse",
+                "data",
+                "events",
+            ]
+        )
 
     return dataset