imap-processing 0.11.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

imap_processing/codice/codice_l1a.py

@@ -26,15 +26,12 @@ from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.codice import constants
 from imap_processing.codice.codice_l0 import decom_packets
 from imap_processing.codice.decompress import decompress
-from imap_processing.codice.utils import CODICEAPID
+from imap_processing.codice.utils import CODICEAPID, CoDICECompression
 from imap_processing.spice.time import met_to_ttj2000ns
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
-# TODO: Determine what should go in event data CDF and how it should be
-#       structured.
-
 
 class CoDICEL1aPipeline:
     """
@@ -74,6 +71,8 @@ class CoDICEL1aPipeline:
         Retrieve the acquisition times via the Lo stepping table.
     get_energy_table()
         Retrieve the ESA sweep values.
+    get_hi_energy_table_data(species)
+        Retrieve energy table data for CoDICE-Hi products
     reshape_data()
         Reshape the data arrays based on the data product being made.
     set_data_product_config()
@@ -149,7 +148,7 @@ class CoDICEL1aPipeline:
         """
         self.coords = {}
 
-        coord_names = ["epoch", *list(self.config["dims"].keys())]
+        coord_names = ["epoch", *list(self.config["output_dims"].keys())]
 
         # These are labels unique to lo-counters products coordinates
         if self.config["dataset_name"] in [
@@ -158,11 +157,19 @@ class CoDICEL1aPipeline:
         ]:
             coord_names.append("spin_sector_pairs_label")
 
+        # Define the values for the coordinates
         for name in coord_names:
             if name == "epoch":
                 values = self.calculate_epoch_values()
-            elif name in ["esa_step", "inst_az", "spin_sector", "spin_sector_pairs"]:
-                values = np.arange(self.config["dims"][name])
+            elif name in [
+                "esa_step",
+                "inst_az",
+                "spin_sector",
+                "spin_sector_pairs",
+                "spin_sector_index",
+                "ssd_index",
+            ]:
+                values = np.arange(self.config["output_dims"][name])
             elif name == "spin_sector_pairs_label":
                 values = np.array(
                     [
@@ -174,10 +181,6 @@ class CoDICEL1aPipeline:
                         "150-180 deg",
                     ]
                 )
-            else:
-                # TODO: May need to implement other types of coords for Hi
-                #       and/or event data products
-                continue
 
             coord = xr.DataArray(
                 values,
@@ -226,9 +229,20 @@ class CoDICEL1aPipeline:
             cdf_attrs_key = f"{descriptor}-{variable_name}"
             attrs = self.cdf_attrs.get_variable_attributes(cdf_attrs_key)
 
-            # The final CDF dimensions always has "epoch" as the first dimension,
-            # followed by the dimensions for the specific data product
-            dims = ["epoch", *list(self.config["dims"].keys())]
+            # For most products, the final CDF dimensions always has "epoch" as
+            # the first dimension followed by the dimensions for the specific
+            # data product
+            dims = ["epoch", *list(self.config["output_dims"].keys())]
+
+            # However, CoDICE-Hi products use specific energy bins for the
+            # energy dimension
+            # TODO: This will be expanded to all CoDICE-Hi products once I
+            #       can validate them. For now, just operate on hi-sectored
+            if self.config["dataset_name"] == "imap_codice_l1a_hi-sectored":
+                dims = [
+                    f"energy_{variable_name}" if item == "esa_step" else item
+                    for item in dims
+                ]
 
             # Create the CDF data variable
             dataset[variable_name] = xr.DataArray(
@@ -241,6 +255,13 @@ class CoDICEL1aPipeline:
         # Add support data variables based on data product
         dataset = self.define_support_variables(dataset)
 
+        # For CoDICE-Hi products, since energy dimension was replaced, we no
+        # longer need the "esa_step" coordinate
+        # TODO: This will be expanded to all CoDICE-Hi products once I
+        #       can validate them. For now, just operate on hi-sectored
+        if self.config["dataset_name"] == "imap_codice_l1a_hi-sectored":
+            dataset = dataset.drop_vars("esa_step")
+
         return dataset
 
     def define_support_variables(self, dataset: xr.Dataset) -> xr.Dataset:
@@ -269,45 +290,88 @@ class CoDICEL1aPipeline:
             "st_bias_gain_mode",
         ]
 
+        hi_energy_table_variables = [
+            "energy_h",
+            "energy_he3",
+            "energy_he4",
+            "energy_c",
+            "energy_o",
+            "energy_ne_mg_si",
+            "energy_fe",
+            "energy_uh",
+            "energy_junk",
+            "energy_he3he4",
+            "energy_cno",
+        ]
+
         for variable_name in self.config["support_variables"]:
-            # These variables require reading in external tables
-            if variable_name == "energy_table":
-                variable_data = self.get_energy_table()
-                dims = ["esa_step"]
-                attrs = self.cdf_attrs.get_variable_attributes("energy_table")
-
-            elif variable_name == "acquisition_time_per_step":
-                variable_data = self.get_acquisition_times()
-                dims = ["esa_step"]
-                attrs = self.cdf_attrs.get_variable_attributes(
-                    "acquisition_time_per_step"
+            # CoDICE-Hi energy tables are treated differently because values
+            # are binned and we need to record the energies _and_ their deltas
+            if variable_name in hi_energy_table_variables:
+                centers, deltas = self.get_hi_energy_table_data(
+                    variable_name.split("energy_")[-1]
                 )
 
-            elif variable_name in packet_data_variables:
-                variable_data = self.dataset[variable_name].data
-                dims = ["epoch"]
-                attrs = self.cdf_attrs.get_variable_attributes(variable_name)
-
-            # Data quality is named differently in packet data and needs to be
-            # treated slightly differently
-            elif variable_name == "data_quality":
-                variable_data = self.dataset.suspect.data
-                dims = ["epoch"]
-                attrs = self.cdf_attrs.get_variable_attributes("data_quality")
-
-            # Spin period requires the application of a conversion factor
-            # See Table B.5 in the algorithm document
-            elif variable_name == "spin_period":
-                variable_data = self.dataset.spin_period.data * 0.00032
-                dims = ["epoch"]
-                attrs = self.cdf_attrs.get_variable_attributes("spin_period")
+                # Add bin centers and deltas to the dataset
+                dataset[variable_name] = xr.DataArray(
+                    centers,
+                    dims=[variable_name],
+                    attrs=self.cdf_attrs.get_variable_attributes(
+                        f"{self.config['dataset_name'].split('_')[-1]}-{variable_name}"
+                    ),
+                )
+                dataset[f"{variable_name}_delta"] = xr.DataArray(
+                    deltas,
+                    dims=[f"{variable_name}_delta"],
+                    attrs=self.cdf_attrs.get_variable_attributes(
+                        f"{self.config['dataset_name'].split('_')[-1]}-{variable_name}_delta"
+                    ),
+                )
 
-            # Add variable to the dataset
-            dataset[variable_name] = xr.DataArray(
-                variable_data,
-                dims=dims,
-                attrs=attrs,
-            )
+            # Otherwise, support variable data can be gathered from nominal
+            # lookup tables or packet data
+            else:
+                # These variables require reading in external tables
+                if variable_name == "energy_table":
+                    variable_data = self.get_energy_table()
+                    dims = ["esa_step"]
+                    attrs = self.cdf_attrs.get_variable_attributes("energy_table")
+
+                elif variable_name == "acquisition_time_per_step":
+                    variable_data = self.get_acquisition_times()
+                    dims = ["esa_step"]
+                    attrs = self.cdf_attrs.get_variable_attributes(
+                        "acquisition_time_per_step"
+                    )
+
+                # These variables can be gathered straight from the packet data
+                elif variable_name in packet_data_variables:
+                    variable_data = self.dataset[variable_name].data
+                    dims = ["epoch"]
+                    attrs = self.cdf_attrs.get_variable_attributes(variable_name)
+
+                # Data quality is named differently in packet data and needs to be
+                # treated slightly differently
+                elif variable_name == "data_quality":
+                    variable_data = self.dataset.suspect.data
+                    dims = ["epoch"]
+                    attrs = self.cdf_attrs.get_variable_attributes("data_quality")
+
+                # Spin period requires the application of a conversion factor
+                # See Table B.5 in the algorithm document
+                elif variable_name == "spin_period":
+                    variable_data = (
+                        self.dataset.spin_period.data * constants.SPIN_PERIOD_CONVERSION
+                    ).astype(np.float32)
+                    dims = ["epoch"]
+                    attrs = self.cdf_attrs.get_variable_attributes("spin_period")
+
+                # Add variable to the dataset
+                dataset[variable_name] = xr.DataArray(
+                    variable_data,
+                    dims=dims,
+                    attrs=attrs,
+                )
 
         return dataset
 
@@ -315,12 +379,11 @@ class CoDICEL1aPipeline:
         """
         Retrieve the acquisition times via the Lo stepping table.
 
-        Get the acquisition times from the data file based on the values of
+        Get the acquisition times from the lookup table based on the values of
         ``plan_id`` and ``plan_step``
 
         The Lo stepping table defines how many voltage steps and which steps are
         used during each spacecraft spin. A full cycle takes 16 spins. The table
-        provides the timing for a given energy step, and most importantly
         provides the "acquisition time", which is the acquisition time, in
         milliseconds, for the energy step.
 
@@ -329,37 +392,15 @@ class CoDICEL1aPipeline:
         acquisition_times : list[float]
             The list of acquisition times from the Lo stepping table.
         """
-        # Read in the Lo stepping data table
-        lo_stepping_data_file = Path(
-            f"{imap_module_directory}/codice/data/lo_stepping_values.csv"
-        )
-        lo_stepping_data = pd.read_csv(lo_stepping_data_file)
-
         # Determine which Lo stepping table is needed
         lo_stepping_table_id = constants.LO_STEPPING_TABLE_ID_LOOKUP[
             (self.plan_id, self.plan_step)
         ]
 
-        # Get the appropriate values
-        lo_stepping_values = lo_stepping_data[
-            lo_stepping_data["table_num"] == lo_stepping_table_id
+        acquisition_times: list[float] = constants.ACQUISITION_TIMES[
+            lo_stepping_table_id
         ]
 
-        # Create a list for the acquisition times
-        acquisition_times = []
-
-        # Only need the energy columns from the table
-        energy_steps = lo_stepping_values[
-            ["e1", "e2", "e3", "e4", "e5", "e6", "e7", "e8"]
-        ].astype(str)  # convert to string to avoid confusion with table index value
-
-        # For each energy step (0-127), scan the energy columns and find the row
-        # number, which corresponds to a specific acquisition time, then append
-        # it to the list
-        for step_number in range(128):
-            row_number = np.argmax(energy_steps == str(step_number), axis=1).argmax()
-            acquisition_times.append(lo_stepping_values.acq_time[row_number])
-
         return acquisition_times
 
     def get_energy_table(self) -> NDArray[float]:
@@ -400,39 +441,100 @@ class CoDICEL1aPipeline:
 
         return energy_table
 
+    def get_hi_energy_table_data(
+        self, species: str
+    ) -> tuple[NDArray[float], NDArray[float]]:
+        """
+        Retrieve energy table data for CoDICE-Hi products.
+
+        This includes the centers and deltas of the energy bins for a given
+        species. These data eventually get included in the CoDICE-Hi CDF data
+        products.
+
+        Parameters
+        ----------
+        species : str
+            The species of interest, which determines which lookup table to
+            use (e.g. ``h``).
+
+        Returns
+        -------
+        centers : NDArray[float]
+            An array whose values represent the centers of the energy bins.
+        deltas : NDArray[float]
+            An array whose values represent the deltas of the energy bins.
+        """
+        data_product = self.config["dataset_name"].split("-")[-1].upper()
+        energy_table = getattr(constants, f"{data_product}_ENERGY_TABLE")[species]
+
+        # Find the centers and deltas of the energy bins
+        centers = np.array(
+            [
+                (energy_table[i] + energy_table[i + 1]) / 2
+                for i in range(len(energy_table) - 1)
+            ]
+        )
+        deltas = energy_table[1:] - centers
+
+        return centers, deltas
+
     def reshape_data(self) -> None:
         """
         Reshape the data arrays based on the data product being made.
 
         These data need to be divided up by species or priorities (or
         what I am calling "counters" as a general term), and re-arranged into
-        3D arrays representing dimensions such as spin sectors, positions, and
-        energies (depending on the data product).
+        4D arrays representing dimensions such as time, spin sectors, positions,
+        and energies (depending on the data product).
+
+        However, the existence and order of these dimensions can vary depending
+        on the specific data product, so we define this in the "input_dims"
+        and "output_dims" values configuration dictionary; the "input_dims"
+        defines how the dimensions are written into the packet data, while
+        "output_dims" defines how the dimensions should be written to the final
+        CDF product.
         """
         # This will contain the reshaped data for all counters
         self.data = []
 
-        # Typically, data are a 4D arrays with a shape representing some
-        # combination of <num_counters>, <num_positions>, <num_spin_sectors>,
-        # and <num_energy_steps>. However, the existence and order of these
-        # dimensions can vary depending on the specific data product, so we
-        # define this in the "dims" value configuration dictionary. The number
-        # of counters is the first dimension/axis.
-        reshape_dims = (self.config["num_counters"], *self.config["dims"].values())
+        # First reshape the data based on how it is written to the data array of
+        # the packet data. The number of counters is the first dimension / axis,
+        # with the exception of lo-counters-aggregated which is treated slightly
+        # differently
+        if self.config["dataset_name"] != "imap_codice_l1a_lo-counters-aggregated":
+            reshape_dims = (
+                self.config["num_counters"],
+                *self.config["input_dims"].values(),
+            )
+        else:
+            reshape_dims = (
+                *self.config["input_dims"].values(),
+                self.config["num_counters"],
+            )
+
+        # Then, transpose the data based on how the dimensions should be written
+        # to the CDF file. Since this is specific to each data product, we need
+        # to determine this dynamically based on the "output_dims" config.
+        # Again, lo-counters-aggregated is treated slightly differently
+        input_keys = ["num_counters", *self.config["input_dims"].keys()]
+        output_keys = ["num_counters", *self.config["output_dims"].keys()]
+        if self.config["dataset_name"] != "imap_codice_l1a_lo-counters-aggregated":
+            transpose_axes = [input_keys.index(dim) for dim in output_keys]
+        else:
+            transpose_axes = [1, 2, 0]  # [esa_step, spin_sector_pairs, num_counters]
 
-        # For each packet/epoch, reshape the data along these dimensions
         for packet_data in self.raw_data:
             reshaped_packet_data = np.array(packet_data, dtype=np.uint32).reshape(
                 reshape_dims
             )
-            self.data.append(reshaped_packet_data)
+            reshaped_cdf_data = np.transpose(reshaped_packet_data, axes=transpose_axes)
+
+            self.data.append(reshaped_cdf_data)
 
         # No longer need to keep the raw data around
         del self.raw_data
 
-    def set_data_product_config(
-        self, apid: int, dataset: xr.Dataset, data_version: str
-    ) -> None:
+    def set_data_product_config(self, apid: int, dataset: xr.Dataset) -> None:
         """
         Set the various settings for defining the data products.
 
@@ -442,79 +544,139 @@ class CoDICEL1aPipeline:
             The APID of interest.
         dataset : xarray.Dataset
             The dataset for the APID of interest.
-        data_version : str
-            Version of the data product being created.
         """
         # Set the packet dataset so that it can be easily called from various
         # methods
         self.dataset = dataset
 
         # Set various configurations of the data product
-        self.config: dict[str, Any] = constants.DATA_PRODUCT_CONFIGURATIONS.get(apid)  # type: ignore
+        self.config: dict[str, Any] = constants.DATA_PRODUCT_CONFIGURATIONS[apid]
 
         # Gather and set the CDF attributes
         self.cdf_attrs = ImapCdfAttributes()
         self.cdf_attrs.add_instrument_global_attrs("codice")
         self.cdf_attrs.add_instrument_variable_attrs("codice", "l1a")
-        self.cdf_attrs.add_global_attribute("Data_version", data_version)
 
 
-def create_event_dataset(
-    apid: int, packet: xr.Dataset, data_version: str
-) -> xr.Dataset:
+def create_direct_event_dataset(apid: int, packets: xr.Dataset) -> xr.Dataset:
     """
-    Create dataset for event data.
+    Create dataset for direct event data.
+
+    For direct event data, the raw data from the spacecraft is organized first
+    by epoch, then by priority, then by events. For example, for a CoDICE-Lo
+    dataset with 10 epochs, we expect the length of the `event_data` field to
+    be (10 epochs * 8 priorities) = 80 items, with each item being a compressed
+    byte object representing a variable number of events (up to 10000 events).
+    Each compressed byte object is comprised of several fields with specific
+    bit lengths/positions, described by the constants.[LO|HI]_DE_BIT_STRUCTURE
+    dictionary. Padding is added to any fields that have less than 10000 events.
+
+    In order to process these data, we must take the decommed raw data, group
+    the packets appropriately based on their `seq_flgs`, decompress the data,
+    then arrange the data into CDF data variables for each priority and bit
+    field. For example, P2_SpinAngle represents the spin angles for the 2nd
+    priority data.
 
     Parameters
     ----------
     apid : int
         The APID of the packet.
-    packet : xarray.Dataset
-        The packet to process.
-    data_version : str
-        Version of the data product being created.
+    packets : xarray.Dataset
+        The packets to process..
 
     Returns
     -------
     dataset : xarray.Dataset
-        Xarray dataset containing the event data.
+        Xarray dataset containing the direct event data.
     """
+    # Set some useful variables unique to CoDICE-Lo and CoDICE-Hi
     if apid == CODICEAPID.COD_LO_PHA:
-        dataset_name = "imap_codice_l1a_lo-pha"
+        num_priorities = 8
+        cdf_fields = [
+            "NumEvents",
+            "DataQuality",
+            "APDGain",
+            "APD_ID",
+            "APDEnergy",
+            "TOF",
+            "MultiFlag",
+            "PHAType",
+            "SpinAngle",
+            "EnergyStep",
+        ]
     elif apid == CODICEAPID.COD_HI_PHA:
-        dataset_name = "imap_codice_l1a_hi-pha"
+        num_priorities = 6
+        cdf_fields = [
+            "NumEvents",
+            "DataQuality",
+            "SSDEnergy0,TOF",
+            "SSD_ID",
+            "ERGE",
+            "MultiFlag",
+            "Type",
+            "SpinAngle",
+            "SpinNumber",
+        ]
+
+    # Group and decompress the data
+    grouped_data = group_data(packets)
+    decompressed_data = [
+        decompress(group, CoDICECompression.LOSSLESS) for group in grouped_data
+    ]
 
-    # Extract the data
-    # event_data = packet.event_data.data (Currently turned off, see TODO)
+    # Reshape the packet data into CDF-ready variables
+    data = reshape_de_data(packets, decompressed_data, num_priorities)
 
+    # Gather the CDF attributes
     cdf_attrs = ImapCdfAttributes()
     cdf_attrs.add_instrument_global_attrs("codice")
     cdf_attrs.add_instrument_variable_attrs("codice", "l1a")
-    cdf_attrs.add_global_attribute("Data_version", data_version)
 
     # Define coordinates
+    # For epoch, we take the first epoch from each priority set
     epoch = xr.DataArray(
-        packet.epoch,
+        packets.epoch[::num_priorities],
         name="epoch",
         dims=["epoch"],
         attrs=cdf_attrs.get_variable_attributes("epoch"),
     )
+    event_num = xr.DataArray(
+        np.arange(10000),
+        name="event_num",
+        dims=["event_num"],
+        attrs=cdf_attrs.get_variable_attributes("event_num"),
+    )
 
     # Create the dataset to hold the data variables
+    if apid == CODICEAPID.COD_LO_PHA:
+        attrs = cdf_attrs.get_global_attributes("imap_codice_l1a_lo-pha")
+    elif apid == CODICEAPID.COD_HI_PHA:
+        attrs = cdf_attrs.get_global_attributes("imap_codice_l1a_hi-pha")
     dataset = xr.Dataset(
-        coords={
-            "epoch": epoch,
-        },
-        attrs=cdf_attrs.get_global_attributes(dataset_name),
+        coords={"epoch": epoch, "event_num": event_num},
+        attrs=attrs,
     )
 
+    # Create the CDF data variables for each Priority and Field
+    for i in range(num_priorities):
+        for field in cdf_fields:
+            variable_name = f"P{i}_{field}"
+            attrs = cdf_attrs.get_variable_attributes(variable_name)
+            if field in ["NumEvents", "DataQuality"]:
+                dims = ["epoch"]
+            else:
+                dims = ["epoch", "event_num"]
+            dataset[variable_name] = xr.DataArray(
+                np.array(data[variable_name]),
+                name=variable_name,
+                dims=dims,
+                attrs=attrs,
+            )
+
     return dataset
 
 
-def create_hskp_dataset(
-    packet: xr.Dataset,
-    data_version: str,
-) -> xr.Dataset:
+def create_hskp_dataset(packet: xr.Dataset) -> xr.Dataset:
     """
     Create dataset for each metadata field for housekeeping data.
 
@@ -522,8 +684,6 @@ def create_hskp_dataset(
     ----------
     packet : xarray.Dataset
         The packet to process.
-    data_version : str
-        Version of the data product being created.
 
     Returns
     -------
@@ -533,7 +693,6 @@ def create_hskp_dataset(
     cdf_attrs = ImapCdfAttributes()
     cdf_attrs.add_instrument_global_attrs("codice")
     cdf_attrs.add_instrument_variable_attrs("codice", "l1a")
-    cdf_attrs.add_global_attribute("Data_version", data_version)
 
     epoch = xr.DataArray(
         packet.epoch,
@@ -547,7 +706,23 @@ def create_hskp_dataset(
         attrs=cdf_attrs.get_global_attributes("imap_codice_l1a_hskp"),
     )
 
+    # These variables don't need to carry over from L0 to L1a
+    exclude_variables = [
+        "spare_1",
+        "spare_2",
+        "spare_3",
+        "spare_4",
+        "spare_5",
+        "spare_6",
+        "spare_62",
+        "spare_68",
+        "chksum",
+    ]
+
     for variable in packet:
+        if variable in exclude_variables:
+            continue
+
         attrs = cdf_attrs.get_variable_attributes(variable)
 
         dataset[variable] = xr.DataArray(
@@ -596,6 +771,69 @@ def get_params(dataset: xr.Dataset) -> tuple[int, int, int, int]:
     return table_id, plan_id, plan_step, view_id
 
 
+def group_data(packets: xr.Dataset) -> list[bytes]:
+    """
+    Organize continuation packets into appropriate groups.
+
+    Some packets are continuation packets, as in, they are packets that are
+    part of a group of packets. These packets are marked by the `seq_flgs` field
+    in the CCSDS header of the packet. For CoDICE, the values are defined as
+    follows:
+
+    3 = Packet is not part of a group
+    1 = Packet is the first packet of the group
+    0 = Packet is in the middle of the group
+    2 = Packet is the last packet of the group
+
+    For packets that are part of a group, the byte count associated with the
+    first packet of the group signifies the byte count for the entire group.
+
+    Parameters
+    ----------
+    packets : xarray.Dataset
+        Dataset containing the packets to group.
+
+    Returns
+    -------
+    grouped_data : list[bytes]
+        The packet data, converted to bytes and grouped appropriately.
+    """
+    grouped_data = []  # Holds the properly grouped data to be decompressed
+    current_group = bytearray()  # Temporary storage for current group
+    group_byte_count = None  # Temporary storage for current group byte count
+
+    for packet_data, group_code, byte_count in zip(
+        packets.event_data.data, packets.seq_flgs.data, packets.byte_count.data
+    ):
+        # If the group code is 3, this means the data is not part of a group
+        # and can be decompressed as-is
+        if group_code == 3:
+            values_to_decompress = packet_data[:byte_count]
+            grouped_data.append(values_to_decompress)
+
+        # If the group code is 1, this means the data is the first data in a
+        # group. Also, set the byte count for the group
+        elif group_code == 1:
+            group_byte_count = byte_count
+            current_group = packet_data
+
+        # If the group code is 0, this means the data is part of the middle of
+        # the group
+        elif group_code == 0:
+            current_group += packet_data
+
+        # If the group code is 2, this means the data is the last data in the
+        # group
+        elif group_code == 2:
+            current_group += packet_data
+            values_to_decompress = current_group[:group_byte_count]
+            grouped_data.append(values_to_decompress)
+            current_group = bytearray()
+            group_byte_count = None
+
+    return grouped_data
+
+
 def log_dataset_info(datasets: dict[int, xr.Dataset]) -> None:
     """
     Log info about the input data to help with tracking and/or debugging.
@@ -607,9 +845,9 @@ def log_dataset_info(datasets: dict[int, xr.Dataset]) -> None:
     """
     launch_time = np.datetime64("2010-01-01T00:01:06.184", "ns")
     logger.info("\nThis input file contains the following APIDs:\n")
-    for apid in datasets:
-        num_packets = len(datasets[apid].epoch.data)
-        time_deltas = [np.timedelta64(item, "ns") for item in datasets[apid].epoch.data]
+    for apid, ds in datasets.items():
+        num_packets = len(ds.epoch.data)
+        time_deltas = [np.timedelta64(item, "ns") for item in ds.epoch.data]
         times = [launch_time + delta for delta in time_deltas]
         start = np.datetime_as_string(times[0])
         end = np.datetime_as_string(times[-1])
@@ -618,7 +856,102 @@ def log_dataset_info(datasets: dict[int, xr.Dataset]) -> None:
         )
 
 
-def process_codice_l1a(file_path: Path, data_version: str) -> list[xr.Dataset]:
+def reshape_de_data(
+    packets: xr.Dataset, decompressed_data: list[list[int]], num_priorities: int
+) -> dict[str, np.ndarray]:
+    """
+    Reshape the decompressed direct event data into CDF-ready arrays.
+
+    Parameters
+    ----------
+    packets : xarray.Dataset
+        Dataset containing the packets, needed to determine priority order
+        and data quality.
+    decompressed_data : list[list[int]]
+        The decompressed data to reshape, in the format <epoch>[<priority>[<event>]].
+    num_priorities : int
+        The number of priorities in the data product (differs between CoDICE-Lo
+        and CoDICE-Hi).
+
+    Returns
+    -------
+    data : dict[str, numpy.ndarray]
+        The reshaped, CDF-ready arrays. The keys of the dictionary represent the
+        CDF variable names, and the values represent the data.
+    """
+    # Dictionary to hold all the (soon to be restructured) direct event data
+    data: dict[str, np.ndarray] = {}
+
+    # Determine the number of epochs to help with data array initialization
+    # There is one epoch per set of priorities
+    num_epochs = len(packets.epoch.data) // num_priorities
+
+    # Initialize data arrays for each priority and field to store the data
+    # We also need arrays to hold number of events and data quality
+    for priority_num in range(num_priorities):
+        for field in constants.LO_DE_BIT_STRUCTURE:
+            if field not in ["Priority", "Spare"]:
+                data[f"P{priority_num}_{field}"] = np.full(
+                    (num_epochs, 10000), 255, dtype=np.uint16
+                )
+        data[f"P{priority_num}_NumEvents"] = np.full(num_epochs, 255, dtype=np.uint16)
+        data[f"P{priority_num}_DataQuality"] = np.full(num_epochs, 255, dtype=np.uint16)
+
+    # decompressed_data is one large list of values of length
+    # (<number of epochs> * <8 priorities>)
+    # Chunk the data into each epoch
+    for epoch_index in range(num_epochs):
+        # Determine the starting and ending indices of the epoch
+        epoch_start = epoch_index * num_priorities
+        epoch_end = epoch_start + num_priorities
+
+        # Extract the data for the epoch
+        epoch_data = decompressed_data[epoch_start:epoch_end]
+
+        # The order of the priorities and data quality flags are unique to each
+        # epoch and can be gathered from the packet data
+        priority_order = packets.priority[epoch_start:epoch_end].data
+        data_quality = packets.suspect[epoch_start:epoch_end].data
+
+        # For each epoch/priority combo, iterate over each event
+        for i, priority_num in enumerate(priority_order):
+            priority_data = epoch_data[i]
+
+            # Number of events and data quality can be determined at this stage
+            num_events = len(priority_data) // num_priorities
+            data[f"P{priority_num}_NumEvents"][epoch_index] = num_events
+            data[f"P{priority_num}_DataQuality"][epoch_index] = data_quality[i]
+
+            # Iterate over each event
+            for event_index in range(num_events):
+                event_start = event_index * num_priorities
+                event_end = event_start + num_priorities
+                event = priority_data[event_start:event_end]
+                # Separate out each individual field from the bit string
+                # The fields are packed into the bit string in reverse order, so
+                # we need to back them out in reverse order
+                bit_string = (
+                    f"{int.from_bytes(event, byteorder='big'):0{len(event) * 8}b}"
+                )
+                bit_position = 0
+                for field_name, bit_length in reversed(
+                    constants.LO_DE_BIT_STRUCTURE.items()
+                ):
+                    if field_name in ["Priority", "Spare"]:
+                        bit_position += bit_length
+                        continue
+                    value = int(bit_string[bit_position : bit_position + bit_length], 2)
+                    data[f"P{priority_num}_{field_name}"][epoch_index, event_index] = (
+                        value
+                    )
+                    bit_position += bit_length
+
+    # TODO: Implement specific np.dtype and fill_val per field
+
+    return data
+
+
+def process_codice_l1a(file_path: Path) -> list[xr.Dataset]:
     """
     Will process CoDICE l0 data to create l1a data products.
 
@@ -626,8 +959,6 @@ def process_codice_l1a(file_path: Path, data_version: str) -> list[xr.Dataset]:
     ----------
     file_path : pathlib.Path | str
         Path to the CoDICE L0 file to process.
-    data_version : str
-        Version of the data product being created.
 
     Returns
     -------
@@ -651,14 +982,19 @@ def process_codice_l1a(file_path: Path, data_version: str) -> list[xr.Dataset]:
 
         # Housekeeping data
         if apid == CODICEAPID.COD_NHK:
-            processed_dataset = create_hskp_dataset(dataset, data_version)
+            processed_dataset = create_hskp_dataset(dataset)
             logger.info(f"\nFinal data product:\n{processed_dataset}\n")
 
         # Event data
-        elif apid in [CODICEAPID.COD_LO_PHA, CODICEAPID.COD_HI_PHA]:
-            processed_dataset = create_event_dataset(apid, dataset, data_version)
+        elif apid == CODICEAPID.COD_LO_PHA:
+            processed_dataset = create_direct_event_dataset(apid, dataset)
             logger.info(f"\nFinal data product:\n{processed_dataset}\n")
 
+        # TODO: Still need to implement
+        elif apid == CODICEAPID.COD_HI_PHA:
+            logger.info("\tStill need to properly implement")
+            processed_dataset = None
+
         # Everything else
         elif apid in constants.APIDS_FOR_SCIENCE_PROCESSING:
             # Extract the data
@@ -669,7 +1005,7 @@ def process_codice_l1a(file_path: Path, data_version: str) -> list[xr.Dataset]:
 
             # Run the pipeline to create a dataset for the product
             pipeline = CoDICEL1aPipeline(table_id, plan_id, plan_step, view_id)
-            pipeline.set_data_product_config(apid, dataset, data_version)
+            pipeline.set_data_product_config(apid, dataset)
             pipeline.decompress_data(science_values)
             pipeline.reshape_data()
             pipeline.define_coordinates()
@@ -677,9 +1013,8 @@ def process_codice_l1a(file_path: Path, data_version: str) -> list[xr.Dataset]:
 
             logger.info(f"\nFinal data product:\n{processed_dataset}\n")
 
-        # TODO: Still need to implement I-ALiRT and hi-priorities data products
+        # TODO: Still need to implement I-ALiRT data products
         elif apid in [
-            CODICEAPID.COD_HI_INST_COUNTS_PRIORITIES,
             CODICEAPID.COD_HI_IAL,
             CODICEAPID.COD_LO_IAL,
         ]: