imap-processing 0.18.0__py3-none-any.whl → 0.19.2__py3-none-any.whl

imap_processing/codice/codice_l1a.py

@@ -54,6 +54,8 @@ class CoDICEL1aPipeline:
 
     Methods
     -------
+    apply_despinning()
+        Apply the despinning algorithm to lo- angular and priority products.
     decompress_data(science_values)
         Perform decompression on the data.
     define_coordinates()
@@ -87,6 +89,73 @@ class CoDICEL1aPipeline:
         self.plan_step = plan_step
         self.view_id = view_id
 
+    def apply_despinning(self) -> None:
+        """
+        Apply the despinning algorithm to lo- angular and priority products.
+
+        This only applies to CoDICE-Lo angular and priority data products. See
+        sections 9.3.4 and 9.3.5 of the algorithm document for more details.
+        """
+        # Determine the appropriate dimensions for the despun data
+        num_energies = self.config["dims"]["esa_step"]
+        num_spin_sectors = self.config["dims"]["spin_sector"]
+        num_spins = num_spin_sectors * 2
+        num_counters = self.config["num_counters"]
+        num_positions = self.config["dims"].get(
+            "inst_az"
+        )  # Defaults to None if not present
+
+        # The dimensions are dependent on the specific data product
+        if "angular" in self.config["dataset_name"]:
+            despun_dims: tuple[int, ...] = (
+                num_energies,
+                num_positions,
+                num_spins,
+                num_counters,
+            )
+        elif "priority" in self.config["dataset_name"]:
+            despun_dims = (num_energies, num_spins, num_counters)
+
+        # Placeholder for finalized despun data
+        self.data: list[np.ndarray]  # Needed to appease mypy
+        despun_data = [np.zeros(despun_dims) for _ in range(len(self.data))]
+
+        # Iterate over the energy and spin sector indices, and determine the
+        # appropriate pixel orientation. The combination of the pixel
+        # orientation and the azimuth determine which spin sector the data
+        # gets stored in.
+        # TODO: All these nested for-loops are bad. Try to find a better
+        #       solution. See GitHub issue #2136.
+        for i, epoch_data in enumerate(self.data):
+            for energy_index in range(num_energies):
+                pixel_orientation = constants.PIXEL_ORIENTATIONS[energy_index]
+                for spin_sector_index in range(num_spin_sectors):
+                    for azimuth_index in range(num_spins):
+                        if pixel_orientation == "A" and azimuth_index < 12:
+                            despun_spin_sector = spin_sector_index
+                        elif pixel_orientation == "A" and azimuth_index >= 12:
+                            despun_spin_sector = spin_sector_index + 12
+                        elif pixel_orientation == "B" and azimuth_index < 12:
+                            despun_spin_sector = spin_sector_index + 12
+                        elif pixel_orientation == "B" and azimuth_index >= 12:
+                            despun_spin_sector = spin_sector_index
+
+                        if "angular" in self.config["dataset_name"]:
+                            spin_data = epoch_data[
+                                energy_index, :, spin_sector_index, :
+                            ]  # (5, 4)
+                            despun_data[i][energy_index, :, despun_spin_sector, :] = (
+                                spin_data
+                            )
+                        elif "priority" in self.config["dataset_name"]:
+                            spin_data = epoch_data[energy_index, spin_sector_index, :]
+                            despun_data[i][energy_index, despun_spin_sector, :] = (
+                                spin_data
+                            )
+
+        # Replace original data
+        self.data = despun_data
+
     def decompress_data(self, science_values: list[NDArray[str]] | list[str]) -> None:
         """
         Perform decompression on the data.
@@ -122,7 +191,7 @@ class CoDICEL1aPipeline:
 
         else:
             for packet_data, byte_count in zip(
-                science_values, self.dataset.byte_count.data
+                science_values, self.dataset.byte_count.data, strict=False
             ):
                 # Convert from numpy array to byte object
                 values = packet_data[()]
@@ -134,11 +203,14 @@ class CoDICEL1aPipeline:
                 decompressed_values = decompress(values, compression_algorithm)
                 self.raw_data.append(decompressed_values)
 
-    def define_coordinates(self) -> None:
+    def define_coordinates(self) -> None:  # noqa: PLR0912 (too many branches)
         """
         Create ``xr.DataArrays`` for the coords needed in the final dataset.
 
         The coordinates for the dataset depend on the data product being made.
+
+        # TODO: Split this function up or simplify it to avoid too many branches
+        #       error.
         """
         self.coords = {}
 
@@ -169,13 +241,18 @@ class CoDICEL1aPipeline:
             if name in [
                 "esa_step",
                 "inst_az",
-                "spin_sector",
                 "spin_sector_pairs",
                 "spin_sector_index",
                 "ssd_index",
             ]:
                 values = np.arange(self.config["dims"][name])
                 dims = [name]
+            elif name == "spin_sector":
+                if self.config["dataset_name"] in constants.REQUIRES_DESPINNING:
+                    values = np.arange(24)
+                else:
+                    values = np.arange(self.config["dims"][name])
+                dims = [name]
             elif name == "spin_sector_pairs_label":
                 values = np.array(
                     [
@@ -197,7 +274,6 @@ class CoDICEL1aPipeline:
                     values = np.arange(self.config["dims"]["inst_az"]).astype(str)
                 dims = ["inst_az"]
             elif name in [
-                "spin_sector_label",
                 "esa_step_label",
                 "spin_sector_index_label",
                 "ssd_index_label",
@@ -205,6 +281,13 @@ class CoDICEL1aPipeline:
                 key = name.removesuffix("_label")
                 values = np.arange(self.config["dims"][key]).astype(str)
                 dims = [key]
+            elif name == "spin_sector_label":
+                key = name.removesuffix("_label")
+                dims = [key]
+                if self.config["dataset_name"] in constants.REQUIRES_DESPINNING:
+                    values = np.arange(24).astype(str)
+                else:
+                    values = np.arange(self.config["dims"][key]).astype(str)
 
             coord = xr.DataArray(
                 values,
@@ -243,7 +326,7 @@ class CoDICEL1aPipeline:
         # the num_counters dimension to isolate the data for each counter so
         # each counter's data can be placed in a separate CDF data variable.
         for counter, variable_name in zip(
-            range(all_data.shape[-1]), self.config["variable_names"]
+            range(all_data.shape[-1]), self.config["variable_names"], strict=False
         ):
             # Extract the counter data
             counter_data = all_data[..., counter]
@@ -262,7 +345,7 @@ class CoDICEL1aPipeline:
             # energy dimension
             # TODO: This bit of code may no longer be needed once I can figure
             #       out how to run hi-sectored product through the
-            #       create_binned_dataset function
+            #       create_binned_dataset function. See GitHub issue #2137.
             if self.config["dataset_name"] == "imap_codice_l1a_hi-sectored":
                 dims = [
                     f"energy_{variable_name}" if item == "esa_step" else item
@@ -284,7 +367,7 @@ class CoDICEL1aPipeline:
         # longer need the "esa_step" coordinate
         # TODO: This bit of code may no longer be needed once I can figure
         #       out how to run hi-sectored product through the
-        #       create_binned_dataset function
+        #       create_binned_dataset function. See GitHub issue #2137.
         if self.config["dataset_name"] == "imap_codice_l1a_hi-sectored":
             for species in self.config["energy_table"]:
                 dataset = self.define_energy_bins(dataset, species)
@@ -313,7 +396,7 @@ class CoDICEL1aPipeline:
             ``xarray`` dataset for the data product, with added energy variables.
         """
         energy_bin_name = f"energy_{species}"
-        centers, deltas = self.get_hi_energy_table_data(
+        centers, deltas_minus, deltas_plus = self.get_hi_energy_table_data(
             energy_bin_name.split("energy_")[-1]
         )
 
@@ -326,11 +409,19 @@ class CoDICEL1aPipeline:
                 check_schema=False,
             ),
         )
-        dataset[f"{energy_bin_name}_delta"] = xr.DataArray(
-            deltas,
-            dims=[f"{energy_bin_name}_delta"],
+        dataset[f"{energy_bin_name}_minus"] = xr.DataArray(
+            deltas_minus,
+            dims=[f"{energy_bin_name}_minus"],
             attrs=self.cdf_attrs.get_variable_attributes(
-                f"{self.config['dataset_name'].split('_')[-1]}-{energy_bin_name}_delta",
+                f"{self.config['dataset_name'].split('_')[-1]}-{energy_bin_name}_minus",
+                check_schema=False,
+            ),
+        )
+        dataset[f"{energy_bin_name}_plus"] = xr.DataArray(
+            deltas_plus,
+            dims=[f"{energy_bin_name}_plus"],
+            attrs=self.cdf_attrs.get_variable_attributes(
+                f"{self.config['dataset_name'].split('_')[-1]}-{energy_bin_name}_plus",
                 check_schema=False,
             ),
         )
@@ -488,7 +579,7 @@ class CoDICEL1aPipeline:
 
     def get_hi_energy_table_data(
         self, species: str
-    ) -> tuple[NDArray[float], NDArray[float]]:
+    ) -> tuple[NDArray[float], NDArray[float], NDArray[float]]:
         """
         Retrieve energy table data for CoDICE-Hi products.
 
@@ -506,22 +597,25 @@ class CoDICEL1aPipeline:
         -------
         centers : NDArray[float]
             An array whose values represent the centers of the energy bins.
-        deltas : NDArray[float]
-            An array whose values represent the deltas of the energy bins.
+        deltas_minus : NDArray[float]
+            An array whose values represent the minus deltas of the energy bins.
+        deltas_plus : NDArray[float]
+            An array whose values represent the plus deltas of the energy bins.
         """
         data_product = self.config["dataset_name"].split("-")[-1].upper()
-        energy_table = getattr(constants, f"{data_product}_ENERGY_TABLE")[species]
-
-        # Find the centers and deltas of the energy bins
-        centers = np.array(
-            [
-                (energy_table[i] + energy_table[i + 1]) / 2
-                for i in range(len(energy_table) - 1)
-            ]
+        energy_table = np.array(
+            getattr(constants, f"{data_product}_ENERGY_TABLE")[species]
         )
-        deltas = energy_table[1:] - centers
 
-        return centers, deltas
+        # Find the geometric centers and deltas of the energy bins
+        # The delta minus is the difference between the center of the bin
+        # and the 'left edge' of the bin. The delta plus is the difference
+        # between the 'right edge' of the bin and the center of the bin
+        centers = np.sqrt(energy_table[:-1] * energy_table[1:])
+        deltas_minus = centers - energy_table[:-1]
+        deltas_plus = energy_table[1:] - centers
+
+        return centers, deltas_minus, deltas_plus
 
     def reshape_binned_data(self, dataset: xr.Dataset) -> dict[str, list]:
         """
@@ -624,6 +718,10 @@ class CoDICEL1aPipeline:
             )
             self.data.append(reshaped_packet_data)
 
+        # Apply despinning if necessary
+        if self.config["dataset_name"] in constants.REQUIRES_DESPINNING:
+            self.apply_despinning()
+
         # No longer need to keep the raw data around
         del self.raw_data
 
@@ -724,9 +822,6 @@ def group_ialirt_data(
 
     # Workaround to get this function working for both I-ALiRT spacecraft
     # data and CoDICE-specific I-ALiRT test data from Joey
-    # TODO: Once CoDICE I-ALiRT processing is more established, we can probably
-    #       do away with processing the test data from Joey and just use the
-    #       I-ALiRT data that is constructed closer to what we expect in-flight.
     if hasattr(packets, "acquisition_time"):
         time_key = "acquisition_time"
         counter_key = "counter"
@@ -782,7 +877,7 @@ def create_binned_dataset(
         Xarray dataset containing the final processed dataset.
     """
     # TODO: hi-sectored data product should be processed similar to hi-omni,
-    #       so I should be able to use this method.
+    #       so I should be able to use this method. See GitHub issue #2137.
 
     # Get the four "main" parameters for processing
     table_id, plan_id, plan_step, view_id = get_params(dataset)
@@ -803,7 +898,7 @@ def create_binned_dataset(
         attrs=pipeline.cdf_attrs.get_variable_attributes("epoch", check_schema=False),
     )
     # TODO: Figure out how to calculate epoch centers and deltas and store them
-    #       in variables here
+    #       in variables here. See GitHub issue #1501.
     dataset = xr.Dataset(
         coords={"epoch": coord},
         attrs=pipeline.cdf_attrs.get_global_attributes(pipeline.config["dataset_name"]),
@@ -843,7 +938,7 @@ def create_binned_dataset(
     return dataset
 
 
-def create_direct_event_dataset(apid: int, packets: xr.Dataset) -> xr.Dataset:
+def create_direct_event_dataset(apid: int, unpacked_dataset: xr.Dataset) -> xr.Dataset:
     """
     Create dataset for direct event data.
 
@@ -857,7 +952,7 @@ def create_direct_event_dataset(apid: int, packets: xr.Dataset) -> xr.Dataset:
     dictionary. Padding is added to any fields that have less than 10000 events.
 
     In order to process these data, we must take the decommed raw data, group
-    the packets appropriately based on their `seq_flgs`, decompress the data,
+    the unpacked_dataset appropriately based on their `seq_flgs`, decompress the data,
     then arrange the data into CDF data variables for each priority and bit
     field. For example, P2_SpinAngle represents the spin angles for the 2nd
     priority data.
@@ -866,8 +961,8 @@ def create_direct_event_dataset(apid: int, packets: xr.Dataset) -> xr.Dataset:
     ----------
     apid : int
         The APID of the packet.
-    packets : xarray.Dataset
-        The packets to process.
+    unpacked_dataset : xarray.Dataset
+        The unpacked dataset to process.
 
     Returns
     -------
@@ -875,13 +970,13 @@ def create_direct_event_dataset(apid: int, packets: xr.Dataset) -> xr.Dataset:
         Xarray dataset containing the direct event data.
     """
     # Group and decompress the data
-    grouped_data = group_data(packets)
+    grouped_data = group_data(unpacked_dataset)
     decompressed_data = [
         decompress(group, CoDICECompression.LOSSLESS) for group in grouped_data
     ]
 
     # Reshape the packet data into CDF-ready variables
-    data = reshape_de_data(packets, decompressed_data, apid)
+    reshaped_de_data = reshape_de_data(unpacked_dataset, decompressed_data, apid)
 
     # Gather the CDF attributes
     cdf_attrs = ImapCdfAttributes()
@@ -891,11 +986,11 @@ def create_direct_event_dataset(apid: int, packets: xr.Dataset) -> xr.Dataset:
     # Determine the epochs to use in the dataset, which are the epochs whenever
     # there is a start of a segment and the priority is 0
     epoch_indices = np.where(
-        ((packets.seq_flgs.data == 3) | (packets.seq_flgs.data == 1))
-        & (packets.priority.data == 0)
+        ((unpacked_dataset.seq_flgs.data == 3) | (unpacked_dataset.seq_flgs.data == 1))
+        & (unpacked_dataset.priority.data == 0)
     )[0]
-    acq_start_seconds = packets.acq_start_seconds[epoch_indices]
-    acq_start_subseconds = packets.acq_start_subseconds[epoch_indices]
+    acq_start_seconds = unpacked_dataset.acq_start_seconds[epoch_indices]
+    acq_start_subseconds = unpacked_dataset.acq_start_subseconds[epoch_indices]
 
     # Calculate epoch variables
     epochs, epochs_delta_minus, epochs_delta_plus = calculate_epoch_values(
@@ -953,20 +1048,19 @@ def create_direct_event_dataset(apid: int, packets: xr.Dataset) -> xr.Dataset:
     )
 
     # Create the CDF data variables for each Priority and Field
-    for i in range(constants.DE_DATA_PRODUCT_CONFIGURATIONS[apid]["num_priorities"]):
-        for field in constants.DE_DATA_PRODUCT_CONFIGURATIONS[apid]["cdf_fields"]:
-            variable_name = f"P{i}_{field}"
-            attrs = cdf_attrs.get_variable_attributes(variable_name)
-            if field in ["NumEvents", "DataQuality"]:
-                dims = ["epoch"]
-            else:
-                dims = ["epoch", "event_num"]
-            dataset[variable_name] = xr.DataArray(
-                np.array(data[variable_name]),
-                name=variable_name,
-                dims=dims,
-                attrs=attrs,
-            )
+    for field in constants.DE_DATA_PRODUCT_CONFIGURATIONS[apid]["cdf_fields"]:
+        if field in ["num_events", "data_quality"]:
+            attrs = cdf_attrs.get_variable_attributes("de_2d_attrs")
+            dims = ["epoch", "priority"]
+        else:
+            attrs = cdf_attrs.get_variable_attributes("de_3d_attrs")
+            dims = ["epoch", "priority", "event_num"]
+        dataset[field] = xr.DataArray(
+            np.array(reshaped_de_data[field]),
+            name=field,
+            dims=dims,
+            attrs=attrs,
+        )
 
     return dataset
 
@@ -1392,7 +1486,7 @@ def reshape_de_data(
         CDF variable names, and the values represent the data.
     """
     # Dictionary to hold all the (soon to be restructured) direct event data
-    data: dict[str, np.ndarray] = {}
+    de_data: dict[str, np.ndarray] = {}
 
     # Extract some useful variables
     num_priorities = constants.DE_DATA_PRODUCT_CONFIGURATIONS[apid]["num_priorities"]
@@ -1412,16 +1506,20 @@ def reshape_de_data(
 
     # Initialize data arrays for each priority and field to store the data
     # We also need arrays to hold number of events and data quality
-    for priority_num in range(num_priorities):
-        for field in bit_structure:
-            if field not in ["Priority", "Spare"]:
-                data[f"P{priority_num}_{field}"] = np.full(
-                    (num_epochs, 10000),
-                    bit_structure[field]["fillval"],
-                    dtype=bit_structure[field]["dtype"],
-                )
-        data[f"P{priority_num}_NumEvents"] = np.full(num_epochs, 65535, dtype=np.uint16)
-        data[f"P{priority_num}_DataQuality"] = np.full(num_epochs, 255, dtype=np.uint8)
+    for field in bit_structure:
+        # if these two, no need to store
+        if field not in ["Priority", "Spare"]:
+            de_data[f"{field}"] = np.full(
+                (num_epochs, num_priorities, 10000),
+                bit_structure[field]["fillval"],
+                dtype=bit_structure[field]["dtype"],
+            )
+    # Add other additional fields of l1a
+    de_data["num_events"] = np.full(
+        (num_epochs, num_priorities), 65535, dtype=np.uint16
+    )
+
+    de_data["data_quality"] = np.full((num_epochs, num_priorities), 255, dtype=np.uint8)
 
     # decompressed_data is one large list of values of length
     # (<number of epochs> * <number of priorities>)
@@ -1445,8 +1543,8 @@ def reshape_de_data(
 
             # Number of events and data quality can be determined at this stage
             num_events = num_events_arr[epoch_start:epoch_end][i]
-            data[f"P{priority_num}_NumEvents"][epoch_index] = num_events
-            data[f"P{priority_num}_DataQuality"][epoch_index] = data_quality[i]
+            de_data["num_events"][epoch_index, priority_num] = num_events
+            de_data["data_quality"][epoch_index, priority_num] = data_quality[i]
 
             # Iterate over each event
             for event_index in range(num_events):
@@ -1477,12 +1575,12 @@ def reshape_de_data(
                     )
 
                     # Set the value into the data array
-                    data[f"P{priority_num}_{field_name}"][epoch_index, event_index] = (
+                    de_data[f"{field_name}"][epoch_index, priority_num, event_index] = (
                         value
                     )
                     bit_position += field_components["bit_length"]
 
-    return data
+    return de_data
 
 
 def process_codice_l1a(file_path: Path) -> list[xr.Dataset]:

imap_processing/codice/codice_l1b.py

@@ -9,18 +9,18 @@ from imap_processing.codice.codice_l1b import process_codice_l1b
 dataset = process_codice_l1b(l1a_filenanme)
 """
 
-# TODO: Figure out how to convert hi-priority data product. Need an updated
-#       algorithm document that describes this.
-
 import logging
 from pathlib import Path
 
 import numpy as np
 import xarray as xr
 
+from imap_processing import imap_module_directory
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.cdf.utils import load_cdf
 from imap_processing.codice import constants
+from imap_processing.codice.utils import CODICEAPID
+from imap_processing.utils import packet_file_to_datasets
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -49,9 +49,6 @@ def convert_to_rates(
     rates_data : np.ndarray
         The converted data array.
     """
-    # TODO: Temporary workaround to create CDFs for SIT-4. Revisit after SIT-4.
-    acq_times = 1
-
     if descriptor in [
         "lo-counters-aggregated",
         "lo-counters-singles",
@@ -65,6 +62,13 @@ def convert_to_rates(
     ]:
         # Applying rate calculation described in section 10.2 of the algorithm
         # document
+        # In order to divide by acquisition times, we must reshape the acq
+        # time data array to match the data variable shape
+        dims = [1] * dataset[variable_name].data.ndim
+        dims[1] = 128
+        acq_times = dataset.acquisition_time_per_step.data.reshape(dims)
+
+        # Now perform the calculation
         rates_data = dataset[variable_name].data / (
             acq_times
             * 1e-6  # Converting from microseconds to seconds
@@ -83,10 +87,8 @@ def convert_to_rates(
         rates_data = dataset[variable_name].data / (
             constants.L1B_DATA_PRODUCT_CONFIGURATIONS[descriptor]["num_spin_sectors"]
             * constants.L1B_DATA_PRODUCT_CONFIGURATIONS[descriptor]["num_spins"]
-            * acq_times
+            * constants.HI_ACQUISITION_TIME
         )
-    elif descriptor == "hskp":
-        rates_data = dataset[variable_name].data / acq_times
 
     return rates_data
 
@@ -131,35 +133,43 @@ def process_codice_l1b(file_path: Path) -> xr.Dataset:
     # Update the global attributes
     l1b_dataset.attrs = cdf_attrs.get_global_attributes(dataset_name)
 
-    # Determine which variables need to be converted from counts to rates
-    # TODO: Figure out exactly which hskp variables need to be converted
-    # Housekeeping and binned datasets are treated a bit differently since
-    # not all variables need to be converted
+    # TODO: This was thrown together quickly and should be double-checked
     if descriptor == "hskp":
-        # TODO: Check with Joey if any housekeeping data needs to be converted
-        variables_to_convert = []
-    elif descriptor == "hi-sectored":
-        variables_to_convert = ["h", "he3he4", "cno", "fe"]
-    elif descriptor == "hi-omni":
-        variables_to_convert = ["h", "he3", "he4", "c", "o", "ne_mg_si", "fe", "uh"]
-    elif descriptor == "hi-ialirt":
-        variables_to_convert = ["h"]
+        xtce_filename = "codice_packet_definition.xml"
+        xtce_packet_definition = Path(
+            f"{imap_module_directory}/codice/packet_definitions/{xtce_filename}"
+        )
+        packet_file = (
+            imap_module_directory
+            / "tests"
+            / "codice"
+            / "data"
+            / "imap_codice_l0_raw_20241110_v001.pkts"
+        )
+        datasets: dict[int, xr.Dataset] = packet_file_to_datasets(
+            packet_file, xtce_packet_definition, use_derived_value=True
+        )
+        l1b_dataset = datasets[CODICEAPID.COD_NHK]
+
+        # TODO: Drop the same variables as we do in L1a? (see line 1103 in
+        #       codice_l1a.py
+
     else:
         variables_to_convert = getattr(
             constants, f"{descriptor.upper().replace('-', '_')}_VARIABLE_NAMES"
         )
 
-    # Apply the conversion to rates
-    for variable_name in variables_to_convert:
-        l1b_dataset[variable_name].data = convert_to_rates(
-            l1b_dataset, descriptor, variable_name
-        )
-
-        # Set the variable attributes
-        cdf_attrs_key = f"{descriptor}-{variable_name}"
-        l1b_dataset[variable_name].attrs = cdf_attrs.get_variable_attributes(
-            cdf_attrs_key, check_schema=False
-        )
+        # Apply the conversion to rates
+        for variable_name in variables_to_convert:
+            l1b_dataset[variable_name].data = convert_to_rates(
+                l1b_dataset, descriptor, variable_name
+            )
+
+            # Set the variable attributes
+            cdf_attrs_key = f"{descriptor}-{variable_name}"
+            l1b_dataset[variable_name].attrs = cdf_attrs.get_variable_attributes(
+                cdf_attrs_key, check_schema=False
+            )
 
     logger.info(f"\nFinal data product:\n{l1b_dataset}\n")