imap-processing 0.16.1__py3-none-any.whl → 0.17.0__py3-none-any.whl

imap_processing/lo/l1a/lo_l1a.py

@@ -15,6 +15,7 @@ from imap_processing.lo.l0.lo_science import (
     parse_events,
     parse_histogram,
 )
+from imap_processing.lo.l0.lo_star_sensor import process_star_sensor
 from imap_processing.utils import convert_to_binary_string, packet_file_to_datasets
 
 logger = logging.getLogger(__name__)
@@ -50,60 +51,59 @@ def lo_l1a(dependency: Path) -> list[xr.Dataset]:
     attr_mgr.add_instrument_global_attrs(instrument="lo")
     attr_mgr.add_instrument_variable_attrs(instrument="lo", level="l1a")
 
+    datasets_to_return = []
+
     if LoAPID.ILO_SPIN in datasets_by_apid:
         logger.info(
             f"\nProcessing {LoAPID(LoAPID.ILO_SPIN).name} "
             f"packet (APID: {LoAPID.ILO_SPIN.value})"
         )
         logical_source = "imap_lo_l1a_spin"
-        datasets_by_apid[LoAPID.ILO_SPIN] = organize_spin_data(
-            datasets_by_apid[LoAPID.ILO_SPIN], attr_mgr
-        )
-
-        datasets_by_apid[LoAPID.ILO_SPIN] = add_dataset_attrs(
-            datasets_by_apid[LoAPID.ILO_SPIN], attr_mgr, logical_source
-        )
+        ds = datasets_by_apid[LoAPID.ILO_SPIN]
+        ds = organize_spin_data(ds, attr_mgr)
+        ds = add_dataset_attrs(ds, attr_mgr, logical_source)
+        datasets_to_return.append(ds)
     if LoAPID.ILO_SCI_CNT in datasets_by_apid:
         logger.info(
             f"\nProcessing {LoAPID(LoAPID.ILO_SCI_CNT).name} "
             f"packet (APID: {LoAPID.ILO_SCI_CNT.value})"
         )
         logical_source = "imap_lo_l1a_histogram"
-        datasets_by_apid[LoAPID.ILO_SCI_CNT] = parse_histogram(
-            datasets_by_apid[LoAPID.ILO_SCI_CNT], attr_mgr
-        )
-        datasets_by_apid[LoAPID.ILO_SCI_CNT] = add_dataset_attrs(
-            datasets_by_apid[LoAPID.ILO_SCI_CNT], attr_mgr, logical_source
-        )
+        ds = datasets_by_apid[LoAPID.ILO_SCI_CNT]
+        ds = parse_histogram(ds, attr_mgr)
+        ds = add_dataset_attrs(ds, attr_mgr, logical_source)
+        datasets_to_return.append(ds)
     if LoAPID.ILO_SCI_DE in datasets_by_apid:
         logger.info(
             f"\nProcessing {LoAPID(LoAPID.ILO_SCI_DE).name} "
             f"packet (APID: {LoAPID.ILO_SCI_DE.value})"
         )
         logical_source = "imap_lo_l1a_de"
-        datasets_by_apid[LoAPID.ILO_SCI_DE]["data"] = xr.DataArray(
-            [
-                convert_to_binary_string(data)
-                for data in datasets_by_apid[LoAPID.ILO_SCI_DE]["data"].values
-            ],
-            dims=datasets_by_apid[LoAPID.ILO_SCI_DE]["data"].dims,
-            attrs=datasets_by_apid[LoAPID.ILO_SCI_DE]["data"].attrs,
-        )
-
-        datasets_by_apid[LoAPID.ILO_SCI_DE] = combine_segmented_packets(
-            datasets_by_apid[LoAPID.ILO_SCI_DE]
+        ds = datasets_by_apid[LoAPID.ILO_SCI_DE]
+        # Process the "data" array into a string
+        ds["data"] = xr.DataArray(
+            [convert_to_binary_string(data) for data in ds["data"].values],
+            dims=ds["data"].dims,
+            attrs=ds["data"].attrs,
         )
 
-        datasets_by_apid[LoAPID.ILO_SCI_DE] = parse_events(
-            datasets_by_apid[LoAPID.ILO_SCI_DE], attr_mgr
-        )
-        datasets_by_apid[LoAPID.ILO_SCI_DE] = add_dataset_attrs(
-            datasets_by_apid[LoAPID.ILO_SCI_DE], attr_mgr, logical_source
+        ds = combine_segmented_packets(ds)
+        ds = parse_events(ds, attr_mgr)
+        ds = add_dataset_attrs(ds, attr_mgr, logical_source)
+        datasets_to_return.append(ds)
+    if LoAPID.ILO_STAR in datasets_by_apid:
+        logger.info(
+            f"\nProcessing {LoAPID(LoAPID.ILO_STAR).name} "
+            f"packet (APID: {LoAPID.ILO_STAR.value})"
         )
+        logical_source = "imap_lo_l1a_star"
+        ds = datasets_by_apid[LoAPID.ILO_STAR]
+        ds = process_star_sensor(ds)
+        ds = add_dataset_attrs(ds, attr_mgr, logical_source)
+        datasets_to_return.append(ds)
 
-    good_apids = [LoAPID.ILO_SPIN, LoAPID.ILO_SCI_CNT, LoAPID.ILO_SCI_DE]
-    logger.info(f"\nReturning datasets: {[LoAPID(apid) for apid in good_apids]}")
-    return [datasets_by_apid[good_apid] for good_apid in good_apids]
+    logger.info(f"Returning [{len(datasets_to_return)}] datasets")
+    return datasets_to_return
 
 
 def add_dataset_attrs(

imap_processing/mag/l0/decom_mag.py

@@ -43,8 +43,9 @@ def decom_packets(packet_file_path: str | Path) -> dict[str, list[MagL0]]:
 
     packet_definition = definitions.XtcePacketDefinition(xtce_document)
 
-    norm_data = []
-    burst_data = []
+    # Store in a dict for de-duplication. Only the keys are returned as a list.
+    norm_dict: dict[MagL0, None] = {}
+    burst_dict: dict[MagL0, None] = {}
 
     with open(packet_file_path, "rb") as binary_data:
         mag_packets = packet_definition.packet_generator(binary_data)
@@ -53,12 +54,14 @@ def decom_packets(packet_file_path: str | Path) -> dict[str, list[MagL0]]:
             apid = packet["PKT_APID"]
             if apid in (Mode.BURST, Mode.NORMAL):
                 values = [item.raw_value for item in packet.user_data.values()]
+                mag_l0 = MagL0(CcsdsData(packet.header), *values)
                 if apid == Mode.NORMAL:
-                    norm_data.append(MagL0(CcsdsData(packet.header), *values))
-                else:
-                    burst_data.append(MagL0(CcsdsData(packet.header), *values))
+                    if mag_l0 not in norm_dict:
+                        norm_dict[mag_l0] = None
+                elif mag_l0 not in burst_dict:
+                    burst_dict[mag_l0] = None
 
-    return {"norm": norm_data, "burst": burst_data}
+    return {"norm": list(norm_dict.keys()), "burst": list(burst_dict.keys())}
 
 
 def generate_dataset(

imap_processing/mag/l0/mag_l0_data.py

@@ -31,6 +31,9 @@ class MagL0:
     """
     Data class for MAG Level 0 data.
 
+    No attributes should be updated after creation. This class acts as a snapshot of the
+    packet data, and is read-only.
+
     Attributes
     ----------
     ccsds_header: CcsdsData
@@ -116,3 +119,46 @@ class MagL0:
 
         self.PRI_VECSEC = 2**self.PRI_VECSEC
         self.SEC_VECSEC = 2**self.SEC_VECSEC
+
+    def __eq__(self, other: object) -> bool:
+        """
+        Compare two MagL0 objects for equality.
+
+        Two objects are said to be equal if the SHCOARSE, APID, and SRC_SEQ_CTR are
+        equal.
+
+        Parameters
+        ----------
+        other : object
+            The other MagL0 object to compare against.
+
+        Returns
+        -------
+        bool
+            True if the objects are equal, False otherwise.
+        """
+        if not isinstance(other, MagL0):
+            return NotImplemented
+
+        return (
+            self.SHCOARSE == other.SHCOARSE
+            and self.ccsds_header.PKT_APID == other.ccsds_header.PKT_APID
+            and self.ccsds_header.SRC_SEQ_CTR == other.ccsds_header.SRC_SEQ_CTR
+        )
+
+    def __hash__(self) -> int:
+        """
+        Return a hash of the MagL0 object for use in sets.
+
+        Returns
+        -------
+        int
+            The hash value of the MagL0 object.
+        """
+        return hash(
+            (
+                self.SHCOARSE,
+                self.ccsds_header.PKT_APID,
+                self.ccsds_header.SRC_SEQ_CTR,
+            )
+        )

imap_processing/swapi/l1/swapi_l1.py

@@ -558,8 +558,11 @@ def process_swapi_science(
     # Step 3: Create xarray.Dataset
     # ===================================================================
 
-    # epoch time. Should be same dimension as number of good sweeps
-    epoch_values = good_sweep_sci["epoch"].data.reshape(total_full_sweeps, 12)[:, 0]
+    # epoch time. Should be same dimension as number of good sweeps.
+    # Use center time for epoch to line up with mission requests. Center time
+    # of SWAPI is time of 7th packet(aka SEQ_NUMBER == 6) creation time at the
+    # beginning of 7th packet.
+    epoch_values = good_sweep_sci["epoch"].data.reshape(total_full_sweeps, 12)[:, 6]
 
     epoch_time = xr.DataArray(
         epoch_values,
@@ -628,6 +631,13 @@ def process_swapi_science(
         dims=["epoch"],
         attrs=cdf_manager.get_variable_attributes("plan_id"),
     )
+    # Store start time for L3 purposes per SWAPI requests
+    dataset["sci_start_time"] = xr.DataArray(
+        good_sweep_sci["epoch"].data.reshape(total_full_sweeps, 12)[:, 0],
+        name="sci_start_time",
+        dims=["epoch"],
+        attrs=cdf_manager.get_variable_attributes("sci_start_time"),
+    )
     # Add ESA_LVL5 for L2 and L3 purposes.
     # We need to store ESA_LVL5 at SEQ_NUMBER==11
     # which is 71 energy step's ESA_LVL5 value. ESA_LVL5 gets

imap_processing/swapi/l2/swapi_l2.py

@@ -149,15 +149,16 @@ def swapi_l2(
     # Copy over only certain variables from L1 to L2 dataset
     l1_data_keys = [
         "epoch",
+        "esa_lvl5",
         "esa_step",
         "esa_step_label",
-        "swp_l1a_flags",
-        "sweep_table",
-        "plan_id",
-        "lut_choice",
-        "fpga_type",
         "fpga_rev",
-        "esa_lvl5",
+        "fpga_type",
+        "lut_choice",
+        "plan_id",
+        "sci_start_time",
+        "sweep_table",
+        "swp_l1a_flags",
     ]
     l2_dataset = l1_dataset[l1_data_keys]
 

imap_processing/swe/l1b/swe_l1b.py

@@ -761,6 +761,10 @@ def swe_l1b_science(dependencies: ProcessingInputCollection) -> xr.Dataset:
 
     count_rate = convert_counts_to_rate(inflight_applied_count, acq_duration)
 
+    # Statistical uncertainty is sqrt(decompressed counts)
+    # TODO: Update this if SWE like to include deadtime correciton.
+    counts_stat_uncert = np.sqrt(populated_data["science_data"])
+
     # Store ESA energies of full cycle for L2 purposes.
     esa_energies = get_esa_energy_pattern(esa_lut_files[0])
     # Repeat energies to be in the same shape as the science data
@@ -894,6 +898,11 @@ def swe_l1b_science(dependencies: ProcessingInputCollection) -> xr.Dataset:
         dims=["epoch", "esa_step", "spin_sector", "cem_id"],
         attrs=cdf_attrs.get_variable_attributes("science_data"),
     )
+    science_dataset["counts_stat_uncert"] = xr.DataArray(
+        counts_stat_uncert,
+        dims=["epoch", "esa_step", "spin_sector", "cem_id"],
+        attrs=cdf_attrs.get_variable_attributes("counts_stat_uncert"),
+    )
     science_dataset["acquisition_time"] = xr.DataArray(
         acq_time,
         dims=["epoch", "esa_step", "spin_sector"],

imap_processing/swe/l2/swe_l2.py

@@ -14,9 +14,11 @@ from imap_processing.spice.spin import get_instrument_spin_phase, get_spin_angle
 from imap_processing.swe.utils import swe_constants
 
 
-def calculate_phase_space_density(l1b_dataset: xr.Dataset) -> npt.NDArray:
+def calculate_phase_space_density(
+    data: np.ndarray, particle_energy_data: np.ndarray
+) -> npt.NDArray:
     """
-    Convert counts to phase space density.
+    Convert counts or uncertainty data to phase space density.
 
     Calculate phase space density is represented by this symbol, fv.
     Its unit is s^3/ (cm^6 * ster).
@@ -49,8 +51,11 @@ def calculate_phase_space_density(l1b_dataset: xr.Dataset) -> npt.NDArray:
 
     Parameters
     ----------
-    l1b_dataset : xarray.Dataset
-        The L1B dataset to process.
+    data : numpy.ndarray
+        The data to process. Two expected inputs are counts or uncertainty data.
+    particle_energy_data : numpy.ndarray
+        The energy values in eV. This is the energy values from the
+        "esa_energy" variable in the L1B dataset.
 
     Returns
     -------
@@ -58,18 +63,14 @@ def calculate_phase_space_density(l1b_dataset: xr.Dataset) -> npt.NDArray:
         Phase space density. We need to call this phase space density because
         there will be density in L3 processing.
     """
-    # Get energy values.
-    particle_energy_data = l1b_dataset["esa_energy"].values
-
     # Calculate phase space density using formula:
-    #   2 * (C/tau) / (G * 1.237e31 * eV^2)
+    #   2 * ((C/tau) or uncertainty data) / (G * 1.237e31 * eV^2)
     # See doc string for more details.
-    density = (2 * l1b_dataset["science_data"]) / (
+    phase_space_density = (2 * data) / (
         swe_constants.GEOMETRIC_FACTORS[np.newaxis, np.newaxis, np.newaxis, :]
         * swe_constants.VELOCITY_CONVERSION_FACTOR
         * particle_energy_data[:, :, :, np.newaxis] ** 2
     )
-    phase_space_density = density.data
 
     return phase_space_density
 
@@ -114,7 +115,7 @@ def calculate_flux(
     Parameters
     ----------
     phase_space_density : numpy.ndarray
-        The phase space density.
+        The phase space density of counts or uncertainty data.
     esa_energy : numpy.ndarray
         The energy values in eV.
 
@@ -131,6 +132,70 @@ def calculate_flux(
     return flux
 
 
+def put_uncertainty_into_angle_bins(
+    data: np.ndarray, angle_bin_indices: npt.NDArray[np.int_]
+) -> npt.NDArray:
+    """
+    Put uncertainty data in its angle bins.
+
+    This function bins uncertainty data into 30 predefined angle bins
+    while preserving the original energy step structure.
+
+    Since multiple data points can fall into the same angle bin,
+    this function computes the combined uncertainty for the bin.
+
+    Parameters
+    ----------
+    data : numpy.ndarray
+        Uncertainty data to put in bins. Shape:
+        (full_cycle_data, N_ESA_STEPS, N_ANGLE_BINS, N_CEMS).
+    angle_bin_indices : numpy.ndarray
+        Indices of angle bins to put data in. Shape:
+        (full_cycle_data, N_ESA_STEPS, N_ANGLE_BINS).
+
+    Returns
+    -------
+    numpy.ndarray
+        Data in bins. Shape:
+        (full_cycle_data, N_ESA_STEPS, N_ANGLE_BINS, N_CEMS).
+    """
+    # Initialize with zeros instead of NaN because np.add.at() does not
+    # work with nan values. It results in nan + value = nan
+    binned_data = np.zeros(
+        (
+            data.shape[0],
+            swe_constants.N_ESA_STEPS,
+            swe_constants.N_ANGLE_BINS,
+            swe_constants.N_CEMS,
+        ),
+        dtype=np.float64,
+    )
+
+    time_indices = np.arange(data.shape[0])[:, None, None]
+    energy_indices = np.arange(swe_constants.N_ESA_STEPS)[None, :, None]
+
+    # Calculate new uncertainty of each uncertainty data in the bins.
+    # Per SWE instruction:
+    #   At L1B, 'data' is result from sqrt(counts). Now in L2, average
+    #   uncertainty data using this formula:
+    #   sqrt(
+    #       sum(
+    #           (unc_1) ** 2 + (unc_2) ** 2 + ... + (unc_n) ** 2
+    #       )
+    #   )
+    # TODO: SWE want to add more defined formula based on spin data and
+    # counts uncertainty from it in the future.
+
+    # Use np.add.at() to put values into bins and add values in the bins into one.
+    # Here, we are applying power of 2 to each data point before summing them.
+    np.add.at(
+        binned_data,
+        (time_indices, energy_indices, angle_bin_indices),
+        data**2,
+    )
+    return np.sqrt(binned_data)
+
+
 def put_data_into_angle_bins(
     data: np.ndarray, angle_bin_indices: npt.NDArray[np.int_]
 ) -> npt.NDArray:
@@ -142,10 +207,8 @@ def put_data_into_angle_bins(
     full cycle, it assigns data to the corresponding angle bin
     based on the provided indices.
 
-    Since multiple data points may fall into the same angle bin,
-    the function accumulates values and computes the average across
-    all 7 CEMs, ensuring that each bin contains a representative
-    mean value while maintaining the 7 CEM structure.
+    Since multiple data points can fall into the same angle bin,
+    this function computes the combined averages.
 
     Parameters
     ----------
@@ -177,7 +240,7 @@ def put_data_into_angle_bins(
     time_indices = np.arange(data.shape[0])[:, None, None]
     energy_indices = np.arange(swe_constants.N_ESA_STEPS)[None, :, None]
 
-    # Use np.add.at() to accumulate values into bins
+    # Use np.add.at() to put values into bins and add values in the bins into one.
     np.add.at(binned_data, (time_indices, energy_indices, angle_bin_indices), data)
 
     # Count occurrences in each bin to compute the mean.
@@ -343,7 +406,9 @@ def swe_l2(l1b_dataset: xr.Dataset) -> xr.Dataset:
     # Calculate phase space density and flux. Store data in shape
     # (epoch, esa_step, spin_sector, cem_id). This is for L3 purposes.
     ############################################################
-    phase_space_density = calculate_phase_space_density(l1b_dataset)
+    phase_space_density = calculate_phase_space_density(
+        l1b_dataset["science_data"].data, l1b_dataset["esa_energy"].data
+    )
     dataset["phase_space_density_spin_sector"] = xr.DataArray(
         phase_space_density,
         name="phase_space_density_spin_sector",
@@ -419,4 +484,33 @@ def swe_l2(l1b_dataset: xr.Dataset) -> xr.Dataset:
         attrs=cdf_attributes.get_variable_attributes("phase_space_density"),
     )
 
+    #######################################################
+    # Calculate flux and phase space density of uncertainty data.
+    # Put uncertainty data in its angle bins.
+    #######################################################
+    # Calculate phase space density for uncertainty data.
+    phase_space_density_uncert = calculate_phase_space_density(
+        l1b_dataset["counts_stat_uncert"].data, l1b_dataset["esa_energy"].data
+    )
+    # Put uncertainty data into its spin angle bins and calculate new uncertainty
+    phase_space_density_uncert = put_uncertainty_into_angle_bins(
+        phase_space_density_uncert, spin_angle_bins_indices
+    )
+    dataset["psd_stat_uncert"] = xr.DataArray(
+        phase_space_density_uncert,
+        name="psd_stat_uncert",
+        dims=["epoch", "esa_step", "spin_sector", "cem_id"],
+        attrs=cdf_attributes.get_variable_attributes("psd_stat_uncert"),
+    )
+    # Calculate flux for uncertainty data.
+    flux_uncert = calculate_flux(
+        phase_space_density_uncert, l1b_dataset["esa_energy"].data
+    )
+    flux_uncert = put_uncertainty_into_angle_bins(flux_uncert, spin_angle_bins_indices)
+    dataset["flux_stat_uncert"] = xr.DataArray(
+        flux_uncert,
+        name="flux_stat_uncert",
+        dims=["epoch", "esa_step", "spin_sector", "cem_id"],
+        attrs=cdf_attributes.get_variable_attributes("flux_stat_uncert"),
+    )
     return dataset

imap_processing/ultra/l0/decom_tools.py

@@ -53,15 +53,15 @@ def log_decompression(value: int, mantissa_bit_length: int) -> int:
     """
     Perform logarithmic decompression on an integer.
 
-    Supports both 16-bit and 8-bit formats based on the specified
+    Supports 16-bit, 10-bit, and 8-bit formats based on the specified
     mantissa bit length.
 
     Parameters
     ----------
     value : int
-        An integer comprised of a 4-bit exponent followed by a variable-length mantissa.
+        An integer comprised of an exponent followed by a mantissa.
     mantissa_bit_length : int
-        The bit length of the mantissa (default is 12 for 16-bit format).
+        The bit length of the mantissa.
 
     Returns
     -------
@@ -72,6 +72,9 @@ def log_decompression(value: int, mantissa_bit_length: int) -> int:
     if mantissa_bit_length == 12:
         base_value = 4096
         mantissa_mask = 0xFFF
+    elif mantissa_bit_length == 5:
+        base_value = 32
+        mantissa_mask = 0x1F
     elif mantissa_bit_length == 4:
         base_value = 16
         mantissa_mask = 0x0F
@@ -239,6 +242,7 @@ def decompress_image(
 def read_image_raw_events_binary(
     event_data: bytes,
     count: int,
+    field_ranges: dict,
 ) -> NDArray:
     """
     Convert contents of binary string 'EVENTDATA' into values.
@@ -249,6 +253,8 @@ def read_image_raw_events_binary(
         Event data.
     count : int
         Number of events.
+    field_ranges : dict
+        Field ranges for the event data.
 
     Returns
     -------
@@ -256,15 +262,16 @@ def read_image_raw_events_binary(
         Event data.
     """
     binary = convert_to_binary_string(event_data)
-    # 166 bits per event
-    event_length = 166 if count else 0
+    length = max(end for (_, end) in field_ranges.values())
+    # bits per event
+    event_length = length if count else 0
     event_data_list = []
 
     # For all packets with event data, parses the binary string
     for i in range(count):
         start_index = i * event_length
         event_binary = binary[start_index : start_index + event_length]
-        parsed_event = parse_event(event_binary)
+        parsed_event = parse_event(event_binary, field_ranges)
         event_data_list.append(parsed_event)
 
     return np.array(event_data_list)