imap-processing 0.16.2__py3-none-any.whl → 0.18.0__py3-none-any.whl

imap_processing/swapi/l2/swapi_l2.py

@@ -16,27 +16,73 @@ TIME_PER_BIN = 0.167  # seconds
 
 
 def solve_full_sweep_energy(
-    esa_lvl5_data: np.ndarray, esa_table_df: pd.DataFrame, lut_notes_df: pd.DataFrame
+    esa_lvl5_data: np.ndarray,
+    sweep_table: np.ndarray,
+    esa_table_df: pd.DataFrame,
+    lut_notes_df: pd.DataFrame,
+    data_time: npt.NDArray[np.datetime64],
 ) -> npt.NDArray:
     """
     Calculate the energy of each full sweep data.
 
+    Get the fixed energy values for steps 0-62 using the
+    esa_table_df information. It's important to ensure
+    that the correct fixed energy values are selected for
+    the specified time, as the sweep table can contain
+    different values depending on the operational phase
+    (e.g., I+T, pre-launch, post-launch). There may be
+    more fixed energy added in the future. TODO: add
+    document section once SWAPI document is updated.
+
+    Now, find the last 9 fine energy values using steps
+    noted in the section x in the algorithm document.
+
     Parameters
     ----------
     esa_lvl5_data : numpy.ndarray
         The L1 data input.
+    sweep_table : numpy.ndarray
+        Sweep table information.
     esa_table_df : pandas.DataFrame
         The ESA unit conversion table that contains first 63 energies.
     lut_notes_df : pandas.DataFrame
         The LUT notes table that contains the last 9 fine energies.
+    data_time : numpy.ndarray
+        The collection time of the data.
 
     Returns
     -------
     energy : numpy.ndarray
         The energy of each full sweep data.
     """
-    # Read 0 - 62 energy steps' fixed energy value
-    fixed_energy_values = esa_table_df["Energy"].values[:63]
+    # Convert timestamp from string to datetime
+    # and to the same format as data_time
+    esa_table_df["timestamp"] = pd.to_datetime(
+        esa_table_df["timestamp"], format="%m/%d/%Y %H:%M"
+    )
+    esa_table_df["timestamp"] = esa_table_df["timestamp"].to_numpy(
+        dtype="datetime64[ns]"
+    )
+
+    first_63_energies = []
+
+    for time, sweep_id in zip(data_time, sweep_table):
+        # Find the sweep's ESA data for the given time and sweep_id
+        subset = esa_table_df[
+            (esa_table_df["timestamp"] <= time) & (esa_table_df["Sweep #"] == sweep_id)
+        ]
+        if subset.empty:
+            first_63_energies.append(np.full(63, np.nan, dtype=np.float64))
+            continue
+
+        # Subset data can contain multiple 72 energy values with last 9 fine energies
+        # with 'Solve' value. We need to sort by time and ESA step to maintain correct
+        # order. Then take the last group of 72 steps values and select first 63
+        # values only.
+        subset = subset.sort_values(["timestamp", "ESA Step #"])
+        grouped = subset["Energy"].values.reshape(-1, 72)
+        first_63 = grouped[-1, :63]
+        first_63_energies.append(first_63)
 
     # Find last 9 fine energy values of all sweeps data
     # -------------------------------------------------
@@ -96,13 +142,9 @@ def solve_full_sweep_energy(
     # order it should be in:
     #  [64, 65, 66, 67, 68, 69, 70, 71, 72]
     energy_values = np.flip(energy_values, axis=1)
-    # Expand to match the number of rows in energy_values
-    first_63_values = np.tile(
-        fixed_energy_values, (energy_values.shape[0], 1)
-    )  # (epoch, 63)
 
     # Append the first_63_values in front of energy_values
-    sweeps_energy_value = np.hstack((first_63_values, energy_values))
+    sweeps_energy_value = np.hstack([first_63_energies, energy_values])
 
     return sweeps_energy_value
 
@@ -149,15 +191,16 @@ def swapi_l2(
     # Copy over only certain variables from L1 to L2 dataset
     l1_data_keys = [
         "epoch",
+        "esa_lvl5",
         "esa_step",
         "esa_step_label",
-        "swp_l1a_flags",
-        "sweep_table",
-        "plan_id",
-        "lut_choice",
-        "fpga_type",
         "fpga_rev",
-        "esa_lvl5",
+        "fpga_type",
+        "lut_choice",
+        "plan_id",
+        "sci_start_time",
+        "sweep_table",
+        "swp_l1a_flags",
     ]
     l2_dataset = l1_dataset[l1_data_keys]
 
@@ -168,8 +211,10 @@ def swapi_l2(
     esa_lvl5_hex = np.vectorize(lambda x: format(x, "X"))(l1_dataset["esa_lvl5"].values)
     esa_energy = solve_full_sweep_energy(
         esa_lvl5_hex,
+        l1_dataset["sweep_table"].data,
         esa_table_df=esa_table_df,
         lut_notes_df=lut_notes_df,
+        data_time=np.array(l1_dataset["epoch"].data, dtype="datetime64[ns]"),
     )
 
     l2_dataset["swp_esa_energy"] = xr.DataArray(

imap_processing/swapi/swapi_utils.py

@@ -51,7 +51,7 @@ def read_swapi_lut_table(file_path: Path) -> pd.DataFrame:
         .astype(str)
         .str.replace(",", "", regex=False)
         .replace("Solve", -1)
-        .astype(np.int64)
+        .astype(np.float64)
     )
 
     return df

imap_processing/swe/l1b/swe_l1b.py

@@ -761,12 +761,14 @@ def swe_l1b_science(dependencies: ProcessingInputCollection) -> xr.Dataset:
 
     count_rate = convert_counts_to_rate(inflight_applied_count, acq_duration)
 
+    # Statistical uncertainty is sqrt(decompressed counts)
+    # TODO: Update this if SWE like to include deadtime correciton.
+    counts_stat_uncert = np.sqrt(populated_data["science_data"])
+
     # Store ESA energies of full cycle for L2 purposes.
     esa_energies = get_esa_energy_pattern(esa_lut_files[0])
-    # Repeat energies to be in the same shape as the science data
-    esa_energies = np.repeat(esa_energies, total_packets // 4).reshape(
-        -1, swe_constants.N_ESA_STEPS, swe_constants.N_ANGLE_SECTORS
-    )
+    # Repeat the (24, 30) energy pattern n_cycles times along a new first axis
+    esa_energies = np.repeat(esa_energies[np.newaxis, :, :], total_packets // 4, axis=0)
     # Convert voltage to electron energy in eV by apply conversion factor
     esa_energies = esa_energies * swe_constants.ENERGY_CONVERSION_FACTOR
     # ------------------------------------------------------------------
@@ -894,6 +896,11 @@ def swe_l1b_science(dependencies: ProcessingInputCollection) -> xr.Dataset:
         dims=["epoch", "esa_step", "spin_sector", "cem_id"],
         attrs=cdf_attrs.get_variable_attributes("science_data"),
     )
+    science_dataset["counts_stat_uncert"] = xr.DataArray(
+        counts_stat_uncert,
+        dims=["epoch", "esa_step", "spin_sector", "cem_id"],
+        attrs=cdf_attrs.get_variable_attributes("counts_stat_uncert"),
+    )
     science_dataset["acquisition_time"] = xr.DataArray(
         acq_time,
         dims=["epoch", "esa_step", "spin_sector"],

imap_processing/swe/l2/swe_l2.py

@@ -14,9 +14,11 @@ from imap_processing.spice.spin import get_instrument_spin_phase, get_spin_angle
 from imap_processing.swe.utils import swe_constants
 
 
-def calculate_phase_space_density(l1b_dataset: xr.Dataset) -> npt.NDArray:
+def calculate_phase_space_density(
+    data: np.ndarray, particle_energy_data: np.ndarray
+) -> npt.NDArray:
     """
-    Convert counts to phase space density.
+    Convert counts or uncertainty data to phase space density.
 
     Calculate phase space density is represented by this symbol, fv.
     Its unit is s^3/ (cm^6 * ster).
@@ -49,8 +51,11 @@ def calculate_phase_space_density(l1b_dataset: xr.Dataset) -> npt.NDArray:
 
     Parameters
     ----------
-    l1b_dataset : xarray.Dataset
-        The L1B dataset to process.
+    data : numpy.ndarray
+        The data to process. Two expected inputs are counts or uncertainty data.
+    particle_energy_data : numpy.ndarray
+        The energy values in eV. This is the energy values from the
+        "esa_energy" variable in the L1B dataset.
 
     Returns
     -------
@@ -58,18 +63,14 @@ def calculate_phase_space_density(l1b_dataset: xr.Dataset) -> npt.NDArray:
         Phase space density. We need to call this phase space density because
         there will be density in L3 processing.
     """
-    # Get energy values.
-    particle_energy_data = l1b_dataset["esa_energy"].values
-
     # Calculate phase space density using formula:
-    #   2 * (C/tau) / (G * 1.237e31 * eV^2)
+    #   2 * ((C/tau) or uncertainty data) / (G * 1.237e31 * eV^2)
     # See doc string for more details.
-    density = (2 * l1b_dataset["science_data"]) / (
+    phase_space_density = (2 * data) / (
         swe_constants.GEOMETRIC_FACTORS[np.newaxis, np.newaxis, np.newaxis, :]
         * swe_constants.VELOCITY_CONVERSION_FACTOR
         * particle_energy_data[:, :, :, np.newaxis] ** 2
     )
-    phase_space_density = density.data
 
     return phase_space_density
 
@@ -114,7 +115,7 @@ def calculate_flux(
     Parameters
     ----------
     phase_space_density : numpy.ndarray
-        The phase space density.
+        The phase space density of counts or uncertainty data.
     esa_energy : numpy.ndarray
         The energy values in eV.
 
@@ -131,6 +132,70 @@ def calculate_flux(
     return flux
 
 
+def put_uncertainty_into_angle_bins(
+    data: np.ndarray, angle_bin_indices: npt.NDArray[np.int_]
+) -> npt.NDArray:
+    """
+    Put uncertainty data in its angle bins.
+
+    This function bins uncertainty data into 30 predefined angle bins
+    while preserving the original energy step structure.
+
+    Since multiple data points can fall into the same angle bin,
+    this function computes the combined uncertainty for the bin.
+
+    Parameters
+    ----------
+    data : numpy.ndarray
+        Uncertainty data to put in bins. Shape:
+        (full_cycle_data, N_ESA_STEPS, N_ANGLE_BINS, N_CEMS).
+    angle_bin_indices : numpy.ndarray
+        Indices of angle bins to put data in. Shape:
+        (full_cycle_data, N_ESA_STEPS, N_ANGLE_BINS).
+
+    Returns
+    -------
+    numpy.ndarray
+        Data in bins. Shape:
+        (full_cycle_data, N_ESA_STEPS, N_ANGLE_BINS, N_CEMS).
+    """
+    # Initialize with zeros instead of NaN because np.add.at() does not
+    # work with nan values. It results in nan + value = nan
+    binned_data = np.zeros(
+        (
+            data.shape[0],
+            swe_constants.N_ESA_STEPS,
+            swe_constants.N_ANGLE_BINS,
+            swe_constants.N_CEMS,
+        ),
+        dtype=np.float64,
+    )
+
+    time_indices = np.arange(data.shape[0])[:, None, None]
+    energy_indices = np.arange(swe_constants.N_ESA_STEPS)[None, :, None]
+
+    # Calculate new uncertainty of each uncertainty data in the bins.
+    # Per SWE instruction:
+    #   At L1B, 'data' is result from sqrt(counts). Now in L2, average
+    #   uncertainty data using this formula:
+    #   sqrt(
+    #       sum(
+    #           (unc_1) ** 2 + (unc_2) ** 2 + ... + (unc_n) ** 2
+    #       )
+    #   )
+    # TODO: SWE want to add more defined formula based on spin data and
+    # counts uncertainty from it in the future.
+
+    # Use np.add.at() to put values into bins and add values in the bins into one.
+    # Here, we are applying power of 2 to each data point before summing them.
+    np.add.at(
+        binned_data,
+        (time_indices, energy_indices, angle_bin_indices),
+        data**2,
+    )
+    return np.sqrt(binned_data)
+
+
 def put_data_into_angle_bins(
     data: np.ndarray, angle_bin_indices: npt.NDArray[np.int_]
 ) -> npt.NDArray:
@@ -142,10 +207,8 @@ def put_data_into_angle_bins(
     full cycle, it assigns data to the corresponding angle bin
     based on the provided indices.
 
-    Since multiple data points may fall into the same angle bin,
-    the function accumulates values and computes the average across
-    all 7 CEMs, ensuring that each bin contains a representative
-    mean value while maintaining the 7 CEM structure.
+    Since multiple data points can fall into the same angle bin,
+    this function computes the combined averages.
 
     Parameters
     ----------
@@ -177,7 +240,7 @@ def put_data_into_angle_bins(
     time_indices = np.arange(data.shape[0])[:, None, None]
     energy_indices = np.arange(swe_constants.N_ESA_STEPS)[None, :, None]
 
-    # Use np.add.at() to accumulate values into bins
+    # Use np.add.at() to put values into bins and add values in the bins into one.
     np.add.at(binned_data, (time_indices, energy_indices, angle_bin_indices), data)
 
     # Count occurrences in each bin to compute the mean.
@@ -343,7 +406,9 @@ def swe_l2(l1b_dataset: xr.Dataset) -> xr.Dataset:
     # Calculate phase space density and flux. Store data in shape
     # (epoch, esa_step, spin_sector, cem_id). This is for L3 purposes.
     ############################################################
-    phase_space_density = calculate_phase_space_density(l1b_dataset)
+    phase_space_density = calculate_phase_space_density(
+        l1b_dataset["science_data"].data, l1b_dataset["esa_energy"].data
+    )
     dataset["phase_space_density_spin_sector"] = xr.DataArray(
         phase_space_density,
         name="phase_space_density_spin_sector",
@@ -419,4 +484,33 @@ def swe_l2(l1b_dataset: xr.Dataset) -> xr.Dataset:
         attrs=cdf_attributes.get_variable_attributes("phase_space_density"),
     )
 
+    #######################################################
+    # Calculate flux and phase space density of uncertainty data.
+    # Put uncertainty data in its angle bins.
+    #######################################################
+    # Calculate phase space density for uncertainty data.
+    phase_space_density_uncert = calculate_phase_space_density(
+        l1b_dataset["counts_stat_uncert"].data, l1b_dataset["esa_energy"].data
+    )
+    # Put uncertainty data into its spin angle bins and calculate new uncertainty
+    phase_space_density_uncert = put_uncertainty_into_angle_bins(
+        phase_space_density_uncert, spin_angle_bins_indices
+    )
+    dataset["psd_stat_uncert"] = xr.DataArray(
+        phase_space_density_uncert,
+        name="psd_stat_uncert",
+        dims=["epoch", "esa_step", "spin_sector", "cem_id"],
+        attrs=cdf_attributes.get_variable_attributes("psd_stat_uncert"),
+    )
+    # Calculate flux for uncertainty data.
+    flux_uncert = calculate_flux(
+        phase_space_density_uncert, l1b_dataset["esa_energy"].data
+    )
+    flux_uncert = put_uncertainty_into_angle_bins(flux_uncert, spin_angle_bins_indices)
+    dataset["flux_stat_uncert"] = xr.DataArray(
+        flux_uncert,
+        name="flux_stat_uncert",
+        dims=["epoch", "esa_step", "spin_sector", "cem_id"],
+        attrs=cdf_attributes.get_variable_attributes("flux_stat_uncert"),
+    )
     return dataset

imap_processing/ultra/constants.py

@@ -80,4 +80,52 @@ class UltraConstants:
     CULLING_RPM_MAX = 6.0
 
     # Thresholds for culling based on counts.
-    CULLING_ENERGY_BIN_EDGES: ClassVar[list] = [0, 10, 20, 1e5]
+    CULLING_ENERGY_BIN_EDGES: ClassVar[list] = [
+        3.385,
+        4.13722222222222,
+        4.13722222222222,
+        5.05660493827161,
+        5.05660493827161,
+        6.18029492455419,
+        6.18029492455419,
+        7.55369379667734,
+        7.55369379667734,
+        9.23229241816119,
+        9.23229241816119,
+        11.2839129555303,
+        11.2839129555303,
+        13.7914491678704,
+        13.7914491678704,
+        16.8562156496194,
+        16.8562156496194,
+        20.6020413495348,
+        20.6020413495348,
+        25.1802727605426,
+        25.1802727605426,
+        30.775888929552,
+        30.775888929552,
+        37.6149753583414,
+        37.6149753583414,
+        45.9738587713061,
+        45.9738587713061,
+        56.1902718315964,
+        56.1902718315964,
+        68.6769989052845,
+        68.6769989052845,
+        83.93855421757,
+        83.93855421757,
+        102.591566265919,
+        102.591566265919,
+        125.38969210279,
+        125.38969210279,
+        153.254068125632,
+        153.254068125632,
+        187.310527709106,
+        187.310527709106,
+        228.93508942224,
+        228.93508942224,
+        279.809553738294,
+        279.809553738294,
+        341.989454569026,
+        1e5,
+    ]

imap_processing/ultra/l0/decom_tools.py

@@ -4,6 +4,7 @@ import numpy as np
 from numpy.typing import NDArray
 
 from imap_processing.ultra.l0.ultra_utils import (
+    PacketProperties,
     parse_event,
 )
 from imap_processing.utils import convert_to_binary_string
@@ -53,15 +54,15 @@ def log_decompression(value: int, mantissa_bit_length: int) -> int:
     """
     Perform logarithmic decompression on an integer.
 
-    Supports both 16-bit and 8-bit formats based on the specified
+    Supports 16-bit, 10-bit, and 8-bit formats based on the specified
     mantissa bit length.
 
     Parameters
     ----------
     value : int
-        An integer comprised of a 4-bit exponent followed by a variable-length mantissa.
+        An integer comprised of an exponent followed by a mantissa.
     mantissa_bit_length : int
-        The bit length of the mantissa (default is 12 for 16-bit format).
+        The bit length of the mantissa.
 
     Returns
     -------
@@ -72,6 +73,9 @@ def log_decompression(value: int, mantissa_bit_length: int) -> int:
     if mantissa_bit_length == 12:
         base_value = 4096
         mantissa_mask = 0xFFF
+    elif mantissa_bit_length == 5:
+        base_value = 32
+        mantissa_mask = 0x1F
     elif mantissa_bit_length == 4:
         base_value = 16
         mantissa_mask = 0x0F
@@ -152,8 +156,7 @@ def decompress_binary(
 def decompress_image(
     pixel0: int,
     binary_data: str,
-    width_bit: int,
-    mantissa_bit_length: int,
+    packet_props: PacketProperties,
 ) -> NDArray:
     """
     Will decompress a binary string representing an image into a matrix of pixel values.
@@ -168,10 +171,9 @@ def decompress_image(
         The first, unmodified pixel p0,0.
     binary_data : str
         Binary string.
-    width_bit : int
-        The bit width that describes the width of data in the block.
-    mantissa_bit_length : int
-        The bit length of the mantissa.
+    packet_props : PacketProperties
+        Properties of the packet, including width bit, mantissa bit length and pixel
+        window dimensions.
 
     Returns
     -------
@@ -184,10 +186,18 @@ def decompress_image(
     This process is described starting on page 168 in IMAP-Ultra Flight
     Software Specification document.
     """
-    rows = 54
-    cols = 180
+    rows = packet_props.pixel_window_rows
+    cols = packet_props.pixel_window_columns
+    width_bit = packet_props.width
+    mantissa_bit_length = packet_props.mantissa_bit_length
     pixels_per_block = 15
 
+    if width_bit is None or rows is None or cols is None or mantissa_bit_length is None:
+        raise ValueError(
+            "Packet properties must specify pixel window dimensions, "
+            "width bit, and mantissa bit length for this packet type."
+        )
+
     blocks_per_row = cols // pixels_per_block
 
     # Compressed pixel matrix
@@ -239,6 +249,7 @@ def decompress_image(
 def read_image_raw_events_binary(
     event_data: bytes,
     count: int,
+    field_ranges: dict,
 ) -> NDArray:
     """
     Convert contents of binary string 'EVENTDATA' into values.
@@ -249,6 +260,8 @@ def read_image_raw_events_binary(
         Event data.
     count : int
         Number of events.
+    field_ranges : dict
+        Field ranges for the event data.
 
     Returns
     -------
@@ -256,15 +269,16 @@ def read_image_raw_events_binary(
         Event data.
     """
     binary = convert_to_binary_string(event_data)
-    # 166 bits per event
-    event_length = 166 if count else 0
+    length = max(end for (_, end) in field_ranges.values())
+    # bits per event
+    event_length = length if count else 0
     event_data_list = []
 
     # For all packets with event data, parses the binary string
     for i in range(count):
         start_index = i * event_length
         event_binary = binary[start_index : start_index + event_length]
-        parsed_event = parse_event(event_binary)
+        parsed_event = parse_event(event_binary, field_ranges)
         event_data_list.append(parsed_event)
 
     return np.array(event_data_list)