imap-processing 0.14.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

imap_processing/swe/l1a/swe_l1a.py

@@ -58,13 +58,13 @@ def swe_l1a(packet_file: str) -> xr.Dataset:
 
     if SWEAPID.SWE_APP_HK in datasets_by_apid:
         logger.info("Processing SWE housekeeping data.")
-        hk_ds = datasets_by_apid[SWEAPID.SWE_APP_HK]
-        hk_ds.attrs.update(imap_attrs.get_global_attributes("imap_swe_l1a_hk"))
-        hk_ds["epoch"].attrs.update(epoch_attrs)
+        l1a_hk_ds = datasets_by_apid[SWEAPID.SWE_APP_HK]
+        l1a_hk_ds.attrs.update(imap_attrs.get_global_attributes("imap_swe_l1a_hk"))
+        l1a_hk_ds["epoch"].attrs.update(epoch_attrs)
         # Add attrs to HK data variables
-        for var_name in hk_ds.data_vars:
-            hk_ds[var_name].attrs.update(non_science_attrs)
-        processed_data.append(hk_ds)
+        for var_name in l1a_hk_ds.data_vars:
+            l1a_hk_ds[var_name].attrs.update(non_science_attrs)
+        processed_data.append(l1a_hk_ds)
 
     if SWEAPID.SWE_CEM_RAW in datasets_by_apid:
         logger.info("Processing SWE CEM raw data.")

imap_processing/swe/l1b/swe_l1b.py

@@ -10,6 +10,7 @@ import pandas as pd
 import xarray as xr
 from imap_data_access.processing_input import ProcessingInputCollection
 
+from imap_processing import imap_module_directory
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.cdf.utils import load_cdf
 from imap_processing.spice.time import met_to_ttj2000ns
@@ -20,7 +21,7 @@ from imap_processing.swe.utils.swe_utils import (
     combine_acquisition_time,
     read_lookup_table,
 )
-from imap_processing.utils import convert_raw_to_eu
+from imap_processing.utils import convert_raw_to_eu, packet_file_to_datasets
 
 logger = logging.getLogger(__name__)
 
@@ -628,7 +629,7 @@ def filter_full_cycle_data(
     return l1a_data
 
 
-def swe_l1b(dependencies: ProcessingInputCollection) -> xr.Dataset:
+def swe_l1b_science(dependencies: ProcessingInputCollection) -> xr.Dataset:
     """
     SWE l1b science processing.
 
@@ -641,7 +642,7 @@ def swe_l1b(dependencies: ProcessingInputCollection) -> xr.Dataset:
     Returns
     -------
     dataset : xarray.Dataset
-        Processed l1b data.
+        Processed l1b science data.
     """
     # Read science data
     science_files = dependencies.get_file_paths(descriptor="sci")
@@ -872,8 +873,8 @@ def swe_l1b(dependencies: ProcessingInputCollection) -> xr.Dataset:
     #   4 rows --> metadata for each full cycle. Each element of 4 maps to
     #              metadata of one quarter cycle.
 
-    # Create the dataset
-    dataset = xr.Dataset(
+    # Create the science dataset
+    science_dataset = xr.Dataset(
         coords={
             "epoch": epoch_time,
             "esa_step": esa_step,
@@ -888,23 +889,23 @@ def swe_l1b(dependencies: ProcessingInputCollection) -> xr.Dataset:
         attrs=cdf_attrs.get_global_attributes("imap_swe_l1b_sci"),
     )
 
-    dataset["science_data"] = xr.DataArray(
+    science_dataset["science_data"] = xr.DataArray(
         count_rate,
         dims=["epoch", "esa_step", "spin_sector", "cem_id"],
         attrs=cdf_attrs.get_variable_attributes("science_data"),
     )
-    dataset["acquisition_time"] = xr.DataArray(
+    science_dataset["acquisition_time"] = xr.DataArray(
         acq_time,
         dims=["epoch", "esa_step", "spin_sector"],
         attrs=cdf_attrs.get_variable_attributes("acquisition_time"),
     )
-    dataset["acq_duration"] = xr.DataArray(
+    science_dataset["acq_duration"] = xr.DataArray(
         acq_duration,
         dims=["epoch", "esa_step", "spin_sector"],
         attrs=cdf_attrs.get_variable_attributes("acq_duration"),
     )
 
-    dataset["esa_energy"] = xr.DataArray(
+    science_dataset["esa_energy"] = xr.DataArray(
         esa_energies,
         dims=["epoch", "esa_step", "spin_sector"],
         attrs=cdf_attrs.get_variable_attributes("esa_energy"),
@@ -915,11 +916,69 @@ def swe_l1b(dependencies: ProcessingInputCollection) -> xr.Dataset:
         if key in ["science_data", "acq_duration"]:
             continue
         varname = key.lower()
-        dataset[varname] = xr.DataArray(
+        science_dataset[varname] = xr.DataArray(
             value.data.reshape(-1, swe_constants.N_QUARTER_CYCLES),
             dims=["epoch", "cycle"],
             attrs=cdf_attrs.get_variable_attributes(varname),
         )
 
     logger.info("SWE L1b science processing completed")
-    return [dataset]
+    return science_dataset
+
+
+def swe_l1b(dependencies: ProcessingInputCollection) -> list[xr.Dataset]:
+    """
+    SWE L1B processing.
+
+    Parameters
+    ----------
+    dependencies : ProcessingInputCollection
+        Object containing lists of dependencies that CLI dependency
+        parameter received.
+
+    Returns
+    -------
+    list[xarray.Dataset]
+        List of processed datasets.
+    """
+    processed_datasets = []
+    has_science_data = dependencies.get_file_paths(descriptor="sci")
+    if has_science_data:
+        # Process science data to L1B
+        science_dataset = swe_l1b_science(dependencies)
+        processed_datasets.append(science_dataset)
+
+    # Process HK data using L0 file
+    l0_files = dependencies.get_file_paths(descriptor="raw")
+    if l0_files:
+        xtce_document = (
+            f"{imap_module_directory}/swe/packet_definitions/swe_packet_definition.xml"
+        )
+        datasets_by_apid = packet_file_to_datasets(
+            l0_files[0], xtce_document, use_derived_value=True
+        )
+        if SWEAPID.SWE_APP_HK in datasets_by_apid:
+            # Define minimal CDF attrs for the HK dataset
+            imap_attrs = ImapCdfAttributes()
+            imap_attrs.add_instrument_global_attrs("swe")
+            imap_attrs.add_instrument_variable_attrs("swe", "l1b")
+            hk_attrs = imap_attrs.get_variable_attributes("l1b_hk_attrs")
+            hk_str_attrs = imap_attrs.get_variable_attributes("l1b_hk_string_attrs")
+            epoch_attrs = imap_attrs.get_variable_attributes(
+                "epoch", check_schema=False
+            )
+
+            l1b_hk_dataset = datasets_by_apid[SWEAPID.SWE_APP_HK]
+            # Update CDF attrs
+            l1b_hk_dataset["epoch"].attrs.update(epoch_attrs)
+            l1b_hk_dataset.attrs.update(
+                imap_attrs.get_global_attributes("imap_swe_l1b_hk")
+            )
+            for hk_var in l1b_hk_dataset.data_vars:
+                if isinstance(l1b_hk_dataset[hk_var].data[0], str):
+                    l1b_hk_dataset[hk_var].attrs.update(hk_str_attrs)
+                else:
+                    l1b_hk_dataset[hk_var].attrs.update(hk_attrs)
+            processed_datasets.append(l1b_hk_dataset)
+
+    return processed_datasets

imap_processing/ultra/l0/decom_ultra.py

@@ -189,7 +189,7 @@ def process_ultra_events(ds: xr.Dataset) -> xr.Dataset:
     }
 
     # Add the event data to the expanded dataset.
-    for key in event_data_list[0]:
+    for key in EVENT_FIELD_RANGES:
         expanded_data[key] = np.array([event[key] for event in all_events])
 
     event_ids = get_event_id(expanded_data["shcoarse"])

imap_processing/ultra/l0/ultra_utils.py

@@ -65,7 +65,6 @@ ULTRA_HK = PacketProperties(
     apid=[
         866,
         867,
-        868,
         869,
         870,
         873,
@@ -74,7 +73,6 @@ ULTRA_HK = PacketProperties(
         877,
         930,
         931,
-        932,
         933,
         934,
         937,
@@ -85,7 +83,6 @@ ULTRA_HK = PacketProperties(
     logical_source=[
         "imap_ultra_l1a_45sensor-alarm",
         "imap_ultra_l1a_45sensor-memchecksum",
-        "imap_ultra_l1a_45sensor-memdump",
         "imap_ultra_l1a_45sensor-status",
         "imap_ultra_l1a_45sensor-bootstatus",
         "imap_ultra_l1a_45sensor-monitorlimits",
@@ -94,7 +91,6 @@ ULTRA_HK = PacketProperties(
         "imap_ultra_l1a_45sensor-imgparams",
         "imap_ultra_l1a_90sensor-alarm",
         "imap_ultra_l1a_90sensor-memchecksum",
-        "imap_ultra_l1a_90sensor-memdump",
         "imap_ultra_l1a_90sensor-status",
         "imap_ultra_l1a_90sensor-bootstatus",
         "imap_ultra_l1a_90sensor-monitorlimits",

imap_processing/ultra/l1b/badtimes.py

@@ -4,7 +4,7 @@ import numpy as np
 import xarray as xr
 from numpy.typing import NDArray
 
-from imap_processing.ultra.utils.ultra_l1_utils import create_dataset
+from imap_processing.ultra.utils.ultra_l1_utils import create_dataset, extract_data_dict
 
 FILLVAL_UINT16 = 65535
 FILLVAL_FLOAT64 = -1.0e31
@@ -36,10 +36,14 @@ def calculate_badtimes(
     culled_spins = np.setdiff1d(
         extendedspin_dataset["spin_number"].values, cullingmask_spins
     )
-
+    extendedspin_dataset = extendedspin_dataset.assign_coords(
+        epoch=("spin_number", extendedspin_dataset["epoch"].values)
+    )
     filtered_dataset = extendedspin_dataset.sel(spin_number=culled_spins)
 
-    badtimes_dataset = create_dataset(filtered_dataset, name, "l1b")
+    data_dict = extract_data_dict(filtered_dataset)
+
+    badtimes_dataset = create_dataset(data_dict, name, "l1b")
 
     if badtimes_dataset["spin_number"].size == 0:
         badtimes_dataset = badtimes_dataset.drop_dims("spin_number")

imap_processing/ultra/l1b/cullingmask.py

@@ -4,7 +4,7 @@ import numpy as np
 import xarray as xr
 
 from imap_processing.quality_flags import ImapAttitudeUltraFlags, ImapRatesUltraFlags
-from imap_processing.ultra.utils.ultra_l1_utils import create_dataset
+from imap_processing.ultra.utils.ultra_l1_utils import create_dataset, extract_data_dict
 
 FILLVAL_UINT16 = 65535
 FILLVAL_FLOAT64 = -1.0e31
@@ -44,11 +44,16 @@ def calculate_cullingmask(extendedspin_dataset: xr.Dataset, name: str) -> xr.Dat
             == 0
         ).all(dim="energy_bin_geometric_mean")
     )
+    extendedspin_dataset = extendedspin_dataset.assign_coords(
+        epoch=("spin_number", extendedspin_dataset["epoch"].values)
+    )
     filtered_dataset = extendedspin_dataset.sel(
         spin_number=extendedspin_dataset["spin_number"][good_mask]
     )
 
-    cullingmask_dataset = create_dataset(filtered_dataset, name, "l1b")
+    data_dict = extract_data_dict(filtered_dataset)
+
+    cullingmask_dataset = create_dataset(data_dict, name, "l1b")
 
     if cullingmask_dataset["spin_number"].size == 0:
         cullingmask_dataset = cullingmask_dataset.drop_dims("spin_number")

imap_processing/ultra/l1b/de.py

@@ -34,7 +34,9 @@ FILLVAL_UINT8 = 255
 FILLVAL_FLOAT32 = -1.0e31
 
 
-def calculate_de(de_dataset: xr.Dataset, name: str) -> xr.Dataset:
+def calculate_de(
+    de_dataset: xr.Dataset, name: str, ancillary_files: dict
+) -> xr.Dataset:
     """
     Create dataset with defined datatypes for Direct Event Data.
 
@@ -44,6 +46,8 @@ def calculate_de(de_dataset: xr.Dataset, name: str) -> xr.Dataset:
         L1a dataset containing direct event data.
     name : str
         Name of the l1a dataset.
+    ancillary_files : dict
+        Ancillary files.
 
     Returns
     -------
@@ -106,7 +110,11 @@ def calculate_de(de_dataset: xr.Dataset, name: str) -> xr.Dataset:
     energy = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32)
     species_bin = np.full(len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8)
     t2 = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32)
-    event_times = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float64)
+    event_times = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32)
+    shape = (len(de_dataset["epoch"]), 3)
+    sc_velocity = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32)
+    sc_dps_velocity = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32)
+    helio_velocity = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32)
     spin_starts = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float64)
     spin_period_sec = np.full(
         len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float64
@@ -224,13 +232,21 @@ def calculate_de(de_dataset: xr.Dataset, name: str) -> xr.Dataset:
 
     # Annotated Events.
     ultra_frame = getattr(SpiceFrame, f"IMAP_ULTRA_{sensor}")
-    sc_velocity, sc_dps_velocity, helio_velocity = get_annotated_particle_velocity(
-        event_times,
-        de_dict["direct_event_velocity"],
-        ultra_frame,
-        SpiceFrame.IMAP_DPS,
-        SpiceFrame.IMAP_SPACECRAFT,
-    )
+
+    # Account for counts=0 (event times have FILL value)
+    valid_events = event_times != FILLVAL_FLOAT32
+    if np.any(valid_events):
+        (
+            sc_velocity[valid_events],
+            sc_dps_velocity[valid_events],
+            helio_velocity[valid_events],
+        ) = get_annotated_particle_velocity(
+            event_times[valid_events],
+            de_dict["direct_event_velocity"][valid_events],
+            ultra_frame,
+            SpiceFrame.IMAP_DPS,
+            SpiceFrame.IMAP_SPACECRAFT,
+        )
 
     de_dict["velocity_sc"] = sc_velocity
     de_dict["velocity_dps_sc"] = sc_dps_velocity
@@ -247,9 +263,7 @@ def calculate_de(de_dataset: xr.Dataset, name: str) -> xr.Dataset:
         de_dict["theta"],
     )
     de_dict["event_efficiency"] = get_efficiency(
-        de_dict["tof_energy"],
-        de_dict["phi"],
-        de_dict["theta"],
+        de_dict["tof_energy"], de_dict["phi"], de_dict["theta"], ancillary_files
     )
 
     dataset = create_dataset(de_dict, name, "l1b")

imap_processing/ultra/l1b/lookup_utils.py

@@ -206,23 +206,24 @@ def get_angular_profiles(start_type: str, sensor: str) -> pd.DataFrame:
     return lookup_table
 
 
-def get_energy_efficiencies() -> pd.DataFrame:
+def get_energy_efficiencies(ancillary_files: dict) -> pd.DataFrame:
     """
     Lookup table for efficiencies for theta and phi.
 
     Further description is available starting on
     page 18 of the Algorithm Document.
 
+    Parameters
+    ----------
+    ancillary_files : dict[Path]
+        Ancillary files.
+
     Returns
     -------
     lookup_table : DataFrame
         Efficiencies lookup table for a given sensor.
     """
-    # TODO: Move this out of tests directory once we have the aux api
-    # TODO: ultra90 efficiencies
-    path = imap_module_directory / "tests" / "ultra" / "data" / "l1"
-    lookup_table = pd.read_csv(
-        path / "Ultra_efficiencies_45_combined_logistic_interpolation.csv"
-    )
+    # TODO: add sensor to input when new lookup tables are available.
+    lookup_table = pd.read_csv(ancillary_files["l1b-45sensor-logistic-interpolation"])
 
     return lookup_table

imap_processing/ultra/l1b/ultra_l1b.py

@@ -8,7 +8,7 @@ from imap_processing.ultra.l1b.de import calculate_de
 from imap_processing.ultra.l1b.extendedspin import calculate_extendedspin
 
 
-def ultra_l1b(data_dict: dict) -> list[xr.Dataset]:
+def ultra_l1b(data_dict: dict, ancillary_files: dict) -> list[xr.Dataset]:
     """
     Will process ULTRA L1A data into L1B CDF files at output_filepath.
 
@@ -16,6 +16,8 @@ def ultra_l1b(data_dict: dict) -> list[xr.Dataset]:
     ----------
     data_dict : dict
         The data itself and its dependent data.
+    ancillary_files : dict
+        Ancillary files.
 
     Returns
     -------
@@ -30,54 +32,63 @@ def ultra_l1b(data_dict: dict) -> list[xr.Dataset]:
     3. l1b extended, culling, badtimes created here
     """
     output_datasets = []
-    instrument_id = 45 if any("45" in key for key in data_dict.keys()) else 90
 
-    # L1b de data will be created if L1a de data is available
-    if f"imap_ultra_l1a_{instrument_id}sensor-de" in data_dict:
-        de_dataset = calculate_de(
-            data_dict[f"imap_ultra_l1a_{instrument_id}sensor-de"],
-            f"imap_ultra_l1b_{instrument_id}sensor-de",
-        )
-        output_datasets.append(de_dataset)
-    # L1b extended data will be created if L1a hk, rates,
-    # aux, params, and l1b de data are available
-    elif (
-        f"imap_ultra_l1b_{instrument_id}sensor-de" in data_dict
-        and f"imap_ultra_l1a_{instrument_id}sensor-rates" in data_dict
-        and f"imap_ultra_l1a_{instrument_id}sensor-aux" in data_dict
-        and f"imap_ultra_l1a_{instrument_id}sensor-params" in data_dict
-    ):
-        extendedspin_dataset = calculate_extendedspin(
-            {
-                f"imap_ultra_l1a_{instrument_id}sensor-aux": data_dict[
-                    f"imap_ultra_l1a_{instrument_id}sensor-aux"
-                ],
-                f"imap_ultra_l1a_{instrument_id}sensor-hk": data_dict[
-                    f"imap_ultra_l1a_{instrument_id}sensor-hk"
-                ],
-                f"imap_ultra_l1a_{instrument_id}sensor-rates": data_dict[
-                    f"imap_ultra_l1a_{instrument_id}sensor-rates"
-                ],
-                f"imap_ultra_l1b_{instrument_id}sensor-de": data_dict[
-                    f"imap_ultra_l1b_{instrument_id}sensor-de"
-                ],
-            },
-            f"imap_ultra_l1b_{instrument_id}sensor-extendedspin",
-            instrument_id,
-        )
-        cullingmask_dataset = calculate_cullingmask(
-            extendedspin_dataset,
-            f"imap_ultra_l1b_{instrument_id}sensor-cullingmask",
-        )
-        badtimes_dataset = calculate_badtimes(
-            extendedspin_dataset,
-            cullingmask_dataset["spin_number"].values,
-            f"imap_ultra_l1b_{instrument_id}sensor-badtimes",
-        )
-        output_datasets.extend(
-            [extendedspin_dataset, cullingmask_dataset, badtimes_dataset]
-        )
-    else:
+    # Account for possibility of having 45 and 90 in dictionary.
+    for instrument_id in [45, 90]:
+        # L1b de data will be created if L1a de data is available
+        if f"imap_ultra_l1a_{instrument_id}sensor-de" in data_dict:
+            de_dataset = calculate_de(
+                data_dict[f"imap_ultra_l1a_{instrument_id}sensor-de"],
+                f"imap_ultra_l1b_{instrument_id}sensor-de",
+                ancillary_files,
+            )
+            output_datasets.append(de_dataset)
+        # L1b extended data will be created if L1a hk, rates,
+        # aux, params, and l1b de data are available
+        elif (
+            f"imap_ultra_l1b_{instrument_id}sensor-de" in data_dict
+            and f"imap_ultra_l1a_{instrument_id}sensor-rates" in data_dict
+            and f"imap_ultra_l1a_{instrument_id}sensor-aux" in data_dict
+            and f"imap_ultra_l1a_{instrument_id}sensor-params" in data_dict
+        ):
+            extendedspin_dataset = calculate_extendedspin(
+                {
+                    f"imap_ultra_l1a_{instrument_id}sensor-aux": data_dict[
+                        f"imap_ultra_l1a_{instrument_id}sensor-aux"
+                    ],
+                    f"imap_ultra_l1a_{instrument_id}sensor-params": data_dict[
+                        f"imap_ultra_l1a_{instrument_id}sensor-params"
+                    ],
+                    f"imap_ultra_l1a_{instrument_id}sensor-rates": data_dict[
+                        f"imap_ultra_l1a_{instrument_id}sensor-rates"
+                    ],
+                    f"imap_ultra_l1b_{instrument_id}sensor-de": data_dict[
+                        f"imap_ultra_l1b_{instrument_id}sensor-de"
+                    ],
+                },
+                f"imap_ultra_l1b_{instrument_id}sensor-extendedspin",
+                instrument_id,
+            )
+            output_datasets.append(extendedspin_dataset)
+        elif (
+            f"imap_ultra_l1b_{instrument_id}sensor-extendedspin" in data_dict
+            and f"imap_ultra_l1b_{instrument_id}sensor-cullingmask" in data_dict
+        ):
+            badtimes_dataset = calculate_badtimes(
+                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-extendedspin"],
+                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-cullingmask"][
+                    "spin_number"
+                ].values,
+                f"imap_ultra_l1b_{instrument_id}sensor-badtimes",
+            )
+            output_datasets.append(badtimes_dataset)
+        elif f"imap_ultra_l1b_{instrument_id}sensor-extendedspin" in data_dict:
+            cullingmask_dataset = calculate_cullingmask(
+                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-extendedspin"],
+                f"imap_ultra_l1b_{instrument_id}sensor-cullingmask",
+            )
+            output_datasets.append(cullingmask_dataset)
+    if not output_datasets:
         raise ValueError("Data dictionary does not contain the expected keys.")
 
     return output_datasets

imap_processing/ultra/l1b/ultra_l1b_culling.py

@@ -1,5 +1,7 @@
 """Culls Events for ULTRA L1b."""
 
+import logging
+
 import numpy as np
 import pandas as pd
 import xarray as xr
@@ -9,6 +11,11 @@ from imap_processing.quality_flags import ImapAttitudeUltraFlags, ImapRatesUltra
 from imap_processing.spice.spin import get_spin_data
 from imap_processing.ultra.constants import UltraConstants
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+SPIN_DURATION = 15  # Default spin duration in seconds.
+
 
 def get_energy_histogram(
     spin_number: NDArray, energy: NDArray
@@ -38,7 +45,8 @@ def get_energy_histogram(
     """
     spin_df = get_spin_data()
 
-    spin_edges = np.unique(spin_number)
+    unique_spin_number = np.unique(spin_number)
+    spin_edges = unique_spin_number.astype(np.uint16)
     spin_edges = np.append(spin_edges, spin_edges.max() + 1)
 
     # Counts per spin at each energy bin.
@@ -52,9 +60,17 @@ def get_energy_histogram(
 
     # Count rate per spin at each energy bin.
     for i in range(hist.shape[1]):
-        spin_duration = spin_df.spin_period_sec[spin_df.spin_number == i]
-        hist[:, i] /= spin_duration.values[0]
-        total_spin_duration += spin_duration.sum()
+        matched_spins = spin_df.spin_number == unique_spin_number[i]
+        if not np.any(matched_spins):
+            # TODO: we might throw an exception here instead.
+            logger.info(f"Unmatched spin number: {unique_spin_number[i]}")
+            spin_duration = SPIN_DURATION  # Default to 15 seconds if no match found
+        else:
+            spin_duration = spin_df.spin_period_sec[
+                spin_df.spin_number == unique_spin_number[i]
+            ].values[0]
+        hist[:, i] /= spin_duration
+        total_spin_duration += spin_duration
 
     mean_duration = total_spin_duration / hist.shape[1]
 
@@ -96,7 +112,7 @@ def flag_attitude(
     )
 
     quality_flags = np.full(
-        spin_rates.shape, ImapAttitudeUltraFlags.NONE.value, dtype=np.uint16
+        spins.shape, ImapAttitudeUltraFlags.NONE.value, dtype=np.uint16
     )
     quality_flags[bad_spin_rate_indices] |= ImapAttitudeUltraFlags.SPINRATE.value
     mismatch_indices = compare_aux_univ_spin_table(aux_dataset, spins, spin_df)
@@ -201,26 +217,42 @@ def compare_aux_univ_spin_table(
     mismatch_indices : np.ndarray
         Boolean array indicating which spins have mismatches.
     """
-    univ_mask = np.isin(spin_df["spin_number"].values, spins)
-    aux_mask = np.isin(aux_dataset["SPINNUMBER"].values, spins)
-
-    filtered_univ = spin_df[univ_mask]
-    filtered_aux = {field: aux_dataset[field].values[aux_mask] for field in aux_dataset}
+    # Identify valid spin matches
+    univ_spins = spin_df["spin_number"].values
+    aux_spins = aux_dataset["spinnumber"].values
+    present_in_both = np.intersect1d(univ_spins, aux_spins)
+
+    # Filter and align by spin number
+    df_univ = spin_df.set_index("spin_number").loc[present_in_both]
+    df_aux = (
+        pd.DataFrame({field: aux_dataset[field].values for field in aux_dataset})
+        .groupby("spinnumber", as_index=True)
+        .first()
+        .loc[present_in_both]
+    )
 
     mismatch_indices = np.zeros(len(spins), dtype=bool)
 
     fields_to_compare = [
-        ("TIMESPINSTART", "spin_start_sec_sclk"),
-        ("TIMESPINSTARTSUB", "spin_start_subsec_sclk"),
-        ("DURATION", "spin_period_sec"),
-        ("TIMESPINDATA", "spin_start_met"),
-        ("SPINPERIOD", "spin_period_sec"),
+        ("timespinstart", "spin_start_sec_sclk"),
+        ("timespinstartsub", "spin_start_subsec_sclk"),
+        ("duration", "spin_period_sec"),
+        ("timespindata", "spin_start_met"),
+        ("spinperiod", "spin_period_sec"),
     ]
 
+    # Compare fields
+    mismatch = np.zeros(len(df_aux), dtype=bool)
     for aux_field, spin_field in fields_to_compare:
-        aux_values = filtered_aux[aux_field]
-        spin_values = filtered_univ[spin_field].values
+        mismatch |= df_aux[aux_field].values != df_univ[spin_field].values
+
+    # Get spin numbers where mismatch is True
+    mismatched_spin_numbers = present_in_both[mismatch]
+    # Find indices in `spins` that correspond to these mismatched spins
+    mismatch_indices[np.isin(spins, mismatched_spin_numbers)] = True
 
-        mismatch_indices |= aux_values != spin_values
+    # Also flag any spins not present in the intersection
+    missing_spin_mask = ~np.isin(spins, present_in_both)
+    mismatch_indices[missing_spin_mask] = True
 
     return mismatch_indices

imap_processing/ultra/l1b/ultra_l1b_extended.py

@@ -941,9 +941,7 @@ def get_fwhm(
 
 
 def get_efficiency(
-    energy: NDArray,
-    phi_inst: NDArray,
-    theta_inst: NDArray,
+    energy: NDArray, phi_inst: NDArray, theta_inst: NDArray, ancillary_files: dict
 ) -> NDArray:
     """
     Interpolate efficiency values for each event.
@@ -956,13 +954,15 @@ def get_efficiency(
         Instrument-frame azimuth angle for each event.
     theta_inst : NDArray
         Instrument-frame elevation angle for each event.
+    ancillary_files : dict
+        Ancillary files.
 
     Returns
     -------
     efficiency : NDArray
         Interpolated efficiency values.
     """
-    lookup_table = get_energy_efficiencies()
+    lookup_table = get_energy_efficiencies(ancillary_files)
 
     theta_vals = np.sort(lookup_table["theta (deg)"].unique())
     phi_vals = np.sort(lookup_table["phi (deg)"].unique())