imap-processing 0.16.1__py3-none-any.whl → 0.17.0__py3-none-any.whl

imap_processing/ultra/l0/decom_ultra.py

@@ -15,8 +15,19 @@ from imap_processing.ultra.l0.decom_tools import (
     read_image_raw_events_binary,
 )
 from imap_processing.ultra.l0.ultra_utils import (
+    CMD_ECHO_MAP,
+    ENERGY_EVENT_FIELD_RANGES,
+    ENERGY_RATES_KEYS,
     EVENT_FIELD_RANGES,
     RATES_KEYS,
+    ULTRA_ENERGY_EVENTS,
+    ULTRA_ENERGY_RATES,
+    ULTRA_ENERGY_SPECTRA,
+    ULTRA_EVENTS,
+    ULTRA_PRI_1_EVENTS,
+    ULTRA_PRI_2_EVENTS,
+    ULTRA_PRI_3_EVENTS,
+    ULTRA_PRI_4_EVENTS,
     ULTRA_RATES,
     ULTRA_TOF,
 )
@@ -136,7 +147,7 @@ def get_event_id(shcoarse: NDArray) -> NDArray:
     return np.array(event_ids, dtype=np.int64)
 
 
-def process_ultra_events(ds: xr.Dataset) -> xr.Dataset:
+def process_ultra_events(ds: xr.Dataset, apid: int) -> xr.Dataset:
     """
     Unpack and decode Ultra EVENTS packets.
 
@@ -144,12 +155,28 @@ def process_ultra_events(ds: xr.Dataset) -> xr.Dataset:
     ----------
     ds : xarray.Dataset
         Events dataset.
+    apid : int
+        APID of the events dataset.
 
     Returns
     -------
     ds : xarray.Dataset
         Dataset containing the decoded and decompressed data.
     """
+    all_event_apids = set(
+        ULTRA_EVENTS.apid
+        + ULTRA_PRI_1_EVENTS.apid
+        + ULTRA_PRI_2_EVENTS.apid
+        + ULTRA_PRI_3_EVENTS.apid
+        + ULTRA_PRI_4_EVENTS.apid
+    )
+    if apid in all_event_apids:
+        field_ranges = EVENT_FIELD_RANGES
+    elif apid in ULTRA_ENERGY_EVENTS.apid:
+        field_ranges = ENERGY_EVENT_FIELD_RANGES
+    else:
+        raise ValueError(f"APID {apid} not recognized for Ultra events processing.")
+
     all_events = []
     all_indices = []
 
@@ -160,7 +187,7 @@ def process_ultra_events(ds: xr.Dataset) -> xr.Dataset:
         field: attrs.get_variable_attributes(field).get(
             "FILLVAL", np.iinfo(np.int64).min
         )
-        for field in EVENT_FIELD_RANGES
+        for field in field_ranges
     }
 
     counts = ds["count"].values
@@ -173,7 +200,9 @@ def process_ultra_events(ds: xr.Dataset) -> xr.Dataset:
         else:
             # Here there are multiple images in a single packet,
             # so we need to loop through each image and decompress it.
-            event_data_list = read_image_raw_events_binary(eventdata_array[i], count)
+            event_data_list = read_image_raw_events_binary(
+                eventdata_array[i], count, field_ranges
+            )
             all_events.extend(event_data_list)
             # Keep track of how many times does the event occurred at this epoch.
             all_indices.extend([i] * count)
@@ -189,7 +218,7 @@ def process_ultra_events(ds: xr.Dataset) -> xr.Dataset:
     }
 
     # Add the event data to the expanded dataset.
-    for key in EVENT_FIELD_RANGES:
+    for key in field_ranges:
         expanded_data[key] = np.array([event[key] for event in all_events])
 
     event_ids = get_event_id(expanded_data["shcoarse"])
@@ -242,3 +271,160 @@ def process_ultra_rates(ds: xr.Dataset) -> xr.Dataset:
         ds[key] = xr.DataArray(np.array(values), dims=["epoch"])
 
     return ds
+
+
+def process_ultra_energy_rates(ds: xr.Dataset) -> xr.Dataset:
+    """
+    Unpack and decode Ultra ENERGY RATES packets.
+
+    Parameters
+    ----------
+    ds : xarray.Dataset
+       Energy rates dataset.
+
+    Returns
+    -------
+    dataset : xarray.Dataset
+        Dataset containing the decoded and decompressed data.
+    """
+    decom_data = defaultdict(list)
+
+    for rate in ds["ratedata"]:
+        raw_binary_string = convert_to_binary_string(rate.item())
+        decompressed_data = decompress_binary(
+            raw_binary_string,
+            cast(int, ULTRA_ENERGY_RATES.width),
+            cast(int, ULTRA_ENERGY_RATES.block),
+            cast(int, ULTRA_ENERGY_RATES.len_array),
+            cast(int, ULTRA_ENERGY_RATES.mantissa_bit_length),
+        )
+
+        for index in range(cast(int, ULTRA_ENERGY_RATES.len_array)):
+            decom_data[ENERGY_RATES_KEYS[index]].append(decompressed_data[index])
+
+    for key, values in decom_data.items():
+        ds[key] = xr.DataArray(np.array(values), dims=["epoch"])
+
+    return ds
+
+
+def process_ultra_energy_spectra(ds: xr.Dataset) -> xr.Dataset:
+    """
+    Unpack and decode Ultra ENERGY SPECTRA packets.
+
+    Parameters
+    ----------
+    ds : xarray.Dataset
+       Energy rates dataset.
+
+    Returns
+    -------
+    dataset : xarray.Dataset
+        Dataset containing the decoded and decompressed data.
+    """
+    energy_spectra = []
+
+    for rate in ds["compdata"]:
+        raw_binary_string = convert_to_binary_string(rate.item())
+        decompressed_data = decompress_binary(
+            raw_binary_string,
+            cast(int, ULTRA_ENERGY_SPECTRA.width),
+            cast(int, ULTRA_ENERGY_SPECTRA.block),
+            cast(int, ULTRA_ENERGY_SPECTRA.len_array),
+            cast(int, ULTRA_ENERGY_SPECTRA.mantissa_bit_length),
+        )
+
+        energy_spectra.append(decompressed_data)
+
+    energy_spectra = np.array(energy_spectra)
+
+    ds["ssd_sum"] = xr.DataArray(
+        energy_spectra,
+        dims=["epoch", "energyspectrastate"],
+        coords={"epoch": ds["epoch"], "energyspectrastate": np.arange(16)},
+    )
+
+    return ds
+
+
+def process_ultra_cmd_echo(ds: xr.Dataset) -> xr.Dataset:
+    """
+    Unpack and decode Ultra CMD ECHO packets.
+
+    Parameters
+    ----------
+    ds : xarray.Dataset
+       Energy rates dataset.
+
+    Returns
+    -------
+    dataset : xarray.Dataset
+        Dataset containing the decoded and decompressed data.
+    """
+    descriptions = []
+
+    fill = 0xFF
+    max_len = 10
+    arg_array = np.full((len(ds["epoch"]), max_len), fill, dtype=np.uint8)
+
+    for i, arg in enumerate(ds["args"].values):
+        # Converts to the numeric representations of each byte.
+        arg_array[i, : len(arg)] = np.frombuffer(arg, dtype=np.uint8)
+
+    # Default to "FILL" for unlisted values
+    for result in ds["result"].values:
+        descriptions.append(CMD_ECHO_MAP.get(result, "FILL"))
+
+    ds["arguments"] = xr.DataArray(
+        arg_array,
+        dims=["epoch", "arg_index"],
+        coords={
+            "epoch": ds["epoch"],
+            "arg_index": np.arange(10),
+        },
+    )
+
+    ds["result_description"] = xr.DataArray(
+        np.array(descriptions),
+        dims=["epoch"],
+        coords={"epoch": ds["epoch"]},
+    )
+
+    ds = ds.drop_vars(["args", "result"])
+
+    return ds
+
+
+def process_ultra_macros_checksum(ds: xr.Dataset) -> xr.Dataset:
+    """
+    Unpack and decode Ultra MACROS CHECKSUM packets.
+
+    Parameters
+    ----------
+    ds : xarray.Dataset
+        Dataset containing macro checksums.
+
+    Returns
+    -------
+    dataset : xarray.Dataset
+        Dataset with unpacked and decoded checksum values.
+    """
+    # big endian uint16
+    packed_dtype = np.dtype(">u2")
+    fill = np.iinfo(packed_dtype).max
+    n_epochs = ds.sizes["epoch"]
+    max_len = 256
+
+    checksum_array = np.full((n_epochs, max_len), fill)
+
+    for i, checksum in enumerate(ds["checksums"]):
+        checksum_array[i, :] = np.frombuffer(checksum.item(), dtype=packed_dtype)
+
+    ds["checksum"] = xr.DataArray(
+        checksum_array,
+        dims=["epoch", "checksum_index"],
+        coords={"epoch": ds["epoch"], "checksum_index": np.arange(max_len)},
+    )
+    ds = ds.drop_vars(["checksums"])
+
+    return ds

imap_processing/ultra/l0/ultra_utils.py

@@ -40,6 +40,30 @@ ULTRA_RATES = PacketProperties(
     len_array=48,
     mantissa_bit_length=12,
 )
+ULTRA_ENERGY_RATES = PacketProperties(
+    apid=[882, 946],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-energy-rates",
+        "imap_ultra_l1a_90sensor-energy-rates",
+    ],
+    addition_to_logical_desc="Energy Rates",
+    width=5,
+    block=16,
+    len_array=11,
+    mantissa_bit_length=12,
+)
+ULTRA_ENERGY_SPECTRA = PacketProperties(
+    apid=[889, 953],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-energy-spectra",
+        "imap_ultra_l1a_90sensor-energy-spectra",
+    ],
+    addition_to_logical_desc="Energy Spectra",
+    width=4,
+    block=16,
+    len_array=1,
+    mantissa_bit_length=5,
+)
 ULTRA_TOF = PacketProperties(
     apid=[883, 947],
     logical_source=[
@@ -61,20 +85,96 @@ ULTRA_EVENTS = PacketProperties(
     len_array=None,
     mantissa_bit_length=None,
 )
+ULTRA_ENERGY_EVENTS = PacketProperties(
+    apid=[897, 961],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-energy-de",
+        "imap_ultra_l1a_90sensor-energy-de",
+    ],
+    addition_to_logical_desc="Single Energy Events",
+    width=None,
+    block=None,
+    len_array=None,
+    mantissa_bit_length=None,
+)
+ULTRA_MACROS_CHECKSUM = PacketProperties(
+    apid=[872, 936],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-macroschecksum",
+        "imap_ultra_l1a_90sensor-macroschecksum",
+    ],
+    addition_to_logical_desc="Macros Checksum",
+    width=None,
+    block=None,
+    len_array=None,
+    mantissa_bit_length=None,
+)
+ULTRA_PRI_1_EVENTS = PacketProperties(
+    apid=[898, 962],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-priority-1-de",
+        "imap_ultra_l1a_90sensor-priority-1-de",
+    ],
+    addition_to_logical_desc="Primary 1 Events",
+    width=None,
+    block=None,
+    len_array=None,
+    mantissa_bit_length=None,
+)
+ULTRA_PRI_2_EVENTS = PacketProperties(
+    apid=[899, 963],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-priority-2-de",
+        "imap_ultra_l1a_90sensor-priority-2-de",
+    ],
+    addition_to_logical_desc="Primary 2 Events",
+    width=None,
+    block=None,
+    len_array=None,
+    mantissa_bit_length=None,
+)
+ULTRA_PRI_3_EVENTS = PacketProperties(
+    apid=[900, 964],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-priority-3-de",
+        "imap_ultra_l1a_90sensor-priority-3-de",
+    ],
+    addition_to_logical_desc="Primary 3 Events",
+    width=None,
+    block=None,
+    len_array=None,
+    mantissa_bit_length=None,
+)
+ULTRA_PRI_4_EVENTS = PacketProperties(
+    apid=[901, 965],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-priority-4-de",
+        "imap_ultra_l1a_90sensor-priority-4-de",
+    ],
+    addition_to_logical_desc="Primary 4 Events",
+    width=None,
+    block=None,
+    len_array=None,
+    mantissa_bit_length=None,
+)
 ULTRA_HK = PacketProperties(
     apid=[
         866,
         867,
+        868,
         869,
         870,
+        871,
         873,
         874,
         876,
         877,
         930,
         931,
+        932,
         933,
         934,
+        935,
         937,
         938,
         940,
@@ -83,16 +183,20 @@ ULTRA_HK = PacketProperties(
     logical_source=[
         "imap_ultra_l1a_45sensor-alarm",
         "imap_ultra_l1a_45sensor-memchecksum",
+        "imap_ultra_l1a_45sensor-memdump",
         "imap_ultra_l1a_45sensor-status",
         "imap_ultra_l1a_45sensor-bootstatus",
+        "imap_ultra_l1a_45sensor-macrodump",
         "imap_ultra_l1a_45sensor-monitorlimits",
         "imap_ultra_l1a_45sensor-params",
         "imap_ultra_l1a_45sensor-scauto",
         "imap_ultra_l1a_45sensor-imgparams",
         "imap_ultra_l1a_90sensor-alarm",
         "imap_ultra_l1a_90sensor-memchecksum",
+        "imap_ultra_l1a_90sensor-memdump",
         "imap_ultra_l1a_90sensor-status",
         "imap_ultra_l1a_90sensor-bootstatus",
+        "imap_ultra_l1a_90sensor-macrodump",
         "imap_ultra_l1a_90sensor-monitorlimits",
         "imap_ultra_l1a_90sensor-params",
         "imap_ultra_l1a_90sensor-scauto",
@@ -119,7 +223,21 @@ ULTRA_CMD_TEXT = PacketProperties(
     len_array=None,
     mantissa_bit_length=None,
 )
-
+ULTRA_CMD_ECHO = PacketProperties(
+    apid=[
+        865,
+        929,
+    ],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-cmdecho",
+        "imap_ultra_l1a_90sensor-cmdecho",
+    ],
+    addition_to_logical_desc="Command echo",
+    width=None,
+    block=None,
+    len_array=None,
+    mantissa_bit_length=None,
+)
 
 # Module-level constant for event field ranges
 EVENT_FIELD_RANGES = {
@@ -206,6 +324,20 @@ EVENT_FIELD_RANGES = {
     "phase_angle": (156, 166),
 }
 
+# Module-level constant for event field ranges
+ENERGY_EVENT_FIELD_RANGES = {
+    # Stop Type
+    "stop_type": (0, 4),
+    # Energy/Pulse Height
+    "energy_ph": (4, 16),
+    # Pulse Width
+    "pulse_width": (16, 27),
+    # Bin
+    "bin": (27, 31),
+    # Phase Angle
+    "phase_angle": (31, 41),
+}
+
 
 RATES_KEYS = [
     # Start Right Full Constant Fraction Discriminator (CFD) Pulses
@@ -323,8 +455,55 @@ RATES_KEYS = [
     # "discarded_events"
 ]
 
+ENERGY_RATES_KEYS = [
+    # SSD0 Energy LED
+    "ssd0_energy_led",
+    # SSD1 Energy LED
+    "ssd1_energy_led",
+    # SSD2 Energy LED
+    "ssd2_energy_led",
+    # SSD3 Energy LED
+    "ssd3_energy_led",
+    # SSD4 Energy LED
+    "ssd4_energy_led",
+    # SSD5 Energy LED
+    "ssd5_energy_led",
+    # SSD6 Energy LED
+    "ssd6_energy_led",
+    # SSD7 Energy LED
+    "ssd7_energy_led",
+    # Event Active Time
+    "event_active_time",
+    # FIFO Valid Events
+    "fifo_valid_events",
+    # Processed Events
+    "processed_events",
+]
+
+ENERGY_SPECTRA_KEYS = [
+    # Sum of the 8 SSDs
+    "ssd_sum",
+]
+
+# Map of command echo fields
+CMD_ECHO_MAP = {
+    0x00: "No error command executed",
+    0x01: "No error command appended to macro",
+    0x02: "Unknown opcode or insufficient arguments",
+    0x03: "Bad argument",
+    0x04: "Cannot run macro; no contexts",
+    0x05: "Cannot be used outside of a macro",
+    0x06: "Macro compilation error",
+    0x07: "Macro not killed (not running?)",
+    0x08: "Cannot boot program; bad checksum",
+    0x09: "Cannot restore macros; bad checksum",
+    0x0A: "Cannot load memory; write disabled",
+    0x10: "HV goal greater than limit",
+    0x11: "Shutter deployment disabled",
+}
+
 
-def parse_event(event_binary: str) -> dict:
+def parse_event(event_binary: str, field_ranges: dict) -> dict:
     """
     Parse a binary string representing a single event.
 
@@ -332,6 +511,8 @@ def parse_event(event_binary: str) -> dict:
     ----------
     event_binary : str
         Event binary string.
+    field_ranges : dict
+        The field ranges for the event data.
 
     Returns
     -------
@@ -339,7 +520,7 @@ def parse_event(event_binary: str) -> dict:
         Dict of the fields for a single event.
     """
     fields_dict = {}
-    for field, (start, end) in EVENT_FIELD_RANGES.items():
+    for field, (start, end) in field_ranges.items():
         field_value = int(event_binary[start:end], 2)
         fields_dict[field] = field_value
     return fields_dict

imap_processing/ultra/l1a/ultra_l1a.py

@@ -8,15 +8,28 @@ import xarray as xr
 from imap_processing import imap_module_directory
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.ultra.l0.decom_ultra import (
+    process_ultra_cmd_echo,
+    process_ultra_energy_rates,
+    process_ultra_energy_spectra,
     process_ultra_events,
+    process_ultra_macros_checksum,
     process_ultra_rates,
     process_ultra_tof,
 )
 from imap_processing.ultra.l0.ultra_utils import (
     ULTRA_AUX,
+    ULTRA_CMD_ECHO,
     ULTRA_CMD_TEXT,
+    ULTRA_ENERGY_EVENTS,
+    ULTRA_ENERGY_RATES,
+    ULTRA_ENERGY_SPECTRA,
     ULTRA_EVENTS,
     ULTRA_HK,
+    ULTRA_MACROS_CHECKSUM,
+    ULTRA_PRI_1_EVENTS,
+    ULTRA_PRI_2_EVENTS,
+    ULTRA_PRI_3_EVENTS,
+    ULTRA_PRI_4_EVENTS,
     ULTRA_RATES,
     ULTRA_TOF,
 )
@@ -25,7 +38,9 @@ from imap_processing.utils import packet_file_to_datasets
 logger = logging.getLogger(__name__)
 
 
-def ultra_l1a(packet_file: str, apid_input: Optional[int] = None) -> list[xr.Dataset]:
+def ultra_l1a(  # noqa: PLR0912
+    packet_file: str, apid_input: Optional[int] = None
+) -> list[xr.Dataset]:
     """
     Will process ULTRA L0 data into L1A CDF files at output_filepath.
 
@@ -59,6 +74,19 @@ def ultra_l1a(packet_file: str, apid_input: Optional[int] = None) -> list[xr.Dat
     else:
         apids = list(datasets_by_apid.keys())
 
+    all_event_apids = {
+        apid: group.logical_source[i]
+        for group in [
+            ULTRA_EVENTS,
+            ULTRA_ENERGY_EVENTS,
+            ULTRA_PRI_1_EVENTS,
+            ULTRA_PRI_2_EVENTS,
+            ULTRA_PRI_3_EVENTS,
+            ULTRA_PRI_4_EVENTS,
+        ]
+        for i, apid in enumerate(group.apid)
+    }
+
     # Update dataset global attributes
     attr_mgr = ImapCdfAttributes()
     attr_mgr.add_instrument_global_attrs("ultra")
@@ -75,12 +103,29 @@ def ultra_l1a(packet_file: str, apid_input: Optional[int] = None) -> list[xr.Dat
             decom_ultra_dataset = process_ultra_rates(datasets_by_apid[apid])
             decom_ultra_dataset = decom_ultra_dataset.drop_vars("fastdata_00")
             gattr_key = ULTRA_RATES.logical_source[ULTRA_RATES.apid.index(apid)]
-        elif apid in ULTRA_EVENTS.apid:
-            decom_ultra_dataset = process_ultra_events(datasets_by_apid[apid])
-            gattr_key = ULTRA_EVENTS.logical_source[ULTRA_EVENTS.apid.index(apid)]
+        elif apid in ULTRA_ENERGY_RATES.apid:
+            decom_ultra_dataset = process_ultra_energy_rates(datasets_by_apid[apid])
+            decom_ultra_dataset = decom_ultra_dataset.drop_vars("ratedata")
+            gattr_key = ULTRA_ENERGY_RATES.logical_source[
+                ULTRA_ENERGY_RATES.apid.index(apid)
+            ]
+        elif apid in all_event_apids:
+            decom_ultra_dataset = process_ultra_events(datasets_by_apid[apid], apid)
+            gattr_key = all_event_apids[apid]
             # Add coordinate attributes
             attrs = attr_mgr.get_variable_attributes("event_id")
             decom_ultra_dataset.coords["event_id"].attrs.update(attrs)
+        elif apid in ULTRA_ENERGY_SPECTRA.apid:
+            decom_ultra_dataset = process_ultra_energy_spectra(datasets_by_apid[apid])
+            decom_ultra_dataset = decom_ultra_dataset.drop_vars("compdata")
+            gattr_key = ULTRA_ENERGY_SPECTRA.logical_source[
+                ULTRA_ENERGY_SPECTRA.apid.index(apid)
+            ]
+        elif apid in ULTRA_MACROS_CHECKSUM.apid:
+            decom_ultra_dataset = process_ultra_macros_checksum(datasets_by_apid[apid])
+            gattr_key = ULTRA_MACROS_CHECKSUM.logical_source[
+                ULTRA_MACROS_CHECKSUM.apid.index(apid)
+            ]
         elif apid in ULTRA_HK.apid:
             decom_ultra_dataset = datasets_by_apid[apid]
             gattr_key = ULTRA_HK.logical_source[ULTRA_HK.apid.index(apid)]
@@ -97,6 +142,9 @@ def ultra_l1a(packet_file: str, apid_input: Optional[int] = None) -> list[xr.Dat
                 coords={"epoch": decom_ultra_dataset["epoch"]},
             )
             gattr_key = ULTRA_CMD_TEXT.logical_source[ULTRA_CMD_TEXT.apid.index(apid)]
+        elif apid in ULTRA_CMD_ECHO.apid:
+            decom_ultra_dataset = process_ultra_cmd_echo(datasets_by_apid[apid])
+            gattr_key = ULTRA_CMD_ECHO.logical_source[ULTRA_CMD_ECHO.apid.index(apid)]
         else:
             logger.error(f"APID {apid} not recognized.")
             continue

imap_processing/ultra/packet_definitions/ULTRA_SCI_COMBINED.xml

@@ -4726,15 +4726,15 @@
 			<xtce:IntegerParameterType name="U45_IMG_ENA_PHXTOF_HI_TIME.SPIN" signed="false">
 				<xtce:IntegerDataEncoding sizeInBits="8" encoding="unsigned" />
 			</xtce:IntegerParameterType>
+			<xtce:IntegerParameterType name="U45_IMG_ENA_PHXTOF_HI_TIME.ABORTFLAG" signed="false">
+				<xtce:IntegerDataEncoding sizeInBits="1" encoding="unsigned" />
+			</xtce:IntegerParameterType>
 			<xtce:IntegerParameterType name="U45_IMG_ENA_PHXTOF_HI_TIME.STARTDELAY" signed="false">
 				<xtce:IntegerDataEncoding sizeInBits="15" encoding="unsigned" />
 			</xtce:IntegerParameterType>
 			<xtce:IntegerParameterType name="U45_IMG_ENA_PHXTOF_HI_TIME.P00" signed="false">
 				<xtce:IntegerDataEncoding sizeInBits="8" encoding="unsigned" />
 			</xtce:IntegerParameterType>
-			<xtce:IntegerParameterType name="U45_IMG_ENA_PHXTOF_HI_TIME.ABORTFLAG" signed="false">
-				<xtce:IntegerDataEncoding sizeInBits="8" encoding="unsigned" />
-			</xtce:IntegerParameterType>
 			<xtce:BinaryParameterType name="U45_IMG_ENA_PHXTOF_HI_TIME.PACKETDATA">
 				<xtce:BinaryDataEncoding bitOrder="mostSignificantBitFirst">
 					<xtce:SizeInBits>

imap_processing/utils.py

@@ -15,48 +15,6 @@ from imap_processing.spice.time import met_to_ttj2000ns
 logger = logging.getLogger(__name__)
 
 
-def sort_by_time(packets: list, time_key: str) -> list:
-    """
-    Sort packets by specified key.
-
-    Parameters
-    ----------
-    packets : list
-        Decom data packets.
-    time_key : str
-        Key to sort by. Must be a key in the packets data dictionary.
-        e.g. "SHCOARSE" or "MET_TIME" or "ACQ_START_COARSE".
-
-    Returns
-    -------
-    sorted_packets : list
-        Sorted packets.
-    """
-    sorted_packets = sorted(packets, key=lambda x: x[time_key])
-    return sorted_packets
-
-
-def group_by_apid(packets: list) -> dict:
-    """
-    Group data by apid.
-
-    Parameters
-    ----------
-    packets : list
-        Packet list.
-
-    Returns
-    -------
-    grouped_packets : dict
-        Grouped data by apid.
-    """
-    grouped_packets: dict[list] = collections.defaultdict(list)
-    for packet in packets:
-        apid = packet["PKT_APID"]
-        grouped_packets.setdefault(apid, []).append(packet)
-    return grouped_packets
-
-
 def convert_raw_to_eu(
     dataset: xr.Dataset,
     conversion_table_path: str,
@@ -347,6 +305,26 @@ def packet_file_to_datasets(
             coords={"epoch": time_data},
         )
         ds = ds.sortby("epoch")
+        # We may get duplicate packets within the packet file if packets were
+        # ingested multiple times by the POC. We want to drop packets where
+        # apid, epoch, and src_seq_ctr are the same.
+
+        # xarray only supports dropping duplicates by index, so we instead go
+        # to pandas multi-index dataframe to identify the unique positions
+        unique_indices = (
+            ds[["src_seq_ctr"]]
+            .to_dataframe()
+            .reset_index()
+            .drop_duplicates()
+            .index.values
+        )
+        nduplicates = len(ds["epoch"]) - len(unique_indices)
+        if nduplicates != 0:
+            logger.warning(
+                f"Found [{nduplicates}] duplicate packets for APID {apid}. "
+                "Dropping duplicate packets and continuing processing."
+            )
+            ds = ds.isel(epoch=unique_indices)
 
         # Strip any leading characters before "." from the field names which was due
         # to the packet_name being a part of the variable name in the XTCE definition