imap-processing 0.16.2__py3-none-any.whl → 0.18.0__py3-none-any.whl

imap_processing/ultra/l0/ultra_utils.py

@@ -10,15 +10,23 @@ class PacketProperties(NamedTuple):
     logical_source: list  # List of logical sources
     addition_to_logical_desc: str  # Description of the logical source
     width: Union[int, None]  # Width of binary data (could be None).
+    # Block, image_planes, pixel_window_rows, and pixel_window_columns are important for
+    # decompressing the images and a description is available on page 171 of IMAP-Ultra
+    # Flight Software Specification document (7523-9009_Rev_-.pdf).
     block: Union[int, None]  # Number of values in each block (could be None).
-    # This is important for decompressing the images and
-    # a description is available on page 171 of IMAP-Ultra Flight
-    # Software Specification document (7523-9009_Rev_-.pdf).
     len_array: Union[
         int, None
     ]  # Length of the array to be decompressed (could be None).
     mantissa_bit_length: Union[int, None]  # used to determine the level of
     # precision that can be recovered from compressed data (could be None).
+    image_planes: Union[int, None] = None
+    # number of images. See table 11 in the FSSD.
+    pixel_window_rows: Union[int, None] = None
+    # number of rows in each image. See table 49 in the FSSD.
+    pixel_window_columns: Union[int, None] = None
+    # number of columns in each image. See table 49 in the FSSD.
+    image_planes_per_packet: Union[int, None] = None
+    # number of image planes in each packet. See table 52 in the FSSD.
 
 
 # Define PacketProperties instances directly in the module namespace
@@ -40,15 +48,123 @@ ULTRA_RATES = PacketProperties(
     len_array=48,
     mantissa_bit_length=12,
 )
-ULTRA_TOF = PacketProperties(
+ULTRA_ENERGY_RATES = PacketProperties(
+    apid=[882, 946],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-energy-rates",
+        "imap_ultra_l1a_90sensor-energy-rates",
+    ],
+    addition_to_logical_desc="Energy Rates",
+    width=5,
+    block=16,
+    len_array=11,
+    mantissa_bit_length=12,
+)
+ULTRA_ENERGY_SPECTRA = PacketProperties(
+    apid=[889, 953],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-energy-spectra",
+        "imap_ultra_l1a_90sensor-energy-spectra",
+    ],
+    addition_to_logical_desc="Energy Spectra",
+    width=4,
+    block=16,
+    len_array=1,
+    mantissa_bit_length=5,
+)
+ULTRA_PHXTOF_HIGH_ANGULAR = PacketProperties(
     apid=[883, 947],
     logical_source=[
         "imap_ultra_l1a_45sensor-histogram-ena-phxtof-hi-ang",
         "imap_ultra_l1a_90sensor-histogram-ena-phxtof-hi-ang",
     ],
-    addition_to_logical_desc="Time of Flight Images",
+    addition_to_logical_desc="Pulse Height Time of Flight High Angular Images",
     width=4,
     block=15,
+    image_planes=8,
+    pixel_window_rows=54,
+    pixel_window_columns=180,
+    image_planes_per_packet=1,
+    len_array=None,
+    mantissa_bit_length=4,
+)
+ULTRA_PHXTOF_HIGH_ENERGY = PacketProperties(
+    apid=[884, 948],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-histogram-ena-phxtof-hi-nrg",
+        "imap_ultra_l1a_90sensor-histogram-ena-phxtof-hi-nrg",
+    ],
+    addition_to_logical_desc="Pulse Height By Time of Flight High Energy Images",
+    width=4,
+    block=15,
+    image_planes=28,
+    pixel_window_rows=27,
+    pixel_window_columns=90,
+    image_planes_per_packet=1,
+    len_array=None,
+    mantissa_bit_length=4,
+)
+ULTRA_PHXTOF_HIGH_TIME = PacketProperties(
+    apid=[885, 949],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-histogram-ena-phxtof-hi-time",
+        "imap_ultra_l1a_90sensor-histogram-ena-phxtof-hi-time",
+    ],
+    addition_to_logical_desc="Time of Flight High Time Images",
+    width=4,
+    block=15,
+    image_planes=8,
+    pixel_window_rows=18,
+    pixel_window_columns=60,
+    image_planes_per_packet=2,
+    len_array=None,
+    mantissa_bit_length=4,
+)
+ULTRA_EXTOF_HIGH_ANGULAR = PacketProperties(
+    apid=[886, 950],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-histogram-ena-extof-hi-ang",
+        "imap_ultra_l1a_90sensor-histogram-ena-extof-hi-ang",
+    ],
+    addition_to_logical_desc="Energy By Time of Flight High Angular Images",
+    width=4,
+    block=15,
+    image_planes=12,
+    pixel_window_rows=18,
+    pixel_window_columns=60,
+    image_planes_per_packet=2,
+    len_array=None,
+    mantissa_bit_length=4,
+)
+ULTRA_EXTOF_HIGH_TIME = PacketProperties(
+    apid=[888, 952],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-histogram-ena-extof-hi-time",
+        "imap_ultra_l1a_90sensor-histogram-ena-extof-hi-time",
+    ],
+    addition_to_logical_desc="Energy By Time of Flight High Time Images",
+    width=4,
+    block=15,
+    image_planes=4,
+    pixel_window_rows=9,
+    pixel_window_columns=30,
+    image_planes_per_packet=8,
+    len_array=None,
+    mantissa_bit_length=4,
+)
+ULTRA_EXTOF_HIGH_ENERGY = PacketProperties(
+    apid=[887, 951],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-histogram-ena-extof-hi-nrg",
+        "imap_ultra_l1a_90sensor-histogram-ena-extof-hi-nrg",
+    ],
+    addition_to_logical_desc="Energy By Time of Flight High Energy Images",
+    width=4,
+    block=15,
+    image_planes=44,
+    pixel_window_rows=9,
+    pixel_window_columns=30,
+    image_planes_per_packet=8,
     len_array=None,
     mantissa_bit_length=4,
 )
@@ -61,20 +177,96 @@ ULTRA_EVENTS = PacketProperties(
     len_array=None,
     mantissa_bit_length=None,
 )
+ULTRA_ENERGY_EVENTS = PacketProperties(
+    apid=[897, 961],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-energy-de",
+        "imap_ultra_l1a_90sensor-energy-de",
+    ],
+    addition_to_logical_desc="Single Energy Events",
+    width=None,
+    block=None,
+    len_array=None,
+    mantissa_bit_length=None,
+)
+ULTRA_MACROS_CHECKSUM = PacketProperties(
+    apid=[872, 936],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-macroschecksum",
+        "imap_ultra_l1a_90sensor-macroschecksum",
+    ],
+    addition_to_logical_desc="Macros Checksum",
+    width=None,
+    block=None,
+    len_array=None,
+    mantissa_bit_length=None,
+)
+ULTRA_PRI_1_EVENTS = PacketProperties(
+    apid=[898, 962],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-priority-1-de",
+        "imap_ultra_l1a_90sensor-priority-1-de",
+    ],
+    addition_to_logical_desc="Primary 1 Events",
+    width=None,
+    block=None,
+    len_array=None,
+    mantissa_bit_length=None,
+)
+ULTRA_PRI_2_EVENTS = PacketProperties(
+    apid=[899, 963],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-priority-2-de",
+        "imap_ultra_l1a_90sensor-priority-2-de",
+    ],
+    addition_to_logical_desc="Primary 2 Events",
+    width=None,
+    block=None,
+    len_array=None,
+    mantissa_bit_length=None,
+)
+ULTRA_PRI_3_EVENTS = PacketProperties(
+    apid=[900, 964],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-priority-3-de",
+        "imap_ultra_l1a_90sensor-priority-3-de",
+    ],
+    addition_to_logical_desc="Primary 3 Events",
+    width=None,
+    block=None,
+    len_array=None,
+    mantissa_bit_length=None,
+)
+ULTRA_PRI_4_EVENTS = PacketProperties(
+    apid=[901, 965],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-priority-4-de",
+        "imap_ultra_l1a_90sensor-priority-4-de",
+    ],
+    addition_to_logical_desc="Primary 4 Events",
+    width=None,
+    block=None,
+    len_array=None,
+    mantissa_bit_length=None,
+)
 ULTRA_HK = PacketProperties(
     apid=[
         866,
         867,
+        868,
         869,
         870,
+        871,
         873,
         874,
         876,
         877,
         930,
         931,
+        932,
         933,
         934,
+        935,
         937,
         938,
         940,
@@ -83,16 +275,20 @@ ULTRA_HK = PacketProperties(
     logical_source=[
         "imap_ultra_l1a_45sensor-alarm",
         "imap_ultra_l1a_45sensor-memchecksum",
+        "imap_ultra_l1a_45sensor-memdump",
         "imap_ultra_l1a_45sensor-status",
         "imap_ultra_l1a_45sensor-bootstatus",
+        "imap_ultra_l1a_45sensor-macrodump",
         "imap_ultra_l1a_45sensor-monitorlimits",
         "imap_ultra_l1a_45sensor-params",
         "imap_ultra_l1a_45sensor-scauto",
         "imap_ultra_l1a_45sensor-imgparams",
         "imap_ultra_l1a_90sensor-alarm",
         "imap_ultra_l1a_90sensor-memchecksum",
+        "imap_ultra_l1a_90sensor-memdump",
         "imap_ultra_l1a_90sensor-status",
         "imap_ultra_l1a_90sensor-bootstatus",
+        "imap_ultra_l1a_90sensor-macrodump",
         "imap_ultra_l1a_90sensor-monitorlimits",
         "imap_ultra_l1a_90sensor-params",
         "imap_ultra_l1a_90sensor-scauto",
@@ -119,7 +315,21 @@ ULTRA_CMD_TEXT = PacketProperties(
     len_array=None,
     mantissa_bit_length=None,
 )
-
+ULTRA_CMD_ECHO = PacketProperties(
+    apid=[
+        865,
+        929,
+    ],
+    logical_source=[
+        "imap_ultra_l1a_45sensor-cmdecho",
+        "imap_ultra_l1a_90sensor-cmdecho",
+    ],
+    addition_to_logical_desc="Command echo",
+    width=None,
+    block=None,
+    len_array=None,
+    mantissa_bit_length=None,
+)
 
 # Module-level constant for event field ranges
 EVENT_FIELD_RANGES = {
@@ -206,6 +416,20 @@ EVENT_FIELD_RANGES = {
     "phase_angle": (156, 166),
 }
 
+# Module-level constant for event field ranges
+ENERGY_EVENT_FIELD_RANGES = {
+    # Stop Type
+    "stop_type": (0, 4),
+    # Energy/Pulse Height
+    "energy_ph": (4, 16),
+    # Pulse Width
+    "pulse_width": (16, 27),
+    # Bin
+    "bin": (27, 31),
+    # Phase Angle
+    "phase_angle": (31, 41),
+}
+
 
 RATES_KEYS = [
     # Start Right Full Constant Fraction Discriminator (CFD) Pulses
@@ -323,8 +547,55 @@ RATES_KEYS = [
     # "discarded_events"
 ]
 
+ENERGY_RATES_KEYS = [
+    # SSD0 Energy LED
+    "ssd0_energy_led",
+    # SSD1 Energy LED
+    "ssd1_energy_led",
+    # SSD2 Energy LED
+    "ssd2_energy_led",
+    # SSD3 Energy LED
+    "ssd3_energy_led",
+    # SSD4 Energy LED
+    "ssd4_energy_led",
+    # SSD5 Energy LED
+    "ssd5_energy_led",
+    # SSD6 Energy LED
+    "ssd6_energy_led",
+    # SSD7 Energy LED
+    "ssd7_energy_led",
+    # Event Active Time
+    "event_active_time",
+    # FIFO Valid Events
+    "fifo_valid_events",
+    # Processed Events
+    "processed_events",
+]
+
+ENERGY_SPECTRA_KEYS = [
+    # Sum of the 8 SSDs
+    "ssd_sum",
+]
+
+# Map of command echo fields
+CMD_ECHO_MAP = {
+    0x00: "No error command executed",
+    0x01: "No error command appended to macro",
+    0x02: "Unknown opcode or insufficient arguments",
+    0x03: "Bad argument",
+    0x04: "Cannot run macro; no contexts",
+    0x05: "Cannot be used outside of a macro",
+    0x06: "Macro compilation error",
+    0x07: "Macro not killed (not running?)",
+    0x08: "Cannot boot program; bad checksum",
+    0x09: "Cannot restore macros; bad checksum",
+    0x0A: "Cannot load memory; write disabled",
+    0x10: "HV goal greater than limit",
+    0x11: "Shutter deployment disabled",
+}
+
 
-def parse_event(event_binary: str) -> dict:
+def parse_event(event_binary: str, field_ranges: dict) -> dict:
     """
     Parse a binary string representing a single event.
 
@@ -332,6 +603,8 @@ def parse_event(event_binary: str) -> dict:
     ----------
     event_binary : str
         Event binary string.
+    field_ranges : dict
+        The field ranges for the event data.
 
     Returns
     -------
@@ -339,7 +612,7 @@ def parse_event(event_binary: str) -> dict:
         Dict of the fields for a single event.
     """
     fields_dict = {}
-    for field, (start, end) in EVENT_FIELD_RANGES.items():
+    for field, (start, end) in field_ranges.items():
         field_value = int(event_binary[start:end], 2)
         fields_dict[field] = field_value
     return fields_dict

imap_processing/ultra/l1a/ultra_l1a.py

@@ -8,24 +8,44 @@ import xarray as xr
 from imap_processing import imap_module_directory
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.ultra.l0.decom_ultra import (
+    process_ultra_cmd_echo,
+    process_ultra_energy_rates,
+    process_ultra_energy_spectra,
     process_ultra_events,
+    process_ultra_macros_checksum,
     process_ultra_rates,
     process_ultra_tof,
 )
 from imap_processing.ultra.l0.ultra_utils import (
     ULTRA_AUX,
+    ULTRA_CMD_ECHO,
     ULTRA_CMD_TEXT,
+    ULTRA_ENERGY_EVENTS,
+    ULTRA_ENERGY_RATES,
+    ULTRA_ENERGY_SPECTRA,
     ULTRA_EVENTS,
+    ULTRA_EXTOF_HIGH_ANGULAR,
+    ULTRA_EXTOF_HIGH_ENERGY,
+    ULTRA_EXTOF_HIGH_TIME,
     ULTRA_HK,
+    ULTRA_MACROS_CHECKSUM,
+    ULTRA_PHXTOF_HIGH_ANGULAR,
+    ULTRA_PHXTOF_HIGH_ENERGY,
+    ULTRA_PHXTOF_HIGH_TIME,
+    ULTRA_PRI_1_EVENTS,
+    ULTRA_PRI_2_EVENTS,
+    ULTRA_PRI_3_EVENTS,
+    ULTRA_PRI_4_EVENTS,
     ULTRA_RATES,
-    ULTRA_TOF,
 )
 from imap_processing.utils import packet_file_to_datasets
 
 logger = logging.getLogger(__name__)
 
 
-def ultra_l1a(packet_file: str, apid_input: Optional[int] = None) -> list[xr.Dataset]:
+def ultra_l1a(  # noqa: PLR0912
+    packet_file: str, apid_input: Optional[int] = None
+) -> list[xr.Dataset]:
     """
     Will process ULTRA L0 data into L1A CDF files at output_filepath.
 
@@ -59,6 +79,32 @@ def ultra_l1a(packet_file: str, apid_input: Optional[int] = None) -> list[xr.Dat
     else:
         apids = list(datasets_by_apid.keys())
 
+    all_event_apids = {
+        apid: group.logical_source[i]
+        for group in [
+            ULTRA_EVENTS,
+            ULTRA_ENERGY_EVENTS,
+            ULTRA_PRI_1_EVENTS,
+            ULTRA_PRI_2_EVENTS,
+            ULTRA_PRI_3_EVENTS,
+            ULTRA_PRI_4_EVENTS,
+        ]
+        for i, apid in enumerate(group.apid)
+    }
+
+    all_l1a_image_apids = {
+        apid: group
+        for group in [
+            ULTRA_PHXTOF_HIGH_ANGULAR,
+            ULTRA_PHXTOF_HIGH_ENERGY,
+            ULTRA_PHXTOF_HIGH_TIME,
+            ULTRA_EXTOF_HIGH_ANGULAR,
+            ULTRA_EXTOF_HIGH_TIME,
+            ULTRA_EXTOF_HIGH_ENERGY,
+        ]
+        for apid in group.apid
+    }
+
     # Update dataset global attributes
     attr_mgr = ImapCdfAttributes()
     attr_mgr.add_instrument_global_attrs("ultra")
@@ -68,19 +114,39 @@ def ultra_l1a(packet_file: str, apid_input: Optional[int] = None) -> list[xr.Dat
         if apid in ULTRA_AUX.apid:
             decom_ultra_dataset = datasets_by_apid[apid]
             gattr_key = ULTRA_AUX.logical_source[ULTRA_AUX.apid.index(apid)]
-        elif apid in ULTRA_TOF.apid:
-            decom_ultra_dataset = process_ultra_tof(datasets_by_apid[apid])
-            gattr_key = ULTRA_TOF.logical_source[ULTRA_TOF.apid.index(apid)]
+        elif apid in all_l1a_image_apids:
+            packet_props = all_l1a_image_apids[apid]
+            decom_ultra_dataset = process_ultra_tof(
+                datasets_by_apid[apid], packet_props
+            )
+            gattr_key = packet_props.logical_source[packet_props.apid.index(apid)]
         elif apid in ULTRA_RATES.apid:
             decom_ultra_dataset = process_ultra_rates(datasets_by_apid[apid])
             decom_ultra_dataset = decom_ultra_dataset.drop_vars("fastdata_00")
             gattr_key = ULTRA_RATES.logical_source[ULTRA_RATES.apid.index(apid)]
-        elif apid in ULTRA_EVENTS.apid:
-            decom_ultra_dataset = process_ultra_events(datasets_by_apid[apid])
-            gattr_key = ULTRA_EVENTS.logical_source[ULTRA_EVENTS.apid.index(apid)]
+        elif apid in ULTRA_ENERGY_RATES.apid:
+            decom_ultra_dataset = process_ultra_energy_rates(datasets_by_apid[apid])
+            decom_ultra_dataset = decom_ultra_dataset.drop_vars("ratedata")
+            gattr_key = ULTRA_ENERGY_RATES.logical_source[
+                ULTRA_ENERGY_RATES.apid.index(apid)
+            ]
+        elif apid in all_event_apids:
+            decom_ultra_dataset = process_ultra_events(datasets_by_apid[apid], apid)
+            gattr_key = all_event_apids[apid]
             # Add coordinate attributes
             attrs = attr_mgr.get_variable_attributes("event_id")
             decom_ultra_dataset.coords["event_id"].attrs.update(attrs)
+        elif apid in ULTRA_ENERGY_SPECTRA.apid:
+            decom_ultra_dataset = process_ultra_energy_spectra(datasets_by_apid[apid])
+            decom_ultra_dataset = decom_ultra_dataset.drop_vars("compdata")
+            gattr_key = ULTRA_ENERGY_SPECTRA.logical_source[
+                ULTRA_ENERGY_SPECTRA.apid.index(apid)
+            ]
+        elif apid in ULTRA_MACROS_CHECKSUM.apid:
+            decom_ultra_dataset = process_ultra_macros_checksum(datasets_by_apid[apid])
+            gattr_key = ULTRA_MACROS_CHECKSUM.logical_source[
+                ULTRA_MACROS_CHECKSUM.apid.index(apid)
+            ]
         elif apid in ULTRA_HK.apid:
             decom_ultra_dataset = datasets_by_apid[apid]
             gattr_key = ULTRA_HK.logical_source[ULTRA_HK.apid.index(apid)]
@@ -97,6 +163,9 @@ def ultra_l1a(packet_file: str, apid_input: Optional[int] = None) -> list[xr.Dat
                 coords={"epoch": decom_ultra_dataset["epoch"]},
             )
             gattr_key = ULTRA_CMD_TEXT.logical_source[ULTRA_CMD_TEXT.apid.index(apid)]
+        elif apid in ULTRA_CMD_ECHO.apid:
+            decom_ultra_dataset = process_ultra_cmd_echo(datasets_by_apid[apid])
+            gattr_key = ULTRA_CMD_ECHO.logical_source[ULTRA_CMD_ECHO.apid.index(apid)]
         else:
             logger.error(f"APID {apid} not recognized.")
             continue

imap_processing/ultra/l1b/cullingmask.py

@@ -3,7 +3,7 @@
 import numpy as np
 import xarray as xr
 
-from imap_processing.quality_flags import ImapAttitudeUltraFlags, ImapRatesUltraFlags
+from imap_processing.ultra.l1b.quality_flag_filters import QUALITY_FLAG_FILTERS
 from imap_processing.ultra.utils.ultra_l1_utils import create_dataset, extract_data_dict
 
 FILLVAL_UINT16 = 65535
@@ -32,14 +32,14 @@ def calculate_cullingmask(extendedspin_dataset: xr.Dataset, name: str) -> xr.Dat
     good_mask = (
         (
             extendedspin_dataset["quality_attitude"]
-            & ImapAttitudeUltraFlags.SPINRATE.value
+            & sum(flag.value for flag in QUALITY_FLAG_FILTERS["quality_attitude"])
         )
         == 0
     ) & (
         (
             (
                 extendedspin_dataset["quality_ena_rates"]
-                & ImapRatesUltraFlags.HIGHRATES.value
+                & sum(flag.value for flag in QUALITY_FLAG_FILTERS["quality_ena_rates"])
             )
             == 0
         ).all(dim="energy_bin_geometric_mean")