imap-processing 0.11.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

imap_processing/ialirt/packet_definitions/ialirt_mag.xml

@@ -0,0 +1,115 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<xtce:SpaceSystem xmlns:xtce="http://www.omg.org/space/xtce" name="ialirt">
+	<xtce:Header date="2023-08-24T07:53:00MST" version="1.0" author="IMAP SDC" />
+	<xtce:TelemetryMetaData>
+		<xtce:ParameterTypeSet>
+			<!-- This file was manually created using content from:
+			I-ALiRT Packet Definitions: https://lasp.colorado.edu/galaxy/x/44nKCQ
+            -->
+			<xtce:IntegerParameterType name="uint1" signed="false">
+				<xtce:IntegerDataEncoding sizeInBits="1" encoding="unsigned" />
+				<xtce:UnitSet />
+			</xtce:IntegerParameterType>
+			<xtce:IntegerParameterType name="uint2" signed="false">
+				<xtce:IntegerDataEncoding sizeInBits="2" encoding="unsigned" />
+				<xtce:UnitSet />
+			</xtce:IntegerParameterType>
+			<xtce:IntegerParameterType name="uint3" signed="false">
+				<xtce:IntegerDataEncoding sizeInBits="3" encoding="unsigned" />
+				<xtce:UnitSet />
+			</xtce:IntegerParameterType>
+			<xtce:IntegerParameterType name="uint11" signed="false">
+				<xtce:IntegerDataEncoding sizeInBits="11" encoding="unsigned" />
+				<xtce:UnitSet />
+			</xtce:IntegerParameterType>
+			<xtce:IntegerParameterType name="uint14" signed="false">
+				<xtce:IntegerDataEncoding sizeInBits="14" encoding="unsigned" />
+				<xtce:UnitSet />
+			</xtce:IntegerParameterType>
+			<xtce:IntegerParameterType name="uint16" signed="false">
+				<xtce:IntegerDataEncoding sizeInBits="16" encoding="unsigned" />
+				<xtce:UnitSet />
+			</xtce:IntegerParameterType>
+			<xtce:IntegerParameterType name="uint24" signed="false">
+				<xtce:IntegerDataEncoding sizeInBits="24" encoding="unsigned" />
+				<xtce:UnitSet />
+			</xtce:IntegerParameterType>
+			<xtce:IntegerParameterType name="uint32" signed="false">
+				<xtce:IntegerDataEncoding sizeInBits="32" encoding="unsigned" />
+				<xtce:UnitSet />
+			</xtce:IntegerParameterType>
+		</xtce:ParameterTypeSet>
+        <xtce:ParameterSet>
+
+            <!--CCSDS Header Elements-->
+			<xtce:Parameter name="VERSION" parameterTypeRef="uint3">
+				<xtce:LongDescription>CCSDS Packet Version Number (always 0)</xtce:LongDescription>
+			</xtce:Parameter>
+			<xtce:Parameter name="TYPE" parameterTypeRef="uint1">
+				<xtce:LongDescription>CCSDS Packet Type Indicator (0=telemetry)</xtce:LongDescription>
+			</xtce:Parameter>
+			<xtce:Parameter name="SEC_HDR_FLG" parameterTypeRef="uint1">
+				<xtce:LongDescription>CCSDS Packet Secondary Header Flag (always 1)</xtce:LongDescription>
+			</xtce:Parameter>
+			<xtce:Parameter name="PKT_APID" parameterTypeRef="uint11">
+				<xtce:LongDescription>CCSDS Packet Application Process ID</xtce:LongDescription>
+			</xtce:Parameter>
+			<xtce:Parameter name="SEQ_FLGS" parameterTypeRef="uint2">
+				<xtce:LongDescription>CCSDS Packet Grouping Flags (3=not part of group)</xtce:LongDescription>
+			</xtce:Parameter>
+			<xtce:Parameter name="SRC_SEQ_CTR" parameterTypeRef="uint14">
+				<xtce:LongDescription>CCSDS Packet Sequence Count (increments with each new packet)</xtce:LongDescription>
+			</xtce:Parameter>
+			<xtce:Parameter name="PKT_LEN" parameterTypeRef="uint16">
+				<xtce:LongDescription>CCSDS Packet Length (number of bytes after Packet length minus 1)</xtce:LongDescription>
+			</xtce:Parameter>
+			<!-- MAG -->
+			<xtce:Parameter name="MAG_SHCOARSE" parameterTypeRef="uint32">
+				<xtce:LongDescription>MAG MET at End of Data Acquisition</xtce:LongDescription>
+			</xtce:Parameter>
+			<xtce:Parameter name="MAG_ACQ_TM_COARSE" parameterTypeRef="uint32">
+				<xtce:LongDescription>MAG Coarse Acquisition Time</xtce:LongDescription>
+			</xtce:Parameter>
+			<xtce:Parameter name="MAG_ACQ_TM_FINE" parameterTypeRef="uint16">
+				<xtce:LongDescription>MAG Fine Acquisition Time</xtce:LongDescription>
+			</xtce:Parameter>
+			<xtce:Parameter name="MAG_STATUS" parameterTypeRef="uint24">
+				<xtce:LongDescription>MAG Status</xtce:LongDescription>
+			</xtce:Parameter>
+			<xtce:Parameter name="MAG_DATA" parameterTypeRef="uint24">
+				<xtce:LongDescription>MAG Data</xtce:LongDescription>
+			</xtce:Parameter>
+			<!-- MAG -->
+		</xtce:ParameterSet>
+		<!-- End metadata -->
+		<xtce:ContainerSet>
+			<xtce:SequenceContainer name="CCSDSPacket" >
+				<xtce:EntryList>
+					<xtce:ParameterRefEntry parameterRef="VERSION" />
+					<xtce:ParameterRefEntry parameterRef="TYPE" />
+					<xtce:ParameterRefEntry parameterRef="SEC_HDR_FLG" />
+					<xtce:ParameterRefEntry parameterRef="PKT_APID" />
+					<xtce:ParameterRefEntry parameterRef="SEQ_FLGS" />
+					<xtce:ParameterRefEntry parameterRef="SRC_SEQ_CTR" />
+					<xtce:ParameterRefEntry parameterRef="PKT_LEN" />
+				</xtce:EntryList>
+			</xtce:SequenceContainer>
+			<xtce:SequenceContainer name="IALiRTPacket">
+				<xtce:BaseContainer containerRef="CCSDSPacket">
+					<xtce:RestrictionCriteria>
+						<xtce:Comparison parameterRef="PKT_APID" value="1001" useCalibratedValue="false" />
+					</xtce:RestrictionCriteria>
+				</xtce:BaseContainer>
+				<xtce:EntryList>
+					<!-- MAG -->
+					<xtce:ParameterRefEntry parameterRef="MAG_SHCOARSE"/>
+					<xtce:ParameterRefEntry parameterRef="MAG_ACQ_TM_COARSE"/>
+					<xtce:ParameterRefEntry parameterRef="MAG_ACQ_TM_FINE"/>
+					<xtce:ParameterRefEntry parameterRef="MAG_STATUS"/>
+					<xtce:ParameterRefEntry parameterRef="MAG_DATA"/>
+					<!-- MAG -->
+				</xtce:EntryList>
+			</xtce:SequenceContainer>
+		</xtce:ContainerSet>
+	</xtce:TelemetryMetaData>
+</xtce:SpaceSystem>

imap_processing/ialirt/utils/grouping.py

@@ -0,0 +1,114 @@
+"""Common grouping functions for I-ALiRT instruments."""
+
+import logging
+
+import numpy as np
+import xarray as xr
+
+logger = logging.getLogger(__name__)
+
+
+def filter_valid_groups(grouped_data: xr.Dataset) -> xr.Dataset:
+    """
+    Filter out groups where `src_seq_ctr` diff are not 1.
+
+    Parameters
+    ----------
+    grouped_data : xr.Dataset
+        Dataset with a "group" coordinate.
+
+    Returns
+    -------
+    filtered_data : xr.Dataset
+        Filtered dataset with only valid groups remaining.
+    """
+    valid_groups = []
+    unique_groups = np.unique(grouped_data["group"].values)
+
+    for group in unique_groups:
+        src_seq_ctr = grouped_data["src_seq_ctr"][
+            (grouped_data["group"] == group).values
+        ]
+        src_seq_ctr_diff = np.diff(src_seq_ctr) % 16384
+
+        # Accept group only if all diffs are 1.
+        if np.all(src_seq_ctr_diff == 1):
+            valid_groups.append(group)
+        else:
+            logger.info(f"src_seq_ctr_diff != 1 for group {group}.")
+
+    filtered_data = grouped_data.where(
+        xr.DataArray(np.isin(grouped_data["group"], valid_groups), dims="epoch"),
+        drop=True,
+    )
+
+    return filtered_data
+
+
+def find_groups(
+    accumulated_data: xr.Dataset,
+    sequence_range: tuple,
+    sequence_name: str,
+    time_name: str,
+) -> xr.Dataset:
+    """
+    Group data based on time and sequence number values.
+
+    Parameters
+    ----------
+    accumulated_data : xr.Dataset
+        Packets dataset accumulated over 1 min.
+    sequence_range : tuple
+        Tuple of two integers defining the range of group values.
+    sequence_name : str
+        Name of the sequence variable.
+    time_name : str
+        Name of the time variable.
+
+    Returns
+    -------
+    grouped_data : xr.Dataset
+        Filtered data with "group" coordinate.
+
+    Notes
+    -----
+    Filters data based on:
+    1. Time values between the first and last sequence_range values.
+    Take out time values before sequence_range[0] and after sequence_range[-1].
+    2. Sequence values src_seq_ctr between the first and
+    last sequence_range. These must be consecutive.
+    """
+    sorted_data = accumulated_data.sortby(time_name, ascending=True)
+
+    # Use sequence_range == 0 to define the beginning of the group.
+    # Find time at this index and use it as the beginning time for the group.
+    start_times = sorted_data[time_name][
+        (sorted_data[sequence_name] == sequence_range[0])
+    ]
+    start_time = start_times.min()
+    # Use max sequence_range to define the end of the group.
+    end_times = sorted_data[time_name][
+        ([sorted_data[sequence_name] == sequence_range[-1]][-1])
+    ]
+    end_time = end_times.max()
+
+    # Filter data before the sequence_range=0
+    # and after the last value of sequence_range.
+    grouped_data = sorted_data.where(
+        (sorted_data[time_name] >= start_time) & (sorted_data[time_name] <= end_time),
+        drop=True,
+    )
+
+    # Assign labels based on the start_times.
+    group_labels = np.searchsorted(start_times, grouped_data[time_name], side="right")
+    # Example:
+    # grouped_data.coords
+    # Coordinates:
+    #   * epoch    (epoch) int64 7kB 315922822184000000 ... 315923721184000000
+    #     group    (epoch) int64 7kB 1 1 1 1 1 1 1 1 1 ... 15 15 15 15 15 15 15 15 15
+    grouped_data = grouped_data.assign_coords(group=("epoch", group_labels))
+
+    # Filter out groups with non-sequential src_seq_ctr values.
+    filtered_data = filter_valid_groups(grouped_data)
+
+    return filtered_data

imap_processing/ialirt/utils/time.py

@@ -0,0 +1,29 @@
+"""Common time functions for I-ALiRT instruments."""
+
+import xarray as xr
+
+
+def calculate_time(
+    coarse_time: xr.DataArray, fine_time: xr.DataArray, conversion: int
+) -> xr.DataArray:
+    """
+    Calculate the time.
+
+    Parameters
+    ----------
+    coarse_time : xr.DataArray
+        Coarse time.
+    fine_time : xr.DataArray
+        Fine time.
+    conversion : int
+        Fine time units = 1 second.
+
+    Returns
+    -------
+    time_seconds: xr.DataArray
+        Calculated time.
+    """
+    fine_time_fraction = fine_time / conversion
+    time_seconds = coarse_time + fine_time_fraction
+
+    return time_seconds

imap_processing/idex/atomic_masses.csv

@@ -0,0 +1,22 @@
+Mass,Name
+1,H
+2,H2
+12,C
+16,O
+22,Na
+23,Mg24
+24,Mg25
+25,Mg26
+27,Si28
+28,Si29
+29,Si30
+39,K39
+39,Ca40
+40,K41
+41,Ca42
+43,Ca44
+53,Fe54
+55,Fe56
+56,Fe57
+57,Fe58
+196,Au197

imap_processing/idex/decode.py

@@ -174,7 +174,7 @@ def rice_decode(compressed_data: str, nbit10: bool, sample_count: int) -> list[i
 
     Returns
     -------
-    list
+    list[int]
         Decompressed data as a list of integers.
     """
     # Constants:
@@ -186,7 +186,7 @@ def rice_decode(compressed_data: str, nbit10: bool, sample_count: int) -> list[i
     sub_frame_per_frame = frame_size / SUB_FRAME_SIZE
 
     bits = compressed_data
-    out_data = []
+    out_data: list[int] = []
     sub_frame_count = 0
     bp = 0
     # Decode all subframes

imap_processing/idex/idex_constants.py

@@ -1,6 +1,7 @@
 """Contains dataclasses to support IDEX processing."""
 
 from dataclasses import dataclass
+from enum import Enum
 
 
 @dataclass
@@ -25,3 +26,27 @@ class IdexConstants:
     DATA_MAX: int = 4096
     SAMPLE_RATE_MIN: int = -130
     SAMPLE_RATE_MAX: int = 130
+
+
+# FM sampling rate (quartz oscillator)
+# Seconds per sample.
+FM_SAMPLING_RATE = 0.0038466235767167234e-6
+# Nanoseconds to seconds conversion
+NS_TO_S = 1e-9
+# Microseconds to seconds conversion
+US_TO_S = 1e-6
+
+TARGET_HIGH_FREQUENCY_CUTOFF = 100
+
+TARGET_NOISE_FREQUENCY = 7000
+
+
+class ConversionFactors(float, Enum):
+    """Conversion factor values (DN to picocoulombs) for each of the six waveforms."""
+
+    TOF_High = 2.89e-4
+    TOF_Low = 5.14e-4
+    TOF_Mid = 1.13e-2
+    Target_Low = 1.58e1
+    Target_High = 1.63e-1
+    Ion_Grid = 7.46e-4

imap_processing/idex/idex_l1a.py

@@ -162,10 +162,10 @@ def _read_waveform_bits(waveform_raw: str, high_sample: bool = True) -> list[int
 
     Returns
     -------
-    ints : list
+    ints : list[int]
         List of the waveform.
     """
-    ints = []
+    ints: list[int] = []
     if high_sample:
         for i in range(0, len(waveform_raw), 32):
             # 32-bit chunks, divided up into 2, 10, 10, 10
@@ -398,6 +398,7 @@ class RawDustEvent:
         # To extract the high gain bits, the bitwise right shift (>> 20) moves the bits
         # 20 positions to the right, and the mask (0b1111111111) keeps only the least
         # significant 10 bits.
+        # TODO use the delay corresponding to the trigger
         high_gain_delay = (packet["IDX__TXHDRSAMPDELAY"] >> 22) & 0b1111111111
         n_blocks = packet["IDX__TXHDRBLOCKS"]
 
@@ -410,22 +411,20 @@ class RawDustEvent:
         # Bits 13-16 represent the number of high sampling pre-trigger blocks.
         #   We can extract this by shifting right by 16 bits and applying a mask to keep
         #   the last 4 bits.
-
         num_low_sample_pretrigger_blocks = (n_blocks >> 6) & 0b111111
         num_high_sample_pretrigger_blocks = (n_blocks >> 16) & 0b1111
-
         # Calculate the low and high sample trigger times based on the high gain delay
         # and the number of high sample/low sample pretrigger blocks
         self.low_sample_trigger_time = (
             self.LOW_SAMPLE_RATE
             * (num_low_sample_pretrigger_blocks + 1)
             * self.NUMBER_SAMPLES_PER_LOW_SAMPLE_BLOCK
-            - self.HIGH_SAMPLE_RATE * high_gain_delay
         )
         self.high_sample_trigger_time = (
             self.HIGH_SAMPLE_RATE
             * (num_high_sample_pretrigger_blocks + 1)
             * self.NUMBER_SAMPLES_PER_HIGH_SAMPLE_BLOCK
+            - self.HIGH_SAMPLE_RATE * high_gain_delay
         )
 
     def _parse_high_sample_waveform(self, waveform_raw: str) -> list[int]:
@@ -442,7 +441,7 @@ class RawDustEvent:
 
         Returns
         -------
-        ints : list
+        ints : list[int]
             List of the high sample waveform.
         """
         samples = self.MAX_HIGH_BLOCKS * self.NUMBER_SAMPLES_PER_HIGH_SAMPLE_BLOCK
@@ -467,7 +466,7 @@ class RawDustEvent:
 
         Returns
         -------
-        ints : list
+        ints : list[int]
             List of processed low sample waveform.
         """
         samples = self.MAX_LOW_BLOCKS * self.NUMBER_SAMPLES_PER_LOW_SAMPLE_BLOCK

imap_processing/idex/idex_l1b.py

@@ -23,6 +23,7 @@ import xarray as xr
 
 from imap_processing import imap_module_directory
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.idex.idex_constants import ConversionFactors
 from imap_processing.spice.geometry import (
     SpiceBody,
     SpiceFrame,
@@ -38,34 +39,6 @@ from imap_processing.utils import convert_raw_to_eu
 logger = logging.getLogger(__name__)
 
 
-class ConversionFactors(float, Enum):
-    """
-    Enum class for conversion factor values.
-
-    Attributes
-    ----------
-    TOF_High : float
-        Time of flight high conversion factor.
-    TOF_Low : float
-        Time of flight low conversion factor.
-    TOF_Mid : float
-        Time of flight mid conversion factor.
-    Target_Low : float
-        Target Low conversion factor.
-    Target_High : float
-        Target High conversion factor.
-    Ion_Grid : float
-        Ion Grid conversion factor.
-    """
-
-    TOF_High = 2.89e-4
-    TOF_Low = 5.14e-4
-    TOF_Mid = 1.13e-2
-    Target_Low = 1.58e1
-    Target_High = 1.63e-1
-    Ion_Grid = 7.46e-4
-
-
 class TriggerMode(Enum):
     """
     Enum class for data collection trigger Modes.
@@ -183,8 +156,6 @@ def idex_l1b(l1a_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
     for var in vars_to_copy:
         l1b_dataset[var] = l1a_dataset[var].copy()
 
-    # TODO: Spice data?
-
     logger.info("IDEX L1B science data processing completed.")
 
     return l1b_dataset
@@ -216,7 +187,9 @@ def unpack_instrument_settings(
         values are the unpacked xr.DataArrays.
     """
     telemetry_data = {}
-
+    # Unpack each instrument setting only once (remove duplicated rows for segmented
+    # polynomials)
+    var_information_df = var_information_df.drop_duplicates(subset=["mnemonic"])
     for _, row in var_information_df.iterrows():
         unpacked_name = row["mnemonic"]