imap-processing 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

imap_processing/idex/decode.py

@@ -0,0 +1,241 @@
+"""
+Decompress IDEX raw wavelength data.
+
+Originally written by Corinne Wuerthner.
+A compressed data frame starts on a byte boundary, and begins with the 8-bit sequence
+0xF5 to indicate the start of a frame.
+A frame contains one or more subframes of compressed data. Subframes are packed as
+tightly as possible, and are not aligned to any byte or word boundary. The compression
+parameters are varied for each subframe to optimize the data compression. Within a
+subframe the same predictor, and the same Rice parameter is used for all the
+samples in that subframe. The predictor and rice parameter is allowed to vary
+from subframe to subframe.
+
+The start of each subframe consists of the following bits:
+    00: Constant value
+        The predictor assumes that every sample in the frame is equal to the
+        first sample in the frame. In this case, the first sample is stored as
+        the original NBIT binary value and no other samples are included in the
+        frame.  The constant value predictor will only be selected for a
+        subframe where every sample is the same value.
+
+    01: Verbatim
+        The predictor assumes that the data is not well correlated, and simply
+        stores every sample in the frame using the original NBIT binary
+        representation. This predictor minimizes the impact of data expansion
+        for uncorrelated data samples where the golomb-rice coding would
+        actually make the encoded data significantly larger than the original
+        data set.
+
+    10: Linear predictor #1
+        This predictor assumes that each sample will have the same value as the
+        sample immediately before it (sample X(n) = sampleX(n-1)). The predicted
+        value is subtracted from the actual value of the sample and only the
+        error (residual) between the actual value, and the predicted value is
+        stored.  This predictor requires a single uncompressed sample as a
+        “warmup sample” to be used in predicting the other samples in the
+        subframe.
+
+    11: Linear Predictor #2
+        This predictor assumes that sample X(n) = 2*X(n-1)-X(n-2). This
+        predictor calculates the slope of the signal based on the two previous
+        values (slope = X(n-1)-X(n-2)) and predicts the value of X(n) will
+        follow on a straight line drawn between the two previous points.
+        X(n) = X(n-1) + (X(n-1)-X(n-2)) = 2*X(n-1)-X(n-2).
+
+M bits for the rice parameter k. The bit width of k is equal to ceiling(log2(NBITS)).
+For 10 bits, the bit width of k is 4 because log2(10) = 3.32 and ceiling(3.32) = 4.
+The rice parameter k is stored in the bit stream as an unsigned binary value. The rice
+parameter is omitted from the bit stream if the predictor type is set to “00” or '01'.
+The next NBITS contain the raw binary value of the first sample in the subframe.
+"""
+
+# sub_frame_size is the compression block size
+SUB_FRAME_SIZE = 64
+
+
+def _decode_sub_frame(
+    bits: str,
+    bp: int,
+    psel: int,
+    k: int,
+    n_bits: int,
+) -> tuple[list[int], int]:
+    """
+    Decode a subframe of compressed data.
+
+    Parameters
+    ----------
+    bits : str
+        Raw waveform binary string.
+    bp : int
+        Current position to start reading from bits.
+    psel : int
+        Predictor select value.
+    k : int
+        Rice parameter used to divide each re-mapped residual value into two parts.
+    n_bits : int
+        Expected number of bits per sample. Either 10 or 12.
+
+    Returns
+    -------
+    tuple[list, int]
+        Decompressed subframe as a list of integers and the bit position.
+    """
+    sample_count = 0
+    sub_frame_data = []
+
+    while (sample_count < SUB_FRAME_SIZE) and bp < len(bits):
+        if sample_count == 0:
+            # For every subframe, the first sample is always uncompressed.
+            # Read warmup sample
+            d1, bp = read_bits(bits, bp, n_bits)
+
+            sub_frame_data.append(d1)
+            sample_count += 1
+
+            # A 'psel' value of zero assumes that every sample in the frame is equal
+            # to the first sample in the frame. In this case, the first sample is
+            # stored as the original NBIT binary value
+            if psel == 0:
+                sub_frame_data.extend([d1] * (SUB_FRAME_SIZE - 1))
+                sample_count = SUB_FRAME_SIZE
+
+        # A 'psel' value of 1 assumes that the data is not well correlated, and
+        # simply stores every sample in the frame using the original NBIT binary
+        # representation.
+        # A 'psel' value of 3 requires two uncompressed 'warm-up' samples.
+        elif (psel == 1) or ((sample_count == 1) and (psel == 3)):
+            d1, bp = read_bits(bits, bp, n_bits)
+            sub_frame_data.append(d1)
+            sample_count += 1
+
+        else:
+            # The rice parameter (k) is used to divide each re-mapped residual value
+            # into two parts. The least significant k bits of the value are called
+            # the remainder (r). The other part of the value (not included in the
+            # remainder) is called the quotient (q).
+            # The remapped quotient is unary encoded by including a number of 0 bits
+            # equal to the value of the quotient, followed by a single '1' bit.
+            q = bits[bp:].find("1")
+            bp += q + 1
+            # If the value of the quotient is equal to or larger than 47, then a
+            # special symbol is used to denote that this particular residual value
+            # is not rice encoded, but that this special symbol is followed by the
+            # raw binary representation of the residual value using a (N_BITS+2)
+            # bit binary number. This special symbol is simply 47 zeros followed
+            # by a one.
+            if q == 47:
+                d1, bp = read_bits(bits, bp, n_bits + 2, True)
+            else:
+                if q % 2 == 1:
+                    q = int(-((q + 1) / 2))
+                else:
+                    q = int(q / 2)
+
+                r, bp = read_bits(bits, bp, k + 1)
+                d1 = (q << (k + 1)) + r
+
+            if psel == 2:
+                d1 = d1 + sub_frame_data[sample_count - 1]
+            elif (sample_count > 1) and (psel == 3):
+                d1 = (
+                    d1
+                    + 2 * sub_frame_data[(sample_count - 1)]
+                    - sub_frame_data[(sample_count - 2)]
+                )
+
+            if (d1 > 2**n_bits) or (d1 < -(2**n_bits)):
+                raise ValueError(
+                    f"Overflow Error while decoding subframe "
+                    f"{k=}, {q=}, {r=}, {d1=}\n"
+                    f"DataOut = {sub_frame_data}"
+                )
+
+            sub_frame_data.append(d1)
+            sample_count += 1
+
+    return sub_frame_data, bp
+
+
+def rice_decode(compressed_data: str, nbit10: bool, sample_count: int) -> list[int]:
+    """
+    Decode compressed IDEX wavelength data using linear prediction and Golomb-RICE.
+
+    Parameters
+    ----------
+    compressed_data : str
+        Binary string representation of the raw waveform.
+    nbit10 : bool
+        If nbit10 is true, then the samples are expected to be 10 bits each, and if
+        nbit10 is false, then the samples are expected to be 12 bits each.
+    sample_count : int
+        The total number of samples to be decompressed.
+
+    Returns
+    -------
+    list
+        Decompressed data as a list of integers.
+    """
+    # Constants:
+    k_bits = 4
+    n_bits = 10 if nbit10 else 12
+
+    # frame_size is the expected amount of data
+    frame_size = sample_count
+    sub_frame_per_frame = frame_size / SUB_FRAME_SIZE
+
+    bits = compressed_data
+    out_data = []
+    sub_frame_count = 0
+    bp = 0
+    # Decode all subframes
+    while bp < len(bits) and (sub_frame_count < sub_frame_per_frame):
+        # The next two bits are the predictor select bits
+        psel, bp = read_bits(bits, bp, 2)
+        if psel > 1:
+            k, bp = read_bits(bits, bp, k_bits)
+        else:
+            k = 0
+
+        sub_frame_data, bp = _decode_sub_frame(bits, bp, psel, k, n_bits)
+
+        out_data.extend(sub_frame_data)
+        sub_frame_count += 1
+
+    if bp < len(bits) and (len(out_data) < frame_size):
+        raise ValueError("End of file reached before", frame_size, "samples decoded")
+
+    return out_data
+
+
+def read_bits(
+    bits: str, bit_pointer: int, read_num: int, signed: bool = False
+) -> tuple[int, int]:
+    """
+    Read bits from a binary string and convert to an int.
+
+    Parameters
+    ----------
+    bits : str
+       Binary string to read from.
+    bit_pointer : int
+       Current position in binary string.
+    read_num : int
+       Number of bits to read.
+    signed : bool
+       If signed is True, convert bits to a signed int. Default is False.
+
+    Returns
+    -------
+    value : int
+       Value of bits read.
+    bit_pointer : int
+       Bit position after reading.
+    """
+    value = int(bits[bit_pointer : bit_pointer + read_num], 2)
+    if signed and bits[bit_pointer] == "1":
+        # If signed and is negative convert value
+        value = value - 2**read_num
+    bit_pointer += read_num
+    return value, bit_pointer

imap_processing/idex/idex_l1a.py

@@ -25,8 +25,9 @@ import space_packet_parser
 import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.idex.decode import rice_decode
 from imap_processing.idex.idex_l0 import decom_packets
-from imap_processing.spice.time import met_to_j2000ns
+from imap_processing.spice.time import met_to_ttj2000ns
 from imap_processing.utils import convert_to_binary_string
 
 logger = logging.getLogger(__name__)
@@ -76,7 +77,6 @@ class PacketParser:
             Currently assumes one L0 file will generate exactly one L1a file.
         """
         decom_packet_list = decom_packets(packet_file)
-
         dust_events = {}
         for packet in decom_packet_list:
             if "IDX__SCI0TYPE" in packet:
@@ -105,24 +105,88 @@ class PacketParser:
         idex_attrs = get_idex_attrs(data_version)
         self.data.attrs = idex_attrs.get_global_attributes("imap_idex_l1a_sci")
 
+        # Add high and low sample rate coords
+        self.data["time_low_sample_rate_index"] = xr.DataArray(
+            np.arange(len(self.data["time_low_sample_rate"][0])),
+            name="time_low_sample_rate_index",
+            dims=["time_low_sample_rate_index"],
+            attrs=idex_attrs.get_variable_attributes("time_low_sample_rate_index"),
+        )
+
+        self.data["time_high_sample_rate_index"] = xr.DataArray(
+            np.arange(len(self.data["time_high_sample_rate"][0])),
+            name="time_high_sample_rate_index",
+            dims=["time_high_sample_rate_index"],
+            attrs=idex_attrs.get_variable_attributes("time_high_sample_rate_index"),
+        )
         # NOTE: LABL_PTR_1 should be CDF_CHAR.
-        self.data["time_low_sr_label"] = xr.DataArray(
-            self.data.time_low_sr_dim.values.astype(str),
-            name="time_low_sr_label",
-            dims=["time_low_sr_label"],
-            attrs=idex_attrs.get_variable_attributes("time_low_sr_label"),
+        self.data["time_low_sample_rate_label"] = xr.DataArray(
+            self.data.time_low_sample_rate_index.values.astype(str),
+            name="time_low_sample_rate_label",
+            dims=["time_low_sample_rate_index"],
+            attrs=idex_attrs.get_variable_attributes("time_low_sample_rate_label"),
         )
 
-        self.data["time_high_sr_label"] = xr.DataArray(
-            self.data.time_high_sr_dim.values.astype(str),
-            name="time_high_sr_label",
-            dims=["time_high_sr_label"],
-            attrs=idex_attrs.get_variable_attributes("time_high_sr_label"),
+        self.data["time_high_sample_rate_label"] = xr.DataArray(
+            self.data.time_high_sample_rate_index.values.astype(str),
+            name="time_high_sample_rate_label",
+            dims=["time_high_sample_rate_index"],
+            attrs=idex_attrs.get_variable_attributes("time_high_sample_rate_label"),
         )
 
         logger.info("IDEX L1A science data processing completed.")
 
 
+def _read_waveform_bits(waveform_raw: str, high_sample: bool = True) -> list[int]:
+    """
+    Convert the raw waveform binary string to ints.
+
+    Parse a binary string representing a waveform.
+    If the data is a high sample waveform:
+        - Data arrives in 32-bit chunks, divided up into:
+            * 2 bits of padding
+            * 3x10 bits of integer data.
+        - The very last four numbers are usually bad, so remove those.
+    If the data is a low sample waveform:
+        - Data arrives in 32-bit chunks, divided up into:
+            * 8 bits of padding
+            * 2x12 bits of integer data.
+
+    Parameters
+    ----------
+    waveform_raw : str
+        The binary string representing the waveform.
+    high_sample : bool
+        If true, parse the waveform according to the high sample pattern,
+        otherwise use the low sample pattern.
+
+    Returns
+    -------
+    ints : list
+        List of the waveform.
+    """
+    ints = []
+    if high_sample:
+        for i in range(0, len(waveform_raw), 32):
+            # 32-bit chunks, divided up into 2, 10, 10, 10
+            # skip first two bits
+            ints += [
+                int(waveform_raw[i + 2 : i + 12], 2),
+                int(waveform_raw[i + 12 : i + 22], 2),
+                int(waveform_raw[i + 22 : i + 32], 2),
+            ]
+        ints = ints[:-4]  # Remove last 4 numbers
+    else:
+        for i in range(0, len(waveform_raw), 32):
+            # 32-bit chunks, divided up into 8, 12, 12
+            # skip first eight bits
+            ints += [
+                int(waveform_raw[i + 8 : i + 20], 2),
+                int(waveform_raw[i + 20 : i + 32], 2),
+            ]
+    return ints
+
+
 class RawDustEvent:
     """
     Encapsulate IDEX Raw Dust Event.
@@ -147,6 +211,10 @@ class RawDustEvent:
         The number of samples in a "block" of low sample data.
     NUMBER_SAMPLES_PER_HIGH_SAMPLE_BLOCK: int
         The number of samples in a "block" of high sample data.
+    MAX_HIGH_BLOCKS: int
+        The maximum number of "blocks" for high sample data.
+    MAX_LOW_BLOCKS: int
+        The maximum number of "blocks" for low sample data.
 
     Methods
     -------
@@ -180,6 +248,9 @@ class RawDustEvent:
     NUMBER_SAMPLES_PER_HIGH_SAMPLE_BLOCK = (
         512  # The number of samples in a "block" of high sample data
     )
+    # Maximum amount of data
+    MAX_HIGH_BLOCKS = 16
+    MAX_LOW_BLOCKS = 64
 
     def __init__(
         self, header_packet: space_packet_parser.packets.CCSDSPacket, data_version: str
@@ -229,6 +300,7 @@ class RawDustEvent:
         self.Target_High_bits = ""
         self.Ion_Grid_bits = ""
 
+        self.compressed = self.telemetry_items["idx__sci0comp"]
         self.cdf_attrs = get_idex_attrs(data_version)
 
     def _append_raw_data(self, scitype: Scitype, bits: str) -> None:
@@ -287,7 +359,7 @@ class RawDustEvent:
         # Get the datetime of Jan 1 2012 as the start date
         met = seconds_since_launch + microseconds_since_last_second * 1e-6
 
-        self.impact_time = met_to_j2000ns(met)
+        self.impact_time = met_to_ttj2000ns(met)
 
     def _set_sample_trigger_times(
         self, packet: space_packet_parser.packets.CCSDSPacket
@@ -361,11 +433,7 @@ class RawDustEvent:
         Will process the high sample waveform.
 
         Parse a binary string representing a high sample waveform.
-        Data arrives in 32 bit chunks, divided up into:
-            * 2 bits of padding
-            * 3x10 bits of integer data.
-
-        The very last 4 numbers are bad usually, so remove those.
+        If the data has been compressed, decompress using the RICE Golomb algorithm.
 
         Parameters
         ----------
@@ -377,25 +445,20 @@ class RawDustEvent:
         ints : list
             List of the high sample waveform.
         """
-        ints = []
-        for i in range(0, len(waveform_raw), 32):
-            # 32 bit chunks, divided up into 2, 10, 10, 10
-            # skip first two bits
-            ints += [
-                int(waveform_raw[i + 2 : i + 12], 2),
-                int(waveform_raw[i + 12 : i + 22], 2),
-                int(waveform_raw[i + 22 : i + 32], 2),
-            ]
-        return ints[:-4]  # Remove last 4 numbers
+        samples = self.MAX_HIGH_BLOCKS * self.NUMBER_SAMPLES_PER_HIGH_SAMPLE_BLOCK
+        if self.compressed.raw_value == 1:
+            ints = rice_decode(waveform_raw, nbit10=True, sample_count=samples)
+            ints = ints[:-3]
+        else:
+            ints = _read_waveform_bits(waveform_raw, high_sample=True)
+        return ints
 
     def _parse_low_sample_waveform(self, waveform_raw: str) -> list[int]:
         """
         Will process the low sample waveform.
 
         Parse a binary string representing a low sample waveform
-        Data arrives in 32-bit chunks, divided up into:
-            * 8 bits of padding
-            * 2x12 bits of integer data.
+        If the data has been compressed, decompress using the RICE Golomb algorithm.
 
         Parameters
         ----------
@@ -407,12 +470,11 @@ class RawDustEvent:
         ints : list
             List of processed low sample waveform.
         """
-        ints = []
-        for i in range(0, len(waveform_raw), 32):
-            ints += [
-                int(waveform_raw[i + 8 : i + 20], 2),
-                int(waveform_raw[i + 20 : i + 32], 2),
-            ]
+        samples = self.MAX_LOW_BLOCKS * self.NUMBER_SAMPLES_PER_LOW_SAMPLE_BLOCK
+        if self.compressed.raw_value == 1:
+            ints = rice_decode(waveform_raw, nbit10=False, sample_count=samples)
+        else:
+            ints = _read_waveform_bits(waveform_raw, high_sample=False)
         return ints
 
     def _calc_low_sample_resolution(self, num_samples: int) -> npt.NDArray:
@@ -432,14 +494,15 @@ class RawDustEvent:
 
         Returns
         -------
-        time_low_sr_data : numpy.ndarray
+        time_low_sample_rate_data : numpy.ndarray
             Low time sample data array.
         """
-        time_low_sr_init = np.linspace(0, num_samples, num_samples)
-        time_low_sr_data = (
-            self.LOW_SAMPLE_RATE * time_low_sr_init - self.low_sample_trigger_time
+        time_low_sample_rate_init = np.linspace(0, num_samples, num_samples)
+        time_low_sample_rate_data = (
+            self.LOW_SAMPLE_RATE * time_low_sample_rate_init
+            - self.low_sample_trigger_time
         )
-        return time_low_sr_data
+        return time_low_sample_rate_data
 
     def _calc_high_sample_resolution(self, num_samples: int) -> npt.NDArray:
         """
@@ -458,14 +521,15 @@ class RawDustEvent:
 
         Returns
         -------
-        time_high_sr_data : numpy.ndarray
+        time_high_sample_rate_data : numpy.ndarray
             High sample time data array.
         """
-        time_high_sr_init = np.linspace(0, num_samples, num_samples)
-        time_high_sr_data = (
-            self.HIGH_SAMPLE_RATE * time_high_sr_init - self.high_sample_trigger_time
+        time_high_sample_rate_init = np.linspace(0, num_samples, num_samples)
+        time_high_sample_rate_data = (
+            self.HIGH_SAMPLE_RATE * time_high_sample_rate_init
+            - self.high_sample_trigger_time
         )
-        return time_high_sr_data
+        return time_high_sample_rate_data
 
     def _populate_bit_strings(
         self, packet: space_packet_parser.packets.CCSDSPacket
@@ -510,81 +574,79 @@ class RawDustEvent:
             )
 
         # Process the 6 primary data variables
-        tof_high_xr = xr.DataArray(
+        tof_high = xr.DataArray(
             name="TOF_High",
             data=[self._parse_high_sample_waveform(self.TOF_High_bits)],
-            dims=("epoch", "time_high_sr_dim"),
+            dims=("epoch", "time_high_sample_rate_index"),
             attrs=idex_attrs.get_variable_attributes("tof_high_attrs"),
         )
-        tof_low_xr = xr.DataArray(
+        tof_low = xr.DataArray(
             name="TOF_Low",
             data=[self._parse_high_sample_waveform(self.TOF_Low_bits)],
-            dims=("epoch", "time_high_sr_dim"),
+            dims=("epoch", "time_high_sample_rate_index"),
             attrs=idex_attrs.get_variable_attributes("tof_low_attrs"),
         )
-        tof_mid_xr = xr.DataArray(
+        tof_mid = xr.DataArray(
             name="TOF_Mid",
             data=[self._parse_high_sample_waveform(self.TOF_Mid_bits)],
-            dims=("epoch", "time_high_sr_dim"),
+            dims=("epoch", "time_high_sample_rate_index"),
             attrs=idex_attrs.get_variable_attributes("tof_mid_attrs"),
         )
-        target_high_xr = xr.DataArray(
+        target_high = xr.DataArray(
             name="Target_High",
             data=[self._parse_low_sample_waveform(self.Target_High_bits)],
-            dims=("epoch", "time_low_sr_dim"),
+            dims=("epoch", "time_low_sample_rate_index"),
             attrs=idex_attrs.get_variable_attributes("target_high_attrs"),
         )
-        target_low_xr = xr.DataArray(
+        target_low = xr.DataArray(
             name="Target_Low",
             data=[self._parse_low_sample_waveform(self.Target_Low_bits)],
-            dims=("epoch", "time_low_sr_dim"),
+            dims=("epoch", "time_low_sample_rate_index"),
             attrs=idex_attrs.get_variable_attributes("target_low_attrs"),
         )
-        ion_grid_xr = xr.DataArray(
+        ion_grid = xr.DataArray(
             name="Ion_Grid",
             data=[self._parse_low_sample_waveform(self.Ion_Grid_bits)],
-            dims=("epoch", "time_low_sr_dim"),
+            dims=("epoch", "time_low_sample_rate_index"),
             attrs=idex_attrs.get_variable_attributes("ion_grid_attrs"),
         )
 
         # Determine the 3 coordinate variables
-        epoch_xr = xr.DataArray(
+        epoch = xr.DataArray(
             name="epoch",
             data=[self.impact_time],
             dims=("epoch"),
             attrs=idex_attrs.get_variable_attributes("epoch"),
         )
 
-        time_low_sr_xr = xr.DataArray(
-            name="time_low_sr",
-            data=[self._calc_low_sample_resolution(len(target_low_xr[0]))],
-            dims=("epoch", "time_low_sr_dim"),
-            attrs=idex_attrs.get_variable_attributes("low_sr_attrs"),
+        time_low_sample_rate = xr.DataArray(
+            name="time_low_sample_rate",
+            data=[self._calc_low_sample_resolution(len(target_low[0]))],
+            dims=("epoch", "time_low_sample_rate_index"),
+            attrs=idex_attrs.get_variable_attributes("low_sample_rate_attrs"),
         )
 
-        time_high_sr_xr = xr.DataArray(
-            name="time_high_sr",
-            data=[self._calc_high_sample_resolution(len(tof_low_xr[0]))],
-            dims=("epoch", "time_high_sr_dim"),
-            attrs=idex_attrs.get_variable_attributes("high_sr_attrs"),
+        time_high_sample_rate = xr.DataArray(
+            name="time_high_sample_rate",
+            data=[self._calc_high_sample_resolution(len(tof_low[0]))],
+            dims=("epoch", "time_high_sample_rate_index"),
+            attrs=idex_attrs.get_variable_attributes("high_sample_rate_attrs"),
         )
 
         # Combine to return a dataset object
         dataset = xr.Dataset(
             data_vars={
-                "TOF_Low": tof_low_xr,
-                "TOF_High": tof_high_xr,
-                "TOF_Mid": tof_mid_xr,
-                "Target_High": target_high_xr,
-                "Target_Low": target_low_xr,
-                "Ion_Grid": ion_grid_xr,
+                "TOF_Low": tof_low,
+                "TOF_High": tof_high,
+                "TOF_Mid": tof_mid,
+                "Target_High": target_high,
+                "Target_Low": target_low,
+                "Ion_Grid": ion_grid,
+                "time_low_sample_rate": time_low_sample_rate,
+                "time_high_sample_rate": time_high_sample_rate,
             }
             | trigger_vars,
-            coords={
-                "epoch": epoch_xr,
-                "time_low_sr": time_low_sr_xr,
-                "time_high_sr": time_high_sr_xr,
-            },
+            coords={"epoch": epoch},
         )
 
         return dataset