imap-processing 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

imap_processing/mag/l0/decom_mag.py

@@ -16,7 +16,7 @@ from imap_processing.ccsds.ccsds_data import CcsdsData
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.mag.constants import DataMode
 from imap_processing.mag.l0.mag_l0_data import MagL0, Mode
-from imap_processing.spice.time import met_to_j2000ns
+from imap_processing.spice.time import met_to_ttj2000ns
 
 logger = logging.getLogger(__name__)
 
@@ -109,7 +109,7 @@ def generate_dataset(
             )
         vector_data[index, :vector_len] = datapoint.VECTORS
 
-        shcoarse_data[index] = met_to_j2000ns(datapoint.SHCOARSE)
+        shcoarse_data[index] = met_to_ttj2000ns(datapoint.SHCOARSE)
         # Add remaining pieces to arrays
         for key, value in dataclasses.asdict(datapoint).items():
             if key not in ("ccsds_header", "VECTORS", "SHCOARSE"):

imap_processing/mag/l1a/mag_l1a.py

@@ -16,7 +16,10 @@ from imap_processing.mag.l1a.mag_l1a_data import (
     MagL1aPacketProperties,
     TimeTuple,
 )
-from imap_processing.spice.time import J2000_EPOCH, met_to_j2000ns
+from imap_processing.spice.time import (
+    et_to_utc,
+    ttj2000ns_to_et,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -148,15 +151,12 @@ def process_packets(
         mago_is_primary = mag_l0.PRI_SENS == PrimarySensor.MAGO.value
 
         primary_day = (
-            J2000_EPOCH
-            + met_to_j2000ns(primary_start_time.to_seconds()).astype("timedelta64[ns]")
+            et_to_utc(ttj2000ns_to_et([primary_start_time.to_j2000ns()]))[0]
         ).astype("datetime64[D]")
         secondary_day = (
-            J2000_EPOCH
-            + met_to_j2000ns(secondary_start_time.to_seconds()).astype(
-                "timedelta64[ns]"
-            )
+            et_to_utc(ttj2000ns_to_et([secondary_start_time.to_j2000ns()]))[0]
         ).astype("datetime64[D]")
+
         primary_packet_properties = MagL1aPacketProperties(
             mag_l0.SHCOARSE,
             primary_start_time,

imap_processing/mag/l1a/mag_l1a_data.py

@@ -16,7 +16,7 @@ from imap_processing.mag.constants import (
     MAX_FINE_TIME,
     RANGE_BIT_WIDTH,
 )
-from imap_processing.spice.time import J2000_EPOCH, met_to_j2000ns
+from imap_processing.spice.time import met_to_ttj2000ns
 
 
 @dataclass
@@ -24,19 +24,20 @@ class TimeTuple:
     """
     Class for storing fine time/coarse time for MAG data.
 
-    Course time is mission SCLK in seconds. Fine time is 16bit unsigned sub-second
+    Coarse time is MET in seconds. Fine time is 16bit unsigned sub-second
     counter.
 
     Attributes
     ----------
     coarse_time : int
-        Coarse time in seconds.
+        Coarse time in seconds (MET).
     fine_time : int
-        Subsecond.
+        Subsecond counter, equal to fine_time/max_int16 seconds.
 
     Methods
     -------
     to_seconds()
+    to_j2000ns()
     """
 
     coarse_time: int
@@ -79,6 +80,19 @@ class TimeTuple:
         """
         return float(self.coarse_time + self.fine_time / MAX_FINE_TIME)
 
+    def to_j2000ns(self) -> np.int64:
+        """
+        Convert time tuple into J2000ns.
+
+        Returns
+        -------
+        j2000ns : numpy.int64
+            Time in nanoseconds since J2000 epoch.
+        """
+        coarse_j2000ns = np.int64(met_to_ttj2000ns(self.coarse_time))
+        fine_ns = np.int64(self.fine_time / MAX_FINE_TIME * 1e9)
+        return coarse_j2000ns + fine_ns
+
 
 @dataclass
 class MagL1aPacketProperties:
@@ -204,7 +218,7 @@ class MagL1a:
         Sequence number of the most recent packet added to the object
     missing_sequences : list[int]
         List of missing sequence numbers in the day
-    start_time : numpy.datetime64
+    start_time : numpy.int64
         Start time of the day, in ns since J2000 epoch
     compression_flags : np.ndarray
         Array of flags to indication compression and width for all timestamps in the
@@ -233,10 +247,10 @@ class MagL1a:
     shcoarse: int
     vectors: np.ndarray
     starting_packet: InitVar[MagL1aPacketProperties]
-    packet_definitions: dict[np.datetime64, MagL1aPacketProperties] = field(init=False)
+    packet_definitions: dict[np.int64, MagL1aPacketProperties] = field(init=False)
     most_recent_sequence: int = field(init=False)
     missing_sequences: list[int] = field(default_factory=list)
-    start_time: np.datetime64 = field(init=False)
+    start_time: np.int64 = field(init=False)
     compression_flags: np.ndarray | None = field(init=False, default=None)
 
     def __post_init__(self, starting_packet: MagL1aPacketProperties) -> None:
@@ -248,10 +262,7 @@ class MagL1a:
         starting_packet : MagL1aPacketProperties
             The packet properties for the first packet in the day, including start time.
         """
-        # TODO should this be from starting_packet
-        self.start_time = (J2000_EPOCH + met_to_j2000ns(self.shcoarse)).astype(
-            "datetime64[D]"
-        )
+        self.start_time = np.int64(met_to_ttj2000ns(starting_packet.shcoarse))
         self.packet_definitions = {self.start_time: starting_packet}
         # most_recent_sequence is the sequence number of the packet used to initialize
         # the object
@@ -338,8 +349,8 @@ class MagL1a:
             cdf.utils.met_to_j2000ns.
         """
         timedelta = np.timedelta64(int(1 / vectors_per_sec * 1e9), "ns")
-        # TODO: validate that start_time from SHCOARSE is precise enough
-        start_time_ns = met_to_j2000ns(start_time.to_seconds())
+        # TODO: From finetime and coarsetime, depends per packet
+        start_time_ns = start_time.to_j2000ns()
 
         # Calculate time skips for each vector in ns
         times = np.reshape(
@@ -616,7 +627,7 @@ class MagL1a:
             (expected_range_data_length - end_padding) * has_range_data_section
         )
 
-        # Cut off the first vector width and the end range data section if it exists.
+        # Cut off the first vector width and the range data section.
         vector_bits = bit_array[first_vector_width - 1 : end_vector]
 
         # Shift the bit array over one to the left, then sum them up. This is used to
@@ -731,7 +742,7 @@ class MagL1a:
             primary_count,
             uncompressed_vector_size,
             compression_width,
-        )
+        )[0]
 
         # Secondary vector processing
         first_secondary_vector = MagL1a.unpack_one_vector(
@@ -747,8 +758,7 @@ class MagL1a:
         secondary_split_bits = np.split(
             vector_bits[: secondary_boundaries[-1]], secondary_boundaries[:-1]
         )[1:]
-
-        secondary_vectors = MagL1a._process_vector_section(
+        secondary_process_vectors = MagL1a._process_vector_section(
             vector_bits,
             secondary_split_bits,
             secondary_boundaries[-1],
@@ -757,21 +767,28 @@ class MagL1a:
             uncompressed_vector_size,
             compression_width,
         )
+        secondary_vectors = secondary_process_vectors[0]
+
+        # The range data length has 2 bits per vector, minus 2 for the uncompressed
+        # first vectors in the primary and secondary sensors.
+        # Then, the range data length is padded to the nearest 8 bits.
+        end_vector = ((secondary_process_vectors[1] + first_vector_width) + 7) // 8 * 8
 
         # If there is a range data section, it describes all the data, compressed or
         # uncompressed.
         if has_range_data_section:
+            range_bits = bit_array[end_vector:]
             primary_vectors = MagL1a.process_range_data_section(
-                bit_array[end_vector : end_vector + (primary_count - 1) * 2],
+                range_bits[: (primary_count - 1) * 2],
                 primary_vectors,
             )
             secondary_vectors = MagL1a.process_range_data_section(
-                bit_array[
-                    end_vector + (primary_count - 1) * 2 : end_vector
-                    + (primary_count + secondary_count - 2) * 2
+                range_bits[
+                    (primary_count - 1) * 2 : (primary_count + secondary_count - 2) * 2
                 ],
                 secondary_vectors,
             )
+
         return primary_vectors, secondary_vectors
 
     @staticmethod
@@ -783,7 +800,7 @@ class MagL1a:
         vector_count: int,
         uncompressed_vector_size: int,
         compression_width: int,
-    ) -> np.ndarray:
+    ) -> tuple[np.ndarray, int]:
         """
         Generate a section of vector data, primary or secondary.
 
@@ -809,20 +826,33 @@ class MagL1a:
 
         Returns
         -------
-        numpy.ndarray
-            An array of processed vectors.
+        (numpy.ndarray, int)
+            A tuple consisting of: an array of processed vectors, and the index of the
+            end of the last vector. In a fully compressed case, this will be the same
+            as last_index.
         """
+        compressed_count = math.ceil(len(split_bits) / AXIS_COUNT) + 1
+        uncompressed_count = vector_count - compressed_count
+
+        # will have either 0 or 8 extra bits
+        # If we have more splits than vectors, we have included part of the range
+        # data section. Drop those values and update end_vector.
+        if len(split_bits) > (vector_count - 1) * AXIS_COUNT:
+            split_bits = split_bits[: (vector_count - 1) * AXIS_COUNT]
+            last_index -= 8
+
+        end = last_index + uncompressed_vector_size * uncompressed_count
+
         vector_diffs = list(map(MagL1a.decode_fib_zig_zag, split_bits))
+        # if we got more than the expected length, we may have gotten some of the range
+        # data section.
+
         vectors = MagL1a.convert_diffs_to_vectors(
             first_vector, vector_diffs, vector_count
         )
         # If we are missing any vectors from primary_split_bits, we know we have
         # uncompressed vectors to process.
-        compressed_count = math.ceil(len(split_bits) / AXIS_COUNT) + 1
-        uncompressed_count = vector_count - compressed_count
-
         if uncompressed_count:
-            end = last_index + uncompressed_vector_size * uncompressed_count
             uncompressed_vectors = vector_bits[last_index : end + 1]
 
             for i in range(uncompressed_count):
@@ -836,8 +866,11 @@ class MagL1a:
                 )
                 vectors[i + compressed_count] = decoded_vector
                 vectors[i + compressed_count][3] = vectors[0][3]
+            # If there is an extra byte, this is from the range data section.
+            if len(vector_bits[last_index:]) - end == 8:
+                end -= 8
 
-        return vectors
+        return (vectors, end)
 
     @staticmethod
     def process_range_data_section(
@@ -870,7 +903,6 @@ class MagL1a:
                 "Incorrect length for range_data, there should be two bits per vector, "
                 "excluding the first."
             )
-
         updated_vectors: np.ndarray = np.copy(vectors)
         range_str = "".join([str(i) for i in range_data])
         for i in range(len(vectors) - 1):

imap_processing/mag/l1b/mag_l1b.py

@@ -4,12 +4,13 @@ from pathlib import Path
 
 import numpy as np
 import xarray as xr
+from xarray import Dataset
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.cdf.utils import load_cdf
 
 
-def mag_l1b(input_dataset: xr.Dataset, version: str) -> xr.Dataset:
+def mag_l1b(input_dataset: xr.Dataset, version: str) -> Dataset:
     """
     Will process MAG L1B data from L1A data.
 
@@ -28,6 +29,9 @@ def mag_l1b(input_dataset: xr.Dataset, version: str) -> xr.Dataset:
     # TODO:
     # Read in calibration file
     # multiply all vectors by calibration file
+    if "raw" in input_dataset.attrs["Logical_source"]:
+        # Raw files should not be processed in L1B.
+        raise ValueError("Raw L1A file passed into L1B. Unable to process.")
 
     output_dataset = mag_l1b_processing(input_dataset)
     attribute_manager = ImapCdfAttributes()
@@ -64,6 +68,7 @@ def mag_l1b_processing(input_dataset: xr.Dataset) -> xr.Dataset:
     """
     # TODO: There is a time alignment step that will add a lot of complexity.
     # This needs to be done once we have some SPICE time data.
+
     mag_attributes = ImapCdfAttributes()
     mag_attributes.add_instrument_variable_attrs("mag", "l1")
 
@@ -73,6 +78,7 @@ def mag_l1b_processing(input_dataset: xr.Dataset) -> xr.Dataset:
     calibration_dataset = load_cdf(
         Path(__file__).parent / "imap_calibration_mag_20240229_v01.cdf"
     )
+    # TODO: add time shift
     # TODO: Check validity of time range for calibration
     if "mago" in input_dataset.attrs["Logical_source"][0]:
         calibration_matrix = calibration_dataset["MFOTOURFO"]
@@ -137,8 +143,9 @@ def update_vector(
     tuple[numpy.ndarray, numpy.ndarray]
         Updated vector and the same compression flags.
     """
-    vector = calibrate_vector(input_vector, calibration_matrix)
-    return rescale_vector(vector, input_compression), input_compression
+    vector = rescale_vector(input_vector, input_compression)
+    cal_vector = calibrate_vector(vector, calibration_matrix)
+    return cal_vector, input_compression
 
 
 def rescale_vector(
@@ -200,9 +207,7 @@ def calibrate_vector(
         Calibrated vector.
     """
     updated_vector = input_vector.copy()
-
-    updated_vector[:3] = np.matmul(
+    updated_vector[:3] = np.dot(
         calibration_matrix.values[:, :, int(input_vector[3])], input_vector[:3]
     )
-
     return updated_vector

imap_processing/quality_flags.py

@@ -37,14 +37,29 @@ class ENAFlags(FlagNameMixin):
     BADSPIN = 2**2  # bit 2, Bad spin
 
 
-class ImapUltraFlags(FlagNameMixin):
+class ImapHkUltraFlags(FlagNameMixin):
     """IMAP Ultra flags."""
 
     NONE = CommonFlags.NONE
     INF = CommonFlags.INF  # bit 0
     NEG = CommonFlags.NEG  # bit 1
     BADSPIN = ENAFlags.BADSPIN  # bit 2
-    FLAG1 = 2**3  # bit 2
+    FLAG1 = 2**3  # bit 3
+
+
+class ImapAttitudeUltraFlags(FlagNameMixin):
+    """IMAP Ultra flags."""
+
+    NONE = CommonFlags.NONE
+    SPINRATE = 2**0  # bit 0
+
+
+class ImapRatesUltraFlags(FlagNameMixin):
+    """IMAP Ultra Rates flags."""
+
+    NONE = CommonFlags.NONE
+    ZEROCOUNTS = 2**0  # bit 0
+    HIGHRATES = 2**1  # bit 1
 
 
 class ImapLoFlags(FlagNameMixin):
@@ -54,7 +69,7 @@ class ImapLoFlags(FlagNameMixin):
     INF = CommonFlags.INF  # bit 0
     NEG = CommonFlags.NEG  # bit 1
     BADSPIN = ENAFlags.BADSPIN  # bit 2
-    FLAG2 = 2**3  # bit 2
+    FLAG2 = 2**3  # bit 3
 
 
 class HitFlags(