imap-processing 0.16.2__py3-none-any.whl → 0.18.0__py3-none-any.whl

imap_processing/mag/l2/mag_l2_data.py

@@ -7,7 +7,9 @@ import numpy as np
 import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
-from imap_processing.mag.constants import DataMode
+from imap_processing.mag.constants import FILLVAL, DataMode
+from imap_processing.mag.l1b.mag_l1b import calibrate_vector
+from imap_processing.spice.geometry import SpiceFrame, frame_transform
 from imap_processing.spice.time import (
     et_to_ttj2000ns,
     str_to_et,
@@ -17,26 +19,29 @@ from imap_processing.spice.time import (
 class ValidFrames(Enum):
     """SPICE reference frames for output."""
 
-    dsrf = "dsrf"
-    srf = "srf"
-    rtn = "rtn"
-    gse = "gse"
+    MAG = SpiceFrame.IMAP_MAG
+    DSRF = SpiceFrame.IMAP_DPS
+    SRF = SpiceFrame.IMAP_SPACECRAFT
+    # TODO: include RTN and GSE as valid frames
 
 
-@dataclass
-class MagL2:
+@dataclass(kw_only=True)
+class MagL2L1dBase:
     """
-    Dataclass for MAG L2 data.
+    Base class for MAG L2 and L1D data.
 
-    Since L2 and L1D should have the same structure, this can be used for either level.
+    Since these two data levels output identical files, and share some methods, this
+    superclass captures the tools in common, while allowing each subclass to define
+    individual attributes and algorithms.
 
-    Some of the methods are also static, so they can be used in i-ALiRT processing.
+    May also be extended for I-ALiRT.
 
     Attributes
     ----------
     vectors: np.ndarray
         Magnetic field vectors of size (n, 3) where n is the number of vectors.
-        Describes (x, y, z) components of the magnetic field.
+        Describes (x, y, z) components of the magnetic field. This field is the output
+        vectors, which are nominally from the MAGo sensor.
     epoch: np.ndarray
         Time of each vector in J2000 seconds. Should be of length n.
     range: np.ndarray
@@ -48,10 +53,8 @@ class MagL2:
     quality_bitmask: np.ndarray
         Quality bitmask for each vector. Should be of length n. Copied from offset
         file in L2, marked as good always in L1D.
-    magnitude: np.ndarray
-        Magnitude of each vector. Should be of length n. Calculated from L2 vectors.
-    is_l1d: bool
-        Flag to indicate if the data is L1D. Defaults to False.
+    frame:
+        The reference frame of the input vectors. Starts as the MAG instrument frame.
     """
 
     vectors: np.ndarray
@@ -62,120 +65,12 @@ class MagL2:
     quality_bitmask: np.ndarray
     data_mode: DataMode
     magnitude: np.ndarray = field(init=False)
-    is_l1d: bool = False
-    offsets: InitVar[np.ndarray] = None
-    timedelta: InitVar[np.ndarray] = None
-
-    def __post_init__(self, offsets: np.ndarray, timedelta: np.ndarray) -> None:
-        """
-        Calculate the magnitude of the vectors after initialization.
-
-        Parameters
-        ----------
-        offsets : np.ndarray
-            Offsets to apply to the vectors. Should be of shape (n, 3) where n is the
-            number of vectors.
-        timedelta : np.ndarray
-            Time deltas to shift the timestamps by. Should be of length n.
-            Given in seconds.
-        """
-        if offsets is not None:
-            self.vectors = self.apply_offsets(self.vectors, offsets)
-        if timedelta is not None:
-            self.epoch = self.shift_timestamps(self.epoch, timedelta)
-
-        self.magnitude = self.calculate_magnitude(self.vectors)
-
-    @staticmethod
-    def calculate_magnitude(
-        vectors: np.ndarray,
-    ) -> np.ndarray:
-        """
-        Given a list of vectors (x, y, z), calculate the magnitude of each vector.
-
-        For an input list of vectors of size (n, 3) returns a list of magnitudes of
-        size (n,).
-
-        Parameters
-        ----------
-        vectors : np.ndarray
-            Array of vectors to calculate the magnitude of.
-
-        Returns
-        -------
-        np.ndarray
-            Array of magnitudes of the input vectors.
-        """
-        return np.linalg.norm(vectors, axis=1)  # type: ignore
-
-    @staticmethod
-    def apply_offsets(vectors: np.ndarray, offsets: np.ndarray) -> np.ndarray:
-        """
-        Apply the offsets to the vectors by adding them together.
-
-        These offsets are used to shift the vectors in the x, y, and z directions.
-        They can either be provided through a custom offsets datafile, or calculated
-        using a gradiometry algorithm.
-
-        Parameters
-        ----------
-        vectors : np.ndarray
-            Array of vectors to apply the offsets to. Should be of shape (n, 3) where n
-            is the number of vectors.
-        offsets : np.ndarray
-            Array of offsets to apply to the vectors. Should be of shape (n, 3) where n
-            is the number of vectors.
-
-        Returns
-        -------
-        np.ndarray
-            Array of vectors with offsets applied. Should be of shape (n, 3).
-        """
-        if vectors.shape[0] != offsets.shape[0]:
-            raise ValueError("Vectors and offsets must have the same length.")
-
-        offset_vectors: np.ndarray = vectors[:, :3] + offsets
-
-        # TODO: CDF files don't have NaNs. Emailed MAG to ask what this will look like.
-        # Any values where offsets is nan must also be nan
-        offset_vectors[np.isnan(offsets).any(axis=1)] = np.nan
-
-        return offset_vectors
-
-    @staticmethod
-    def shift_timestamps(epoch: np.ndarray, timedelta: np.ndarray) -> np.ndarray:
-        """
-        Shift the timestamps by the given timedelta.
-
-        If timedelta is positive, the epochs are shifted forward in time.
-
-        Parameters
-        ----------
-        epoch : np.ndarray
-            Array of timestamps to shift. Should be of length n.
-        timedelta : np.ndarray
-            Array of time deltas to shift the timestamps by. Should be the same length
-            as epoch. Given in seconds.
-
-        Returns
-        -------
-        np.ndarray
-            Shifted timestamps.
-        """
-        if epoch.shape[0] != timedelta.shape[0]:
-            raise ValueError(
-                "Input Epoch and offsets timedeltas must be the same length."
-            )
-
-        timedelta_ns = timedelta * 1e9
-        shifted_timestamps = epoch + timedelta_ns
-        return shifted_timestamps
+    frame: ValidFrames = ValidFrames.MAG
 
     def generate_dataset(
         self,
         attribute_manager: ImapCdfAttributes,
         day: np.datetime64,
-        frame: ValidFrames = ValidFrames.dsrf,
     ) -> xr.Dataset:
         """
         Generate an xarray dataset from the dataclass.
@@ -189,8 +84,6 @@ class MagL2:
             CDF attributes object for the correct level.
         day : np.datetime64
          The 24 hour day to process, as a numpy datetime format.
-        frame : ValidFrames
-            SPICE reference frame to rotate the data into.
 
         Returns
         -------
@@ -199,7 +92,9 @@ class MagL2:
         """
         self.truncate_to_24h(day)
 
-        logical_source_id = f"imap_mag_l2_{self.data_mode.value.lower()}-{frame.name}"
+        logical_source_id = (
+            f"imap_mag_l2_{self.data_mode.value.lower()}-{self.frame.name.lower()}"
+        )
         direction = xr.DataArray(
             np.arange(3),
             name="direction",
@@ -242,8 +137,8 @@ class MagL2:
         )
 
         quality_bitmask = xr.DataArray(
-            self.quality_flags,
-            name="quality_flags",
+            self.quality_bitmask,
+            name="quality_bitmask",
             dims=["epoch"],
             attrs=attribute_manager.get_variable_attributes("qf"),
         )
@@ -298,7 +193,6 @@ class MagL2:
         """
         if self.epoch.shape[0] != self.vectors.shape[0]:
             raise ValueError("Timestamps and vectors are not the same shape!")
-
         start_timestamp_j2000 = et_to_ttj2000ns(str_to_et(str(timestamp)))
         end_timestamp_j2000 = et_to_ttj2000ns(
             str_to_et(str(timestamp + np.timedelta64(1, "D")))
@@ -313,3 +207,170 @@ class MagL2:
         self.magnitude = self.magnitude[day_start_index:day_end_index]
         self.quality_flags = self.quality_flags[day_start_index:day_end_index]
         self.quality_bitmask = self.quality_bitmask[day_start_index:day_end_index]
+
+    @staticmethod
+    def calculate_magnitude(
+        vectors: np.ndarray,
+    ) -> np.ndarray:
+        """
+        Given a list of vectors (x, y, z), calculate the magnitude of each vector.
+
+        For an input list of vectors of size (n, 3) returns a list of magnitudes of
+        size (n,).
+
+        Parameters
+        ----------
+        vectors : np.ndarray
+            Array of vectors to calculate the magnitude of.
+
+        Returns
+        -------
+        np.ndarray
+            Array of magnitudes of the input vectors.
+        """
+        return np.linalg.norm(vectors, axis=1)
+
+    @staticmethod
+    def apply_calibration(
+        vectors: np.ndarray, calibration_matrix: np.ndarray
+    ) -> np.ndarray:
+        """
+        Apply the calibration matrix to the vectors.
+
+        This works by repeatedly calling the function calibrate_vector on the vectors
+        input.
+
+        Parameters
+        ----------
+        vectors : np.ndarray
+            Array of vectors to apply the calibration to, including x,y,z and range.
+            Should be of shape (n, 4) where n is the number of vectors.
+        calibration_matrix : np.ndarray
+            Calibration matrix to apply to the vectors. Should be of shape (3, 3, 4).
+
+        Returns
+        -------
+        np.ndarray
+            Array of calibrated vectors. Should be of shape (n, 4).
+        """
+        calibrated_vectors = np.apply_along_axis(
+            func1d=calibrate_vector,
+            axis=1,
+            arr=vectors,
+            calibration_matrix=calibration_matrix,
+        )
+
+        return calibrated_vectors
+
+    @staticmethod
+    def shift_timestamps(epoch: np.ndarray, timedelta: np.ndarray) -> np.ndarray:
+        """
+        Shift the timestamps by the given timedelta.
+
+        If timedelta is positive, the epochs are shifted forward in time.
+
+        Parameters
+        ----------
+        epoch : np.ndarray
+            Array of timestamps to shift. Should be of length n.
+        timedelta : np.ndarray
+            Array of time deltas to shift the timestamps by. Should be the same length
+            as epoch. Given in seconds.
+
+        Returns
+        -------
+        np.ndarray
+            Shifted timestamps.
+        """
+        if epoch.shape[0] != timedelta.shape[0]:
+            raise ValueError(
+                "Input Epoch and offsets timedeltas must be the same length."
+            )
+
+        timedelta_ns = timedelta * 1e9
+        shifted_timestamps = epoch + timedelta_ns
+        return shifted_timestamps
+
+    def rotate_frame(self, end_frame: ValidFrames) -> None:
+        """
+        Rotate the vector data in the class to the output frame.
+
+        Parameters
+        ----------
+        end_frame : ValidFrames
+            The frame to rotate the data to. Must be one of the ValidFrames enum
+            values.
+        """
+        self.vectors = frame_transform(
+            self.epoch,
+            self.vectors,
+            from_frame=self.frame.value,
+            to_frame=end_frame.value,
+        )
+        self.frame = end_frame
+
+
+@dataclass(kw_only=True)
+class MagL2(MagL2L1dBase):
+    """
+    Dataclass for MAG L2 data.
+
+    Since L2 and L1D should have the same structure, this can be used for either level.
+
+    Some of the methods are also static, so they can be used in i-ALiRT processing.
+    """
+
+    offsets: InitVar[np.ndarray] = None
+    timedelta: InitVar[np.ndarray] = None
+
+    def __post_init__(self, offsets: np.ndarray, timedelta: np.ndarray) -> None:
+        """
+        Calculate the magnitude of the vectors after initialization.
+
+        Parameters
+        ----------
+        offsets : np.ndarray
+            Offsets to apply to the vectors. Should be of shape (n, 3) where n is the
+            number of vectors.
+        timedelta : np.ndarray
+            Time deltas to shift the timestamps by. Should be of length n.
+            Given in seconds.
+        """
+        if offsets is not None:
+            self.vectors = self.apply_offsets(self.vectors, offsets)
+        if timedelta is not None:
+            self.epoch = self.shift_timestamps(self.epoch, timedelta)
+
+        self.magnitude = self.calculate_magnitude(self.vectors)
+
+    @staticmethod
+    def apply_offsets(vectors: np.ndarray, offsets: np.ndarray) -> np.ndarray:
+        """
+        Apply the offsets to the vectors by adding them together.
+
+        These offsets are used to shift the vectors in the x, y, and z directions.
+        They can either be provided through a custom offsets datafile, or calculated
+        using a gradiometry algorithm.
+
+        Parameters
+        ----------
+        vectors : np.ndarray
+            Array of vectors to apply the offsets to. Should be of shape (n, 3) where n
+            is the number of vectors.
+        offsets : np.ndarray
+            Array of offsets to apply to the vectors. Should be of shape (n, 3) where n
+            is the number of vectors.
+
+        Returns
+        -------
+        np.ndarray
+            Array of vectors with offsets applied. Should be of shape (n, 3).
+        """
+        if vectors.shape[0] != offsets.shape[0]:
+            raise ValueError("Vectors and offsets must have the same length.")
+
+        offset_vectors: np.ndarray = vectors + offsets
+
+        # Any values where offsets is FILLVAL must also be FILLVAL
+        offset_vectors[(offsets == FILLVAL).any(axis=1), :] = FILLVAL
+        return offset_vectors

imap_processing/quality_flags.py

@@ -37,6 +37,14 @@ class ENAFlags(FlagNameMixin):
     BADSPIN = 2**2  # bit 2, Bad spin
 
 
+class ImapDEUltraFlags(FlagNameMixin):
+    """IMAP Ultra flags."""
+
+    NONE = CommonFlags.NONE
+    FOV = 2**0  # bit 0
+    PHCORR = 2**1  # bit 1
+
+
 class ImapHkUltraFlags(FlagNameMixin):
     """IMAP Ultra flags."""
 
@@ -53,14 +61,24 @@ class ImapAttitudeUltraFlags(FlagNameMixin):
     NONE = CommonFlags.NONE
     SPINRATE = 2**0  # bit 0
     AUXMISMATCH = 2**1  # bit 1 # aux packet does not match Universal Spin Table
+    SPINPHASE = 2**2  # bit 2 # spin phase flagged by Universal Spin Table
+    SPINPERIOD = 2**3  # bit 3 # spin period flagged by Universal Spin Table
 
 
 class ImapRatesUltraFlags(FlagNameMixin):
     """IMAP Ultra Rates flags."""
 
     NONE = CommonFlags.NONE
-    ZEROCOUNTS = 2**0  # bit 0
-    HIGHRATES = 2**1  # bit 1
+    HIGHRATES = 2**0  # bit 0
+    FIRSTSPIN = 2**1  # bit 1
+    LASTSPIN = 2**2  # bit 2
+    PARTIALSPIN = 2**2  # bit 2
+
+
+class ImapInstrumentUltraFlags(FlagNameMixin):
+    """IMAP Ultra flags using other instruments."""
+
+    NONE = CommonFlags.NONE
 
 
 class ImapLoFlags(FlagNameMixin):

imap_processing/spice/geometry.py

@@ -27,7 +27,7 @@ class SpiceBody(IntEnum):
     # A subset of IMAP Specific bodies as defined in imap_wkcp.tf
     IMAP = -43
     IMAP_SPACECRAFT = -43000
-    # IMAP Pointing Frame (Despun) as defined in imap_science_0001.tf
+    # IMAP Pointing Frame (Despun) as defined in imap_science_xxx.tf
     IMAP_DPS = -43901
     # Standard NAIF bodies
     SOLAR_SYSTEM_BARYCENTER = spiceypy.bodn2c("SOLAR_SYSTEM_BARYCENTER")
@@ -36,13 +36,13 @@ class SpiceBody(IntEnum):
 
 
 class SpiceFrame(IntEnum):
-    """Enum containing SPICE IDs for reference frames, defined in imap_wkcp.tf."""
+    """SPICE IDs for reference frames in imap_wkcp.tf and imap_science_xxx.tf."""
 
     # Standard SPICE Frames
     J2000 = spiceypy.irfnum("J2000")
     ECLIPJ2000 = spiceypy.irfnum("ECLIPJ2000")
     ITRF93 = 13000
-    # IMAP Pointing Frame (Despun) as defined in imap_science_0001.tf
+    # IMAP Pointing Frame (Despun) as defined in imap_science_xxx.tf
     IMAP_DPS = -43901
     # IMAP specific as defined in imap_wkcp.tf
     IMAP_SPACECRAFT = -43000
@@ -61,6 +61,28 @@ class SpiceFrame(IntEnum):
     IMAP_IDEX = -43700
     IMAP_GLOWS = -43750
 
+    # IMAP Science Frames (new additions from imap_science_xxx.tf)
+    IMAP_OMD = -43900
+    IMAP_EARTHFIXED = -43910
+    IMAP_ECLIPDATE = -43911
+    IMAP_MDI = -43912
+    IMAP_MDR = -43913
+    IMAP_GMC = -43914
+    IMAP_GEI = -43915
+    IMAP_GSE = -43916
+    IMAP_GSM = -43917
+    IMAP_SMD = -43918
+    IMAP_RTN = -43920
+    IMAP_HCI = -43921  # HGI_J2K
+    IMAP_HCD = -43922  # HGI_D
+    IMAP_HGC = -43923  # HGS_D
+    IMAP_HAE = -43924
+    IMAP_HAED = -43925
+    IMAP_HEE = -43926
+    IMAP_HRE = -43927
+    IMAP_HNU = -43928
+    IMAP_GCS = -43929
+
 
 BORESIGHT_LOOKUP = {
     SpiceFrame.IMAP_LO_BASE: np.array([0, -1, 0]),

imap_processing/spice/pointing_frame.py

@@ -200,7 +200,7 @@ def calculate_pointing_attitude_segments(
     - Latest NAIF leapseconds kernel (naif0012.tls)
     - The latest IMAP sclk (imap_sclk_NNNN.tsc)
     - The latest IMAP frame kernel (imap_wkcp.tf)
-    - IMAP DPS frame kernel (imap_science_0001.tf)
+    - IMAP DPS frame kernel (imap_science_100.tf)
     - IMAP historical attitude kernel from which the pointing frame kernel will
     be generated.
     """

imap_processing/spice/spin.py

@@ -197,6 +197,10 @@ def interpolate_spin_data(query_met_times: Union[float, npt.NDArray]) -> pd.Data
     #   spin_period_valid columns.
     invalid_spin_phase_range = (spin_phases < 0) | (spin_phases >= 1)
 
+    # TODO: add optional to filter this if this flag means
+    # that repointing is happening. otherwise, then keep it.
+    # This needs to be discussed and receive guidance at
+    # the project level.
     invalid_spins = (out_df["spin_phase_valid"].values == 0) | (
         out_df["spin_period_valid"].values == 0
     )

imap_processing/spice/time.py

@@ -220,6 +220,57 @@ def et_to_datetime64(
     return np.array(et_to_utc(et), dtype=np.datetime64)[()]
 
 
+@typing.no_type_check
+@ensure_spice
+def et_to_met(
+    et: Union[float, Collection[float]],
+) -> Union[float, np.ndarray]:
+    """
+    Convert ephemeris time to mission elapsed time (MET).
+
+    This function converts ET to spacecraft clock ticks and then to MET seconds.
+    This is the inverse of the MET to ET conversion process.
+
+    Parameters
+    ----------
+    et : Union[float, Collection[float]]
+        Input ephemeris time value(s) to be converted to MET.
+
+    Returns
+    -------
+    met: np.ndarray
+        Mission elapsed time in seconds.
+    """
+    vectorized_sce2c = _vectorize(spiceypy.sce2c, otypes=[float], excluded=[0])
+    sclk_ticks = vectorized_sce2c(IMAP_SC_ID, et)
+    met = np.asarray(sclk_ticks, dtype=float) * TICK_DURATION
+    return met
+
+
+def ttj2000ns_to_met(
+    tt_ns: npt.ArrayLike,
+) -> npt.NDArray[float]:
+    """
+    Convert terrestrial time nanoseconds since J2000 to mission elapsed time (MET).
+
+    This is the inverse of met_to_ttj2000ns. The conversion process is:
+    TTJ2000ns -> ET -> MET
+
+    Parameters
+    ----------
+    tt_ns : float, numpy.ndarray
+        Number of nanoseconds since the J2000 epoch in the TT timescale.
+
+    Returns
+    -------
+    numpy.ndarray[float]
+        The mission elapsed time in seconds.
+    """
+    et = ttj2000ns_to_et(tt_ns)
+    met = et_to_met(et)
+    return met
+
+
 @typing.no_type_check
 @ensure_spice
 def sct_to_et(

imap_processing/swapi/l1/swapi_l1.py

@@ -558,8 +558,11 @@ def process_swapi_science(
     # Step 3: Create xarray.Dataset
     # ===================================================================
 
-    # epoch time. Should be same dimension as number of good sweeps
-    epoch_values = good_sweep_sci["epoch"].data.reshape(total_full_sweeps, 12)[:, 0]
+    # epoch time. Should be same dimension as number of good sweeps.
+    # Use center time for epoch to line up with mission requests. Center time
+    # of SWAPI is time of 7th packet(aka SEQ_NUMBER == 6) creation time at the
+    # beginning of 7th packet.
+    epoch_values = good_sweep_sci["epoch"].data.reshape(total_full_sweeps, 12)[:, 6]
 
     epoch_time = xr.DataArray(
         epoch_values,
@@ -628,6 +631,13 @@ def process_swapi_science(
         dims=["epoch"],
         attrs=cdf_manager.get_variable_attributes("plan_id"),
     )
+    # Store start time for L3 purposes per SWAPI requests
+    dataset["sci_start_time"] = xr.DataArray(
+        good_sweep_sci["epoch"].data.reshape(total_full_sweeps, 12)[:, 0],
+        name="sci_start_time",
+        dims=["epoch"],
+        attrs=cdf_manager.get_variable_attributes("sci_start_time"),
+    )
     # Add ESA_LVL5 for L2 and L3 purposes.
     # We need to store ESA_LVL5 at SEQ_NUMBER==11
     # which is 71 energy step's ESA_LVL5 value. ESA_LVL5 gets