imap-processing 0.19.4__py3-none-any.whl → 1.0.1__py3-none-any.whl

imap_processing/mag/l1c/mag_l1c.py

@@ -63,17 +63,16 @@ def mag_l1c(
     )
 
     interp_function = InterpolationFunction[configuration.L1C_INTERPOLATION_METHOD]
-    if normal_mode_dataset and burst_mode_dataset:
-        full_interpolated_timeline = process_mag_l1c(
-            normal_mode_dataset, burst_mode_dataset, interp_function
+    if burst_mode_dataset is not None:
+        # Only use day_to_process if there is no norm data
+        day_to_process_arg = day_to_process if normal_mode_dataset is None else None
+        full_interpolated_timeline: np.ndarray = process_mag_l1c(
+            normal_mode_dataset, burst_mode_dataset, interp_function, day_to_process_arg
         )
     elif normal_mode_dataset is not None:
-        full_interpolated_timeline = fill_normal_data(
-            normal_mode_dataset, normal_mode_dataset["epoch"].data
-        )
+        full_interpolated_timeline = fill_normal_data(normal_mode_dataset)
     else:
-        # TODO: With only burst data, downsample by retrieving the timeline
-        raise NotImplementedError
+        raise ValueError("At least one of norm or burst dataset must be provided.")
 
     completed_timeline = remove_missing_data(full_interpolated_timeline)
 
@@ -127,12 +126,14 @@ def mag_l1c(
     global_attributes["missing_sequences"] = ""
 
     try:
-        global_attributes["is_mago"] = normal_mode_dataset.attrs["is_mago"]
-        global_attributes["is_active"] = normal_mode_dataset.attrs["is_active"]
+        active_dataset = normal_mode_dataset or burst_mode_dataset
+
+        global_attributes["is_mago"] = active_dataset.attrs["is_mago"]
+        global_attributes["is_active"] = active_dataset.attrs["is_active"]
 
         # Check if all vectors are primary in both normal and burst datasets
-        is_mago = normal_mode_dataset.attrs.get("is_mago", "False") == "True"
-        normal_all_primary = normal_mode_dataset.attrs.get("all_vectors_primary", False)
+        is_mago = active_dataset.attrs.get("is_mago", "False") == "True"
+        normal_all_primary = active_dataset.attrs.get("all_vectors_primary", False)
 
         # Default for missing burst dataset: 1 if MAGO (expected primary), 0 if MAGI
         burst_all_primary = is_mago
@@ -146,14 +147,14 @@ def mag_l1c(
             normal_all_primary and burst_all_primary
         )
 
-        global_attributes["missing_sequences"] = normal_mode_dataset.attrs[
+        global_attributes["missing_sequences"] = active_dataset.attrs[
             "missing_sequences"
         ]
     except KeyError as e:
         logger.info(
             f"Key error when assigning global attributes, attribute not found in "
             f"L1B file with logical source "
-            f"{normal_mode_dataset.attrs['Logical_source']}: {e}"
+            f"{active_dataset.attrs['Logical_source']}: {e}"
         )
 
     global_attributes["interpolation_method"] = interp_function.name
@@ -176,16 +177,24 @@ def mag_l1c(
         attrs=attribute_manager.get_variable_attributes("vector_attrs"),
     )
 
-    output_dataset["vector_magnitude"] = xr.apply_ufunc(
-        lambda x: np.linalg.norm(x[:4]),
-        output_dataset["vectors"],
-        input_core_dims=[["direction"]],
-        output_core_dims=[[]],
-        vectorize=True,
-    )
-    output_dataset[
-        "vector_magnitude"
-    ].attrs = attribute_manager.get_variable_attributes("vector_magnitude_attrs")
+    if len(output_dataset["vectors"]) > 0:
+        output_dataset["vector_magnitude"] = xr.apply_ufunc(
+            lambda x: np.linalg.norm(x[:4]),
+            output_dataset["vectors"],
+            input_core_dims=[["direction"]],
+            output_core_dims=[[]],
+            vectorize=True,
+        )
+        output_dataset[
+            "vector_magnitude"
+        ].attrs = attribute_manager.get_variable_attributes("vector_magnitude_attrs")
+    else:
+        output_dataset["vector_magnitude"] = xr.DataArray(
+            np.empty((0, 1)),
+            name="vector_magnitude",
+            dims=["epoch", "vector_magnitude"],
+            attrs=attribute_manager.get_variable_attributes("vector_magnitude_attrs"),
+        )
 
     output_dataset["compression_flags"] = xr.DataArray(
         completed_timeline[:, 6:8],
@@ -265,7 +274,7 @@ def select_datasets(
 
 
 def process_mag_l1c(
-    normal_mode_dataset: xr.Dataset,
+    normal_mode_dataset: xr.Dataset | None,
     burst_mode_dataset: xr.Dataset,
     interpolation_function: InterpolationFunction,
     day_to_process: np.datetime64 | None = None,
@@ -305,38 +314,51 @@ def process_mag_l1c(
     np.ndarray
         An (n, 8) shaped array containing the completed timeline.
     """
-    norm_epoch = normal_mode_dataset["epoch"].data
-    if "vectors_per_second" in normal_mode_dataset.attrs:
-        normal_vecsec_dict = vectors_per_second_from_string(
-            normal_mode_dataset.attrs["vectors_per_second"]
-        )
-    else:
-        normal_vecsec_dict = None
-
-    output_dataset = normal_mode_dataset.copy(deep=True)
-    output_dataset["sample_interpolated"] = xr.DataArray(
-        np.zeros(len(normal_mode_dataset))
-    )
     day_start_ns = None
     day_end_ns = None
 
     if day_to_process is not None:
-        day_start = day_to_process.astype("datetime64[s]") - np.timedelta64(15, "m")
+        day_start = day_to_process.astype("datetime64[s]") - np.timedelta64(30, "m")
 
-        # get the end of the day plus 15 minutes
+        # get the end of the day plus 30 minutes
         day_end = (
             day_to_process.astype("datetime64[s]")
             + np.timedelta64(1, "D")
-            + np.timedelta64(15, "m")
+            + np.timedelta64(30, "m")
         )
 
         day_start_ns = et_to_ttj2000ns(str_to_et(str(day_start)))
         day_end_ns = et_to_ttj2000ns(str_to_et(str(day_end)))
 
-    gaps = find_all_gaps(norm_epoch, normal_vecsec_dict, day_start_ns, day_end_ns)
+    if normal_mode_dataset:
+        norm_epoch = normal_mode_dataset["epoch"].data
+        if "vectors_per_second" in normal_mode_dataset.attrs:
+            normal_vecsec_dict = vectors_per_second_from_string(
+                normal_mode_dataset.attrs["vectors_per_second"]
+            )
+        else:
+            normal_vecsec_dict = None
+
+        gaps = find_all_gaps(norm_epoch, normal_vecsec_dict, day_start_ns, day_end_ns)
+    else:
+        norm_epoch = [day_start_ns, day_end_ns]
+        gaps = np.array(
+            [
+                [
+                    day_start_ns,
+                    day_end_ns,
+                    VecSec.TWO_VECS_PER_S.value,
+                ]
+            ]
+        )
 
     new_timeline = generate_timeline(norm_epoch, gaps)
-    norm_filled = fill_normal_data(normal_mode_dataset, new_timeline)
+
+    if normal_mode_dataset:
+        norm_filled: np.ndarray = fill_normal_data(normal_mode_dataset, new_timeline)
+    else:
+        norm_filled = generate_empty_norm_array(new_timeline)
+
     interpolated = interpolate_gaps(
         burst_mode_dataset, gaps, norm_filled, interpolation_function
     )
@@ -344,10 +366,32 @@ def process_mag_l1c(
     return interpolated
 
 
+def generate_empty_norm_array(new_timeline: np.ndarray) -> np.ndarray:
+    """
+    Generate an empty Normal mode array with the new timeline.
+
+    Parameters
+    ----------
+    new_timeline : np.ndarray
+        A 1D array of timestamps to fill.
+
+    Returns
+    -------
+    np.ndarray
+        An (n, 8) shaped array containing the timeline filled with `FILLVAL` data.
+    """
+    # TODO: fill with FILLVAL
+    norm_filled: np.ndarray = np.zeros((len(new_timeline), 8))
+    norm_filled[:, 0] = new_timeline
+    # Flags, will also indicate any missed timestamps
+    norm_filled[:, 5] = ModeFlags.MISSING.value
+
+    return norm_filled
+
+
 def fill_normal_data(
     normal_dataset: xr.Dataset,
-    new_timeline: np.ndarray,
-    day_to_process: np.datetime64 | None = None,
+    new_timeline: np.ndarray | None = None,
 ) -> np.ndarray:
     """
     Fill the new timeline with the normal mode data.
@@ -358,26 +402,23 @@ def fill_normal_data(
     ----------
     normal_dataset : xr.Dataset
         The normal mode dataset.
-    new_timeline : np.ndarray
-        A 1D array of timestamps to fill.
-    day_to_process : np.datetime64, optional
-        The day to process, in np.datetime64[D] format. This is used to fill
-        gaps at the beginning or end of the day if needed. If not included, these
-        gaps will not be filled.
+    new_timeline : np.ndarray, optional
+        A 1D array of timestamps to fill. If not provided, the normal mode timestamps
+        will be used.
 
     Returns
     -------
-    np.ndarray
+    filled_timeline : np.ndarray
         An (n, 8) shaped array containing the timeline filled with normal mode data.
         Gaps are marked as -1 in the generated flag column at index 5.
         Indices: 0 - epoch, 1-4 - vector x, y, z, and range, 5 - generated flag,
         6-7 - compression flags.
     """
-    # TODO: fill with FILLVAL
-    filled_timeline: np.ndarray = np.zeros((len(new_timeline), 8))
-    filled_timeline[:, 0] = new_timeline
-    # Flags, will also indicate any missed timestamps
-    filled_timeline[:, 5] = ModeFlags.MISSING.value
+    if new_timeline is None:
+        new_timeline = normal_dataset["epoch"].data
+
+    filled_timeline = generate_empty_norm_array(new_timeline)
+
     for index, timestamp in enumerate(normal_dataset["epoch"].data):
         timeline_index = np.searchsorted(new_timeline, timestamp)
         filled_timeline[timeline_index, 1:5] = normal_dataset["vectors"].data[index]
@@ -463,20 +504,17 @@ def interpolate_gaps(
         ]
 
         short = (gap_timeline >= burst_epochs[burst_start]) & (
-            gap_timeline <= burst_epochs[burst_gap_end]
+            gap_timeline <= burst_epochs[burst_end]
         )
-        if len(gap_timeline) != (short).sum():
-            print(f"Chopping timeline from {len(gap_timeline)} to {short.sum()}")
+        num_short = int(short.sum())
+
+        if len(gap_timeline) != num_short:
+            print(f"Chopping timeline from {len(gap_timeline)} to {num_short}")
 
         # Limit timestamps to only include the areas with burst data
-        gap_timeline = gap_timeline[
-            (
-                (gap_timeline >= burst_epochs[burst_start])
-                & (gap_timeline <= burst_epochs[burst_gap_end])
-            )
-        ]
+        gap_timeline = gap_timeline[short]
         # do not include range
-        gap_fill = interpolation_function(
+        adjusted_gap_timeline, gap_fill = interpolation_function(
             burst_vectors[burst_start:burst_end, :3],
             burst_epochs[burst_start:burst_end],
             gap_timeline,
@@ -485,7 +523,7 @@ def interpolate_gaps(
         )
 
         # gaps should not have data in timeline, still check it
-        for index, timestamp in enumerate(gap_timeline):
+        for index, timestamp in enumerate(adjusted_gap_timeline):
             timeline_index = np.searchsorted(filled_norm_timeline[:, 0], timestamp)
             if sum(
                 filled_norm_timeline[timeline_index, 1:4]
@@ -500,6 +538,18 @@ def interpolate_gaps(
                     "compression_flags"
                 ].data[burst_gap_start + index]
 
+        # for any timestamp that was not filled and is still missing, remove it
+        missing_timeline = np.setdiff1d(gap_timeline, adjusted_gap_timeline)
+
+        for timestamp in missing_timeline:
+            timeline_index = np.searchsorted(filled_norm_timeline[:, 0], timestamp)
+            if filled_norm_timeline[timeline_index, 5] != ModeFlags.MISSING.value:
+                raise RuntimeError(
+                    "Self-inconsistent data. "
+                    "Gaps not included in final timeline should be missing."
+                )
+            np.delete(filled_norm_timeline, timeline_index)
+
     return filled_norm_timeline
 
 

imap_processing/mag/l1d/mag_l1d_data.py

@@ -693,10 +693,12 @@ class MagL1d(MagL2L1dBase):  # type: ignore[misc]
             - gradiometer_offset_magnitude: magnitude of the offset vector
             - quality_flags: quality flags (1 if magnitude > threshold, 0 otherwise)
         """
-        aligned_magi = linear(
+        # TODO: should this extrapolate or should non-overlapping data be removed?
+        _, aligned_magi = linear(
             magi_vectors,
             magi_epoch,
             mago_epoch,
+            extrapolate=True,
         )
 
         diff = aligned_magi - mago_vectors

imap_processing/spice/geometry.py

@@ -21,7 +21,7 @@ from numpy.typing import NDArray
 class SpiceBody(IntEnum):
     """Enum containing SPICE IDs for bodies that we use."""
 
-    # A subset of IMAP Specific bodies as defined in imap_001.tf
+    # A subset of IMAP Specific bodies as defined in imap_xxx.tf
     IMAP = -43
     IMAP_SPACECRAFT = -43000
     # IMAP Pointing Frame (Despun) as defined in imap_science_xxx.tf
@@ -58,7 +58,7 @@ class SpiceFrame(IntEnum):
     IMAP_CODICE = -43400
     IMAP_HIT = -43500
     IMAP_IDEX = -43700
-    IMAP_GLOWS = -43751
+    IMAP_GLOWS = -43750
 
     # IMAP Science Frames (new additions from imap_science_xxx.tf)
     IMAP_OMD = -43900
@@ -85,6 +85,8 @@ class SpiceFrame(IntEnum):
 
 BORESIGHT_LOOKUP = {
     SpiceFrame.IMAP_LO_BASE: np.array([0, -1, 0]),
+    SpiceFrame.IMAP_LO: np.array([0, -1, 0]),
+    SpiceFrame.IMAP_LO_STAR_SENSOR: np.array([0, -1, 0]),
     SpiceFrame.IMAP_HI_45: np.array([0, 1, 0]),
     SpiceFrame.IMAP_HI_90: np.array([0, 1, 0]),
     SpiceFrame.IMAP_ULTRA_45: np.array([0, 0, 1]),
@@ -127,7 +129,7 @@ def imap_state(
     -------
     state : np.ndarray
      The Cartesian state vector representing the position and velocity of the
-     IMAP spacecraft.
+     IMAP spacecraft. Units are km and km/s.
     """
     state, _ = spiceypy.spkezr(
         SpiceBody.IMAP.name, et, ref_frame.name, abcorr, observer.name
@@ -160,7 +162,7 @@ def get_instrument_mounting_az_el(instrument: SpiceFrame) -> np.ndarray:
     # frame that is used to compute the s/c to instrument mounting.
     # Most of these vectors are the same as the instrument boresight vector.
     mounting_normal_vector = {
-        SpiceFrame.IMAP_LO_BASE: np.array([0, -1, 0]),
+        SpiceFrame.IMAP_LO_BASE: np.array([0, 0, -1]),
         SpiceFrame.IMAP_HI_45: np.array([0, 1, 0]),
         SpiceFrame.IMAP_HI_90: np.array([0, 1, 0]),
         SpiceFrame.IMAP_ULTRA_45: np.array([0, 0, 1]),
@@ -190,11 +192,16 @@ def get_spacecraft_to_instrument_spin_phase_offset(instrument: SpiceFrame) -> fl
     """
     Get the spin phase offset from the spacecraft to the instrument.
 
-    For now, the offset is a fixed lookup based on `Table 1: Nominal Instrument
+    Nominal offset values were determined using `Table 1: Nominal Instrument
     to S/C CS Transformations` in document `7516-0011_drw.pdf`. That Table
-    defines the angle from the spacecraft y-axis. We add 90 and take the modulous
-    with 360 in order to get the angle from the spacecraft x-axis. These fixed
-    values will need to be updated based on calibration data.
+    defines the angle from the spacecraft y-axis. We add 90-degrees and take the
+    modulus with 360 to get the angle from the spacecraft x-axis. This math is
+    shown in the comments after each key value pair in the dictionary defined
+    in code. The true values differ slightly from the nominal values. True
+    values are derived from the frame definitions in the IMAP frames kernel
+    which uses ground calibration measurements to define the as-built mounting
+    of each instrument. The function in this module, `get_instrument_mounting_az_el`,
+    was used to retrieve the true azimuth angles from the IMAP frames kernel.
 
     Parameters
     ----------
@@ -207,19 +214,21 @@ def get_spacecraft_to_instrument_spin_phase_offset(instrument: SpiceFrame) -> fl
         The spin phase offset from the spacecraft to the instrument.
     """
     phase_offset_lookup = {
-        SpiceFrame.IMAP_LO_BASE: 60 / 360,  # (330 + 90) % 360 = 60
-        SpiceFrame.IMAP_HI_45: 345 / 360,  # 255 + 90 = 345
-        SpiceFrame.IMAP_HI_90: 15 / 360,  # (285 + 90) % 360 = 15
-        SpiceFrame.IMAP_ULTRA_45: 123 / 360,  # 33 + 90 = 123
-        SpiceFrame.IMAP_ULTRA_90: 300 / 360,  # 210 + 90 = 300
+        # Phase offset values based on imap_100.tf frame kernel
+        # See docstring notes for details on how these values were determined.
+        SpiceFrame.IMAP_LO: 60 / 360,  # (330 + 90) % 360 = 60
+        SpiceFrame.IMAP_HI_45: 344.8264 / 360,  # 255 + 90 = 345
+        SpiceFrame.IMAP_HI_90: 15.1649 / 360,  # (285 + 90) % 360 = 15
+        SpiceFrame.IMAP_ULTRA_45: 122.8642 / 360,  # 33 + 90 = 123
+        SpiceFrame.IMAP_ULTRA_90: 299.9511 / 360,  # 210 + 90 = 300
         SpiceFrame.IMAP_SWAPI: 258 / 360,  # 168 + 90 = 258
-        SpiceFrame.IMAP_IDEX: 180 / 360,  # 90 + 90 = 180
-        SpiceFrame.IMAP_CODICE: 226 / 360,  # 136 + 90 = 226
-        SpiceFrame.IMAP_HIT: 120 / 360,  # 30 + 90 = 120
-        SpiceFrame.IMAP_SWE: 243 / 360,  # 153 + 90 = 243
-        SpiceFrame.IMAP_GLOWS: 217 / 360,  # 127 + 90 = 217
-        SpiceFrame.IMAP_MAG_I: 90 / 360,  # 0 + 90 = 90
-        SpiceFrame.IMAP_MAG_O: 90 / 360,  # 0 + 90 = 90
+        SpiceFrame.IMAP_IDEX: 179.9229 / 360,  # 90 + 90 = 180
+        SpiceFrame.IMAP_CODICE: 225.9086 / 360,  # 136 + 90 = 226
+        SpiceFrame.IMAP_HIT: 119.6452 / 360,  # 30 + 90 = 120
+        SpiceFrame.IMAP_SWE: 243.0155 / 360,  # 153 + 90 = 243
+        SpiceFrame.IMAP_GLOWS: 217.1384 / 360,  # 127 + 90 = 217
+        SpiceFrame.IMAP_MAG_I: 89.9709 / 360,  # 0 + 90 = 90
+        SpiceFrame.IMAP_MAG_O: 89.4077 / 360,  # 0 + 90 = 90
     }
     return phase_offset_lookup[instrument]
 
@@ -314,6 +323,7 @@ def frame_transform_az_el(
         Ephemeris time(s) corresponding to position(s).
     az_el :  np.ndarray
         <azimuth, elevation> vector or array of vectors in reference frame `from_frame`.
+        Azimuth and elevation pairs are always the final dimension of the array.
         There are several possible shapes for the input az_el and et:
         1. A single az_el vector may be provided for multiple `et` query times
         2. A single `et` may be provided for multiple az_el vectors,
@@ -331,15 +341,16 @@ def frame_transform_az_el(
     to_frame_az_el : np.ndarray
         Azimuth/elevation coordinates in reference frame `to_frame`. This
         output coordinate vector will have shape (2,) if a single `az_el` position
-        vector and single `et` time are input. Otherwise, it will have shape (n, 2)
-        where n is the number of input position vector or ephemeris times. The last
-        axis of the output vector contains azimuth in the 0th position and elevation
-        in the 1st position.
+        vector and single `et` time are input. Otherwise, it will have shape (..., 2)
+        where ... matches the leading dimensions of the input position vector or
+        ephemeris times. The last axis of the output vector contains azimuth in
+        the 0th position and elevation in the 1st position.
     """
     # Convert input az/el to Cartesian vectors
-    spherical_coords_in = np.array(
-        [np.ones_like(az_el[..., 0]), az_el[..., 0], az_el[..., 1]]
-    ).T
+    spherical_coords_in = np.stack(
+        [np.ones_like(az_el[..., 0]), az_el[..., 0], az_el[..., 1]],
+        axis=-1,
+    )
     from_frame_cartesian = spherical_to_cartesian(spherical_coords_in)
     # Transform to to_frame
     to_frame_cartesian = frame_transform(et, from_frame_cartesian, from_frame, to_frame)
@@ -522,7 +533,7 @@ def cartesian_to_spherical(
         az = np.degrees(az)
         el = np.degrees(el)
 
-    spherical_coords = np.stack((np.squeeze(magnitude_v), az, el), axis=-1)
+    spherical_coords = np.stack((np.squeeze(magnitude_v, -1), az, el), axis=-1)
 
     return spherical_coords