imap-processing 0.18.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

imap_processing/ialirt/l0/parse_mag.py

@@ -2,11 +2,13 @@
 
 import logging
 from decimal import Decimal
-from typing import Union
 
 import numpy as np
 import xarray as xr
 
+from imap_processing.ialirt.l0.ialirt_spice import (
+    transform_instrument_vectors_to_inertial,
+)
 from imap_processing.ialirt.l0.mag_l0_ialirt_data import (
     Packet0,
     Packet1,
@@ -22,7 +24,13 @@ from imap_processing.mag.l1b.mag_l1b import (
 )
 from imap_processing.mag.l1d.mag_l1d_data import MagL1d
 from imap_processing.mag.l2.mag_l2_data import MagL2L1dBase
-from imap_processing.spice.time import met_to_ttj2000ns, met_to_utc
+from imap_processing.spice.geometry import (
+    SpiceFrame,
+    cartesian_to_spherical,
+    frame_transform,
+    spherical_to_cartesian,
+)
+from imap_processing.spice.time import met_to_ttj2000ns, met_to_utc, ttj2000ns_to_et
 
 logger = logging.getLogger(__name__)
 
@@ -195,7 +203,7 @@ def get_time(
         (grouped_data["group"] == group).values
     ][pkt_counter == 2]
 
-    time_data: dict[str, Union[int, float]] = {
+    time_data: dict[str, int | float] = {
         "pri_coarsetm": int(pri_coarsetm.item()),
         "pri_fintm": int(pri_fintm.item()),
         "sec_coarsetm": int(sec_coarsetm.item()),
@@ -290,7 +298,6 @@ def calculate_l1b(
 
 def calibrate_and_offset_vectors(
     vectors: np.ndarray,
-    range_vals: np.ndarray,
     calibration: np.ndarray,
     offsets: np.ndarray,
     is_magi: bool = False,
@@ -301,9 +308,7 @@ def calibrate_and_offset_vectors(
     Parameters
     ----------
     vectors : np.ndarray
-        Raw magnetic vectors, shape (n, 3).
-    range_vals : np.ndarray
-        Range indices for each vector, shape (n). Values 0–3.
+        Raw magnetic vectors, shape (n, 4).
     calibration : np.ndarray
         Calibration matrix, shape (3, 3, 4).
     offsets : np.ndarray
@@ -319,11 +324,9 @@ def calibrate_and_offset_vectors(
     calibrated_and_offset_vectors : np.ndarray
         Calibrated and offset vectors, shape (n, 3).
     """
-    # Append range as 4th column
-    vec_plus_range = np.concatenate((vectors, range_vals[:, np.newaxis]), axis=1)
-
     # Apply calibration matrix -> (n,4)
-    calibrated = MagL2L1dBase.apply_calibration(vec_plus_range, calibration)
+    # apply_calibration_offset_single_vector
+    calibrated = MagL2L1dBase.apply_calibration(vectors.reshape(1, 4), calibration)
 
     # Apply offsets per vector
     # vec shape (4)
@@ -338,9 +341,193 @@ def calibrate_and_offset_vectors(
     return calibrated[:, :3]
 
 
+def apply_gradiometry_correction(
+    mago_vectors_eclipj2000: np.ndarray,
+    mago_time_data: np.ndarray,
+    magi_vectors_eclipj2000: np.ndarray,
+    magi_time_data: np.ndarray,
+    gradiometer_factor: np.ndarray,
+) -> tuple[np.ndarray, np.ndarray]:
+    """
+    Align MAGi to MAGo timestamps and apply gradiometry correction.
+
+    Parameters
+    ----------
+    mago_vectors_eclipj2000 : np.ndarray
+        MAGo vectors in inertial frame, shape (N, 3).
+    mago_time_data : np.ndarray
+        Time for primary sensor, shape (N, 3).
+    magi_vectors_eclipj2000 : np.ndarray
+        MAGi vectors in inertial frame, shape (M, 3).
+    magi_time_data : np.ndarray
+        Time for secondary sensor, shape (N, 3).
+    gradiometer_factor : np.ndarray
+        A (3,3) element matrix to scale and rotate the gradiometer offsets.
+
+    Returns
+    -------
+    mago_corrected : np.ndarray
+        Corrected MAGo vectors in inertial frame, shape (N, 3).
+    magnitude : np.ndarray
+        Magnitude of corrected MAGo vectors, shape (N,).
+    """
+    gradiometry_offsets = MagL1d.calculate_gradiometry_offsets(
+        mago_vectors_eclipj2000,
+        mago_time_data,
+        magi_vectors_eclipj2000,
+        magi_time_data,
+    )
+    mago_corrected = MagL1d.apply_gradiometry_offsets(
+        gradiometry_offsets, mago_vectors_eclipj2000, gradiometer_factor
+    )
+    magnitude = np.linalg.norm(mago_corrected, axis=-1).squeeze()
+
+    return mago_corrected, magnitude
+
+
+def transform_to_inertial(
+    sc_spin_phase_rad: np.ndarray,
+    sc_inertial_right: np.ndarray,
+    sc_inertial_decline: np.ndarray,
+    attitude_time: np.ndarray,
+    target_time: float,
+    mag_vector: np.ndarray,
+) -> np.ndarray:
+    """
+    Transform vector to ECLIPJ2000.
+
+    Parameters
+    ----------
+    sc_spin_phase_rad : numpy.ndarray
+        Spin phase for 4 packets 0 to 2π radians, shape (4).
+    sc_inertial_right : numpy.ndarray
+        Inertial right ascension for 4 packets 0 to 2π radians, shape (4).
+    sc_inertial_decline : numpy.ndarray
+        Inertial declination for 4 packets -π/2 to π/2 radians, shape (4).
+    attitude_time : np.ndarray
+        Timestamps for the 4 packets.
+        Example: test_met = grouped_data["met"][
+                 (grouped_data["group"] == group).values].
+        ttj2000ns = met_to_ttj2000ns(test_met.values).
+    target_time : float
+        Time at which to apply the transformation.
+        Will be primary_epoch (mago vector) or secondary_epoch (magi vector).
+        Example: time_data['primary_epoch'].
+    mag_vector : numpy.ndarray
+        Vector, shape (3).
+
+    Returns
+    -------
+    inertial_vector : np.ndarray
+        Transformed vector in the ECLIPJ2000 frame, shape (3,).
+
+    Notes
+    -----
+    The MAG vectors are calculated based on 4 packets,
+    each of which contains its own spin phase,
+    inertial right ascension, and inertial decline.
+    """
+    if target_time < attitude_time.min() or target_time > attitude_time.max():
+        logger.warning(
+            f"target_time {target_time} is outside attitude_time bounds "
+            f"[{attitude_time.min()}, {attitude_time.max()}]; using edge values."
+        )
+
+    # Get sort order based on attitude_time
+    sort_idx = np.argsort(attitude_time)
+
+    # Sort all arrays accordingly
+    attitude_time = attitude_time[sort_idx]
+    sc_spin_phase_rad = sc_spin_phase_rad[sort_idx]
+    sc_inertial_right = sc_inertial_right[sort_idx]
+    sc_inertial_decline = sc_inertial_decline[sort_idx]
+
+    # Interpolate spin phase, RA, and Dec at target_time
+    # Convert RA/Dec to unit cartesian vectors
+    spherical_coords = np.stack(
+        [
+            np.ones_like(sc_inertial_right),
+            np.degrees(sc_inertial_right),
+            np.degrees(sc_inertial_decline),
+        ],
+        axis=-1,
+    )
+    vecs = spherical_to_cartesian(spherical_coords)
+
+    # Interpolate in Cartesian space
+    vx = np.interp(target_time, attitude_time, vecs[:, 0])
+    vy = np.interp(target_time, attitude_time, vecs[:, 1])
+    vz = np.interp(target_time, attitude_time, vecs[:, 2])
+    v_interp = np.array([vx, vy, vz])
+    # Normalize vector so that its magnitude is 1.
+    v_interp /= np.linalg.norm(v_interp)
+
+    # Convert back to spherical
+    ra_dec = cartesian_to_spherical(v_interp)
+    ra_deg = ra_dec[1]
+    dec_deg = ra_dec[2]
+
+    # Account for discontinuities in spin phase.
+    spin_phase_unwrapped = np.unwrap(sc_spin_phase_rad)
+    spin_phase_interp = np.interp(target_time, attitude_time, spin_phase_unwrapped)
+    spin_phase_deg = np.degrees(spin_phase_interp) % 360
+
+    # Transform each into ECLIPJ2000
+    inertial_vector = transform_instrument_vectors_to_inertial(
+        np.asarray(mag_vector).reshape(1, 3),
+        np.array([spin_phase_deg]),
+        np.array([ra_deg]),
+        np.array([dec_deg]),
+    )[0]
+
+    return inertial_vector
+
+
+def transform_to_frames(
+    target_time: np.ndarray,
+    inertial_vector: np.ndarray,
+) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
+    """
+    Transform vector to different frames.
+
+    Parameters
+    ----------
+    target_time : np.ndarray
+        Time at which to apply the transformation.
+        Will be primary_epoch (mago vector).
+        Example: time_data['primary_epoch'].
+    inertial_vector : np.ndarray
+        Transformed vector in the ECLIPJ2000 frame, shape (3,).
+
+    Returns
+    -------
+    gse_vector : np.ndarray
+        Transformed vector in the GSE frame, shape (3,).
+    gsm_vector : np.ndarray
+        Transformed vector in the GSM frame, shape (3,).
+    rtn_vector : np.ndarray
+        Transformed vector in the RTN frame, shape (3,).
+    """
+    et_target_time = ttj2000ns_to_et(target_time)
+
+    gse_vector = frame_transform(
+        et_target_time, inertial_vector, SpiceFrame.ECLIPJ2000, SpiceFrame.IMAP_GSE
+    )
+    gsm_vector = frame_transform(
+        et_target_time, inertial_vector, SpiceFrame.ECLIPJ2000, SpiceFrame.IMAP_GSM
+    )
+    rtn_vector = frame_transform(
+        et_target_time, inertial_vector, SpiceFrame.ECLIPJ2000, SpiceFrame.IMAP_RTN
+    )
+
+    return gse_vector, gsm_vector, rtn_vector
+
+
 def process_packet(
-    accumulated_data: xr.Dataset, calibration_dataset: xr.Dataset
-) -> tuple[list[dict], list[dict]]:
+    accumulated_data: xr.Dataset,
+    engineering_calibration_dataset: xr.Dataset,
+    l1d_calibration_dataset: xr.Dataset,
+) -> list[dict]:
     """
     Parse the MAG packets.
 
@@ -348,8 +535,10 @@ def process_packet(
     ----------
     accumulated_data : xr.Dataset
         Packets dataset accumulated over 1 min.
-    calibration_dataset : xr.Dataset
-        Calibration dataset.
+    engineering_calibration_dataset : xr.Dataset
+        Engineering calibration dataset.
+    l1d_calibration_dataset : xr.Dataset
+        L1D calibration dataset.
 
     Returns
     -------
@@ -375,8 +564,12 @@ def process_packet(
     grouped_data = find_groups(accumulated_data, (0, 3), "pkt_counter", "met")
 
     unique_groups = np.unique(grouped_data["group"])
-    l1b_data = []
     mag_data = []
+    met_all = []
+    mago_vectors_all = []
+    mago_times_all = []
+    magi_vectors_all = []
+    magi_times_all = []
 
     for group in unique_groups:
         # Get status values for each group.
@@ -412,7 +605,7 @@ def process_packet(
             pkt_counter,
             science_data,
             status_data,
-            calibration_dataset,
+            engineering_calibration_dataset,
         )
 
         # Note: primary = MAGo, secondary = MAGi.
@@ -423,41 +616,99 @@ def process_packet(
         if status_data["sec_isvalid"] == 0:
             updated_vector_magi = np.full(4, -32768)
 
-        science_data.update(
-            {
-                "calibrated_pri_x": updated_vector_mago[0],
-                "calibrated_pri_y": updated_vector_mago[1],
-                "calibrated_pri_z": updated_vector_mago[2],
-                "calibrated_sec_x": updated_vector_magi[0],
-                "calibrated_sec_y": updated_vector_magi[1],
-                "calibrated_sec_z": updated_vector_magi[2],
-            }
+        mago_calibration = l1d_calibration_dataset["URFTOORFO"][0]
+        magi_calibration = l1d_calibration_dataset["URFTOORFI"][0]
+        offsets = l1d_calibration_dataset["offsets"][0]
+
+        mago_out = calibrate_and_offset_vectors(
+            updated_vector_mago, mago_calibration, offsets, is_magi=False
+        )
+        magi_out = calibrate_and_offset_vectors(
+            updated_vector_magi, magi_calibration, offsets, is_magi=True
+        )
+        sc_spin_phase_rad = grouped_data["sc_spin_phase"][
+            (grouped_data["group"] == group).values
+        ]
+        sc_inertial_right = grouped_data["sc_inertial_right"][
+            (grouped_data["group"] == group).values
+        ]
+        sc_inertial_decline = grouped_data["sc_inertial_decline"][
+            (grouped_data["group"] == group).values
+        ]
+
+        attitude_time = met_to_ttj2000ns(
+            grouped_data["met"][(grouped_data["group"] == group).values]
         )
 
-        l1b_data.append({**status_data, **science_data, **time_data})
+        # Convert to ECLIPJ2000 frame.
+        mago_inertial_vector = transform_to_inertial(
+            sc_spin_phase_rad.values,
+            sc_inertial_right.values,
+            sc_inertial_decline.values,
+            attitude_time,
+            time_data["primary_epoch"],
+            mago_out,
+        )
+        magi_inertial_vector = transform_to_inertial(
+            sc_spin_phase_rad.values,
+            sc_inertial_right.values,
+            sc_inertial_decline.values,
+            attitude_time,
+            time_data["secondary_epoch"],
+            magi_out,
+        )
 
-        # Placeholder for real data.
         met = grouped_data["met"][(grouped_data["group"] == group).values]
+        met_all.append(met.values[0])
+        mago_times_all.append(time_data["primary_epoch"])
+        mago_vectors_all.append(mago_inertial_vector)
+        magi_vectors_all.append(magi_inertial_vector)
+        magi_times_all.append(time_data["secondary_epoch"])
+
+    mago_corrected, magnitude = apply_gradiometry_correction(
+        np.array(mago_vectors_all),
+        np.array(mago_times_all),
+        np.array(magi_vectors_all),
+        np.array(magi_times_all),
+        l1d_calibration_dataset["gradiometer_factor"].values.squeeze(),
+    )
+
+    gse_vector, gsm_vector, rtn_vector = transform_to_frames(
+        np.array(mago_times_all), mago_corrected
+    )
+
+    spherical = cartesian_to_spherical(gsm_vector)
+    phi_gsm = spherical[:, 1]
+    theta_gsm = spherical[:, 2]
+
+    spherical = cartesian_to_spherical(gse_vector)
+    phi_gse = spherical[:, 1]
+    theta_gse = spherical[:, 2]
+
+    # Omit the first value since we expect it to be extrapolated.
+    for i in range(len(mago_corrected)):
+        if i == 0:
+            continue
+
         mag_data.append(
             {
                 "apid": 478,
-                "met": int(met.values.min()),
-                "met_in_utc": met_to_utc(met.values.min()).split(".")[0],
-                "ttj2000ns": int(met_to_ttj2000ns(met.values.min())),
-                # TODO: Placeholder for mag_epoch
-                "mag_epoch": int(met.values.min()),
-                "mag_B_GSE": [Decimal("0.0") for _ in range(3)],
-                "mag_B_GSM": [Decimal("0.0") for _ in range(3)],
-                "mag_B_RTN": [Decimal("0.0") for _ in range(3)],
-                "mag_B_magnitude": Decimal("0.0"),
-                "mag_phi_B_GSM": Decimal("0.0"),
-                "mag_theta_B_GSM": Decimal("0.0"),
-                "mag_phi_B_GSE": Decimal("0.0"),
-                "mag_theta_B_GSE": Decimal("0.0"),
+                "met": int(met_all[i]),
+                "met_in_utc": met_to_utc(met_all[i]).split(".")[0],
+                "ttj2000ns": int(met_to_ttj2000ns(met_all[i])),
+                "mag_epoch": int(mago_times_all[i]),
+                "mag_B_GSE": [Decimal(str(v)) for v in gse_vector[i]],
+                "mag_B_GSM": [Decimal(str(v)) for v in gsm_vector[i]],
+                "mag_B_RTN": [Decimal(str(v)) for v in rtn_vector[i]],
+                "mag_B_magnitude": Decimal(str(magnitude[i])),
+                "mag_phi_B_GSM": Decimal(str(phi_gsm[i])),
+                "mag_theta_B_GSM": Decimal(str(theta_gsm[i])),
+                "mag_phi_B_GSE": Decimal(str(phi_gse[i])),
+                "mag_theta_B_GSE": Decimal(str(theta_gse[i])),
             }
         )
 
-    return mag_data, l1b_data
+    return mag_data
 
 
 def retrieve_matrix_from_single_l1b_calibration(

imap_processing/ialirt/l0/process_hit.py

@@ -91,18 +91,20 @@ def create_l1(
     fast_rate_1_dict = {
         prefix: value
         for prefix, value in zip(
-            HIT_PREFIX_TO_RATE_TYPE["FAST_RATE_1"], fast_rate_1.data
+            HIT_PREFIX_TO_RATE_TYPE["FAST_RATE_1"], fast_rate_1.data, strict=False
         )
     }
     fast_rate_2_dict = {
         prefix: value
         for prefix, value in zip(
-            HIT_PREFIX_TO_RATE_TYPE["FAST_RATE_2"], fast_rate_2.data
+            HIT_PREFIX_TO_RATE_TYPE["FAST_RATE_2"], fast_rate_2.data, strict=False
         )
     }
     slow_rate_dict = {
         prefix: value
-        for prefix, value in zip(HIT_PREFIX_TO_RATE_TYPE["SLOW_RATE"], slow_rate.data)
+        for prefix, value in zip(
+            HIT_PREFIX_TO_RATE_TYPE["SLOW_RATE"], slow_rate.data, strict=False
+        )
     }
 
     l1 = {**fast_rate_1_dict, **fast_rate_2_dict, **slow_rate_dict}

imap_processing/ialirt/l0/process_swapi.py

@@ -2,7 +2,6 @@
 
 import logging
 from decimal import Decimal
-from typing import Optional
 
 import numpy as np
 import pandas as pd
@@ -10,13 +9,12 @@ import xarray as xr
 from scipy.optimize import curve_fit
 from scipy.special import erf
 
-from imap_processing import imap_module_directory
 from imap_processing.ialirt.constants import IalirtSwapiConstants as Consts
 from imap_processing.ialirt.utils.grouping import find_groups
 from imap_processing.ialirt.utils.time import calculate_time
 from imap_processing.spice.time import met_to_ttj2000ns, met_to_utc
 from imap_processing.swapi.l1.swapi_l1 import process_sweep_data
-from imap_processing.swapi.l2.swapi_l2 import TIME_PER_BIN
+from imap_processing.swapi.l2.swapi_l2 import SWAPI_LIVETIME
 
 logger = logging.getLogger(__name__)
 
@@ -70,7 +68,7 @@ def count_rate(
 def optimize_pseudo_parameters(
     count_rates: np.ndarray,
     count_rate_error: np.ndarray,
-    energy_passbands: Optional[np.ndarray] = None,
+    energy_passbands: np.ndarray,
 ) -> (dict)[str, list[float]]:
     """
     Find the pseudo speed (u), density (n) and temperature (T) of solar wind particles.
@@ -84,7 +82,7 @@ def optimize_pseudo_parameters(
     count_rate_error : np.ndarray
         Standard deviation of the coincidence count rates parameter.
     energy_passbands : np.ndarray, default None
-        Energy passbands, passed in only for testing purposes.
+        Energy values, taken from the SWAPI lookup table.
 
     Returns
     -------
@@ -92,21 +90,6 @@ def optimize_pseudo_parameters(
         Dictionary containing the optimized speed, density, and temperature values for
         each sweep included in the input count_rates array.
     """
-    if not energy_passbands:
-        # Read in energy passbands
-        energy_data = pd.read_csv(
-            f"{imap_module_directory}/tests/swapi/lut/imap_swapi_esa-unit"
-            f"-conversion_20250626_v001.csv"
-        )
-        energy_passbands = (
-            energy_data["Energy"][0:63]
-            .replace(",", "", regex=True)
-            .to_numpy()
-            .astype(float)
-        )
-
-    # Initial guess pulled from page 52 of the IMAP SWAPI Instrument Algorithms Document
-    initial_param_guess = np.array([550, 5.27, 1e5])
     solution_dict = {  # type: ignore
         "pseudo_speed": [],
         "pseudo_density": [],
@@ -116,8 +99,16 @@ def optimize_pseudo_parameters(
     for sweep in np.arange(count_rates.shape[0]):
         current_sweep_count_rates = count_rates[sweep, :]
         current_sweep_count_rate_errors = count_rate_error[sweep, :]
-        # Find the max count rate, and use the 6 points surrounding it (inclusive)
+        # Find the max count rate, and use the 5 points surrounding it
         max_index = np.argmax(current_sweep_count_rates)
+        initial_speed_guess = np.sqrt(energy_passbands[max_index]) * Consts.speed_coeff
+        initial_param_guess = np.array(
+            [
+                initial_speed_guess,
+                5 * (400 / initial_speed_guess) ** 2,
+                60000 * (initial_speed_guess / 400) ** 2,
+            ]
+        )
         sol = curve_fit(
             f=count_rate,
             xdata=energy_passbands.take(
@@ -138,7 +129,9 @@ def optimize_pseudo_parameters(
     return solution_dict
 
 
-def process_swapi_ialirt(unpacked_data: xr.Dataset) -> list[dict]:
+def process_swapi_ialirt(
+    unpacked_data: xr.Dataset, calibration_lut_table: pd.DataFrame
+) -> list[dict]:
     """
     Extract I-ALiRT variables and calculate coincidence count rate.
 
@@ -146,6 +139,8 @@ def process_swapi_ialirt(unpacked_data: xr.Dataset) -> list[dict]:
     ----------
     unpacked_data : xr.Dataset
         SWAPI I-ALiRT data that has been parsed from the spacecraft packet.
+    calibration_lut_table : pd.DataFrame
+        DataFrame containing the contents of the SWAPI esa-unit-conversion lookup table.
 
     Returns
     -------
@@ -191,10 +186,31 @@ def process_swapi_ialirt(unpacked_data: xr.Dataset) -> list[dict]:
             continue
 
     raw_coin_count = process_sweep_data(grouped_dataset, "swapi_coin_cnt")
-    raw_coin_rate = raw_coin_count / TIME_PER_BIN
-    count_rate_error = np.sqrt(raw_coin_count) / TIME_PER_BIN
+    raw_coin_rate = raw_coin_count / SWAPI_LIVETIME
+    count_rate_error = np.sqrt(raw_coin_count) / SWAPI_LIVETIME
 
-    solution = optimize_pseudo_parameters(raw_coin_rate, count_rate_error)
+    # Extract energy values from the calibration lookup table file
+    calibration_lut_table["timestamp"] = pd.to_datetime(
+        calibration_lut_table["timestamp"], format="%m/%d/%Y %H:%M"
+    )
+    calibration_lut_table["timestamp"] = calibration_lut_table["timestamp"].to_numpy(
+        dtype="datetime64[ns]"
+    )
+
+    # Find the sweep's energy data for the latest time, where sweep_id == 2
+    subset = calibration_lut_table[
+        (calibration_lut_table["timestamp"] == calibration_lut_table["timestamp"].max())
+        & (calibration_lut_table["Sweep #"] == 2)
+    ]
+    if subset.empty:
+        energy_passbands = np.full(63, np.nan, dtype=np.float64)
+    else:
+        subset = subset.sort_values(["timestamp", "ESA Step #"])
+        energy_passbands = subset["Energy"][0:63].to_numpy().astype(float)
+
+    solution = optimize_pseudo_parameters(
+        raw_coin_rate, count_rate_error, energy_passbands
+    )
 
     swapi_data = []