imap-processing 0.11.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

imap_processing/ultra/l1b/ultra_l1b.py

@@ -23,26 +23,51 @@ def ultra_l1b(data_dict: dict, data_version: str) -> list[xr.Dataset]:
     -------
     output_datasets : list[xarray.Dataset]
         List of xarray.Dataset.
+
+    Notes
+    -----
+    General flow:
+    1. l1a data products are created (upstream to this code)
+    2. l1b de is created here and dropped in s3 kicking off processing again
+    3. l1b extended, culling, badtimes created here
     """
     output_datasets = []
     instrument_id = 45 if any("45" in key for key in data_dict.keys()) else 90
 
-    if (
-        f"imap_ultra_l1a_{instrument_id}sensor-hk" in data_dict
-        and f"imap_ultra_l1a_{instrument_id}sensor-de" in data_dict
-        and f"imap_ultra_l1a_{instrument_id}sensor-rates" in data_dict
-    ):
+    # L1b de data will be created if L1a de data is available
+    if f"imap_ultra_l1a_{instrument_id}sensor-de" in data_dict:
         de_dataset = calculate_de(
             data_dict[f"imap_ultra_l1a_{instrument_id}sensor-de"],
             f"imap_ultra_l1b_{instrument_id}sensor-de",
             data_version,
         )
+        output_datasets.append(de_dataset)
+    # L1b extended data will be created if L1a hk, rates,
+    # aux and l1b de data are available
+    elif (
+        f"imap_ultra_l1a_{instrument_id}sensor-hk" in data_dict
+        and f"imap_ultra_l1b_{instrument_id}sensor-de" in data_dict
+        and f"imap_ultra_l1a_{instrument_id}sensor-rates" in data_dict
+        and f"imap_ultra_l1a_{instrument_id}sensor-aux" in data_dict
+    ):
         extendedspin_dataset = calculate_extendedspin(
-            data_dict[f"imap_ultra_l1a_{instrument_id}sensor-hk"],
-            data_dict[f"imap_ultra_l1a_{instrument_id}sensor-rates"],
-            de_dataset,
+            {
+                f"imap_ultra_l1a_{instrument_id}sensor-aux": data_dict[
+                    f"imap_ultra_l1a_{instrument_id}sensor-aux"
+                ],
+                f"imap_ultra_l1a_{instrument_id}sensor-hk": data_dict[
+                    f"imap_ultra_l1a_{instrument_id}sensor-hk"
+                ],
+                f"imap_ultra_l1a_{instrument_id}sensor-rates": data_dict[
+                    f"imap_ultra_l1a_{instrument_id}sensor-rates"
+                ],
+                f"imap_ultra_l1b_{instrument_id}sensor-de": data_dict[
+                    f"imap_ultra_l1b_{instrument_id}sensor-de"
+                ],
+            },
             f"imap_ultra_l1b_{instrument_id}sensor-extendedspin",
             data_version,
+            instrument_id,
         )
         cullingmask_dataset = calculate_cullingmask(
             extendedspin_dataset,
@@ -56,7 +81,7 @@ def ultra_l1b(data_dict: dict, data_version: str) -> list[xr.Dataset]:
             data_version,
         )
         output_datasets.extend(
-            [de_dataset, extendedspin_dataset, cullingmask_dataset, badtimes_dataset]
+            [extendedspin_dataset, cullingmask_dataset, badtimes_dataset]
         )
     else:
         raise ValueError("Data dictionary does not contain the expected keys.")

imap_processing/ultra/l1b/ultra_l1b_culling.py

@@ -1,31 +1,15 @@
 """Culls Events for ULTRA L1b."""
 
 import numpy as np
+import pandas as pd
+import xarray as xr
 from numpy.typing import NDArray
 
 from imap_processing.quality_flags import ImapAttitudeUltraFlags, ImapRatesUltraFlags
-from imap_processing.spice.spin import get_spin_data, interpolate_spin_data
+from imap_processing.spice.spin import get_spin_data
 from imap_processing.ultra.constants import UltraConstants
 
 
-def get_spin(eventtimes_met: NDArray) -> NDArray:
-    """
-    Get spin number for each event.
-
-    Parameters
-    ----------
-    eventtimes_met : NDArray
-        Event Times in Mission Elapsed Time.
-
-    Returns
-    -------
-    spin_number : NDArray
-        Spin number at each event derived the from Universal Spin Table.
-    """
-    spin_df = interpolate_spin_data(eventtimes_met)
-    return spin_df["spin_number"].values
-
-
 def get_energy_histogram(
     spin_number: NDArray, energy: NDArray
 ) -> tuple[NDArray, NDArray, NDArray, float]:
@@ -77,14 +61,18 @@ def get_energy_histogram(
     return hist, spin_edges, counts, mean_duration
 
 
-def flag_attitude(eventtimes_met: NDArray) -> tuple[NDArray, NDArray, NDArray, NDArray]:
+def flag_attitude(
+    spin_number: NDArray, aux_dataset: xr.Dataset
+) -> tuple[NDArray, NDArray, NDArray, NDArray]:
     """
     Flag data based on attitude.
 
     Parameters
     ----------
-    eventtimes_met : NDArray
-        Event Times in Mission Elapsed Time.
+    spin_number : NDArray
+        Spin number at each direct event.
+    aux_dataset : xarray.Dataset
+        Auxiliary dataset.
 
     Returns
     -------
@@ -97,7 +85,7 @@ def flag_attitude(eventtimes_met: NDArray) -> tuple[NDArray, NDArray, NDArray, N
     spin_starttime : NDArray
         Spin start time.
     """
-    spins = np.unique(get_spin(eventtimes_met))  # Get unique spins
+    spins = np.unique(spin_number)  # Get unique spins
     spin_df = get_spin_data()  # Load spin data
 
     spin_period = spin_df.loc[spin_df.spin_number.isin(spins), "spin_period_sec"]
@@ -111,6 +99,8 @@ def flag_attitude(eventtimes_met: NDArray) -> tuple[NDArray, NDArray, NDArray, N
         spin_rates.shape, ImapAttitudeUltraFlags.NONE.value, dtype=np.uint16
     )
     quality_flags[bad_spin_rate_indices] |= ImapAttitudeUltraFlags.SPINRATE.value
+    mismatch_indices = compare_aux_univ_spin_table(aux_dataset, spins, spin_df)
+    quality_flags[mismatch_indices] |= ImapAttitudeUltraFlags.AUXMISMATCH.value
 
     return quality_flags, spin_rates, spin_period, spin_starttime
 
@@ -144,15 +134,15 @@ def get_n_sigma(count_rates: NDArray, mean_duration: float, sigma: int = 6) -> N
 
 
 def flag_spin(
-    eventtimes_met: NDArray, energy: NDArray, sigma: int = 6
+    spin_number: NDArray, energy: NDArray, sigma: int = 6
 ) -> tuple[NDArray, NDArray, NDArray, NDArray]:
     """
     Flag data based on counts and negative energies.
 
     Parameters
     ----------
-    eventtimes_met : NDArray
-        Event Times in Mission Elapsed Time.
+    spin_number : NDArray
+        Spin number at each direct event.
     energy : NDArray
         Energy data.
     sigma : int (default=6)
@@ -169,8 +159,9 @@ def flag_spin(
     n_sigma_per_energy_reshape : NDArray
         N sigma per energy.
     """
-    spin = get_spin(eventtimes_met)
-    count_rates, spin_edges, counts, duration = get_energy_histogram(spin, energy)
+    count_rates, spin_edges, counts, duration = get_energy_histogram(
+        spin_number, energy
+    )
     quality_flags = np.full(
         count_rates.shape, ImapRatesUltraFlags.NONE.value, dtype=np.uint16
     )
@@ -181,10 +172,55 @@ def flag_spin(
 
     bin_edges = np.array(UltraConstants.CULLING_ENERGY_BIN_EDGES)
     energy_midpoints = np.sqrt(bin_edges[:-1] * bin_edges[1:])
-    spin = np.unique(spin)
+    spin = np.unique(spin_number)
 
     # Indices where the counts exceed the threshold
     indices_n_sigma = count_rates > threshold[:, np.newaxis]
     quality_flags[indices_n_sigma] |= ImapRatesUltraFlags.HIGHRATES.value
 
     return quality_flags, spin, energy_midpoints, threshold
+
+
+def compare_aux_univ_spin_table(
+    aux_dataset: xr.Dataset, spins: NDArray, spin_df: pd.DataFrame
+) -> NDArray:
+    """
+    Compare the auxiliary and Universal Spin Table.
+
+    Parameters
+    ----------
+    aux_dataset : xarray.Dataset
+        Auxiliary dataset.
+    spins : np.ndarray
+        Array of spin numbers to compare.
+    spin_df : pd.DataFrame
+        Universal Spin Table.
+
+    Returns
+    -------
+    mismatch_indices : np.ndarray
+        Boolean array indicating which spins have mismatches.
+    """
+    univ_mask = np.isin(spin_df["spin_number"].values, spins)
+    aux_mask = np.isin(aux_dataset["SPINNUMBER"].values, spins)
+
+    filtered_univ = spin_df[univ_mask]
+    filtered_aux = {field: aux_dataset[field].values[aux_mask] for field in aux_dataset}
+
+    mismatch_indices = np.zeros(len(spins), dtype=bool)
+
+    fields_to_compare = [
+        ("TIMESPINSTART", "spin_start_sec"),
+        ("TIMESPINSTARTSUB", "spin_start_subsec"),
+        ("DURATION", "spin_period_sec"),
+        ("TIMESPINDATA", "spin_start_time"),
+        ("SPINPERIOD", "spin_period_sec"),
+    ]
+
+    for aux_field, spin_field in fields_to_compare:
+        aux_values = filtered_aux[aux_field]
+        spin_values = filtered_univ[spin_field].values
+
+        mismatch_indices |= aux_values != spin_values
+
+    return mismatch_indices

imap_processing/ultra/l1b/ultra_l1b_extended.py

@@ -10,7 +10,7 @@ import xarray
 from numpy import ndarray
 from numpy.typing import NDArray
 
-from imap_processing.spice.geometry import cartesian_to_spherical
+from imap_processing.spice.spin import get_spin_data
 from imap_processing.ultra.constants import UltraConstants
 from imap_processing.ultra.l1b.lookup_utils import (
     get_back_position,
@@ -751,31 +751,76 @@ def determine_species(tof: np.ndarray, path_length: np.ndarray, type: str) -> ND
     return species_bin
 
 
-def get_de_az_el(v: NDArray) -> tuple[NDArray, NDArray]:
+def get_phi_theta(
+    front_position: tuple, back_position: tuple, d: np.ndarray
+) -> tuple[NDArray, NDArray]:
     """
-    Compute azimuth (phi) angles and elevation (theta).
+    Compute the instrument angles with range -90 -> 90 degrees.
+
+    Further description is available on page 18 of
+    the Ultra Algorithm Theoretical Basis Document.
 
     Parameters
     ----------
-    v : np.ndarray
-        A NumPy array with shape (n, 3) where each
-        row represents a vector
-        with x, y, z-components.
+    front_position : tuple of floats
+        Front position (xf,yf) (hundredths of a millimeter).
+    back_position : tuple of floats
+        Back position (xb,yb) (hundredths of a millimeter).
+    d : np.ndarray
+        Distance from slit to foil (hundredths of a millimeter).
 
     Returns
     -------
-    spherical_coords : np.ndarray
-        A NumPy array with shape (n, 3), where each row contains
-        the spherical coordinates (r, azimuth, elevation):
+    phi : np.array
+        Ultra instrument frame event azimuth.
+    theta : np.array
+        Ultra instrument frame event elevation.
+    """
+    path_length = get_path_length(front_position, back_position, d)
+
+    phi = np.arctan((front_position[1] - back_position[1]) / d)
+    theta = np.arcsin((front_position[0] - back_position[0]) / path_length)
+
+    return np.degrees(phi), np.degrees(theta)
+
 
-        - azimuth : angle in the xy-plane
-          In radians:
-          output range=[0, 2*pi].
-        - elevation : angle from the xy-plane
-          In radians:
-          output range=[-pi/2, pi/2].
+def get_eventtimes(
+    spin: NDArray, phase_angle: NDArray
+) -> tuple[NDArray, NDArray, NDArray]:
     """
-    # Compute azimuth (phi) angles and elevation (theta)
-    spherical_coords = cartesian_to_spherical(v, degrees=False)
+    Get the event times.
+
+    Parameters
+    ----------
+    spin : np.ndarray
+        Spin number.
+    phase_angle : np.ndarray
+        Phase angle.
+
+    Returns
+    -------
+    event_times : np.ndarray
+        Event times.
+    spin_starts : np.ndarray
+        Spin start times.
+    spin_period_sec : np.ndarray
+        Spin period in seconds.
+
+    Notes
+    -----
+    Equation for event time:
+    t = t_(spin start) + t_(spin start sub)/1000 +
+    t_spin_period_sec * phase_angle/720
+    """
+    spin_df = get_spin_data()
+    index = np.searchsorted(spin_df["spin_number"].values, spin)
+    spin_starts = (
+        spin_df["spin_start_sec"].values[index]
+        + spin_df["spin_start_subsec"].values[index] / 1000
+    )
+
+    spin_period_sec = spin_df["spin_period_sec"].values[index]
+
+    event_times = spin_starts + spin_period_sec * (phase_angle / 720)
 
-    return spherical_coords[:, 1], spherical_coords[:, 2]
+    return event_times, spin_starts, spin_period_sec

imap_processing/ultra/l1c/spacecraft_pset.py

@@ -0,0 +1,86 @@
+"""Calculate Pointing Set Grids."""
+
+import numpy as np
+import pandas as pd
+import xarray as xr
+
+from imap_processing import imap_module_directory
+from imap_processing.ultra.l1c.ultra_l1c_pset_bins import (
+    build_energy_bins,
+    get_background_rates,
+    get_spacecraft_exposure_times,
+    get_spacecraft_histogram,
+)
+from imap_processing.ultra.utils.ultra_l1_utils import create_dataset
+
+# TODO: This is a placeholder for the API lookup table directory.
+TEST_PATH = imap_module_directory / "tests" / "ultra" / "data" / "l1"
+
+
+def calculate_spacecraft_pset(
+    de_dataset: xr.Dataset,
+    extendedspin_dataset: xr.Dataset,
+    cullingmask_dataset: xr.Dataset,
+    name: str,
+    data_version: str,
+) -> xr.Dataset:
+    """
+    Create dictionary with defined datatype for Pointing Set Grid Data.
+
+    Parameters
+    ----------
+    de_dataset : xarray.Dataset
+        Dataset containing de data.
+    extendedspin_dataset : xarray.Dataset
+        Dataset containing extendedspin data.
+    cullingmask_dataset : xarray.Dataset
+        Dataset containing cullingmask data.
+    name : str
+        Name of the dataset.
+    data_version : str
+        Version of the data.
+
+    Returns
+    -------
+    dataset : xarray.Dataset
+        Dataset containing the data.
+    """
+    pset_dict: dict[str, np.ndarray] = {}
+
+    v_mag_dps_spacecraft = np.linalg.norm(de_dataset["velocity_dps_sc"].values, axis=1)
+    vhat_dps_spacecraft = (
+        de_dataset["velocity_dps_sc"].values / v_mag_dps_spacecraft[:, np.newaxis]
+    )
+
+    intervals, _, energy_bin_geometric_means = build_energy_bins()
+    counts, latitude, longitude, n_pix = get_spacecraft_histogram(
+        vhat_dps_spacecraft,
+        de_dataset["energy_spacecraft"].values,
+        intervals,
+        nside=128,
+    )
+    healpix = np.arange(n_pix)
+
+    # calculate background rates
+    background_rates = get_background_rates()
+
+    # TODO: calculate sensitivity and interpolate based on energy.
+
+    # Calculate exposure
+    constant_exposure = TEST_PATH / "ultra_90_dps_exposure.csv"
+    df_exposure = pd.read_csv(constant_exposure)
+    exposure_pointing = get_spacecraft_exposure_times(df_exposure)
+
+    # For ISTP, epoch should be the center of the time bin.
+    pset_dict["epoch"] = de_dataset.epoch.data[0].astype(np.int64)
+    pset_dict["counts"] = counts
+    pset_dict["latitude"] = latitude
+    pset_dict["longitude"] = longitude
+    pset_dict["energy_bin_geometric_mean"] = energy_bin_geometric_means
+    pset_dict["background_rates"] = background_rates
+    pset_dict["exposure_factor"] = exposure_pointing
+    pset_dict["healpix"] = healpix
+
+    dataset = create_dataset(pset_dict, name, "l1c", data_version)
+
+    return dataset

imap_processing/ultra/l1c/ultra_l1c.py

@@ -3,7 +3,7 @@
 import xarray as xr
 
 from imap_processing.ultra.l1c.histogram import calculate_histogram
-from imap_processing.ultra.l1c.pset import calculate_pset
+from imap_processing.ultra.l1c.spacecraft_pset import calculate_spacecraft_pset
 
 
 def ultra_l1c(data_dict: dict, data_version: str) -> list[xr.Dataset]:
@@ -39,12 +39,15 @@ def ultra_l1c(data_dict: dict, data_version: str) -> list[xr.Dataset]:
         and f"imap_ultra_l1b_{instrument_id}sensor-de" in data_dict
         and f"imap_ultra_l1b_{instrument_id}sensor-extendedspin" in data_dict
     ):
-        pset_dataset = calculate_pset(
+        spacecraft_pset = calculate_spacecraft_pset(
             data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"],
-            f"imap_ultra_l1c_{instrument_id}sensor-pset",
+            data_dict[f"imap_ultra_l1b_{instrument_id}sensor-extendedspin"],
+            data_dict[f"imap_ultra_l1b_{instrument_id}sensor-cullingmask"],
+            f"imap_ultra_l1c_{instrument_id}sensor-spacecraftpset",
             data_version,
         )
-        output_datasets = [pset_dataset]
+        # TODO: add calculate_helio_pset here
+        output_datasets = [spacecraft_pset]
     else:
         raise ValueError("Data dictionary does not contain the expected keys.")