imap-processing 0.17.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

imap_processing/ultra/l1b/lookup_utils.py

@@ -4,43 +4,12 @@ import numpy as np
 import numpy.typing as npt
 import pandas as pd
 import xarray as xr
+from numpy.typing import NDArray
 
-from imap_processing import imap_module_directory
-
-BASE_PATH = imap_module_directory / "ultra" / "lookup_tables"
-
-_YADJUST_DF = pd.read_csv(BASE_PATH / "yadjust.csv").set_index("dYLUT")
-_TDC_NORM_DF_ULTRA45 = pd.read_csv(
-    BASE_PATH / "ultra45_tdc_norm.csv", header=1, index_col="Index"
-)
-_TDC_NORM_DF_ULTRA90 = pd.read_csv(
-    BASE_PATH / "ultra90_tdc_norm.csv", header=1, index_col="Index"
-)
-_BACK_POS_DF_ULTRA45 = pd.read_csv(
-    BASE_PATH / "ultra45_back-pos-luts.csv", index_col="Index_offset"
-)
-_BACK_POS_DF_ULTRA90 = pd.read_csv(
-    BASE_PATH / "ultra90_back-pos-luts.csv", index_col="Index_offset"
-)
-_ENERGY_NORM_DF = pd.read_csv(BASE_PATH / "EgyNorm.mem.csv")
-_IMAGE_PARAMS_DF = {
-    "ultra45": pd.read_csv(BASE_PATH / "FM45_Startup1_ULTRA_IMGPARAMS_20240719.csv"),
-    "ultra90": pd.read_csv(BASE_PATH / "FM90_Startup1_ULTRA_IMGPARAMS_20240719.csv"),
-}
-
-_FWHM_TABLES = {
-    ("left", "ultra45"): pd.read_csv(BASE_PATH / "Angular_Profiles_FM45_LeftSlit.csv"),
-    ("right", "ultra45"): pd.read_csv(
-        BASE_PATH / "Angular_Profiles_FM45_RightSlit.csv"
-    ),
-    ("left", "ultra90"): pd.read_csv(BASE_PATH / "Angular_Profiles_FM90_LeftSlit.csv"),
-    ("right", "ultra90"): pd.read_csv(
-        BASE_PATH / "Angular_Profiles_FM90_RightSlit.csv"
-    ),
-}
-
-
-def get_y_adjust(dy_lut: np.ndarray) -> npt.NDArray:
+from imap_processing.quality_flags import ImapDEOutliersUltraFlags
+
+
+def get_y_adjust(dy_lut: np.ndarray, ancillary_files: dict) -> npt.NDArray:
     """
     Adjust the front yf position based on the particle's trajectory.
 
@@ -52,16 +21,21 @@ def get_y_adjust(dy_lut: np.ndarray) -> npt.NDArray:
     ----------
     dy_lut : np.ndarray
         Change in y direction used for the lookup table (mm).
+    ancillary_files : dict[Path]
+        Ancillary files containing the lookup tables.
 
     Returns
     -------
     yadj : np.ndarray
         Y adjustment (mm).
     """
-    return _YADJUST_DF["dYAdj"].iloc[dy_lut].values
+    yadjust_df = pd.read_csv(ancillary_files["l1b-yadjust-lookup"]).set_index("dYLUT")
+    return yadjust_df["dYAdj"].iloc[dy_lut].values
 
 
-def get_norm(dn: xr.DataArray, key: str, file_label: str) -> npt.NDArray:
+def get_norm(
+    dn: xr.DataArray, key: str, file_label: str, ancillary_files: dict
+) -> npt.NDArray:
     """
     Correct mismatches between the stop Time to Digital Converters (TDCs).
 
@@ -82,6 +56,8 @@ def get_norm(dn: xr.DataArray, key: str, file_label: str) -> npt.NDArray:
         BtSpNNorm, BtSpSNorm, BtSpENorm, or BtSpWNorm.
     file_label : str
         Instrument (ultra45 or ultra90).
+    ancillary_files : dict[Path]
+        Ancillary files containing the lookup tables.
 
     Returns
     -------
@@ -89,16 +65,22 @@ def get_norm(dn: xr.DataArray, key: str, file_label: str) -> npt.NDArray:
         Normalized DNs.
     """
     if file_label == "ultra45":
-        tdc_norm_df = _TDC_NORM_DF_ULTRA45
+        tdc_norm_df = pd.read_csv(
+            ancillary_files["l1b-45sensor-tdc-norm-lookup"], header=1, index_col="Index"
+        )
     else:
-        tdc_norm_df = _TDC_NORM_DF_ULTRA90
+        tdc_norm_df = pd.read_csv(
+            ancillary_files["l1b-90sensor-tdc-norm-lookup"], header=1, index_col="Index"
+        )
 
     dn_norm = tdc_norm_df[key].iloc[dn].values
 
     return dn_norm
 
 
-def get_back_position(back_index: np.ndarray, key: str, file_label: str) -> npt.NDArray:
+def get_back_position(
+    back_index: np.ndarray, key: str, file_label: str, ancillary_files: dict
+) -> npt.NDArray:
     """
     Convert normalized TDC values using lookup tables.
 
@@ -117,6 +99,8 @@ def get_back_position(back_index: np.ndarray, key: str, file_label: str) -> npt.
         XBkTp, YBkTp, XBkBt, or YBkBt.
     file_label : str
         Instrument (ultra45 or ultra90).
+    ancillary_files : dict[Path]
+        Ancillary files containing the lookup tables.
 
     Returns
     -------
@@ -124,14 +108,20 @@ def get_back_position(back_index: np.ndarray, key: str, file_label: str) -> npt.
         Converted DNs to Units of hundredths of a millimeter.
     """
     if file_label == "ultra45":
-        back_pos_df = _BACK_POS_DF_ULTRA45
+        back_pos_df = pd.read_csv(
+            ancillary_files["l1b-45sensor-back-pos-lookup"], index_col="Index_offset"
+        )
     else:
-        back_pos_df = _BACK_POS_DF_ULTRA90
+        back_pos_df = pd.read_csv(
+            ancillary_files["l1b-90sensor-back-pos-lookup"], index_col="Index_offset"
+        )
 
     return back_pos_df[key].values[back_index]
 
 
-def get_energy_norm(ssd: np.ndarray, composite_energy: np.ndarray) -> npt.NDArray:
+def get_energy_norm(
+    ssd: np.ndarray, composite_energy: np.ndarray, ancillary_files: dict
+) -> npt.NDArray:
     """
     Normalize composite energy per SSD using a lookup table.
 
@@ -146,6 +136,8 @@ def get_energy_norm(ssd: np.ndarray, composite_energy: np.ndarray) -> npt.NDArra
         Acts as index 1.
     composite_energy : np.ndarray
         Acts as index 2.
+    ancillary_files : dict[Path]
+        Ancillary files containing the lookup tables.
 
     Returns
     -------
@@ -153,11 +145,11 @@ def get_energy_norm(ssd: np.ndarray, composite_energy: np.ndarray) -> npt.NDArra
         Normalized composite energy.
     """
     row_number = ssd * 4096 + composite_energy
-
-    return _ENERGY_NORM_DF["NormEnergy"].iloc[row_number]
+    norm_lookup = pd.read_csv(ancillary_files["l1b-egynorm-lookup"])
+    return norm_lookup["NormEnergy"].iloc[row_number]
 
 
-def get_image_params(image: str, sensor: str) -> np.float64:
+def get_image_params(image: str, sensor: str, ancillary_files: dict) -> np.float64:
     """
     Lookup table for image parameters.
 
@@ -171,18 +163,26 @@ def get_image_params(image: str, sensor: str) -> np.float64:
         The column name to lookup in the CSV file, e.g., 'XFTLTOFF' or 'XFTRTOFF'.
     sensor : str
         Sensor name: "ultra45" or "ultra90".
+    ancillary_files : dict[Path]
+        Ancillary files containing the lookup tables.
 
     Returns
     -------
     value : np.float64
         Image parameter value from the CSV file.
     """
-    lookup_table = _IMAGE_PARAMS_DF[sensor]
+    if sensor == "ultra45":
+        lookup_table = pd.read_csv(ancillary_files["l1b-45sensor-imgparams-lookup"])
+    else:
+        lookup_table = pd.read_csv(ancillary_files["l1b-90sensor-imgparams-lookup"])
+
     value: np.float64 = lookup_table[image].values[0]
     return value
 
 
-def get_angular_profiles(start_type: str, sensor: str) -> pd.DataFrame:
+def get_angular_profiles(
+    start_type: str, sensor: str, ancillary_files: dict
+) -> pd.DataFrame:
     """
     Lookup table for FWHM for theta and phi.
 
@@ -195,13 +195,16 @@ def get_angular_profiles(start_type: str, sensor: str) -> pd.DataFrame:
        Start Type: Left, Right.
     sensor : str
         Sensor name: "ultra45" or "ultra90".
+    ancillary_files : dict[Path]
+        Ancillary files.
 
     Returns
     -------
     lookup_table : DataFrame
         Angular profile lookup table for a given start_type and sensor.
     """
-    lookup_table = _FWHM_TABLES[(start_type.lower(), sensor)]
+    lut_descriptor = f"l1b-{sensor[-2:]}sensor-{start_type.lower()}slit-lookup"
+    lookup_table = pd.read_csv(ancillary_files[lut_descriptor])
 
     return lookup_table
 
@@ -227,3 +230,374 @@ def get_energy_efficiencies(ancillary_files: dict) -> pd.DataFrame:
     lookup_table = pd.read_csv(ancillary_files["l1b-45sensor-logistic-interpolation"])
 
     return lookup_table
+
+
+def load_geometric_factor_tables(
+    ancillary_files: dict,
+    filename: str,
+) -> dict:
+    """
+    Lookup tables for geometric factor.
+
+    Parameters
+    ----------
+    ancillary_files : dict[Path]
+        Ancillary files.
+    filename : str
+        Name of the file in ancillary_files to use.
+
+    Returns
+    -------
+    geometric_factor_tables : dict
+        Geometric factor lookup tables.
+    """
+    gf_table = pd.read_csv(
+        ancillary_files[filename], header=None, skiprows=6, nrows=301
+    ).to_numpy(dtype=float)
+    theta_table = pd.read_csv(
+        ancillary_files[filename], header=None, skiprows=308, nrows=301
+    ).to_numpy(dtype=float)
+    phi_table = pd.read_csv(
+        ancillary_files[filename], header=None, skiprows=610, nrows=301
+    ).to_numpy(dtype=float)
+
+    return {
+        "gf_table": gf_table,
+        "theta_table": theta_table,
+        "phi_table": phi_table,
+    }
+
+
+def get_geometric_factor(
+    phi: NDArray,
+    theta: NDArray,
+    quality_flag: NDArray,
+    ancillary_files: dict | None = None,
+    filename: str | None = None,
+    geometric_factor_tables: dict | None = None,
+) -> tuple[NDArray, NDArray]:
+    """
+    Lookup table for geometric factor using nearest neighbor.
+
+    Parameters
+    ----------
+    phi : NDArray
+        Azimuth angles in degrees.
+    theta : NDArray
+        Elevation angles in degrees.
+    quality_flag : NDArray
+        Quality flag to set when geometric factor is zero.
+    ancillary_files : dict[Path], optional
+        Ancillary files.
+    filename : str, optional
+        Name of the file in ancillary_files to use.
+    geometric_factor_tables : dict, optional
+        Preloaded geometric factor lookup tables. If not provided, will load.
+
+    Returns
+    -------
+    geometric_factor : NDArray
+        Geometric factor.
+    """
+    if geometric_factor_tables is None:
+        if ancillary_files is None or filename is None:
+            raise ValueError(
+                "ancillary_files and filename must be provided if "
+                "geometric_factor_tables is not supplied."
+            )
+        geometric_factor_tables = load_geometric_factor_tables(
+            ancillary_files, filename
+        )
+    # Assume uniform grids: extract 1D arrays from first row/col
+    theta_vals = geometric_factor_tables["theta_table"][0, :]  # columns represent theta
+    phi_vals = geometric_factor_tables["phi_table"][:, 0]  # rows represent phi
+
+    # Find nearest index in table for each input value
+    phi_idx = np.abs(phi_vals[:, None] - phi).argmin(axis=0)
+    theta_idx = np.abs(theta_vals[:, None] - theta).argmin(axis=0)
+
+    # Fetch geometric factor values at nearest (phi, theta) pairs
+    geometric_factor = geometric_factor_tables["gf_table"][phi_idx, theta_idx]
+
+    outside_fov = ~is_inside_fov(np.deg2rad(phi), np.deg2rad(theta))
+    quality_flag[outside_fov] |= ImapDEOutliersUltraFlags.FOV.value
+
+    return geometric_factor
+
+
+def load_scattering_lookup_tables(ancillary_files: dict, instrument_id: int) -> dict:
+    """
+    Load scattering coefficient lookup tables for the specified instrument.
+
+    Parameters
+    ----------
+    ancillary_files : dict
+        Ancillary files.
+    instrument_id : int
+        Instrument ID, either 45 or 90.
+
+    Returns
+    -------
+    dict
+        Dictionary containing arrays for theta_grid, phi_grid, a_theta, g_theta,
+         a_phi, g_phi.
+    """
+    # TODO remove the line below when the 45 sensor scattering coefficients are
+    #  delivered.
+    instrument_id = 90
+    descriptor = f"l1b-{instrument_id}sensor-scattering-calibration"
+    theta_grid = pd.read_csv(
+        ancillary_files[descriptor], header=None, skiprows=7, nrows=241
+    ).to_numpy(dtype=float)
+    phi_grid = pd.read_csv(
+        ancillary_files[descriptor], header=None, skiprows=249, nrows=241
+    ).to_numpy(dtype=float)
+    a_theta = pd.read_csv(
+        ancillary_files[descriptor], header=None, skiprows=491, nrows=241
+    ).to_numpy(dtype=float)
+    g_theta = pd.read_csv(
+        ancillary_files[descriptor], header=None, skiprows=733, nrows=241
+    ).to_numpy(dtype=float)
+    a_phi = pd.read_csv(
+        ancillary_files[descriptor], header=None, skiprows=975, nrows=241
+    ).to_numpy(dtype=float)
+    g_phi = pd.read_csv(
+        ancillary_files[descriptor], header=None, skiprows=1217, nrows=241
+    ).to_numpy(dtype=float)
+    return {
+        "theta_grid": theta_grid,
+        "phi_grid": phi_grid,
+        "a_theta": a_theta,
+        "g_theta": g_theta,
+        "a_phi": a_phi,
+        "g_phi": g_phi,
+    }
+
+
+def get_scattering_coefficients(
+    theta: NDArray,
+    phi: NDArray,
+    lookup_tables: dict | None = None,
+    ancillary_files: dict | None = None,
+    instrument_id: int | None = None,
+) -> tuple[NDArray, NDArray]:
+    """
+    Get a and g coefficients for theta and phi to compute scattering FWHM.
+
+    Parameters
+    ----------
+    theta : NDArray
+        Elevation angles in degrees.
+    phi : NDArray
+        Azimuth angles in degrees.
+    lookup_tables : dict, optional
+        Preloaded lookup tables. If not provided, will load using ancillary_files and
+         instrument_id.
+    ancillary_files : dict, optional
+        Ancillary files, required if lookup_tables is not provided.
+    instrument_id : int, optional
+        Instrument ID, required if lookup_tables is not provided.
+
+    Returns
+    -------
+    tuple
+        Scattering a and g values corresponding to the given theta and phi values.
+    """
+    if lookup_tables is None:
+        if ancillary_files is None or instrument_id is None:
+            raise ValueError(
+                "ancillary_files and instrument_id must be provided if lookup_tables "
+                "is not supplied."
+            )
+        lookup_tables = load_scattering_lookup_tables(ancillary_files, instrument_id)
+
+    theta_grid = lookup_tables["theta_grid"]
+    phi_grid = lookup_tables["phi_grid"]
+    a_theta = lookup_tables["a_theta"]
+    g_theta = lookup_tables["g_theta"]
+    a_phi = lookup_tables["a_phi"]
+    g_phi = lookup_tables["g_phi"]
+
+    theta_vals = theta_grid[0, :]  # columns represent theta
+    phi_vals = phi_grid[:, 0]  # rows represent phi
+
+    phi_idx = np.abs(phi_vals[:, None] - phi).argmin(axis=0)
+    theta_idx = np.abs(theta_vals[:, None] - theta).argmin(axis=0)
+
+    a_theta_val = a_theta[phi_idx, theta_idx]
+    g_theta_val = g_theta[phi_idx, theta_idx]
+    a_phi_val = a_phi[phi_idx, theta_idx]
+    g_phi_val = g_phi[phi_idx, theta_idx]
+
+    return np.column_stack([a_theta_val, g_theta_val]), np.column_stack(
+        [a_phi_val, g_phi_val]
+    )
+
+
+def is_inside_fov(phi: np.ndarray, theta: np.ndarray) -> np.ndarray:
+    """
+    Determine angles in the field of view (FOV).
+
+    This function is used in the deadtime correction to determine whether a given
+    (theta, phi) angle is within the instrument's Field of View (FOV).
+    Only pixels inside the FOV are considered for time accumulation. The FOV boundary
+    is defined by equation 19 in the Ultra Algorithm Document.
+
+    Parameters
+    ----------
+    phi : np.ndarray
+        Azimuth angles in radians.
+    theta : np.ndarray
+        Elevation angles in radians.
+
+    Returns
+    -------
+    numpy.ndarray
+        Boolean array indicating if the angle is in the FOV, False otherwise.
+    """
+    numerator = 5.0 * np.cos(phi)
+    denominator = 1 + 2.80 * np.cos(phi)
+    # Equation 19 in the Ultra Algorithm Document.
+    theta_nom = np.arctan(numerator / denominator)
+    return np.abs(theta) <= theta_nom
+
+
+def get_ph_corrected(
+    sensor: str,
+    location: str,
+    ancillary_files: dict,
+    xlut: NDArray,
+    ylut: NDArray,
+    quality_flag: NDArray,
+) -> tuple[NDArray, NDArray]:
+    """
+    PH correction for stop anodes, top and bottom.
+
+    Further description is available starting on
+    page 207 of the Ultra Flight Software Document.
+
+    Parameters
+    ----------
+    sensor : str
+        Sensor name: "ultra45" or "ultra90".
+    location : str
+        Location: "tp" or "bt".
+    ancillary_files : dict[Path]
+        Ancillary files.
+    xlut : NDArray
+        X lookup index for PH correction.
+    ylut : NDArray
+        Y lookup index for PH correction.
+    quality_flag : NDArray
+        Quality flag to set when there is an outlier.
+
+    Returns
+    -------
+    ph_correction : NDArray
+        Correction for pulse height.
+    quality_flag : NDArray
+        Quality flag updated with PH correction flags.
+    """
+    ph_correct = pd.read_csv(
+        ancillary_files[f"l1b-{sensor[-2:]}sensor-sp{location}phcorr"], header=None
+    )
+    ph_correct_array = ph_correct.to_numpy()
+
+    max_x, max_y = ph_correct_array.shape[0] - 1, ph_correct_array.shape[1] - 1
+
+    # Clamp indices to nearest valid value
+    xlut_clamped = np.clip(xlut.astype(int), 0, max_x)
+    ylut_clamped = np.clip(ylut.astype(int), 0, max_y)
+
+    # Flag where clamping occurred
+    flagged_mask = (xlut != xlut_clamped) | (ylut != ylut_clamped)
+    quality_flag[flagged_mask] |= ImapDEOutliersUltraFlags.PHCORR.value
+
+    ph_correction = ph_correct_array[xlut_clamped, ylut_clamped]
+
+    return ph_correction, quality_flag
+
+
+def get_ebins(
+    lut: str,
+    energy: NDArray,
+    ctof: NDArray,
+    ebins: NDArray,
+    ancillary_files: dict,
+) -> NDArray:
+    """
+    Get energy bins from the lookup table.
+
+    Parameters
+    ----------
+    lut : str
+        Lookup table name, e.g., "l1b-tofxpht".
+    energy : NDArray
+        Energy from the event (keV).
+    ctof : NDArray
+        Corrected TOF (tenths of a ns).
+    ebins : NDArray
+        Energy bins to fill with values.
+    ancillary_files : dict[Path]
+        Ancillary files.
+
+    Returns
+    -------
+    ebins : NDArray
+        Energy bins from the lookup table.
+    """
+    with open(ancillary_files[lut]) as f:
+        all_lines = f.readlines()
+        pixel_text = "".join(all_lines[4:])
+
+    lut_array = np.fromstring(pixel_text, sep=" ", dtype=int).reshape((2048, 4096))
+    # Note that the LUT is indexed [energy, ctof] for l1b-tofxph
+    # and [ctof, energy] for everything else.
+    if lut == "l1b-tofxph":
+        energy_lookup = (2048 - np.floor(energy)).astype(int)
+        ctof_lookup = np.floor(ctof).astype(int)
+        valid = (
+            (energy_lookup >= 0)
+            & (energy_lookup < 2048)
+            & (ctof_lookup >= 0)
+            & (ctof_lookup < 4096)
+        )
+        ebins[valid] = lut_array[energy_lookup[valid], ctof_lookup[valid]]
+    else:
+        energy_lookup = np.floor(energy).astype(int)
+        ctof_lookup = (2048 - np.floor(ctof)).astype(int)
+        valid = (
+            (energy_lookup >= 0)
+            & (energy_lookup < 4096)
+            & (ctof_lookup >= 0)
+            & (ctof_lookup < 2048)
+        )
+        ebins[valid] = lut_array[ctof_lookup[valid], energy_lookup[valid]]
+
+    return ebins
+
+
+def get_scattering_thresholds(ancillary_files: dict) -> dict:
+    """
+    Load scattering culling thresholds as a function of energy from a lookup table.
+
+    Parameters
+    ----------
+    ancillary_files : dict[Path]
+        Ancillary files.
+
+    Returns
+    -------
+    threshold_dict
+         Dictionary containing energy ranges and the corresponding scattering culling
+          threshold.
+    """
+    # Culling FWHM Scattering values as a function of energy.
+    thresholds = pd.read_csv(
+        ancillary_files["l1b-scattering-thresholds-per-energy"], header=None, skiprows=1
+    ).to_numpy(dtype=np.float64)
+    # The first two columns represent the energy range (min, max) in keV, and the
+    # value is the FWHM scattering threshold in degrees
+    threshold_dict = {(row[0], row[1]): row[2] for row in thresholds}
+
+    return threshold_dict

imap_processing/ultra/l1b/quality_flag_filters.py

@@ -0,0 +1,23 @@
+"""Contains list of QFs to use for filtering."""
+
+from imap_processing.quality_flags import (
+    FlagNameMixin,
+    ImapDEOutliersUltraFlags,
+    ImapDEScatteringUltraFlags,
+    ImapRatesUltraFlags,
+)
+
+SPIN_QUALITY_FLAG_FILTERS: dict[str, list[FlagNameMixin]] = {
+    "quality_attitude": [],
+    "quality_ena_rates": [
+        ImapRatesUltraFlags.FIRSTSPIN,
+        ImapRatesUltraFlags.LASTSPIN,
+    ],
+}
+
+DE_QUALITY_FLAG_FILTERS: dict[str, list[FlagNameMixin]] = {
+    "quality_outliers": [ImapDEOutliersUltraFlags.FOV],
+    "quality_scattering": [
+        ImapDEScatteringUltraFlags.ABOVE_THRESHOLD,
+    ],
+}