imap-processing 0.18.0__py3-none-any.whl → 0.19.2__py3-none-any.whl

imap_processing/ultra/l1c/spacecraft_pset.py

@@ -1,27 +1,46 @@
 """Calculate Pointing Set Grids."""
 
+import logging
+
+import astropy_healpix.healpy as hp
 import numpy as np
-import pandas as pd
 import xarray as xr
 
+from imap_processing.cdf.utils import parse_filename_like
+from imap_processing.quality_flags import ImapPSETUltraFlags
+from imap_processing.spice.repoint import get_pointing_times
+from imap_processing.spice.time import (
+    et_to_met,
+    met_to_ttj2000ns,
+)
+from imap_processing.ultra.l1b.ultra_l1b_culling import get_de_rejection_mask
+from imap_processing.ultra.l1c.l1c_lookup_utils import (
+    calculate_fwhm_spun_scattering,
+    get_spacecraft_pointing_lookup_tables,
+)
+from imap_processing.ultra.l1c.ultra_l1c_culling import compute_culling_mask
 from imap_processing.ultra.l1c.ultra_l1c_pset_bins import (
     build_energy_bins,
+    get_efficiencies_and_geometric_function,
+    get_energy_delta_minus_plus,
     get_spacecraft_background_rates,
     get_spacecraft_exposure_times,
     get_spacecraft_histogram,
-    interpolate_sensitivity,
 )
 from imap_processing.ultra.utils.ultra_l1_utils import create_dataset
 
+logger = logging.getLogger(__name__)
+
 
 def calculate_spacecraft_pset(
     de_dataset: xr.Dataset,
-    extendedspin_dataset: xr.Dataset,
-    cullingmask_dataset: xr.Dataset,
+    goodtimes_dataset: xr.Dataset,
     rates_dataset: xr.Dataset,
     params_dataset: xr.Dataset,
     name: str,
     ancillary_files: dict,
+    instrument_id: int,
+    species_id: list,
 ) -> xr.Dataset:
     """
     Create dictionary with defined datatype for Pointing Set Grid Data.
@@ -30,10 +49,8 @@ def calculate_spacecraft_pset(
     ----------
     de_dataset : xarray.Dataset
         Dataset containing de data.
-    extendedspin_dataset : xarray.Dataset
-        Dataset containing extendedspin data.
-    cullingmask_dataset : xarray.Dataset
-        Dataset containing cullingmask data.
+    goodtimes_dataset : xarray.Dataset
+        Dataset containing goodtimes data.
     rates_dataset : xarray.Dataset
         Dataset containing image rates data.
     params_dataset : xarray.Dataset
@@ -42,6 +59,10 @@ def calculate_spacecraft_pset(
         Name of the dataset.
     ancillary_files : dict
         Ancillary files.
+    instrument_id : int
+        Instrument ID, either 45 or 90.
+    species_id : List
+        Species ID.
 
     Returns
     -------
@@ -50,50 +71,140 @@ def calculate_spacecraft_pset(
     """
     pset_dict: dict[str, np.ndarray] = {}
 
-    v_mag_dps_spacecraft = np.linalg.norm(de_dataset["velocity_dps_sc"].values, axis=1)
+    sensor = parse_filename_like(name)["sensor"][0:2]
+    indices = np.where(np.isin(de_dataset["e_bin"].values, species_id))[0]
+    species_dataset = de_dataset.isel(epoch=indices)
+
+    # If there are no species return None.
+    if indices.size == 0:
+        return None
+
+    # Before we use the de_dataset to calculate the pointing set grid we need to filter.
+    rejected = get_de_rejection_mask(
+        species_dataset["quality_scattering"].values,
+        species_dataset["quality_outliers"].values,
+    )
+    species_dataset = species_dataset.isel(epoch=~rejected)
+
+    v_mag_dps_spacecraft = np.linalg.norm(
+        species_dataset["velocity_dps_sc"].values, axis=1
+    )
     vhat_dps_spacecraft = (
-        de_dataset["velocity_dps_sc"].values / v_mag_dps_spacecraft[:, np.newaxis]
+        species_dataset["velocity_dps_sc"].values / v_mag_dps_spacecraft[:, np.newaxis]
     )
 
     intervals, _, energy_bin_geometric_means = build_energy_bins()
+
+    # Get lookup table for FOR indices by spin phase step
+    (
+        for_indices_by_spin_phase,
+        theta_vals,
+        phi_vals,
+        ra_and_dec,
+        boundary_scale_factors,
+    ) = get_spacecraft_pointing_lookup_tables(ancillary_files, instrument_id)
+
+    logger.info("calculating spun FWHM scattering values.")
+    pixels_below_scattering, scattering_theta, scattering_phi, scattering_thresholds = (
+        calculate_fwhm_spun_scattering(
+            for_indices_by_spin_phase,
+            theta_vals,
+            phi_vals,
+            ancillary_files,
+            instrument_id,
+        )
+    )
+    # Determine nside from the lookup table
+    nside = hp.npix2nside(len(for_indices_by_spin_phase))
     counts, latitude, longitude, n_pix = get_spacecraft_histogram(
         vhat_dps_spacecraft,
-        de_dataset["energy_spacecraft"].values,
+        species_dataset["energy_spacecraft"].values,
         intervals,
-        nside=128,
+        nside=nside,
     )
     healpix = np.arange(n_pix)
 
-    # calculate background rates
-    background_rates = get_spacecraft_background_rates()
+    logger.info("Calculating spun efficiencies and geometric function.")
+    # calculate efficiency and geometric function as a function of energy
+    efficiencies, geometric_function = get_efficiencies_and_geometric_function(
+        pixels_below_scattering,
+        boundary_scale_factors,
+        theta_vals,
+        phi_vals,
+        n_pix,
+        ancillary_files,
+    )
+    sensitivity = efficiencies * geometric_function
 
-    efficiencies = ancillary_files["l1c-90sensor-efficiencies"]
-    geometric_function = ancillary_files["l1c-90sensor-gf"]
+    # Calculate exposure times
+    logger.info("Calculating spacecraft exposure times with deadtime correction.")
+    exposure_pointing, deadtime_ratios = get_spacecraft_exposure_times(
+        rates_dataset,
+        params_dataset,
+        pixels_below_scattering,
+        boundary_scale_factors,
+        n_pix=n_pix,
+    )
+    logger.info("Calculating background rates.")
+    # Calculate background rates
+    background_rates = get_spacecraft_background_rates(
+        rates_dataset,
+        sensor,
+        ancillary_files,
+        intervals,
+        goodtimes_dataset["spin_number"].values,
+        nside=nside,
+    )
+    spacecraft_pset_quality_flags = np.full(
+        n_pix, ImapPSETUltraFlags.NONE.value, dtype=np.uint16
+    )
 
-    df_efficiencies = pd.read_csv(efficiencies)
-    df_geometric_function = pd.read_csv(geometric_function)
-    sensitivity = interpolate_sensitivity(df_efficiencies, df_geometric_function)
+    start: float = np.min(species_dataset["event_times"].values)
+    end: float = np.max(species_dataset["event_times"].values)
 
-    # Calculate exposure
-    constant_exposure = ancillary_files["l1c-90sensor-dps-exposure"]
-    df_exposure = pd.read_csv(constant_exposure)
-    exposure_pointing = get_spacecraft_exposure_times(
-        df_exposure, rates_dataset, params_dataset
-    )
+    # Time bins in 30 minute intervals
+    time_bins = np.arange(start, end + 1800, 1800)
 
-    # For ISTP, epoch should be the center of the time bin.
-    pset_dict["epoch"] = de_dataset.epoch.data[:1].astype(np.int64)
+    # Compute mask for culling the Earth
+    compute_culling_mask(
+        time_bins,
+        6378.1,  # Earth radius
+        spacecraft_pset_quality_flags,
+        nside=nside,
+    )
+    # Get pointing start and stop times and convert to ttj2000ns
+    pointing_start, pointing_stop = get_pointing_times(
+        float(et_to_met(species_dataset["event_times"].data[0]))
+    )
+    pointing_start = met_to_ttj2000ns(pointing_start)
+    # Epoch should be the start of the pointing
+    pset_dict["epoch"] = np.atleast_1d(pointing_start).astype(np.int64)
     pset_dict["counts"] = counts[np.newaxis, ...]
     pset_dict["latitude"] = latitude[np.newaxis, ...]
     pset_dict["longitude"] = longitude[np.newaxis, ...]
     pset_dict["energy_bin_geometric_mean"] = energy_bin_geometric_means
     pset_dict["background_rates"] = background_rates[np.newaxis, ...]
-    pset_dict["exposure_factor"] = exposure_pointing.to_numpy()[np.newaxis, ...]
+    pset_dict["exposure_factor"] = exposure_pointing
     pset_dict["pixel_index"] = healpix
     pset_dict["energy_bin_delta"] = np.diff(intervals, axis=1).squeeze()[
         np.newaxis, ...
     ]
-    pset_dict["sensitivity"] = sensitivity[np.newaxis, ...]
+    pset_dict["quality_flags"] = spacecraft_pset_quality_flags[np.newaxis, ...]
+
+    pset_dict["sensitivity"] = sensitivity
+    pset_dict["efficiency"] = efficiencies
+    pset_dict["geometric_function"] = geometric_function
+    pset_dict["dead_time_ratio"] = deadtime_ratios
+    pset_dict["spin_phase_step"] = np.arange(len(deadtime_ratios))
+
+    pset_dict["scatter_theta"] = scattering_theta
+    pset_dict["scatter_phi"] = scattering_phi
+    pset_dict["scatter_threshold"] = scattering_thresholds
+
+    # Add the energy delta plus/minus to the dataset
+    energy_delta_minus, energy_delta_plus = get_energy_delta_minus_plus()
+    pset_dict["energy_delta_minus"] = energy_delta_minus
+    pset_dict["energy_delta_plus"] = energy_delta_plus
 
     dataset = create_dataset(pset_dict, name, "l1c")
 

imap_processing/ultra/l1c/ultra_l1c.py

@@ -2,13 +2,13 @@
 
 import xarray as xr
 
+from imap_processing.ultra.constants import UltraConstants
 from imap_processing.ultra.l1c.helio_pset import calculate_helio_pset
-from imap_processing.ultra.l1c.histogram import calculate_histogram
 from imap_processing.ultra.l1c.spacecraft_pset import calculate_spacecraft_pset
 
 
 def ultra_l1c(
-    data_dict: dict, ancillary_files: dict, has_spice: bool
+    data_dict: dict, ancillary_files: dict, imap_frames: bool
 ) -> list[xr.Dataset]:
     """
     Will process ULTRA L1A and L1B data into L1C CDF files at output_filepath.
@@ -19,8 +19,8 @@ def ultra_l1c(
         The data itself and its dependent data.
     ancillary_files : dict
         Ancillary files.
-    has_spice : bool
-        Whether to use SPICE data.
+    imap_frames : bool
+        Whether to use IMAP frames.
 
     Returns
     -------
@@ -32,43 +32,52 @@ def ultra_l1c(
     # Account for possibility of having 45 and 90 in dictionary.
     for instrument_id in [45, 90]:
         if (
-            f"imap_ultra_l1a_{instrument_id}sensor-histogram" in data_dict
-            and f"imap_ultra_l1b_{instrument_id}sensor-cullingmask" in data_dict
-        ):
-            histogram_dataset = calculate_histogram(
-                data_dict[f"imap_ultra_l1a_{instrument_id}sensor-histogram"],
-                f"imap_ultra_l1c_{instrument_id}sensor-histogram",
-            )
-            output_datasets = [histogram_dataset]
-        elif (
-            f"imap_ultra_l1b_{instrument_id}sensor-cullingmask" in data_dict
+            f"imap_ultra_l1b_{instrument_id}sensor-goodtimes" in data_dict
             and f"imap_ultra_l1b_{instrument_id}sensor-de" in data_dict
-            and f"imap_ultra_l1b_{instrument_id}sensor-extendedspin" in data_dict
-            and has_spice
+            and f"imap_ultra_l1a_{instrument_id}sensor-rates" in data_dict
+            and f"imap_ultra_l1a_{instrument_id}sensor-params" in data_dict
+            and imap_frames
         ):
             helio_pset = calculate_helio_pset(
                 data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"],
-                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-extendedspin"],
-                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-cullingmask"],
+                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-goodtimes"],
+                data_dict[f"imap_ultra_l1a_{instrument_id}sensor-rates"],
+                data_dict[f"imap_ultra_l1a_{instrument_id}sensor-params"],
                 f"imap_ultra_l1c_{instrument_id}sensor-heliopset",
                 ancillary_files,
+                instrument_id,
+                UltraConstants.TOFXPH_SPECIES_GROUPS["proton"],
             )
             output_datasets = [helio_pset]
         elif (
-            f"imap_ultra_l1b_{instrument_id}sensor-cullingmask" in data_dict
+            f"imap_ultra_l1b_{instrument_id}sensor-goodtimes" in data_dict
             and f"imap_ultra_l1b_{instrument_id}sensor-de" in data_dict
-            and f"imap_ultra_l1b_{instrument_id}sensor-extendedspin" in data_dict
+            and f"imap_ultra_l1a_{instrument_id}sensor-rates" in data_dict
+            and f"imap_ultra_l1a_{instrument_id}sensor-params" in data_dict
         ):
             spacecraft_pset = calculate_spacecraft_pset(
                 data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"],
-                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-extendedspin"],
-                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-cullingmask"],
-                data_dict[f"imap_ultra_{instrument_id}sensor-rates"],
-                data_dict[f"imap_ultra_{instrument_id}sensor-params"],
+                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-goodtimes"],
+                data_dict[f"imap_ultra_l1a_{instrument_id}sensor-rates"],
+                data_dict[f"imap_ultra_l1a_{instrument_id}sensor-params"],
                 f"imap_ultra_l1c_{instrument_id}sensor-spacecraftpset",
                 ancillary_files,
+                instrument_id,
+                UltraConstants.TOFXPH_SPECIES_GROUPS["proton"],
             )
             output_datasets = [spacecraft_pset]
+            spacecraft_pset_non_proton = calculate_spacecraft_pset(
+                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"],
+                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-goodtimes"],
+                data_dict[f"imap_ultra_l1a_{instrument_id}sensor-rates"],
+                data_dict[f"imap_ultra_l1a_{instrument_id}sensor-params"],
+                f"imap_ultra_l1c_{instrument_id}sensor-spacecraftpset-nonproton",
+                ancillary_files,
+                instrument_id,
+                UltraConstants.TOFXPH_SPECIES_GROUPS["non_proton"],
+            )
+            if spacecraft_pset_non_proton is not None:
+                output_datasets.append(spacecraft_pset_non_proton)
     if not output_datasets:
         raise ValueError("Data dictionary does not contain the expected keys.")
 

imap_processing/ultra/l1c/ultra_l1c_culling.py

@@ -0,0 +1,92 @@
+"""Culling for ULTRA L1c."""
+
+import astropy_healpix.healpy as hp
+import numpy as np
+from numpy.typing import NDArray
+
+from imap_processing.quality_flags import ImapPSETUltraFlags
+from imap_processing.spice.geometry import (
+    SpiceBody,
+    SpiceFrame,
+    imap_state,
+)
+
+
+def compute_culling_mask(
+    et: NDArray,
+    keepout_radius_km: float,
+    pset_quality_flags: NDArray,
+    observer: SpiceBody = SpiceBody.EARTH,
+    nside: int = 128,
+    nested: bool = False,
+) -> tuple[NDArray, NDArray]:
+    """
+    Compute a mask for HEALPix pixels within a keep-out radius of the target body.
+
+    Parameters
+    ----------
+    et : NDArray
+        Ephemeris times in TDB seconds past J2000.
+    keepout_radius_km : float
+        Radius (in km) within which HEALPix pixels will be excluded.
+    pset_quality_flags : NDArray,
+        Quality flag to set when HEALPIX pixels are within a
+        keep-out radius of the target body.
+    observer : SpiceBody, optional
+        Body from which IMAP is observed.
+    nside : int, optional
+        HEALPix NSIDE resolution. Default is 128.
+    nested : bool, optional
+        Whether to use NESTED indexing.
+
+    Returns
+    -------
+    mask : tuple[NDArray, NDArray]
+        Boolean array of shape (len(et), npix).
+    unit_target_vecs : NDArray
+        Unit vectors from IMAP to the target body
+        (e.g., Earth), shape (len(et), 3).
+    """
+    # Compute number of HEALPix pixels
+    npix = hp.nside2npix(nside)
+
+    # Compute IMAP to Earth position in the pointing frame.
+    state = imap_state(et, ref_frame=SpiceFrame.IMAP_DPS, observer=observer)
+    # Flip to get vector from IMAP to Earth
+    # position.shape = (len(et), 3)
+    position = -state[:, :3]
+
+    # Distance from IMAP to target (e.g. Earth) (km):
+    # distance.shape = (len(et),)
+    distance = np.linalg.norm(position, axis=1)  # shape (len(et),)
+
+    # Calculate the keepout angle (radians).
+    # keepout_angle.shape = (len(et),)
+    keepout_angle = np.arcsin(keepout_radius_km / distance)  # radians
+
+    # Calculate the direction from IMAP to Earth. (shape: [N, 3])
+    # unit_target_vecs.shape = (len(et), 3)
+    unit_target_vecs = position / distance[:, np.newaxis]
+
+    # Get pixel unit vectors pointing from the center of the
+    # HEALPix sphere to the center of each pixel on the sky.
+    pixel_vecs = np.column_stack(
+        hp.pix2vec(nside, np.arange(npix), nest=nested)
+    )  # shape: (npix, 3)
+
+    # Returns cos(theta) where theta is the separation angle between:
+    # (1) vector from IMAP to Earth
+    # (2) vector from IMAP to HEALPix pixel center
+    # If theta is within the keepout angle, then the pixel is culled.
+    cos_sep = np.dot(unit_target_vecs, pixel_vecs.T)  # shape (N, npix)
+    cos_sep = np.clip(cos_sep, -1.0, 1.0)
+    # Get theta here.
+    sep_angle = np.arccos(cos_sep)
+
+    # Exclude pixels within the keepout angle.
+    # mask.shape = (len(et), npix)
+    mask = sep_angle > keepout_angle[:, np.newaxis]
+    culled_any_time = np.any(~mask, axis=0)  # shape: (npix,)
+    pset_quality_flags[culled_any_time] |= ImapPSETUltraFlags.EARTH_FOV.value
+
+    return mask, unit_target_vecs