imap-processing 0.19.0__py3-none-any.whl → 0.19.2__py3-none-any.whl

imap_processing/ultra/l1c/l1c_lookup_utils.py

@@ -21,7 +21,7 @@ def mask_below_fwhm_scattering_threshold(
     phi_coeffs: np.ndarray,
     energy: np.ndarray,
     scattering_thresholds: np.ndarray,
-) -> np.ndarray:
+) -> tuple[NDArray, NDArray, NDArray]:
     """
     Determine indices of theta and phi values below the FWHM scattering threshold.
 
@@ -43,8 +43,12 @@ def mask_below_fwhm_scattering_threshold(
 
     Returns
     -------
-    numpy.ndarray
+    scattering_mask : numpy.ndarray
         Boolean array indicating indices below the scattering threshold.
+    fwhm_theta : numpy.ndarray
+        Calculated FWHM values for theta.
+    fwhm_phi : numpy.ndarray
+        Calculated FWHM values for phi.
     """
     # Calculate FWHM for all pixels and all energies
     fwhm_theta = theta_coeffs[..., 0:1] * (
@@ -58,18 +62,21 @@ def mask_below_fwhm_scattering_threshold(
 
     # Combine conditions for both theta and phi.
     # shape = (npix, energy.shape[1])
-    return np.logical_and(fwhm_theta <= thresholds, fwhm_phi <= thresholds)
+    scattering_mask = np.logical_and(fwhm_theta <= thresholds, fwhm_phi <= thresholds)
+    return scattering_mask, fwhm_theta, fwhm_phi
 
 
-def calculate_pixels_within_scattering_threshold(
+def calculate_fwhm_spun_scattering(
     for_indices_by_spin_phase: np.ndarray,
     theta_vals: np.ndarray,
     phi_vals: np.ndarray,
     ancillary_files: dict,
     instrument_id: int,
-) -> list:
+) -> tuple[list, NDArray, NDArray, NDArray]:
     """
-    Calculate pixels within the FWHM scattering threshold for each spin phase step.
+    Calculate FWHM scattering values for each pixel, energy bin, and spin phase step.
+
+    This function also calculates a mask for pixels that are below the FWHM threshold.
 
     Parameters
     ----------
@@ -93,6 +100,14 @@ def calculate_pixels_within_scattering_threshold(
         The outer list indicates spin phase steps, the middle list indicates energy
         bins, and the inner arrays contain indices indicating pixels that are below
         the FWHM scattering threshold.
+    scattering_fwhm_theta : NDArray
+        Calculated FWHM scatting values for theta at each energy bin and averaged
+        over spin phase.
+    scattering_fwhm_phi : NDArray
+        Calculated FWHM scatting values for theta at each energy bin and averaged
+        over spin phase.
+    scattering_thresholds_for_energy_mean : NDArray
+        Scattering thresholds corresponding to each energy bin.
     """
     # Load scattering coefficient lookup table
     scattering_luts = load_scattering_lookup_tables(ancillary_files, instrument_id)
@@ -103,6 +118,13 @@ def calculate_pixels_within_scattering_threshold(
     scattering_thresholds_for_energy_mean = get_scattering_thresholds_for_energy(
         energy_bin_geometric_means, ancillary_files
     )
+    # Initialize arrays to accumulate FWHM values for averaging
+    fwhm_theta_sum = np.zeros(
+        (len(energy_bin_geometric_means), for_indices_by_spin_phase.shape[0])
+    )
+    fwhm_phi_sum = np.zeros_like(fwhm_theta_sum)
+    sample_count = np.zeros_like(fwhm_theta_sum)
+
     steps = for_indices_by_spin_phase.shape[1]
     energies = energy_bin_geometric_means[np.newaxis, :]
     # The "for_indices_by_spin_phase" lookup table contains the boolean values of each
@@ -132,7 +154,7 @@ def calculate_pixels_within_scattering_threshold(
             theta, phi, lookup_tables=scattering_luts
         )
         # Get a mask for pixels below the FWHM scattering threshold
-        scattering_mask = mask_below_fwhm_scattering_threshold(
+        scattering_mask, fwhm_theta, fwhm_phi = mask_below_fwhm_scattering_threshold(
             theta_coeffs,
             phi_coeffs,
             energies,
@@ -147,8 +169,19 @@ def calculate_pixels_within_scattering_threshold(
             pixels_below_scattering_for_energy.append(for_pixel_indices[valid_pixels])
 
         pixels_below_scattering.append(pixels_below_scattering_for_energy)
+        # Accumulate FWHM values for averaging
+        fwhm_theta_sum[:, for_inds] += fwhm_theta.T
+        fwhm_phi_sum[:, for_inds] += fwhm_phi.T
+        sample_count[:, for_inds] += 1
 
-    return pixels_below_scattering
+    fwhm_phi_avg = np.divide(fwhm_theta_sum, sample_count, where=sample_count != 0)
+    fwhm_theta_avg = np.divide(fwhm_theta_sum, sample_count, where=sample_count != 0)
+    return (
+        pixels_below_scattering,
+        fwhm_theta_avg,
+        fwhm_phi_avg,
+        scattering_thresholds_for_energy_mean,
+    )
 
 
 def get_spacecraft_pointing_lookup_tables(
@@ -184,10 +217,10 @@ def get_spacecraft_pointing_lookup_tables(
         A 2D array of boundary scale factors for each HEALPix pixel at each spin phase
         step.
     """
-    theta_descriptor = f"l1c-{instrument_id}sensor-sc-pointing-theta-n32"
-    phi_descriptor = f"l1c-{instrument_id}sensor-sc-pointing-phi-n32"
-    index_descriptor = f"l1c-{instrument_id}sensor-sc-pointing-index-n32"
-    bsf_descriptor = f"l1c-{instrument_id}sensor-sc-pointing-bsf-n32"
+    theta_descriptor = f"l1c-{instrument_id}sensor-sc-pointing-theta"
+    phi_descriptor = f"l1c-{instrument_id}sensor-sc-pointing-phi"
+    index_descriptor = f"l1c-{instrument_id}sensor-sc-pointing-index"
+    bsf_descriptor = f"l1c-{instrument_id}sensor-sc-pointing-bsf"
 
     theta_vals = pd.read_csv(
         ancillary_files[theta_descriptor], header=None, skiprows=1

imap_processing/ultra/l1c/spacecraft_pset.py

@@ -2,19 +2,27 @@
 
 import logging
 
+import astropy_healpix.healpy as hp
 import numpy as np
-import pandas as pd
 import xarray as xr
 
 from imap_processing.cdf.utils import parse_filename_like
+from imap_processing.quality_flags import ImapPSETUltraFlags
+from imap_processing.spice.repoint import get_pointing_times
+from imap_processing.spice.time import (
+    et_to_met,
+    met_to_ttj2000ns,
+)
 from imap_processing.ultra.l1b.ultra_l1b_culling import get_de_rejection_mask
 from imap_processing.ultra.l1c.l1c_lookup_utils import (
-    calculate_pixels_within_scattering_threshold,
+    calculate_fwhm_spun_scattering,
     get_spacecraft_pointing_lookup_tables,
 )
+from imap_processing.ultra.l1c.ultra_l1c_culling import compute_culling_mask
 from imap_processing.ultra.l1c.ultra_l1c_pset_bins import (
     build_energy_bins,
     get_efficiencies_and_geometric_function,
+    get_energy_delta_minus_plus,
     get_spacecraft_background_rates,
     get_spacecraft_exposure_times,
     get_spacecraft_histogram,
@@ -26,14 +34,13 @@ logger = logging.getLogger(__name__)
 
 def calculate_spacecraft_pset(
     de_dataset: xr.Dataset,
-    extendedspin_dataset: xr.Dataset,
-    cullingmask_dataset: xr.Dataset,
+    goodtimes_dataset: xr.Dataset,
     rates_dataset: xr.Dataset,
     params_dataset: xr.Dataset,
     name: str,
     ancillary_files: dict,
     instrument_id: int,
-    species_id: int = 1,
+    species_id: list,
 ) -> xr.Dataset:
     """
     Create dictionary with defined datatype for Pointing Set Grid Data.
@@ -42,10 +49,8 @@ def calculate_spacecraft_pset(
     ----------
     de_dataset : xarray.Dataset
         Dataset containing de data.
-    extendedspin_dataset : xarray.Dataset
-        Dataset containing extendedspin data.
-    cullingmask_dataset : xarray.Dataset
-        Dataset containing cullingmask data.
+    goodtimes_dataset : xarray.Dataset
+        Dataset containing goodtimes data.
     rates_dataset : xarray.Dataset
         Dataset containing image rates data.
     params_dataset : xarray.Dataset
@@ -56,8 +61,8 @@ def calculate_spacecraft_pset(
         Ancillary files.
     instrument_id : int
         Instrument ID, either 45 or 90.
-    species_id : int
-        Species ID, default of 1 refers to Hydrogen.
+    species_id : List
+        Species ID.
 
     Returns
     -------
@@ -65,11 +70,15 @@ def calculate_spacecraft_pset(
         Dataset containing the data.
     """
     pset_dict: dict[str, np.ndarray] = {}
+
     sensor = parse_filename_like(name)["sensor"][0:2]
-    # Select only the species we are interested in.
-    indices = np.where(de_dataset["species"].values == species_id)[0]
+    indices = np.where(np.isin(de_dataset["e_bin"].values, species_id))[0]
     species_dataset = de_dataset.isel(epoch=indices)
 
+    # If there are no species return None.
+    if indices.size == 0:
+        return None
+
     # Before we use the de_dataset to calculate the pointing set grid we need to filter.
     rejected = get_de_rejection_mask(
         species_dataset["quality_scattering"].values,
@@ -85,13 +94,6 @@ def calculate_spacecraft_pset(
     )
 
     intervals, _, energy_bin_geometric_means = build_energy_bins()
-    counts, latitude, longitude, n_pix = get_spacecraft_histogram(
-        vhat_dps_spacecraft,
-        species_dataset["energy_spacecraft"].values,
-        intervals,
-        nside=128,
-    )
-    healpix = np.arange(n_pix)
 
     # Get lookup table for FOR indices by spin phase step
     (
@@ -101,16 +103,28 @@ def calculate_spacecraft_pset(
         ra_and_dec,
         boundary_scale_factors,
     ) = get_spacecraft_pointing_lookup_tables(ancillary_files, instrument_id)
-    # Check that the number of rows in the lookup table matches the number of pixels
-    if for_indices_by_spin_phase.shape[0] != n_pix:
-        logger.warning(
-            "The lookup table is expected to have the same number of rows as "
-            "the number of HEALPix pixels."
-        )
 
-    pixels_below_scattering = calculate_pixels_within_scattering_threshold(
-        for_indices_by_spin_phase, theta_vals, phi_vals, ancillary_files, instrument_id
+    logger.info("calculating spun FWHM scattering values.")
+    pixels_below_scattering, scattering_theta, scattering_phi, scattering_thresholds = (
+        calculate_fwhm_spun_scattering(
+            for_indices_by_spin_phase,
+            theta_vals,
+            phi_vals,
+            ancillary_files,
+            instrument_id,
+        )
     )
+    # Determine nside from the lookup table
+    nside = hp.npix2nside(len(for_indices_by_spin_phase))
+    counts, latitude, longitude, n_pix = get_spacecraft_histogram(
+        vhat_dps_spacecraft,
+        species_dataset["energy_spacecraft"].values,
+        intervals,
+        nside=nside,
+    )
+    healpix = np.arange(n_pix)
+
+    logger.info("Calculating spun efficiencies and geometric function.")
     # calculate efficiency and geometric function as a function of energy
     efficiencies, geometric_function = get_efficiencies_and_geometric_function(
         pixels_below_scattering,
@@ -122,29 +136,49 @@ def calculate_spacecraft_pset(
     )
     sensitivity = efficiencies * geometric_function
 
-    # Calculate exposure
-    constant_exposure = ancillary_files["l1c-90sensor-dps-exposure"]
-    df_exposure = pd.read_csv(constant_exposure)
-
+    # Calculate exposure times
+    logger.info("Calculating spacecraft exposure times with deadtime correction.")
     exposure_pointing, deadtime_ratios = get_spacecraft_exposure_times(
-        df_exposure,
         rates_dataset,
         params_dataset,
         pixels_below_scattering,
         boundary_scale_factors,
+        n_pix=n_pix,
     )
-
+    logger.info("Calculating background rates.")
     # Calculate background rates
     background_rates = get_spacecraft_background_rates(
         rates_dataset,
         sensor,
         ancillary_files,
         intervals,
-        cullingmask_dataset["spin_number"].values,
+        goodtimes_dataset["spin_number"].values,
+        nside=nside,
+    )
+    spacecraft_pset_quality_flags = np.full(
+        n_pix, ImapPSETUltraFlags.NONE.value, dtype=np.uint16
     )
 
-    # For ISTP, epoch should be the center of the time bin.
-    pset_dict["epoch"] = de_dataset.epoch.data[:1].astype(np.int64)
+    start: float = np.min(species_dataset["event_times"].values)
+    end: float = np.max(species_dataset["event_times"].values)
+
+    # Time bins in 30 minute intervals
+    time_bins = np.arange(start, end + 1800, 1800)
+
+    # Compute mask for culling the Earth
+    compute_culling_mask(
+        time_bins,
+        6378.1,  # Earth radius
+        spacecraft_pset_quality_flags,
+        nside=nside,
+    )
+    # Get pointing start and stop times and convert to ttj2000ns
+    pointing_start, pointing_stop = get_pointing_times(
+        float(et_to_met(species_dataset["event_times"].data[0]))
+    )
+    pointing_start = met_to_ttj2000ns(pointing_start)
+    # Epoch should be the start of the pointing
+    pset_dict["epoch"] = np.atleast_1d(pointing_start).astype(np.int64)
     pset_dict["counts"] = counts[np.newaxis, ...]
     pset_dict["latitude"] = latitude[np.newaxis, ...]
     pset_dict["longitude"] = longitude[np.newaxis, ...]
@@ -155,6 +189,7 @@ def calculate_spacecraft_pset(
     pset_dict["energy_bin_delta"] = np.diff(intervals, axis=1).squeeze()[
         np.newaxis, ...
     ]
+    pset_dict["quality_flags"] = spacecraft_pset_quality_flags[np.newaxis, ...]
 
     pset_dict["sensitivity"] = sensitivity
     pset_dict["efficiency"] = efficiencies
@@ -162,6 +197,15 @@ def calculate_spacecraft_pset(
     pset_dict["dead_time_ratio"] = deadtime_ratios
     pset_dict["spin_phase_step"] = np.arange(len(deadtime_ratios))
 
+    pset_dict["scatter_theta"] = scattering_theta
+    pset_dict["scatter_phi"] = scattering_phi
+    pset_dict["scatter_threshold"] = scattering_thresholds
+
+    # Add the energy delta plus/minus to the dataset
+    energy_delta_minus, energy_delta_plus = get_energy_delta_minus_plus()
+    pset_dict["energy_delta_minus"] = energy_delta_minus
+    pset_dict["energy_delta_plus"] = energy_delta_plus
+
     dataset = create_dataset(pset_dict, name, "l1c")
 
     return dataset

imap_processing/ultra/l1c/ultra_l1c.py

@@ -2,13 +2,13 @@
 
 import xarray as xr
 
+from imap_processing.ultra.constants import UltraConstants
 from imap_processing.ultra.l1c.helio_pset import calculate_helio_pset
-from imap_processing.ultra.l1c.histogram import calculate_histogram
 from imap_processing.ultra.l1c.spacecraft_pset import calculate_spacecraft_pset
 
 
 def ultra_l1c(
-    data_dict: dict, ancillary_files: dict, has_spice: bool
+    data_dict: dict, ancillary_files: dict, imap_frames: bool
 ) -> list[xr.Dataset]:
     """
     Will process ULTRA L1A and L1B data into L1C CDF files at output_filepath.
@@ -19,8 +19,8 @@ def ultra_l1c(
         The data itself and its dependent data.
     ancillary_files : dict
         Ancillary files.
-    has_spice : bool
-        Whether to use SPICE data.
+    imap_frames : bool
+        Whether to use IMAP frames.
 
     Returns
     -------
@@ -32,47 +32,52 @@ def ultra_l1c(
     # Account for possibility of having 45 and 90 in dictionary.
     for instrument_id in [45, 90]:
         if (
-            f"imap_ultra_l1a_{instrument_id}sensor-histogram" in data_dict
-            and f"imap_ultra_l1b_{instrument_id}sensor-cullingmask" in data_dict
-        ):
-            histogram_dataset = calculate_histogram(
-                data_dict[f"imap_ultra_l1a_{instrument_id}sensor-histogram"],
-                f"imap_ultra_l1c_{instrument_id}sensor-histogram",
-            )
-            output_datasets = [histogram_dataset]
-        elif (
-            f"imap_ultra_l1b_{instrument_id}sensor-cullingmask" in data_dict
+            f"imap_ultra_l1b_{instrument_id}sensor-goodtimes" in data_dict
             and f"imap_ultra_l1b_{instrument_id}sensor-de" in data_dict
-            and f"imap_ultra_l1b_{instrument_id}sensor-extendedspin" in data_dict
-            and has_spice
+            and f"imap_ultra_l1a_{instrument_id}sensor-rates" in data_dict
+            and f"imap_ultra_l1a_{instrument_id}sensor-params" in data_dict
+            and imap_frames
         ):
             helio_pset = calculate_helio_pset(
                 data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"],
-                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-extendedspin"],
-                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-cullingmask"],
+                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-goodtimes"],
                 data_dict[f"imap_ultra_l1a_{instrument_id}sensor-rates"],
                 data_dict[f"imap_ultra_l1a_{instrument_id}sensor-params"],
                 f"imap_ultra_l1c_{instrument_id}sensor-heliopset",
                 ancillary_files,
                 instrument_id,
+                UltraConstants.TOFXPH_SPECIES_GROUPS["proton"],
             )
             output_datasets = [helio_pset]
         elif (
-            f"imap_ultra_l1b_{instrument_id}sensor-cullingmask" in data_dict
+            f"imap_ultra_l1b_{instrument_id}sensor-goodtimes" in data_dict
             and f"imap_ultra_l1b_{instrument_id}sensor-de" in data_dict
-            and f"imap_ultra_l1b_{instrument_id}sensor-extendedspin" in data_dict
+            and f"imap_ultra_l1a_{instrument_id}sensor-rates" in data_dict
+            and f"imap_ultra_l1a_{instrument_id}sensor-params" in data_dict
         ):
             spacecraft_pset = calculate_spacecraft_pset(
                 data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"],
-                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-extendedspin"],
-                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-cullingmask"],
+                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-goodtimes"],
                 data_dict[f"imap_ultra_l1a_{instrument_id}sensor-rates"],
                 data_dict[f"imap_ultra_l1a_{instrument_id}sensor-params"],
                 f"imap_ultra_l1c_{instrument_id}sensor-spacecraftpset",
                 ancillary_files,
                 instrument_id,
+                UltraConstants.TOFXPH_SPECIES_GROUPS["proton"],
             )
             output_datasets = [spacecraft_pset]
+            spacecraft_pset_non_proton = calculate_spacecraft_pset(
+                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"],
+                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-goodtimes"],
+                data_dict[f"imap_ultra_l1a_{instrument_id}sensor-rates"],
+                data_dict[f"imap_ultra_l1a_{instrument_id}sensor-params"],
+                f"imap_ultra_l1c_{instrument_id}sensor-spacecraftpset-nonproton",
+                ancillary_files,
+                instrument_id,
+                UltraConstants.TOFXPH_SPECIES_GROUPS["non_proton"],
+            )
+            if spacecraft_pset_non_proton is not None:
+                output_datasets.append(spacecraft_pset_non_proton)
     if not output_datasets:
         raise ValueError("Data dictionary does not contain the expected keys.")
 

imap_processing/ultra/l1c/ultra_l1c_culling.py

@@ -4,6 +4,7 @@ import astropy_healpix.healpy as hp
 import numpy as np
 from numpy.typing import NDArray
 
+from imap_processing.quality_flags import ImapPSETUltraFlags
 from imap_processing.spice.geometry import (
     SpiceBody,
     SpiceFrame,
@@ -14,6 +15,7 @@ from imap_processing.spice.geometry import (
 def compute_culling_mask(
     et: NDArray,
     keepout_radius_km: float,
+    pset_quality_flags: NDArray,
     observer: SpiceBody = SpiceBody.EARTH,
     nside: int = 128,
     nested: bool = False,
@@ -27,6 +29,9 @@ def compute_culling_mask(
         Ephemeris times in TDB seconds past J2000.
     keepout_radius_km : float
         Radius (in km) within which HEALPix pixels will be excluded.
+    pset_quality_flags : NDArray,
+        Quality flag to set when HEALPIX pixels are within a
+        keep-out radius of the target body.
     observer : SpiceBody, optional
         Body from which IMAP is observed.
     nside : int, optional
@@ -81,5 +86,7 @@ def compute_culling_mask(
     # Exclude pixels within the keepout angle.
     # mask.shape = (len(et), npix)
     mask = sep_angle > keepout_angle[:, np.newaxis]
+    culled_any_time = np.any(~mask, axis=0)  # shape: (npix,)
+    pset_quality_flags[culled_any_time] |= ImapPSETUltraFlags.EARTH_FOV.value
 
     return mask, unit_target_vecs