imap-processing 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

imap_processing/ultra/l1b/ultra_l1b_extended.py

@@ -14,7 +14,7 @@ from scipy.interpolate import LinearNDInterpolator, RegularGridInterpolator
 
 from imap_processing.quality_flags import ImapDEOutliersUltraFlags
 from imap_processing.spice.spin import get_spin_data
-from imap_processing.spice.time import sct_to_et
+from imap_processing.spice.time import met_to_ttj2000ns, ttj2000ns_to_et
 from imap_processing.ultra.constants import UltraConstants
 from imap_processing.ultra.l1b.lookup_utils import (
     get_angular_profiles,
@@ -592,7 +592,7 @@ def get_ssd_tof(
     tof : np.ndarray
         Time of flight (tenths of a nanosecond).
     """
-    _, tof_offset, ssd_number = get_ssd_back_position_and_tof_offset(
+    _, tof_offset, _ssd_number = get_ssd_back_position_and_tof_offset(
         de_dataset, sensor, ancillary_files
     )
     indices = np.nonzero(np.isin(de_dataset["stop_type"], [StopType.SSD.value]))[0]
@@ -711,7 +711,7 @@ def get_energy_pulse_height(
     ylut[indices_bottom] = (yb[indices_bottom] / 100 + 82 / 2) * 32 / 82  # mm
 
     ph_correction_top, updated_flags_top = get_ph_corrected(
-        "ultra45",
+        sensor,
         "tp",
         ancillary_files,
         np.round(xlut[indices_top]),
@@ -720,7 +720,7 @@ def get_energy_pulse_height(
     )
     quality_flags[indices_top] = updated_flags_top
     ph_correction_bottom, updated_flags_bottom = get_ph_corrected(
-        "ultra45",
+        sensor,
         "bt",
         ancillary_files,
         np.round(xlut[indices_bottom]),
@@ -996,6 +996,7 @@ def get_eventtimes(
     t_spin_period_sec * phase_angle/720
     """
     spin_df = get_spin_data()
+
     index = np.searchsorted(spin_df["spin_number"].values, spin)
     spin_starts = (
         spin_df["spin_start_sec_sclk"].values[index]
@@ -1003,10 +1004,13 @@ def get_eventtimes(
     )
 
     spin_period_sec = spin_df["spin_period_sec"].values[index]
-
     event_times = spin_starts + spin_period_sec * (phase_angle / 720)
 
-    return sct_to_et(event_times), sct_to_et(spin_starts), spin_period_sec
+    return (
+        ttj2000ns_to_et(met_to_ttj2000ns(event_times)),
+        ttj2000ns_to_et(met_to_ttj2000ns(spin_starts)),
+        spin_period_sec,
+    )
 
 
 def interpolate_fwhm(
@@ -1108,7 +1112,9 @@ def get_fwhm(
     return phi_interp, theta_interp
 
 
-def get_efficiency_interpolator(ancillary_files: dict) -> RegularGridInterpolator:
+def get_efficiency_interpolator(
+    ancillary_files: dict,
+) -> tuple[RegularGridInterpolator, tuple, tuple]:
     """
     Return a callable function that interpolates efficiency values for each event.
 
@@ -1119,8 +1125,12 @@ def get_efficiency_interpolator(ancillary_files: dict) -> RegularGridInterpolato
 
     Returns
     -------
-    efficiency : NDArray
-        Interpolated efficiency values.
+    interpolator : RegularGridInterpolator
+        Callable function to interpolate efficiency values.
+    theta_min_max : tuple
+        Minimum and maximum theta values in the lookup table.
+    phi_min_max : tuple
+        Minimum and maximum phi values in the lookup table.
     """
     lookup_table = get_energy_efficiencies(ancillary_files)
 
@@ -1133,6 +1143,9 @@ def get_efficiency_interpolator(ancillary_files: dict) -> RegularGridInterpolato
     efficiency_grid = efficiency_2d.reshape(
         (len(theta_vals), len(phi_vals), len(energy_vals))
     )
+    # Find the min and max values for theta and phi
+    theta_min_max = (theta_vals.min(), theta_vals.max())
+    phi_min_max = (phi_vals.min(), phi_vals.max())
 
     interpolator = RegularGridInterpolator(
         (theta_vals, phi_vals, energy_vals),
@@ -1141,7 +1154,7 @@ def get_efficiency_interpolator(ancillary_files: dict) -> RegularGridInterpolato
         fill_value=FILLVAL_FLOAT32,
     )
 
-    return interpolator
+    return interpolator, theta_min_max, phi_min_max
 
 
 def get_efficiency(
@@ -1174,7 +1187,7 @@ def get_efficiency(
         Interpolated efficiency values.
     """
     if not interpolator:
-        interpolator = get_efficiency_interpolator(ancillary_files)
+        interpolator, _, _ = get_efficiency_interpolator(ancillary_files)
 
     return interpolator((theta_inst, phi_inst, energy))
 
@@ -1343,7 +1356,7 @@ def is_back_tof_valid(
     From page 33 of the IMAP-Ultra Flight Software Specification document.
     """
     _, _, _, _, tofx, tofy = get_ph_tof_and_back_positions(
-        de_dataset, xf, "ultra45", ancillary_files
+        de_dataset, xf, sensor, ancillary_files
     )
     diff = tofy - tofx
 

imap_processing/ultra/l1c/helio_pset.py

@@ -6,6 +6,7 @@ import astropy_healpix.healpy as hp
 import numpy as np
 import xarray as xr
 
+from imap_processing.cdf.utils import parse_filename_like
 from imap_processing.quality_flags import ImapPSETUltraFlags
 from imap_processing.spice.repoint import get_pointing_times
 from imap_processing.spice.time import (
@@ -24,6 +25,7 @@ from imap_processing.ultra.l1c.ultra_l1c_pset_bins import (
     get_efficiencies_and_geometric_function,
     get_energy_delta_minus_plus,
     get_helio_adjusted_data,
+    get_spacecraft_background_rates,
     get_spacecraft_exposure_times,
     get_spacecraft_histogram,
 )
@@ -69,9 +71,14 @@ def calculate_helio_pset(
     dataset : xarray.Dataset
         Dataset containing the data.
     """
+    sensor = parse_filename_like(name)["sensor"][0:2]
     pset_dict: dict[str, np.ndarray] = {}
     # Select only the species we are interested in.
     indices = np.where(np.isin(de_dataset["ebin"].values, species_id))[0]
+    if indices.size == 0:
+        logger.info(f"No data available for {name}")
+        return None
+
     species_dataset = de_dataset.isel(epoch=indices)
 
     rejected = get_de_rejection_mask(
@@ -120,17 +127,23 @@ def calculate_helio_pset(
     )
     healpix = np.arange(n_pix)
 
+    # Get midpoint timestamp for pointing.
+    pointing_start, pointing_stop = get_pointing_times(
+        et_to_met(species_dataset["event_times"].data[0])
+    )
     logger.info("Calculating spacecraft exposure times with deadtime correction.")
     exposure_time, deadtime_ratios = get_spacecraft_exposure_times(
         rates_dataset,
         params_dataset,
         pixels_below_scattering,
         boundary_scale_factors,
+        pointing_start,
+        pointing_stop,
         n_pix=n_pix,
     )
     logger.info("Calculating spun efficiencies and geometric function.")
     # calculate efficiency and geometric function as a function of energy
-    efficiencies, geometric_function = get_efficiencies_and_geometric_function(
+    geometric_function, efficiencies = get_efficiencies_and_geometric_function(
         pixels_below_scattering,
         boundary_scale_factors,
         theta_vals,
@@ -138,14 +151,23 @@ def calculate_helio_pset(
         n_pix,
         ancillary_files,
     )
-    # Get midpoint timestamp for pointing.
-    pointing_start, pointing_stop = get_pointing_times(
-        et_to_met(species_dataset["event_times"].data[0])
+
+    logger.info("Calculating background rates.")
+    # TODO calculate helio background rates
+    # Calculate background rates
+    background_rates = get_spacecraft_background_rates(
+        rates_dataset,
+        sensor,
+        ancillary_files,
+        intervals,
+        goodtimes_dataset["spin_number"].values,
+        nside=nside,
     )
+
     mid_time = ttj2000ns_to_et(met_to_ttj2000ns((pointing_start + pointing_stop) / 2))
 
     logger.info("Adjusting data for helio frame.")
-    exposure_time, efficiency, geometric_function = get_helio_adjusted_data(
+    exposure_time, _efficiency, geometric_function = get_helio_adjusted_data(
         mid_time,
         exposure_time,
         geometric_function,
@@ -176,7 +198,8 @@ def calculate_helio_pset(
     pset_dict["latitude"] = latitude[np.newaxis, ...]
     pset_dict["longitude"] = longitude[np.newaxis, ...]
     pset_dict["energy_bin_geometric_mean"] = energy_bin_geometric_means
-    pset_dict["helio_exposure_factor"] = exposure_time[np.newaxis, ...]
+    pset_dict["background_rates"] = background_rates[np.newaxis, ...]
+    pset_dict["exposure_factor"] = exposure_time[np.newaxis, ...]
     pset_dict["pixel_index"] = healpix
     pset_dict["energy_bin_delta"] = np.diff(intervals, axis=1).squeeze()[
         np.newaxis, ...

imap_processing/ultra/l1c/l1c_lookup_utils.py

@@ -174,8 +174,10 @@ def calculate_fwhm_spun_scattering(
         fwhm_phi_sum[:, for_inds] += fwhm_phi.T
         sample_count[:, for_inds] += 1
 
-    fwhm_phi_avg = np.divide(fwhm_theta_sum, sample_count, where=sample_count != 0)
-    fwhm_theta_avg = np.divide(fwhm_theta_sum, sample_count, where=sample_count != 0)
+    fwhm_phi_avg = np.zeros_like(fwhm_phi_sum)
+    fwhm_theta_avg = np.zeros_like(fwhm_theta_sum)
+    np.divide(fwhm_phi_sum, sample_count, out=fwhm_phi_avg, where=sample_count != 0)
+    np.divide(fwhm_theta_sum, sample_count, out=fwhm_theta_avg, where=sample_count != 0)
     return (
         pixels_below_scattering,
         fwhm_theta_avg,

imap_processing/ultra/l1c/spacecraft_pset.py

@@ -77,6 +77,7 @@ def calculate_spacecraft_pset(
 
     # If there are no species return None.
     if indices.size == 0:
+        logger.info(f"No data available for {name}")
         return None
 
     # Before we use the de_dataset to calculate the pointing set grid we need to filter.
@@ -100,7 +101,7 @@ def calculate_spacecraft_pset(
         for_indices_by_spin_phase,
         theta_vals,
         phi_vals,
-        ra_and_dec,
+        _ra_and_dec,
         boundary_scale_factors,
     ) = get_spacecraft_pointing_lookup_tables(ancillary_files, instrument_id)
 
@@ -126,7 +127,7 @@ def calculate_spacecraft_pset(
 
     logger.info("Calculating spun efficiencies and geometric function.")
     # calculate efficiency and geometric function as a function of energy
-    efficiencies, geometric_function = get_efficiencies_and_geometric_function(
+    geometric_function, efficiencies = get_efficiencies_and_geometric_function(
         pixels_below_scattering,
         boundary_scale_factors,
         theta_vals,
@@ -136,6 +137,10 @@ def calculate_spacecraft_pset(
     )
     sensitivity = efficiencies * geometric_function
 
+    # Get the start and stop times of the pointing period
+    pointing_start, pointing_stop = get_pointing_times(
+        float(et_to_met(species_dataset["event_times"].data[0]))
+    )
     # Calculate exposure times
     logger.info("Calculating spacecraft exposure times with deadtime correction.")
     exposure_pointing, deadtime_ratios = get_spacecraft_exposure_times(
@@ -143,6 +148,8 @@ def calculate_spacecraft_pset(
         params_dataset,
         pixels_below_scattering,
         boundary_scale_factors,
+        pointing_start,
+        pointing_stop,
         n_pix=n_pix,
     )
     logger.info("Calculating background rates.")
@@ -172,10 +179,7 @@ def calculate_spacecraft_pset(
         spacecraft_pset_quality_flags,
         nside=nside,
     )
-    # Get pointing start and stop times and convert to ttj2000ns
-    pointing_start, pointing_stop = get_pointing_times(
-        float(et_to_met(species_dataset["event_times"].data[0]))
-    )
+    # Convert pointing start time to ttj2000ns
     pointing_start = met_to_ttj2000ns(pointing_start)
     # Epoch should be the start of the pointing
     pset_dict["epoch"] = np.atleast_1d(pointing_start).astype(np.int64)

imap_processing/ultra/l1c/ultra_l1c.py

@@ -8,7 +8,7 @@ from imap_processing.ultra.l1c.spacecraft_pset import calculate_spacecraft_pset
 
 
 def ultra_l1c(
-    data_dict: dict, ancillary_files: dict, imap_frames: bool
+    data_dict: dict, ancillary_files: dict, descriptor: str
 ) -> list[xr.Dataset]:
     """
     Will process ULTRA L1A and L1B data into L1C CDF files at output_filepath.
@@ -19,8 +19,8 @@ def ultra_l1c(
         The data itself and its dependent data.
     ancillary_files : dict
         Ancillary files.
-    imap_frames : bool
-        Whether to use IMAP frames.
+    descriptor : str
+        Job descriptor.
 
     Returns
     -------
@@ -28,15 +28,15 @@ def ultra_l1c(
         List of xarray.Dataset.
     """
     output_datasets = []
-
-    # Account for possibility of having 45 and 90 in dictionary.
+    create_helio_pset = True if "helio" in descriptor else False
+    # Account for the possibility of having 45 and 90 in the dictionary.
     for instrument_id in [45, 90]:
         if (
             f"imap_ultra_l1b_{instrument_id}sensor-goodtimes" in data_dict
             and f"imap_ultra_l1b_{instrument_id}sensor-de" in data_dict
             and f"imap_ultra_l1a_{instrument_id}sensor-rates" in data_dict
             and f"imap_ultra_l1a_{instrument_id}sensor-params" in data_dict
-            and imap_frames
+            and create_helio_pset
         ):
             helio_pset = calculate_helio_pset(
                 data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"],

imap_processing/ultra/l1c/ultra_l1c_pset_bins.py

@@ -13,7 +13,11 @@ from imap_processing.spice.geometry import (
     cartesian_to_spherical,
     imap_state,
 )
-from imap_processing.spice.spin import get_spacecraft_spin_phase, get_spin_angle
+from imap_processing.spice.spin import (
+    get_spacecraft_spin_phase,
+    get_spin_angle,
+    get_spin_data,
+)
 from imap_processing.spice.time import ttj2000ns_to_met
 from imap_processing.ultra.constants import UltraConstants
 from imap_processing.ultra.l1b.lookup_utils import (
@@ -381,7 +385,7 @@ def calculate_exposure_time(
     # Get energy bin geometric means
     energy_bin_geometric_means = build_energy_bins()[2]
     # Exposure time should now be of shape (energy, npix)
-    exposure_pointing = np.zeros((len(energy_bin_geometric_means), n_pix))
+    counts = np.zeros((len(energy_bin_geometric_means), n_pix))
     # nominal spin phase step.
     nominal_ms_step = 15 / len(pixels_below_scattering)  # time step
     # Query the dead-time ratio and apply the nominal exposure time to pixels in the FOR
@@ -396,12 +400,13 @@ def calculate_exposure_time(
                 continue
             # Apply the nominal exposure time (1 ms) scaled by the deadtime ratio to
             # every pixel in the FOR, that is below the FWHM scattering threshold,
-            exposure_pointing[energy_bin_idx, pixels_at_energy_and_spin] += (
-                nominal_ms_step
-                * deadtime_ratios[i]
+            counts[energy_bin_idx, pixels_at_energy_and_spin] += (
+                deadtime_ratios[i]
                 * boundary_scale_factors[pixels_at_energy_and_spin, i]
             )
 
+    # Multiply by the nominal spin step to get the exposure time in ms
+    exposure_pointing = counts * nominal_ms_step
     return exposure_pointing
 
 
@@ -410,6 +415,8 @@ def get_spacecraft_exposure_times(
     params_dataset: xr.Dataset,
     pixels_below_scattering: list[list],
     boundary_scale_factors: NDArray,
+    pointing_start_met: float,
+    pointing_stop_met: float,
     n_pix: int,
 ) -> tuple[NDArray, NDArray]:
     """
@@ -428,6 +435,10 @@ def get_spacecraft_exposure_times(
         below the FWHM scattering threshold.
     boundary_scale_factors : np.ndarray
         Boundary scale factors for each pixel at each spin phase.
+    pointing_start_met : float
+        Start time of the pointing period in mission elapsed time.
+    pointing_stop_met : float
+        Stop time of the pointing period in mission elapsed time.
     n_pix : int
         Number of HEALPix pixels.
 
@@ -440,13 +451,36 @@ def get_spacecraft_exposure_times(
     nominal_deadtime_ratios : np.ndarray
         Deadtime ratios at each spin phase step (1ms res).
     """
-    # TODO: use the universal spin table and
-    #  universal pointing table here to determine actual number of spins
     sectored_rates = get_sectored_rates(rates_dataset, params_dataset)
     nominal_deadtime_ratios = get_deadtime_ratios_by_spin_phase(sectored_rates)
-    exposure_pointing_adjusted = calculate_exposure_time(
+    # The exposure time will be approximately the same per spin, so to save
+    # computation time, calculate the exposure time for a single spin and then scale it
+    # by the number of spins in the pointing. For more information, see section 3.4.3
+    # of the Ultra Algorithm Document.
+    exposure_time = calculate_exposure_time(
         nominal_deadtime_ratios, pixels_below_scattering, boundary_scale_factors, n_pix
     )
+    # Use the universal spin table to determine the actual number of spins
+    nominal_spin_seconds = 15.0
+    spin_data = get_spin_data()
+    # Filter for spins only in pointing
+    spin_data = spin_data[
+        (spin_data["spin_start_met"] >= pointing_start_met)
+        & (spin_data["spin_start_met"] <= pointing_stop_met)
+    ]
+    # Get only valid spin data
+    valid_mask = (spin_data["spin_phase_valid"].values == 1) & (
+        spin_data["spin_period_valid"].values == 1
+    )
+    n_spins_in_pointing: float = np.sum(
+        spin_data[valid_mask].spin_period_sec / nominal_spin_seconds
+    )
+    logger.info(
+        f"Calculated total spins universal spin table. Found {n_spins_in_pointing} "
+        f"valid spins."
+    )
+    # Adjust exposure time by the actual number of valid spins in the pointing
+    exposure_pointing_adjusted = n_spins_in_pointing * exposure_time
     return exposure_pointing_adjusted, nominal_deadtime_ratios
 
 
@@ -483,13 +517,17 @@ def get_efficiencies_and_geometric_function(
 
     Returns
     -------
-    gf_summation : np.ndarray
-        Summation of geometric factors for each pixel and energy bin.
-    eff_summation : np.ndarray
-        Summation of efficiencies for each pixel and energy bin.
+    gf_averaged : np.ndarray
+        Averaged geometric factors across all spin phases.
+        Shape = (n_energy_bins, npix).
+    eff_averaged : np.ndarray
+        Averaged efficiencies across all spin phases.
+        Shape = (n_energy_bins, npix).
     """
     # Load callable efficiency interpolator function
-    eff_interpolator = get_efficiency_interpolator(ancillary_files)
+    eff_interpolator, theta_min_max, phi_min_max = get_efficiency_interpolator(
+        ancillary_files
+    )
     # load geometric factor lookup table
     geometric_lookup_table = load_geometric_factor_tables(
         ancillary_files, "l1b-sensor-gf-blades"
@@ -497,6 +535,24 @@ def get_efficiencies_and_geometric_function(
     # Get energy bin geometric means
     energy_bin_geometric_means = build_energy_bins()[2]
     energy_bins = len(energy_bin_geometric_means)
+    # clip arrays to avoid out of bounds errors
+    logger.info(
+        "Clipping Theta and Phi values to valid ranges for the efficiency "
+        "interpolation. \n"
+        f"Theta valid range: {theta_min_max}, Phi valid range: {phi_min_max}. \n "
+        f"Found "
+        f"{np.sum((theta_vals < theta_min_max[0]) | (theta_vals > theta_min_max[1]))}"
+        f" Theta values out of range. \n"
+        f"Found "
+        f"{np.sum((phi_vals < phi_min_max[0]) | (phi_vals > phi_min_max[1]))}"
+        f" Phi values out of range. \n"
+        f"Theta min and max values before clipping: "
+        f"{theta_vals.min()}, {theta_vals.max()} \n"
+        f"Phi min and max values before clipping:"
+        f" {phi_vals.min()}, {phi_vals.max()} \n"
+    )
+    theta_vals_clipped = np.clip(theta_vals, theta_min_max[0], theta_min_max[1])
+    phi_vals_clipped = np.clip(phi_vals, phi_min_max[0], phi_min_max[1])
     # Initialize summation arrays for geometric factors and efficiencies
     gf_summation = np.zeros((energy_bins, npix))
     eff_summation = np.zeros((energy_bins, npix))
@@ -507,6 +563,8 @@ def get_efficiencies_and_geometric_function(
         # Compute gf and eff for these theta/phi pairs
         theta_at_spin = theta_vals[:, i]
         phi_at_spin = phi_vals[:, i]
+        theta_at_spin_clipped = theta_vals_clipped[:, i]
+        phi_at_spin_clipped = phi_vals_clipped[:, i]
         gf_values = get_geometric_factor(
             phi=phi_at_spin,
             theta=theta_at_spin,
@@ -518,10 +576,12 @@ def get_efficiencies_and_geometric_function(
             if pixel_inds.size == 0:
                 continue
             energy = energy_bin_geometric_means[energy_bin_idx]
+            # Clip energy to calibrated range
+            energy_clipped = np.clip(energy, 3.0, 80.0)
             eff_values = get_efficiency(
-                np.full(phi_at_spin[pixel_inds].shape, energy),
-                phi_at_spin[pixel_inds],
-                theta_at_spin[pixel_inds],
+                np.full(phi_at_spin[pixel_inds].shape, energy_clipped),
+                phi_at_spin_clipped[pixel_inds],
+                theta_at_spin_clipped[pixel_inds],
                 ancillary_files,
                 interpolator=eff_interpolator,
             )
@@ -536,8 +596,10 @@ def get_efficiencies_and_geometric_function(
 
     # return averaged geometric factors and efficiencies across all spin phases
     # These are now energy dependent.
-    gf_averaged = np.divide(gf_summation, sample_count, where=sample_count != 0)
-    eff_averaged = np.divide(eff_summation, sample_count, where=sample_count != 0)
+    gf_averaged = np.zeros_like(gf_summation)
+    eff_averaged = np.zeros_like(eff_summation)
+    np.divide(gf_summation, sample_count, out=gf_averaged, where=sample_count != 0)
+    np.divide(eff_summation, sample_count, out=eff_averaged, where=sample_count != 0)
     return gf_averaged, eff_averaged
 
 
@@ -698,8 +760,8 @@ def get_spacecraft_background_rates(
     """
     pulses = get_pulses_per_spin(rates_dataset)
     # Pulses for the pointing.
-    etof_min = get_image_params("eTOFMin", sensor, ancillary_files)
-    etof_max = get_image_params("eTOFMax", sensor, ancillary_files)
+    etof_min = get_image_params("eTOFMin", f"ultra{sensor}", ancillary_files)
+    etof_max = get_image_params("eTOFMax", f"ultra{sensor}", ancillary_files)
     spin_number, _ = get_spin_and_duration(
         rates_dataset["shcoarse"], rates_dataset["spin"]
     )

imap_processing/ultra/l2/ultra_l2.py

@@ -364,8 +364,8 @@ def generate_ultra_healpix_skymap(  # noqa: PLR0912
     for var in pointing_indep_vars:
         skymap.data_1d[var] = skymap.data_1d[var].squeeze("epoch", drop=True)
 
-    # Background rates must be scaled by the ratio of the solid angles of the
-    # map pixel / pointing set pixel
+    # Background rates must be scaled by
+    # the ratio of the solid angles of the map pixel / pointing set pixel
     skymap.data_1d["background_rates"] *= skymap.solid_angle / pointing_set.solid_angle
 
     # Get the energy bin widths from a PointingSet (they will all be the same)