imap-processing 0.19.0__py3-none-any.whl → 0.19.2__py3-none-any.whl

imap_processing/ultra/l1c/ultra_l1c_pset_bins.py

@@ -1,8 +1,9 @@
 """Module to create pointing sets."""
 
+import logging
+
 import astropy_healpix.healpy as hp
 import numpy as np
-import pandas
 import xarray as xr
 from numpy.typing import NDArray
 from scipy import interpolate
@@ -13,6 +14,7 @@ from imap_processing.spice.geometry import (
     imap_state,
 )
 from imap_processing.spice.spin import get_spacecraft_spin_phase, get_spin_angle
+from imap_processing.spice.time import ttj2000ns_to_met
 from imap_processing.ultra.constants import UltraConstants
 from imap_processing.ultra.l1b.lookup_utils import (
     get_geometric_factor,
@@ -31,6 +33,8 @@ from imap_processing.ultra.l1b.ultra_l1b_extended import (
 # TODO: add species binning.
 FILLVAL_FLOAT32 = -1.0e31
 
+logger = logging.getLogger(__name__)
+
 
 def build_energy_bins() -> tuple[list[tuple[float, float]], np.ndarray, np.ndarray]:
     """
@@ -46,7 +50,7 @@ def build_energy_bins() -> tuple[list[tuple[float, float]], np.ndarray, np.ndarr
         Array of geometric means of energy bins.
     """
     # Create energy bins.
-    energy_bin_edges = np.array(UltraConstants.CULLING_ENERGY_BIN_EDGES)
+    energy_bin_edges = np.array(UltraConstants.PSET_ENERGY_BIN_EDGES)
     energy_midpoints = (energy_bin_edges[:-1] + energy_bin_edges[1:]) / 2
 
     intervals = [
@@ -222,7 +226,6 @@ def get_deadtime_ratios(sectored_rates_ds: xr.Dataset) -> xr.DataArray:
         - sectored_rates_ds.stop_tn
         - sectored_rates_ds.stop_bn
     )
-
     corrected_valid_events = b * np.exp(1e-7 * 8 * coin_stop_nd)
 
     # Compute dead time ratio
@@ -252,21 +255,24 @@ def get_sectored_rates(rates_ds: xr.Dataset, params_ds: xr.Dataset) -> xr.Datase
 
     # This means that data was collected as a function of spin allowing for fine grained
     # rate analysis.
-    sector_mode_start_inds = np.where(params_ds["imageratescadence"] == 3)[0]
+    # Only get unique combinations of epoch and imageratescadence
+    params = params_ds.groupby(["epoch", "imageratescadence"]).first()
+
+    sector_mode_start_inds = np.where(params["imageratescadence"] == 3)[0]
+    if len(sector_mode_start_inds) == 0:
+        raise ValueError("No sector mode data found in the parameters dataset.")
     # get the sector mode start and stop indices
     sector_mode_stop_inds = sector_mode_start_inds + 1
     # get the sector mode start and stop times
-    mode_3_start = params_ds["epoch"].values[sector_mode_start_inds]
-
+    mode_3_start = params["epoch"].values[sector_mode_start_inds]
     # if the last mode is a sector mode, we can assume that the sector data goes through
     # the end of the dataset, so we append np.inf to the end of the last time range.
-    if sector_mode_stop_inds[-1] == len(params_ds["epoch"]):
+    if sector_mode_stop_inds[-1] == len(params["epoch"]):
         mode_3_end = np.append(
-            params_ds["epoch"].values[sector_mode_stop_inds[:-1]], np.inf
+            params["epoch"].values[sector_mode_stop_inds[:-1]], np.inf
         )
     else:
-        mode_3_end = params_ds["epoch"].values[sector_mode_stop_inds]
-
+        mode_3_end = params["epoch"].values[sector_mode_stop_inds]
     # Build a list of conditions for each sector mode time range
     conditions = [
         (rates_ds["epoch"] >= start) & (rates_ds["epoch"] < end)
@@ -295,10 +301,9 @@ def get_deadtime_ratios_by_spin_phase(
     """
     deadtime_ratios = get_deadtime_ratios(sectored_rates)
     # Get the spin phase at the start of each sector rate measurement
+    met_times = ttj2000ns_to_met(sectored_rates.epoch.data)
     spin_phases = np.asarray(
-        get_spin_angle(
-            get_spacecraft_spin_phase(np.array(sectored_rates.epoch.data)), degrees=True
-        )
+        get_spin_angle(get_spacecraft_spin_phase(met_times), degrees=True)
     )
     # Assume the sectored rate data is evenly spaced in time, and find the middle spin
     # phase value for each sector.
@@ -325,11 +330,16 @@ def get_deadtime_ratios_by_spin_phase(
     deadtime_by_spin_phase = deadtime_by_spin_phase.sortby("spin_phase")
     # Group by spin phase and calculate the median dead time ratio for each phase
     deadtime_medians = deadtime_by_spin_phase.groupby("spin_phase").median(skipna=True)
-
     if np.any(np.isnan(deadtime_medians["deadtime_ratio"].values)):
-        raise ValueError(
-            "Dead time ratios contain NaN values, cannot create interpolator."
+        if not np.any(np.isfinite(deadtime_medians["deadtime_ratio"].values)):
+            raise ValueError("All dead time ratios are NaN, cannot interpolate.")
+        logger.warning(
+            "Dead time ratios contain NaN values, filtering data to only include "
+            "finite values."
         )
+    deadtime_medians = deadtime_medians.where(
+        np.isfinite(deadtime_medians["deadtime_ratio"]), drop=True
+    )
     interpolator = interpolate.PchipInterpolator(
         deadtime_medians["spin_phase"].values, deadtime_medians["deadtime_ratio"].values
     )
@@ -340,19 +350,17 @@ def get_deadtime_ratios_by_spin_phase(
     return interpolator(nominal_spin_phases_1ms_res)
 
 
-def apply_deadtime_correction(
-    exposure_pointing: pandas.DataFrame,
+def calculate_exposure_time(
     deadtime_ratios: np.ndarray,
     pixels_below_scattering: list,
     boundary_scale_factors: NDArray,
+    n_pix: int,
 ) -> np.ndarray:
     """
     Adjust the exposure time at each pixel to account for dead time.
 
     Parameters
     ----------
-    exposure_pointing : pandas.DataFrame
-        Exposure data.
     deadtime_ratios : PchipInterpolator
         Interpolating function for dead time ratios.
     pixels_below_scattering : list
@@ -362,6 +370,8 @@ def apply_deadtime_correction(
         the FWHM scattering threshold.
     boundary_scale_factors : np.ndarray
         Boundary scale factors for each pixel at each spin phase.
+    n_pix : int
+        Number of HEALPix pixels.
 
     Returns
     -------
@@ -370,12 +380,8 @@ def apply_deadtime_correction(
     """
     # Get energy bin geometric means
     energy_bin_geometric_means = build_energy_bins()[2]
-    # Exposure time should now be of shape (npix, energy)
-    exposure_pointing = np.repeat(
-        exposure_pointing.to_numpy()[np.newaxis, :],
-        len(energy_bin_geometric_means),
-        axis=0,
-    )
+    # Exposure time should now be of shape (energy, npix)
+    exposure_pointing = np.zeros((len(energy_bin_geometric_means), n_pix))
     # nominal spin phase step.
     nominal_ms_step = 15 / len(pixels_below_scattering)  # time step
     # Query the dead-time ratio and apply the nominal exposure time to pixels in the FOR
@@ -400,19 +406,17 @@ def apply_deadtime_correction(
 
 
 def get_spacecraft_exposure_times(
-    constant_exposure: pandas.DataFrame,
     rates_dataset: xr.Dataset,
     params_dataset: xr.Dataset,
     pixels_below_scattering: list[list],
     boundary_scale_factors: NDArray,
+    n_pix: int,
 ) -> tuple[NDArray, NDArray]:
     """
     Compute exposure times for HEALPix pixels.
 
     Parameters
     ----------
-    constant_exposure : pandas.DataFrame
-        Exposure data.
     rates_dataset : xarray.Dataset
         Dataset containing image rates data.
     params_dataset : xarray.Dataset
@@ -424,6 +428,8 @@ def get_spacecraft_exposure_times(
         below the FWHM scattering threshold.
     boundary_scale_factors : np.ndarray
         Boundary scale factors for each pixel at each spin phase.
+    n_pix : int
+        Number of HEALPix pixels.
 
     Returns
     -------
@@ -438,14 +444,8 @@ def get_spacecraft_exposure_times(
     #  universal pointing table here to determine actual number of spins
     sectored_rates = get_sectored_rates(rates_dataset, params_dataset)
     nominal_deadtime_ratios = get_deadtime_ratios_by_spin_phase(sectored_rates)
-    exposure_pointing = (
-        constant_exposure["Exposure Time"] * 5760
-    )  # 5760 spins per pointing (for now)
-    exposure_pointing_adjusted = apply_deadtime_correction(
-        exposure_pointing,
-        nominal_deadtime_ratios,
-        pixels_below_scattering,
-        boundary_scale_factors,
+    exposure_pointing_adjusted = calculate_exposure_time(
+        nominal_deadtime_ratios, pixels_below_scattering, boundary_scale_factors, n_pix
     )
     return exposure_pointing_adjusted, nominal_deadtime_ratios
 
@@ -664,7 +664,7 @@ def get_spacecraft_background_rates(
     sensor: str,
     ancillary_files: dict,
     energy_bin_edges: list[tuple[float, float]],
-    cullingmask_spin_number: NDArray,
+    goodtimes_spin_number: NDArray,
     nside: int = 128,
 ) -> NDArray:
     """
@@ -680,9 +680,9 @@ def get_spacecraft_background_rates(
         Ancillary files containing the lookup tables.
     energy_bin_edges : list[tuple[float, float]]
         Energy bin edges.
-    cullingmask_spin_number : NDArray
+    goodtimes_spin_number : NDArray
         Goodtime spins.
-        Ex. imap_ultra_l1b_45sensor-cullingmask[0]["spin_number"]
+        Ex. imap_ultra_l1b_45sensor-goodtimes[0]["spin_number"]
         This is used to determine the number of pulses per spin.
     nside : int, optional
         The nside parameter of the Healpix tessellation (default is 128).
@@ -714,7 +714,7 @@ def get_spacecraft_background_rates(
     background_rates = np.zeros((len(energy_bin_edges), n_pix))
 
     # Only select pulses from goodtimes.
-    goodtime_mask = np.isin(spin_number, cullingmask_spin_number)
+    goodtime_mask = np.isin(spin_number, goodtimes_spin_number)
     mean_start_pulses = np.mean(pulses.start_pulses[goodtime_mask])
     mean_stop_pulses = np.mean(pulses.stop_pulses[goodtime_mask])
     mean_coin_pulses = np.mean(pulses.coin_pulses[goodtime_mask])

imap_processing/ultra/l2/ultra_l2.py

@@ -10,6 +10,7 @@ import xarray as xr
 from numpy.typing import NDArray
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.cdf.utils import load_cdf
 from imap_processing.ena_maps import ena_maps
 from imap_processing.ena_maps.utils.coordinates import CoordNames
 from imap_processing.ena_maps.utils.naming import (
@@ -17,6 +18,7 @@ from imap_processing.ena_maps.utils.naming import (
     MapDescriptor,
     ns_to_duration_months,
 )
+from imap_processing.quality_flags import ImapPSETUltraFlags
 from imap_processing.ultra.l1c.ultra_l1c_pset_bins import get_energy_delta_minus_plus
 
 logger = logging.getLogger(__name__)
@@ -58,7 +60,15 @@ REQUIRED_L1C_VARIABLES_PULL = [
     "background_rates",
     "obs_date",
 ]
-
+# These variables are expected but not strictly required. In certain test scenarios,
+# they may be missing, in which case we will raise a warning and continue.
+# All psets must be consistent and either have these variables or not.
+EXPECTED_L1C_VARIABLES_PULL = [
+    "geometric_function",
+    "efficiency",
+    "scatter_theta",
+    "scatter_phi",
+]
 # These variables are projected to the map as the mean of pointing set pixels value,
 # weighted by that pointing set pixel's exposure and solid angle
 VARIABLES_TO_WEIGHT_BY_POINTING_SET_EXPOSURE_TIMES_SOLID_ANGLE = [
@@ -71,12 +81,9 @@ VARIABLES_TO_WEIGHT_BY_POINTING_SET_EXPOSURE_TIMES_SOLID_ANGLE = [
 # calculate ena_intensity and its statistical uncertainty
 # They will not be present in the final map
 VARIABLES_TO_DROP_AFTER_INTENSITY_CALCULATION = [
-    "counts",
-    "background_rates",
     "pointing_set_exposure_times_solid_angle",
     "num_pointing_set_pixel_members",
     "corrected_count_rate",
-    "obs_date_for_std",
     "obs_date_squared_for_std",
 ]
 
@@ -127,6 +134,8 @@ def get_variable_attributes_optional_energy_dependence(
         and (CoordNames.ENERGY_ULTRA_L1C.value not in variable_dims)
     ):
         variable_name = f"{variable_name}_energy_independent"
+    if variable_name == "counts":
+        variable_name = "ena_count"
 
     metadata = cdf_attrs.get_variable_attributes(
         variable_name=variable_name,
@@ -205,7 +214,7 @@ def generate_ultra_healpix_skymap(
     output_map_structure.values_to_push_project.extend(
         [
             "num_pointing_set_pixel_members",
-            "obs_date_for_std",
+            "obs_date_range",
             "obs_date_squared_for_std",
         ]
     )
@@ -235,6 +244,27 @@ def generate_ultra_healpix_skymap(
             f"PUSH Variables: {output_map_structure.values_to_push_project} \n"
             f"PULL Variables: {output_map_structure.values_to_pull_project}"
         )
+    # TODO remove this in the future once all test data includes these variables
+    # Add expected but not required variables to the pull projection list
+    # Log a warning if they are missing from any PSET but continue processing.
+    expected_present_vars = []
+    first_pset = (
+        load_cdf(ultra_l1c_psets[0])
+        if isinstance(ultra_l1c_psets[0], (str, Path))
+        else ultra_l1c_psets[0]
+    )
+    for var in EXPECTED_L1C_VARIABLES_PULL:
+        if var not in first_pset.variables:
+            logger.warning(
+                f"Expected variable {var} not found in the first L1C PSET. "
+                "This variable will not be projected to the map."
+            )
+        else:
+            expected_present_vars.append(var)
+
+    output_map_structure.values_to_pull_project = list(
+        set(output_map_structure.values_to_pull_project + expected_present_vars)
+    )
 
     all_pset_epochs = []
     for ultra_l1c_pset in ultra_l1c_psets:
@@ -248,9 +278,15 @@ def generate_ultra_healpix_skymap(
             "\nThese values will be pull projected: "
             f">> {output_map_structure.values_to_pull_project}",
         )
+        flags_1d = pointing_set.data["quality_flags"].isel(epoch=0)
+        # This is a good pixel mask where zero is when the earth is not in the FOV.
+        good_pixel_mask = (
+            (flags_1d & ImapPSETUltraFlags.EARTH_FOV.value) == 0
+        ).to_numpy()
 
+        # Only count the number of pointing set pixels which are not flagged.
         pointing_set.data["num_pointing_set_pixel_members"] = xr.DataArray(
-            np.ones(pointing_set.num_points, dtype=int),
+            good_pixel_mask.astype(int),
             dims=(CoordNames.HEALPIX_INDEX.value),
         )
 
@@ -261,11 +297,11 @@ def generate_ultra_healpix_skymap(
             fill_value=pointing_set.epoch,
             dtype=np.int64,
         )
-        pointing_set.data["obs_date_for_std"] = pointing_set.data["obs_date"].astype(
+        pointing_set.data["obs_date_range"] = pointing_set.data["obs_date"].astype(
             np.float64
         )
         pointing_set.data["obs_date_squared_for_std"] = (
-            pointing_set.data["obs_date_for_std"] ** 2
+            pointing_set.data["obs_date_range"] ** 2
         )
 
         # Add solid_angle * exposure of pointing set as data_var
@@ -276,15 +312,22 @@ def generate_ultra_healpix_skymap(
 
         # Initial processing for weighted quantities at PSET level
         # Weight the values by exposure and solid angle
+        # Ensure only valid pointing set pixels contribute to the weighted mean.
         pointing_set.data[
             VARIABLES_TO_WEIGHT_BY_POINTING_SET_EXPOSURE_TIMES_SOLID_ANGLE
-        ] *= pointing_set.data["pointing_set_exposure_times_solid_angle"]
+        ] = (
+            pointing_set.data[
+                VARIABLES_TO_WEIGHT_BY_POINTING_SET_EXPOSURE_TIMES_SOLID_ANGLE
+            ]
+            * pointing_set.data["pointing_set_exposure_times_solid_angle"]
+        ).where(good_pixel_mask)
 
         # Project values such as counts via the PUSH method
         skymap.project_pset_values_to_map(
             pointing_set=pointing_set,
             value_keys=output_map_structure.values_to_push_project,
             index_match_method=ena_maps.IndexMatchMethod.PUSH,
+            pset_valid_mask=good_pixel_mask,
         )
 
         # Project values such as exposure_factor via the PULL method
@@ -292,6 +335,7 @@ def generate_ultra_healpix_skymap(
             pointing_set=pointing_set,
             value_keys=output_map_structure.values_to_pull_project,
             index_match_method=ena_maps.IndexMatchMethod.PULL,
+            pset_valid_mask=good_pixel_mask,
         )
 
     # Subsequent processing for weighted quantities at SkyMap level
@@ -347,7 +391,7 @@ def generate_ultra_healpix_skymap(
                 )
                 - (
                     (
-                        skymap.data_1d["obs_date_for_std"]
+                        skymap.data_1d["obs_date_range"]
                         / (skymap.data_1d["num_pointing_set_pixel_members"])
                     )
                     ** 2

imap_processing/ultra/utils/ultra_l1_utils.py

@@ -1,6 +1,5 @@
 """Create dataset."""
 
-import numpy as np
 import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
@@ -32,7 +31,7 @@ def create_dataset(  # noqa: PLR0912
     cdf_manager.add_instrument_global_attrs("ultra")
     cdf_manager.add_instrument_variable_attrs("ultra", level)
 
-    # L1b extended spin, badtimes, and cullingmask data products
+    # L1b extended spin, badtimes, and goodtimes data products
     if "spin_number" in data_dict.keys():
         coords = {
             "spin_number": ("spin_number", data_dict["spin_number"]),
@@ -40,7 +39,6 @@ def create_dataset(  # noqa: PLR0912
                 "energy_bin_geometric_mean",
                 data_dict["energy_bin_geometric_mean"],
             ),
-            "epoch": ("spin_number", np.asarray(data_dict["epoch"])),
         }
         default_dimension = "spin_number"
     # L1c pset data products
@@ -110,7 +108,12 @@ def create_dataset(  # noqa: PLR0912
                 dims=["epoch", "component"],
                 attrs=cdf_manager.get_variable_attributes(key, check_schema=False),
             )
-        elif key == "ena_rates_threshold":
+        elif key in [
+            "ena_rates_threshold",
+            "scatter_threshold",
+            "energy_delta_minus",
+            "energy_delta_plus",
+        ]:
             dataset[key] = xr.DataArray(
                 data,
                 dims=["energy_bin_geometric_mean"],
@@ -134,7 +137,7 @@ def create_dataset(  # noqa: PLR0912
                 dims=["energy_bin_geometric_mean", "spin_number"],
                 attrs=cdf_manager.get_variable_attributes(key, check_schema=False),
             )
-        elif key in {"latitude", "longitude"}:
+        elif key in {"quality_flags", "latitude", "longitude"}:
             dataset[key] = xr.DataArray(
                 data,
                 dims=["epoch", "pixel_index"],
@@ -155,6 +158,8 @@ def create_dataset(  # noqa: PLR0912
             "sensitivity",
             "efficiency",
             "geometric_function",
+            "scatter_theta",
+            "scatter_phi",
         }:
             dataset[key] = xr.DataArray(
                 data,

{imap_processing-0.19.0.dist-info → imap_processing-0.19.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: imap-processing
-Version: 0.19.0
+Version: 0.19.2
 Summary: IMAP Science Operations Center Processing
 License: MIT
 Keywords: IMAP,SDC,SOC,Science Operations