imap-processing 0.14.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

imap_processing/ultra/l1c/helio_pset.py

@@ -0,0 +1,53 @@
+"""Calculate Pointing Set Grids."""
+
+import astropy_healpix.healpy as hp
+import numpy as np
+import xarray as xr
+
+from imap_processing.ultra.l1c.ultra_l1c_pset_bins import build_energy_bins
+from imap_processing.ultra.utils.ultra_l1_utils import create_dataset
+
+
+def calculate_helio_pset(
+    de_dataset: xr.Dataset,
+    extendedspin_dataset: xr.Dataset,
+    cullingmask_dataset: xr.Dataset,
+    name: str,
+    ancillary_files: dict,
+) -> xr.Dataset:
+    """
+    Create dictionary with defined datatype for Pointing Set Grid Data.
+
+    Parameters
+    ----------
+    de_dataset : xarray.Dataset
+        Dataset containing de data.
+    extendedspin_dataset : xarray.Dataset
+        Dataset containing extendedspin data.
+    cullingmask_dataset : xarray.Dataset
+        Dataset containing cullingmask data.
+    name : str
+        Name of the dataset.
+    ancillary_files : dict
+        Ancillary files.
+
+    Returns
+    -------
+    dataset : xarray.Dataset
+        Dataset containing the data.
+    """
+    # TODO: Fill in the rest of this later.
+    pset_dict: dict[str, np.ndarray] = {}
+    healpix = np.arange(hp.nside2npix(128))
+    _, _, energy_bin_geometric_means = build_energy_bins()
+
+    pset_dict["epoch"] = de_dataset.epoch.data[:1].astype(np.int64)
+    pset_dict["pixel_index"] = healpix
+    pset_dict["energy_bin_geometric_mean"] = energy_bin_geometric_means
+    pset_dict["exposure_factor"] = np.zeros(len(healpix), dtype=np.uint8)[
+        np.newaxis, ...
+    ]
+
+    dataset = create_dataset(pset_dict, name, "l1c")
+
+    return dataset

imap_processing/ultra/l1c/spacecraft_pset.py

@@ -4,24 +4,22 @@ import numpy as np
 import pandas as pd
 import xarray as xr
 
-from imap_processing import imap_module_directory
 from imap_processing.ultra.l1c.ultra_l1c_pset_bins import (
     build_energy_bins,
     get_spacecraft_background_rates,
     get_spacecraft_exposure_times,
     get_spacecraft_histogram,
+    interpolate_sensitivity,
 )
 from imap_processing.ultra.utils.ultra_l1_utils import create_dataset
 
-# TODO: This is a placeholder for the API lookup table directory.
-TEST_PATH = imap_module_directory / "tests" / "ultra" / "data" / "l1"
-
 
 def calculate_spacecraft_pset(
     de_dataset: xr.Dataset,
     extendedspin_dataset: xr.Dataset,
     cullingmask_dataset: xr.Dataset,
     name: str,
+    ancillary_files: dict,
 ) -> xr.Dataset:
     """
     Create dictionary with defined datatype for Pointing Set Grid Data.
@@ -36,6 +34,8 @@ def calculate_spacecraft_pset(
         Dataset containing cullingmask data.
     name : str
         Name of the dataset.
+    ancillary_files : dict
+        Ancillary files.
 
     Returns
     -------
@@ -61,23 +61,31 @@ def calculate_spacecraft_pset(
     # calculate background rates
     background_rates = get_spacecraft_background_rates()
 
-    # TODO: calculate sensitivity and interpolate based on energy.
+    efficiencies = ancillary_files["l1c-90sensor-efficiencies"]
+    geometric_function = ancillary_files["l1c-90sensor-gf"]
+
+    df_efficiencies = pd.read_csv(efficiencies)
+    df_geometric_function = pd.read_csv(geometric_function)
+    sensitivity = interpolate_sensitivity(df_efficiencies, df_geometric_function)
 
     # Calculate exposure
-    constant_exposure = TEST_PATH / "ultra_90_dps_exposure.csv"
+    constant_exposure = ancillary_files["l1c-90sensor-dps-exposure"]
     df_exposure = pd.read_csv(constant_exposure)
     exposure_pointing = get_spacecraft_exposure_times(df_exposure)
 
     # For ISTP, epoch should be the center of the time bin.
     pset_dict["epoch"] = de_dataset.epoch.data[:1].astype(np.int64)
-    pset_dict["counts"] = counts
-    pset_dict["latitude"] = latitude
-    pset_dict["longitude"] = longitude
+    pset_dict["counts"] = counts[np.newaxis, ...]
+    pset_dict["latitude"] = latitude[np.newaxis, ...]
+    pset_dict["longitude"] = longitude[np.newaxis, ...]
     pset_dict["energy_bin_geometric_mean"] = energy_bin_geometric_means
-    pset_dict["background_rates"] = background_rates
-    pset_dict["exposure_factor"] = exposure_pointing
+    pset_dict["background_rates"] = background_rates[np.newaxis, ...]
+    pset_dict["exposure_factor"] = exposure_pointing.to_numpy()[np.newaxis, ...]
     pset_dict["pixel_index"] = healpix
-    pset_dict["energy_bin_delta"] = np.diff(intervals, axis=1).squeeze()
+    pset_dict["energy_bin_delta"] = np.diff(intervals, axis=1).squeeze()[
+        np.newaxis, ...
+    ]
+    pset_dict["sensitivity"] = sensitivity[np.newaxis, ...]
 
     dataset = create_dataset(pset_dict, name, "l1c")
 

imap_processing/ultra/l1c/ultra_l1c.py

@@ -2,11 +2,14 @@
 
 import xarray as xr
 
+from imap_processing.ultra.l1c.helio_pset import calculate_helio_pset
 from imap_processing.ultra.l1c.histogram import calculate_histogram
 from imap_processing.ultra.l1c.spacecraft_pset import calculate_spacecraft_pset
 
 
-def ultra_l1c(data_dict: dict) -> list[xr.Dataset]:
+def ultra_l1c(
+    data_dict: dict, ancillary_files: dict, has_spice: bool
+) -> list[xr.Dataset]:
     """
     Will process ULTRA L1A and L1B data into L1C CDF files at output_filepath.
 
@@ -14,37 +17,57 @@ def ultra_l1c(data_dict: dict) -> list[xr.Dataset]:
     ----------
     data_dict : dict
         The data itself and its dependent data.
+    ancillary_files : dict
+        Ancillary files.
+    has_spice : bool
+        Whether to use SPICE data.
 
     Returns
     -------
     output_datasets : list[xarray.Dataset]
         List of xarray.Dataset.
     """
-    instrument_id = 45 if any("45" in key for key in data_dict.keys()) else 90
-
-    if (
-        f"imap_ultra_l1a_{instrument_id}sensor-histogram" in data_dict
-        and f"imap_ultra_l1b_{instrument_id}sensor-cullingmask" in data_dict
-    ):
-        histogram_dataset = calculate_histogram(
-            data_dict[f"imap_ultra_l1a_{instrument_id}sensor-histogram"],
-            f"imap_ultra_l1c_{instrument_id}sensor-histogram",
-        )
-        output_datasets = [histogram_dataset]
-    elif (
-        f"imap_ultra_l1b_{instrument_id}sensor-cullingmask" in data_dict
-        and f"imap_ultra_l1b_{instrument_id}sensor-de" in data_dict
-        and f"imap_ultra_l1b_{instrument_id}sensor-extendedspin" in data_dict
-    ):
-        spacecraft_pset = calculate_spacecraft_pset(
-            data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"],
-            data_dict[f"imap_ultra_l1b_{instrument_id}sensor-extendedspin"],
-            data_dict[f"imap_ultra_l1b_{instrument_id}sensor-cullingmask"],
-            f"imap_ultra_l1c_{instrument_id}sensor-spacecraftpset",
-        )
-        # TODO: add calculate_helio_pset here
-        output_datasets = [spacecraft_pset]
-    else:
+    output_datasets = []
+
+    # Account for possibility of having 45 and 90 in dictionary.
+    for instrument_id in [45, 90]:
+        if (
+            f"imap_ultra_l1a_{instrument_id}sensor-histogram" in data_dict
+            and f"imap_ultra_l1b_{instrument_id}sensor-cullingmask" in data_dict
+        ):
+            histogram_dataset = calculate_histogram(
+                data_dict[f"imap_ultra_l1a_{instrument_id}sensor-histogram"],
+                f"imap_ultra_l1c_{instrument_id}sensor-histogram",
+            )
+            output_datasets = [histogram_dataset]
+        elif (
+            f"imap_ultra_l1b_{instrument_id}sensor-cullingmask" in data_dict
+            and f"imap_ultra_l1b_{instrument_id}sensor-de" in data_dict
+            and f"imap_ultra_l1b_{instrument_id}sensor-extendedspin" in data_dict
+            and has_spice
+        ):
+            helio_pset = calculate_helio_pset(
+                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"],
+                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-extendedspin"],
+                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-cullingmask"],
+                f"imap_ultra_l1c_{instrument_id}sensor-heliopset",
+                ancillary_files,
+            )
+            output_datasets = [helio_pset]
+        elif (
+            f"imap_ultra_l1b_{instrument_id}sensor-cullingmask" in data_dict
+            and f"imap_ultra_l1b_{instrument_id}sensor-de" in data_dict
+            and f"imap_ultra_l1b_{instrument_id}sensor-extendedspin" in data_dict
+        ):
+            spacecraft_pset = calculate_spacecraft_pset(
+                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"],
+                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-extendedspin"],
+                data_dict[f"imap_ultra_l1b_{instrument_id}sensor-cullingmask"],
+                f"imap_ultra_l1c_{instrument_id}sensor-spacecraftpset",
+                ancillary_files,
+            )
+            output_datasets = [spacecraft_pset]
+    if not output_datasets:
         raise ValueError("Data dictionary does not contain the expected keys.")
 
     return output_datasets

imap_processing/ultra/l1c/ultra_l1c_pset_bins.py

@@ -15,6 +15,7 @@ from imap_processing.spice.geometry import (
 from imap_processing.ultra.constants import UltraConstants
 
 # TODO: add species binning.
+FILLVAL_FLOAT32 = -1.0e31
 
 
 def build_energy_bins() -> tuple[list[tuple[float, float]], np.ndarray, np.ndarray]:
@@ -478,8 +479,6 @@ def grid_sensitivity(
         Efficiencies at different energy levels.
     geometric_function : pandas.DataFrame
         Geometric function.
-        energy : np.ndarray
-        The particle energy.
     energy : float
         Energy to which we are interpolating.
 
@@ -503,10 +502,49 @@ def grid_sensitivity(
 
     # Interpolate to energy
     interpolated = interp_func(energy)
+    interpolated = np.where(np.isnan(interpolated), FILLVAL_FLOAT32, interpolated)
 
     return interpolated
 
 
+def interpolate_sensitivity(
+    efficiencies: pd.DataFrame,
+    geometric_function: pd.DataFrame,
+    nside: int = 128,
+) -> NDArray:
+    """
+    Interpolate the sensitivity and bin it in HEALPix space.
+
+    Parameters
+    ----------
+    efficiencies : pandas.DataFrame
+        Efficiencies at different energy levels.
+    geometric_function : pandas.DataFrame
+        Geometric function.
+    nside : int, optional
+        Healpix nside resolution (default is 128).
+
+    Returns
+    -------
+    interpolated_sensitivity : np.ndarray
+        Array of shape (n_energy_bins, n_healpix_pixels).
+    """
+    _, _, energy_bin_geometric_means = build_energy_bins()
+    npix = hp.nside2npix(nside)
+
+    interpolated_sensitivity = np.full(
+        (len(energy_bin_geometric_means), npix), FILLVAL_FLOAT32
+    )
+
+    for i, energy in enumerate(energy_bin_geometric_means):
+        pixel_sensitivity = grid_sensitivity(
+            efficiencies, geometric_function, energy
+        ).flatten()
+        interpolated_sensitivity[i, :] = pixel_sensitivity
+
+    return interpolated_sensitivity
+
+
 def get_helio_sensitivity(
     time: np.ndarray,
     efficiencies: pandas.DataFrame,

imap_processing/ultra/l2/ultra_l2.py

@@ -77,12 +77,18 @@ VARIABLES_TO_DROP_AFTER_INTENSITY_CALCULATION = [
     "pointing_set_exposure_times_solid_angle",
     "num_pointing_set_pixel_members",
     "corrected_count_rate",
+    "obs_date_for_std",
+    "obs_date_squared_for_std",
 ]
 
 # These variables may or may not be energy dependent, depending on the
 # input data. They must be handled slightly differently when it comes to adding
 # metadata to the map dataset.
-INCONSISTENTLY_ENERGY_DEPENDENT_VARIABLES = ["obs_date", "exposure_factor"]
+INCONSISTENTLY_ENERGY_DEPENDENT_VARIABLES = [
+    "obs_date",
+    "exposure_factor",
+    "obs_date_range",
+]
 
 
 def get_variable_attributes_optional_energy_dependence(
@@ -200,6 +206,8 @@ def generate_ultra_healpix_skymap(
     output_map_structure.values_to_push_project.extend(
         [
             "num_pointing_set_pixel_members",
+            "obs_date_for_std",
+            "obs_date_squared_for_std",
         ]
     )
     output_map_structure.values_to_pull_project.extend(
@@ -254,6 +262,13 @@ def generate_ultra_healpix_skymap(
             fill_value=pointing_set.epoch,
             dtype=np.int64,
         )
+        pointing_set.data["obs_date_for_std"] = pointing_set.data["obs_date"].astype(
+            np.float64
+        )
+        pointing_set.data["obs_date_squared_for_std"] = (
+            pointing_set.data["obs_date_for_std"] ** 2
+        )
+
         # Add solid_angle * exposure of pointing set as data_var
         # so this quantity is projected to map pixels for use in weighted averaging
         pointing_set.data["pointing_set_exposure_times_solid_angle"] = (
@@ -291,6 +306,10 @@ def generate_ultra_healpix_skymap(
 
     # Get the energy bin widths from a PointingSet (they will all be the same)
     delta_energy = pointing_set.data["energy_bin_delta"]
+    if CoordNames.TIME.value in delta_energy.dims:
+        delta_energy = delta_energy.mean(
+            dim=CoordNames.TIME.value,
+        )
 
     # Core calculations of ena_intensity and its statistical uncertainty for L2
     # Exposure time may contain 0s, producing NaNs in the corrected count rate
@@ -317,6 +336,27 @@ def generate_ultra_healpix_skymap(
             * delta_energy
         )
 
+        # Calculate the standard deviation of the observation date as:
+        # sqrt((sum(obs_date^2) / N) - (sum(obs_date) / N)^2)
+        # where sum here refers to the projection process
+        # summing over N pset pixels across different psets
+        skymap.data_1d["obs_date_range"] = (
+            (
+                (
+                    skymap.data_1d["obs_date_squared_for_std"]
+                    / (skymap.data_1d["num_pointing_set_pixel_members"])
+                )
+                - (
+                    (
+                        skymap.data_1d["obs_date_for_std"]
+                        / (skymap.data_1d["num_pointing_set_pixel_members"])
+                    )
+                    ** 2
+                )
+            )
+            ** 0.5
+        ).astype(np.int64)
+
     # Drop the variables that are no longer needed
     skymap.data_1d = skymap.data_1d.drop_vars(
         VARIABLES_TO_DROP_AFTER_INTENSITY_CALCULATION,
@@ -425,7 +465,6 @@ def ultra_l2(
                 data=healpix_skymap.az_el_points[:, i],
                 dims=(CoordNames.GENERIC_PIXEL.value,),
             )
-
         map_dataset = healpix_skymap.to_dataset()
         # Add attributes related to the map
         map_attrs = {
@@ -517,6 +556,12 @@ def ultra_l2(
                 name=f"{coord_var}_label",
             )
 
+    # Add systematic error as all zeros with shape matching statistical unc
+    # TODO: update once we have information from the instrument team
+    map_dataset["ena_intensity_sys_err"] = xr.zeros_like(
+        map_dataset["ena_intensity_stat_unc"],
+    )
+
     # Add epoch_delta
     map_dataset.coords["epoch_delta"] = xr.DataArray(
         [