imap-processing 0.18.0__py3-none-any.whl → 0.19.2__py3-none-any.whl

imap_processing/hi/hi_l2.py

@@ -2,13 +2,18 @@
 
 import logging
 from pathlib import Path
-from typing import Literal
 
 import numpy as np
+import pandas as pd
 import xarray as xr
 
-from imap_processing.ena_maps.ena_maps import HiPointingSet, RectangularSkyMap
-from imap_processing.spice.geometry import SpiceFrame
+from imap_processing.ena_maps.ena_maps import (
+    AbstractSkyMap,
+    HiPointingSet,
+    RectangularSkyMap,
+)
+from imap_processing.ena_maps.utils.naming import MapDescriptor
+from imap_processing.hi.utils import CalibrationProductConfig
 
 logger = logging.getLogger(__name__)
 
@@ -42,24 +47,36 @@ def hi_l2(
     l2_dataset : list[xarray.Dataset]
         Level 2 IMAP-Hi dataset ready to be written to a CDF file.
     """
-    # TODO: parse descriptor to determine map configuration
-    sensor = "45" if "45" in descriptor else "90"
-    direction: Literal["full"] = "full"
     cg_corrected = False
-    map_spacing = 4
+    map_descriptor = MapDescriptor.from_string(descriptor)
 
-    rect_map = generate_hi_map(
+    sky_map = generate_hi_map(
         psets,
         geometric_factors_path,
         esa_energies_path,
-        direction=direction,
+        spin_phase=map_descriptor.spin_phase,
+        output_map=map_descriptor.to_empty_map(),
         cg_corrected=cg_corrected,
-        map_spacing=map_spacing,
     )
 
     # Get the map dataset with variables/coordinates in the correct shape
     # TODO get the correct descriptor and frame
-    l2_ds = rect_map.build_cdf_dataset("hi", "l2", "sf", descriptor, sensor=sensor)
+
+    if not isinstance(sky_map, RectangularSkyMap):
+        raise NotImplementedError("HEALPix map output not supported for Hi")
+    if not isinstance(map_descriptor.sensor, str):
+        raise ValueError(
+            "Invalid map_descriptor. Sensor attribute must be of type str "
+            "and be either '45' or '90'"
+        )
+
+    l2_ds = sky_map.build_cdf_dataset(
+        "hi",
+        "l2",
+        map_descriptor.frame_descriptor,
+        descriptor,
+        sensor=map_descriptor.sensor,
+    )
 
     return [l2_ds]
 
@@ -68,12 +85,12 @@ def generate_hi_map(
     psets: list[str | Path],
     geometric_factors_path: str | Path,
     esa_energies_path: str | Path,
+    output_map: AbstractSkyMap,
     cg_corrected: bool = False,
-    direction: Literal["ram", "anti-ram", "full"] = "full",
-    map_spacing: int = 4,
-) -> RectangularSkyMap:
+    spin_phase: str = "full",
+) -> AbstractSkyMap:
     """
-    Project Hi PSET data into a rectangular sky map.
+    Project Hi PSET data into a sky map.
 
     Parameters
     ----------
@@ -83,34 +100,28 @@ def generate_hi_map(
         Where to get the geometric factors from.
     esa_energies_path : str or pathlib.Path
         Where to get the energies from.
+    output_map : AbstractSkyMap
+        The map object to collect data into. Determines pixel spacing,
+        coordinate system, etc.
     cg_corrected : bool, Optional
         Whether to apply Compton-Getting correction to the energies. Defaults to
         False.
-    direction : str, Optional
+    spin_phase : str, Optional
         Apply filtering to PSET data include ram or anti-ram or full spin data.
         Defaults to "full".
-    map_spacing : int, Optional
-        Pixel spacing, in degrees, of the output map in degrees. Defaults to 4.
 
     Returns
     -------
-    sky_map : RectangularSkyMap
+    sky_map : AbstractSkyMap
         The sky map with all the PSET data projected into the map.
     """
-    rect_map = RectangularSkyMap(
-        spacing_deg=map_spacing, spice_frame=SpiceFrame.ECLIPJ2000
-    )
-
     # TODO: Implement Compton-Getting correction
     if cg_corrected:
         raise NotImplementedError
-    # TODO: Implement directional filtering
-    if direction != "full":
-        raise NotImplementedError
 
     for pset_path in psets:
         logger.info(f"Processing {pset_path}")
-        pset = HiPointingSet(pset_path)
+        pset = HiPointingSet(pset_path, spin_phase=spin_phase)
 
         # Background rate and uncertainty are exposure time weighted means in
         # the map.
@@ -118,7 +129,7 @@ def generate_hi_map(
             pset.data[var] *= pset.data["exposure_factor"]
 
         # Project (bin) the PSET variables into the map pixels
-        rect_map.project_pset_values_to_map(
+        output_map.project_pset_values_to_map(
             pset,
             ["counts", "exposure_factor", "bg_rates", "bg_rates_unc", "obs_date"],
         )
@@ -127,35 +138,45 @@ def generate_hi_map(
     # Allow divide by zero to fill set pixels with zero exposure time to NaN
     with np.errstate(divide="ignore"):
         for var in VARS_TO_EXPOSURE_TIME_AVERAGE:
-            rect_map.data_1d[var] /= rect_map.data_1d["exposure_factor"]
+            output_map.data_1d[var] /= output_map.data_1d["exposure_factor"]
 
-    rect_map.data_1d.update(calculate_ena_signal_rates(rect_map.data_1d))
-    rect_map.data_1d.update(
+    output_map.data_1d.update(calculate_ena_signal_rates(output_map.data_1d))
+    output_map.data_1d.update(
         calculate_ena_intensity(
-            rect_map.data_1d, geometric_factors_path, esa_energies_path
+            output_map.data_1d, geometric_factors_path, esa_energies_path
         )
     )
 
-    rect_map.data_1d["obs_date"].data = rect_map.data_1d["obs_date"].data.astype(
+    output_map.data_1d["obs_date"].data = output_map.data_1d["obs_date"].data.astype(
         np.int64
     )
     # TODO: Figure out how to compute obs_date_range (stddev of obs_date)
-    rect_map.data_1d["obs_date_range"] = xr.zeros_like(rect_map.data_1d["obs_date"])
+    output_map.data_1d["obs_date_range"] = xr.zeros_like(output_map.data_1d["obs_date"])
 
     # Rename and convert coordinate from esa_energy_step energy
-    # TODO: the correct conversion from esa_energy_step to esa_energy
-    esa_energy_step_conversion = (np.arange(10, dtype=float) + 1) * 1000
-    rect_map.data_1d = rect_map.data_1d.rename({"esa_energy_step": "energy"})
-    rect_map.data_1d = rect_map.data_1d.assign_coords(
-        energy=esa_energy_step_conversion[rect_map.data_1d["energy"].values]
+    esa_df = esa_energy_df(
+        esa_energies_path, output_map.data_1d["esa_energy_step"].data
+    )
+    output_map.data_1d = output_map.data_1d.rename({"esa_energy_step": "energy"})
+    output_map.data_1d = output_map.data_1d.assign_coords(
+        energy=esa_df["nominal_central_energy"].values
+    )
+    # Set the energy_step_delta values to the energy bandpass half-width-half-max
+    energy_delta = esa_df["bandpass_fwhm"].values / 2
+    output_map.data_1d["energy_delta_minus"] = xr.DataArray(
+        energy_delta,
+        name="energy_delta_minus",
+        dims=["energy"],
+    )
+    output_map.data_1d["energy_delta_plus"] = xr.DataArray(
+        energy_delta,
+        name="energy_delta_plus",
+        dims=["energy"],
     )
-    # Set the energy_step_delta values
-    # TODO: get the correct energy delta values (they are set to NaN) in
-    #    rect_map.build_cdf_dataset()
 
-    rect_map.data_1d = rect_map.data_1d.drop("esa_energy_step_label")
+    output_map.data_1d = output_map.data_1d.drop("esa_energy_step_label")
 
-    return rect_map
+    return output_map
 
 
 def calculate_ena_signal_rates(map_ds: xr.Dataset) -> dict[str, xr.DataArray]:
@@ -214,33 +235,31 @@ def calculate_ena_intensity(
     geometric_factors_path : str or pathlib.Path
         Where to get the geometric factors from.
     esa_energies_path : str or pathlib.Path
-        Where to get the energies from.
+        Where to get the esa energies, energy deltas, and geometric factors.
 
     Returns
     -------
     intensity_vars : dict[str, xarray.DataArray]
         ENA Intensity with statistical and systematic uncertainties.
     """
-    # TODO: Implement geometric factor lookup
-    if geometric_factors_path:
-        raise NotImplementedError
-    geometric_factor = xr.DataArray(
-        np.ones((map_ds["esa_energy_step"].size, map_ds["calibration_prod"].size)),
-        coords=[map_ds["esa_energy_step"], map_ds["calibration_prod"]],
+    # read calibration product configuration file
+    cal_prod_df = CalibrationProductConfig.from_csv(geometric_factors_path)
+    # reindex_like removes esa_energy_steps and calibration products not in the
+    # map_ds esa_energy_step and calibration_product coordinates
+    geometric_factor = cal_prod_df.to_xarray().reindex_like(map_ds)["geometric_factor"]
+    geometric_factor = geometric_factor.transpose(
+        *[coord for coord in map_ds.coords if coord in geometric_factor.coords]
     )
-    # TODO: Implement esa energies lookup
-    if esa_energies_path:
-        raise NotImplementedError
-    esa_energy = xr.ones_like(map_ds["esa_energy_step"])
+    energy_df = esa_energy_df(esa_energies_path, map_ds["esa_energy_step"].data)
+    esa_energy = energy_df.to_xarray()["nominal_central_energy"]
 
     # Convert ENA Signal Rate to Flux
+    flux_conversion_divisor = geometric_factor * esa_energy
     intensity_vars = {
-        "ena_intensity": map_ds["ena_signal_rates"] / (geometric_factor * esa_energy),
+        "ena_intensity": map_ds["ena_signal_rates"] / flux_conversion_divisor,
         "ena_intensity_stat_unc": map_ds["ena_signal_rate_stat_unc"]
-        / geometric_factor
-        / esa_energy,
-        "ena_intensity_sys_err": map_ds["bg_rates_unc"]
-        / (geometric_factor * esa_energy),
+        / flux_conversion_divisor,
+        "ena_intensity_sys_err": map_ds["bg_rates_unc"] / flux_conversion_divisor,
     }
 
     # TODO: Correctly implement combining of calibration products. For now, just sum
@@ -259,3 +278,28 @@ def calculate_ena_intensity(
     )
 
     return intensity_vars
+
+
+def esa_energy_df(
+    esa_energies_path: str | Path, esa_energy_steps: np.ndarray
+) -> pd.DataFrame:
+    """
+    Lookup the nominal central energy values for given esa energy steps.
+
+    Parameters
+    ----------
+    esa_energies_path : str or pathlib.Path
+        Location of the calibration csv file containing the lookup data.
+    esa_energy_steps : numpy.ndarray
+        The ESA energy steps to get energies for.
+
+    Returns
+    -------
+    esa_energies_df: pandas.DataFrame
+        Full data frame from the csv file filtered to only include the
+        esa_energy_steps input.
+    """
+    esa_energies_lut = pd.read_csv(
+        esa_energies_path, comment="#", index_col="esa_energy_step"
+    )
+    return esa_energies_lut.loc[esa_energy_steps]

imap_processing/hi/utils.py

@@ -1,12 +1,15 @@
 """IMAP-Hi utils functions."""
 
+from __future__ import annotations
+
 import re
-from collections.abc import Sequence
+from collections.abc import Iterable, Sequence
 from dataclasses import dataclass
 from enum import IntEnum
-from typing import Optional, Union
+from pathlib import Path
 
 import numpy as np
+import pandas as pd
 import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
@@ -15,11 +18,13 @@ from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 class HIAPID(IntEnum):
     """Create ENUM for apid."""
 
+    H45_MEMDMP = 740
     H45_APP_NHK = 754
     H45_SCI_CNT = 769
     H45_SCI_DE = 770
     H45_DIAG_FEE = 772
 
+    H90_MEMDMP = 804
     H90_APP_NHK = 818
     H90_SCI_CNT = 833
     H90_SCI_DE = 834
@@ -100,9 +105,9 @@ def parse_sensor_number(full_string: str) -> int:
 def full_dataarray(
     name: str,
     attrs: dict,
-    coords: Optional[dict[str, xr.DataArray]] = None,
-    shape: Optional[Union[int, Sequence[int]]] = None,
-    fill_value: Optional[float] = None,
+    coords: dict[str, xr.DataArray] | None = None,
+    shape: int | Sequence[int] | None = None,
+    fill_value: float | None = None,
 ) -> xr.DataArray:
     """
     Generate an empty xarray.DataArray with appropriate attributes.
@@ -158,9 +163,9 @@ def full_dataarray(
 
 def create_dataset_variables(
     variable_names: list[str],
-    variable_shape: Optional[Union[int, Sequence[int]]] = None,
-    coords: Optional[dict[str, xr.DataArray]] = None,
-    fill_value: Optional[float] = None,
+    variable_shape: int | Sequence[int] | None = None,
+    coords: dict[str, xr.DataArray] | None = None,
+    fill_value: float | None = None,
     att_manager_lookup_str: str = "{0}",
 ) -> dict[str, xr.DataArray]:
     """
@@ -247,3 +252,252 @@ class CoincidenceBitmap(IntEnum):
         matches = re.findall(pattern, detector_hit_str)
         # Sum the integer value assigned to the detector name for each match
         return sum(CoincidenceBitmap[m] for m in matches)
+
+
+class EsaEnergyStepLookupTable:
+    """Class for holding a esa_step to esa_energy lookup table."""
+
+    def __init__(self) -> None:
+        self.df = pd.DataFrame(
+            columns=["start_met", "end_met", "esa_step", "esa_energy_step"]
+        )
+        self._indexed = False
+
+        # Get the FILLVAL from the CDF attribute manager that will be returned
+        # for queries without matches
+        attr_mgr = ImapCdfAttributes()
+        attr_mgr.add_instrument_global_attrs("hi")
+        attr_mgr.add_instrument_variable_attrs(instrument="hi", level=None)
+        var_attrs = attr_mgr.get_variable_attributes(
+            "hi_de_esa_energy_step", check_schema=False
+        )
+        self._fillval = var_attrs["FILLVAL"]
+        self._esa_energy_step_dtype = var_attrs["dtype"]
+
+    def add_entry(
+        self, start_met: float, end_met: float, esa_step: int, esa_energy_step: int
+    ) -> None:
+        """
+        Add a single entry to the lookup table.
+
+        Parameters
+        ----------
+        start_met : float
+            Start mission elapsed time of the time range.
+        end_met : float
+            End mission elapsed time of the time range.
+        esa_step : int
+            ESA step value.
+        esa_energy_step : int
+            ESA energy step value to be stored.
+        """
+        new_row = pd.DataFrame(
+            {
+                "start_met": [start_met],
+                "end_met": [end_met],
+                "esa_step": [esa_step],
+                "esa_energy_step": [esa_energy_step],
+            }
+        )
+        self.df = pd.concat([self.df, new_row], ignore_index=True)
+        self._indexed = False
+
+    def _ensure_indexed(self) -> None:
+        """
+        Create index for faster queries if not already done.
+
+        Notes
+        -----
+        This method sorts the internal DataFrame by start_met and esa_step
+        for improved query performance.
+        """
+        if not self._indexed:
+            # Sort by start_met and esa_step for better query performance
+            self.df = self.df.sort_values(["start_met", "esa_step"]).reset_index(
+                drop=True
+            )
+            self._indexed = True
+
+    def query(
+        self,
+        query_met: float | Iterable[float],
+        esa_step: int | Iterable[float],
+    ) -> float | np.ndarray:
+        """
+        Query MET(s) and esa_step(s) to retrieve esa_energy_step(s).
+
+        Parameters
+        ----------
+        query_met : float or array_like
+            Mission elapsed time value(s) to query.
+            Can be a single float or array-like of floats.
+        esa_step : int or array_like
+            ESA step value(s) to match. Can be a single int or array-like of ints.
+            Must be same type (scalar or array-like) as query_met.
+
+        Returns
+        -------
+        float or numpy.ndarray
+            - If inputs are scalars: returns float (esa_energy_step)
+            - If inputs are array-like: returns numpy array of esa_energy_steps
+              with same length as inputs.
+              Contains FILLVAL for queries with no matches.
+
+        Raises
+        ------
+        ValueError
+            If one input is scalar and the other is array-like, or if both are
+            array-like but have different lengths.
+
+        Notes
+        -----
+        If multiple entries match a query, returns the first match found.
+        """
+        self._ensure_indexed()
+
+        # Check if inputs are scalars
+        is_scalar_met = np.isscalar(query_met)
+        is_scalar_step = np.isscalar(esa_step)
+
+        # Check for mismatched input types
+        if is_scalar_met != is_scalar_step:
+            raise ValueError(
+                "query_met and esa_step must both be scalars or both be array-like"
+            )
+
+        # Convert to arrays for uniform processing
+        query_mets = np.atleast_1d(query_met)
+        esa_steps = np.atleast_1d(esa_step)
+
+        # Ensure both arrays have the same shape
+        if query_mets.shape != esa_steps.shape:
+            raise ValueError(
+                "query_met and esa_step must have the same "
+                "length when both are array-like"
+            )
+
+        results = np.full_like(query_mets, self._fillval)
+
+        # Lookup esa_energy_steps for queries
+        for i, (qm, es) in enumerate(zip(query_mets, esa_steps, strict=False)):
+            mask = (
+                (self.df["start_met"] <= qm)
+                & (self.df["end_met"] >= qm)
+                & (self.df["esa_step"] == es)
+            )
+
+            matches = self.df[mask]
+            if not matches.empty:
+                results[i] = matches["esa_energy_step"].iloc[0]
+
+        # Return scalar for scalar inputs, array for array inputs
+        if is_scalar_met and is_scalar_step:
+            return results.astype(self._esa_energy_step_dtype)[0]
+        else:
+            return results.astype(self._esa_energy_step_dtype)
+
+
+@pd.api.extensions.register_dataframe_accessor("cal_prod_config")
+class CalibrationProductConfig:
+    """
+    Register custom accessor for calibration product configuration DataFrames.
+
+    Parameters
+    ----------
+    pandas_obj : pandas.DataFrame
+        Object to run validation and use accessor functions on.
+    """
+
+    index_columns = (
+        "calibration_prod",
+        "esa_energy_step",
+    )
+    tof_detector_pairs = ("ab", "ac1", "bc1", "c1c2")
+    required_columns = (
+        "coincidence_type_list",
+        *[
+            f"tof_{det_pair}_{limit}"
+            for det_pair in tof_detector_pairs
+            for limit in ["low", "high"]
+        ],
+    )
+
+    def __init__(self, pandas_obj: pd.DataFrame) -> None:
+        self._validate(pandas_obj)
+        self._obj = pandas_obj
+        self._add_coincidence_values_column()
+
+    def _validate(self, df: pd.DataFrame) -> None:
+        """
+        Validate the current configuration.
+
+        Parameters
+        ----------
+        df : pandas.DataFrame
+            Object to validate.
+
+        Raises
+        ------
+        AttributeError : If the dataframe does not pass validation.
+        """
+        for index_name in self.index_columns:
+            if index_name in df.index:
+                raise AttributeError(
+                    f"Required index {index_name} not present in dataframe."
+                )
+        # Verify that the Dataframe has all the required columns
+        for col in self.required_columns:
+            if col not in df.columns:
+                raise AttributeError(f"Required column {col} not present in dataframe.")
+        # TODO: Verify that the same ESA energy steps exist in all unique calibration
+        #   product numbers
+
+    def _add_coincidence_values_column(self) -> None:
+        """Generate and add the coincidence_type_values column to the dataframe."""
+        # Add a column that consists of the coincidence type strings converted
+        # to integer values
+        self._obj["coincidence_type_values"] = self._obj.apply(
+            lambda row: tuple(
+                CoincidenceBitmap.detector_hit_str_to_int(entry)
+                for entry in row["coincidence_type_list"]
+            ),
+            axis=1,
+        )
+
+    @classmethod
+    def from_csv(cls, path: str | Path) -> pd.DataFrame:
+        """
+        Read configuration CSV file into a pandas.DataFrame.
+
+        Parameters
+        ----------
+        path : str or pathlib.Path
+            Location of the Calibration Product configuration CSV file.
+
+        Returns
+        -------
+        dataframe : pandas.DataFrame
+            Validated calibration product configuration data frame.
+        """
+        df = pd.read_csv(
+            path,
+            index_col=cls.index_columns,
+            converters={"coincidence_type_list": lambda s: tuple(s.split("|"))},
+            comment="#",
+        )
+        # Force the _init_ method to run by using the namespace
+        _ = df.cal_prod_config.number_of_products
+        return df
+
+    @property
+    def number_of_products(self) -> int:
+        """
+        Get the number of calibration products in the current configuration.
+
+        Returns
+        -------
+        number_of_products : int
+            The maximum number of calibration products defined in the list of
+            calibration product definitions.
+        """
+        return len(self._obj.index.unique(level="calibration_prod"))

imap_processing/hit/l0/constants.py

@@ -114,6 +114,9 @@ FLAG_PATTERN = np.array([1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
 # Define size of science frame (num of packets)
 FRAME_SIZE = len(FLAG_PATTERN)
 
+# Mod 10 pattern
+MOD_10_PATTERN = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
+
 # Define the number of bits in the mantissa and exponent for
 # decompressing data
 MANTISSA_BITS = 12

imap_processing/hit/l0/decom_hit.py

@@ -260,12 +260,9 @@ def assemble_science_frames(sci_dataset: xr.Dataset) -> xr.Dataset:
         height event data per valid science frame added as new
         data variables.
     """
-    # TODO: Figure out how to handle partial science frames at the
-    #  beginning and end of CCSDS files. These science frames are split
-    #  across CCSDS files and still need to be processed with packets
-    #  from the previous file. Only discard incomplete science frames
-    #  in the middle of the CCSDS file. The code currently skips all
-    #  incomplete science frames.
+    # TODO: The code currently skips all incomplete science frames.
+    #  Only discard incomplete science frames in the middle of the CCSDS file or
+    #  use fill values?
 
     # Convert sequence flags and counters to NumPy arrays for vectorized operations
     seq_flgs = sci_dataset.seq_flgs.values