imap-processing 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

imap_processing/codice/utils.py

@@ -5,7 +5,40 @@ This module contains utility classes and functions that are used by various
 other CoDICE processing modules.
 """
 
+import json
+from dataclasses import dataclass
 from enum import IntEnum
+from pathlib import Path
+
+import numpy as np
+
+from imap_processing.spice.time import met_to_ttj2000ns
+
+
+@dataclass
+class ViewTabInfo:
+    """
+    Class to hold view table information.
+
+    Attributes
+    ----------
+    apid : int
+        The APID for the packet.
+    collapse_table : int
+        Collapse table id used to determine the collapse pattern.
+    sensor : int
+        Sensor id (0 for LO, 1 for HI).
+    three_d_collapsed : int
+        The 3D collapsed value from the LUT.
+    view_id : int
+        The view identifier from the packet.
+    """
+
+    apid: int
+    collapse_table: int
+    sensor: int
+    three_d_collapsed: int
+    view_id: int
 
 
 class CODICEAPID(IntEnum):
@@ -57,3 +90,240 @@ class CoDICECompression(IntEnum):
     LOSSY_A_LOSSLESS = 4
     LOSSY_B_LOSSLESS = 5
     PACK_24_BIT = 6
+
+
+def read_sci_lut(file_path: Path, table_id: str) -> dict:
+    """
+    Read the SCI-LUT JSON file for a specific table ID.
+
+    Parameters
+    ----------
+    file_path : pathlib.Path
+        Path to the SCI-LUT JSON file.
+    table_id : str
+        Table identifier to extract from the JSON.
+
+    Returns
+    -------
+    dict
+        The SCI-LUT data for the specified table id.
+    """
+    sci_lut_data = json.loads(file_path.read_text()).get(f"{table_id}")
+    if sci_lut_data is None:
+        raise ValueError(f"SCI-LUT file does not have data for table ID {table_id}.")
+    return sci_lut_data
+
+
+def get_view_tab_info(json_data: dict, view_id: int, apid: int) -> dict:
+    """
+    Get the view table information for a specific view and APID.
+
+    Parameters
+    ----------
+    json_data : dict
+        The JSON data loaded from the SCI-LUT file.
+    view_id : int
+        The view ID from the packet.
+    apid : int
+        The APID from the packet.
+
+    Returns
+    -------
+    dict
+        The view table information containing details like sensor,
+        collapse_table, data_product, etc.
+    """
+    apid_hex = f"0x{apid:X}"
+    # This is how we get view information that will be used to get
+    # collapse pattern:
+    #  table_id -> view_tab -> (view_id, apid) -> sensor -> collapse_table
+    view_tab = json_data.get("view_tab").get(f"({view_id}, {apid_hex})")
+    return view_tab
+
+
+def get_collapse_pattern_shape(
+    json_data: dict, sensor_id: int, collapse_table_id: int
+) -> tuple[int, ...]:
+    """
+    Get the collapse pattern for a specific sensor id and collapse table id.
+
+    Parameters
+    ----------
+    json_data : dict
+        The JSON data loaded from the SCI-LUT file.
+    sensor_id : int
+        Sensor identifier (0 for LO, 1 for HI).
+    collapse_table_id : int
+        Collapse table id to look up in the SCI-LUT.
+
+    Returns
+    -------
+    tuple[int, ...]
+        The reduced shape describing the collapsed pattern. Examples:
+        ``(1,)`` for a fully collapsed 1-D pattern or ``(N, M)`` for a
+        reduced 2-D pattern.
+    """
+    sensor = "lo" if sensor_id == 0 else "hi"
+    collapse_matrix = np.array(
+        json_data[f"collapse_{sensor}"][f"{collapse_table_id}"]["matrix"]
+    )
+
+    # Analyze the collapse pattern matrix to determine its reduced shape.
+    # Steps:
+    # - Extract non-zero elements from the matrix.
+    # - Reshape to group unique non-zero rows and columns.
+    # - If all non-zero values are identical, return (1,) for a fully collapsed pattern.
+    # - Otherwise, compute the number of unique rows and columns to describe the
+    #   reduced shape.
+    non_zero_data = np.where(collapse_matrix != 0)
+    non_zero_reformatted = collapse_matrix[non_zero_data].reshape(
+        np.unique(non_zero_data[0]).size, np.unique(non_zero_data[1]).size
+    )
+
+    if np.unique(non_zero_reformatted).size == 1:
+        # all non-zero values are identical means -> fully collapsed
+        return (1,)
+
+    # If not fully collapsed, find repeated patterns in rows and columns
+    # to reduce shape further.
+    unique_rows = np.unique(non_zero_reformatted, axis=0)
+    unique_columns = np.unique(non_zero_reformatted, axis=1)
+    # Unique spin sectors and instrument azimuths to unpack data
+    unique_spin_sectors = unique_columns.shape[1]
+    unique_inst_azs = unique_rows.shape[0]
+    return (unique_spin_sectors, unique_inst_azs)
+
+
+def index_to_position(
+    json_data: dict, sensor_id: int, collapse_table_id: int
+) -> np.ndarray:
+    """
+    Get the indices of non-zero unique rows in the collapse pattern matrix.
+
+    Parameters
+    ----------
+    json_data : dict
+        The JSON data loaded from the SCI-LUT file.
+    sensor_id : int
+        Sensor identifier (0 for LO, 1 for HI).
+    collapse_table_id : int
+        Collapse table id to look up in the SCI-LUT.
+
+    Returns
+    -------
+    np.ndarray
+        Array of indices corresponding to non-zero unique rows.
+    """
+    sensor = "lo" if sensor_id == 0 else "hi"
+    collapse_matrix = np.array(
+        json_data[f"collapse_{sensor}"][f"{collapse_table_id}"]["matrix"]
+    )
+
+    # Find unique non-zero rows and their original indices
+    non_zero_row_mask = np.any(collapse_matrix != 0, axis=1)
+    non_zero_rows = collapse_matrix[non_zero_row_mask]
+    _, unique_indices = np.unique(non_zero_rows, axis=0, return_index=True)
+    non_zero_row_indices = np.flatnonzero(non_zero_row_mask)[unique_indices]
+    return non_zero_row_indices
+
+
+def get_codice_epoch_time(
+    acq_start_seconds: np.ndarray,
+    acq_start_subseconds: np.ndarray,
+    spin_period: np.ndarray,
+    view_tab_obj: ViewTabInfo,
+) -> tuple[np.ndarray, np.ndarray]:
+    """
+    Calculate center time and delta.
+
+    Parameters
+    ----------
+    acq_start_seconds : np.ndarray
+        Array of acquisition start seconds.
+    acq_start_subseconds : np.ndarray
+        Array of acquisition start subseconds.
+    spin_period : np.ndarray
+        Array of spin periods.
+    view_tab_obj : ViewTabInfo
+        The view table information object. It contains information such as sensor ID
+        and three_d_collapsed value and others.
+
+    Returns
+    -------
+    tuple[np.ndarray, np.ndarray]
+        (center_times, delta_times).
+    """
+    # If Lo sensor
+    if view_tab_obj.sensor == 0:
+        # Lo sensor, we need to set spins to be constant.
+        # 32 half spins makes full 16 spins for all non direct event products.
+        # But Lo direct event's spins is also 16 spins. Because of that, we can use
+        # the same calculation for all Lo products.
+        num_spins = 16.0
+    # If Hi sensor and Direct Event product
+    elif view_tab_obj.sensor == 1 and view_tab_obj.apid == CODICEAPID.COD_HI_PHA:
+        # Use constant 16 spins for Hi PHA
+        num_spins = 16.0
+    # If Non-Direct Event Hi product
+    else:
+        # Use 3d_collapsed value from LUT for other Hi products
+        num_spins = view_tab_obj.three_d_collapsed
+
+    # Units of 'spin ticks', where one 'spin tick' equals 320 microseconds.
+    # It takes multiple spins to collect data for a view.
+    spin_period_ns = spin_period.astype(np.float64) * 320 * 1e3  # Convert to ns
+    delta_times = (num_spins * spin_period_ns) / 2
+    # subseconds need to converted to seconds using this formula per CoDICE team:
+    #   subseconds / 65536 gives seconds
+    center_times_seconds = (
+        acq_start_seconds + acq_start_subseconds / 65536 + (delta_times / 1e9)
+    )
+
+    return met_to_ttj2000ns(center_times_seconds), delta_times
+
+
+def calculate_acq_time_per_step(low_stepping_tab: dict) -> np.ndarray:
+    """
+    Calculate acquisition time per step from low stepping table.
+
+    Parameters
+    ----------
+    low_stepping_tab : dict
+        The low stepping table from the SCI-LUT JSON.
+
+    Returns
+    -------
+    np.ndarray
+        Array of acquisition times per step of shape (num_esa_steps,).
+    """
+    # These tunable values are used to calculate acquisition time per step
+    tunable_values = low_stepping_tab["tunable_values"]
+
+    # pre-calculate values
+    sector_time = tunable_values["spin_time_ms"] / tunable_values["num_sectors_ms"]
+    sector_margin_ms = tunable_values["sector_margin_ms"]
+    dwell_fraction = tunable_values["dwell_fraction_percentage"]
+    min_hv_settle_ms = tunable_values["min_hv_settle_ms"]
+    max_hv_settle_ms = tunable_values["max_hv_settle_ms"]
+    num_steps_data = np.array(
+        low_stepping_tab["num_steps"].get("data"), dtype=np.float64
+    )
+    # Total non-acquisition time is in column (BD) of science LUT
+    dwell_fraction_percentage = float(sector_time) * (100.0 - dwell_fraction) / 100.0
+
+    # Calculate HV settle time per step not adjusted for Min/Max.
+    # It's in column (BF) of science LUT.
+    non_adjusted_hv_settle_per_step = (
+        dwell_fraction_percentage - sector_margin_ms
+    ) / num_steps_data
+    hv_settle_per_step = np.minimum(
+        np.maximum(non_adjusted_hv_settle_per_step, min_hv_settle_ms), max_hv_settle_ms
+    )
+
+    # acquisition time per step in milliseconds
+    # sector_time - sector_margin_ms / num_steps - hv_settle_per_step
+    acq_time_per_step = (
+        (sector_time - sector_margin_ms) / num_steps_data
+    ) - hv_settle_per_step
+    # Convert to seconds
+    return acq_time_per_step / 1e3

imap_processing/ena_maps/ena_maps.py

@@ -647,28 +647,21 @@ class HiPointingSet(LoHiBasePointingSet):
     ----------
     dataset : xarray.Dataset | str | Path
         Hi L1C pointing set data loaded in a xarray.DataArray.
-    spin_phase : str
-        Include ENAs from "full", "ram" or "anti-ram" phases of the spin.
     """
 
-    def __init__(self, dataset: xr.Dataset | str | Path, spin_phase: str):
-        super().__init__(dataset, spice_reference_frame=geometry.SpiceFrame.ECLIPJ2000)
+    def __init__(self, dataset: xr.Dataset | str | Path):
+        super().__init__(dataset, spice_reference_frame=geometry.SpiceFrame.IMAP_HAE)
+
+        self.spatial_coords = ("spin_angle_bin",)
 
-        # Filter out ENAs from non-selected portions of the spin.
-        if spin_phase not in ["full", "ram", "anti"]:
-            raise ValueError(f"Unrecognized spin_phase value: {spin_phase}.")
+        # Naively generate the ram_mask variable assuming spacecraft frame
+        # binning. The ram_mask variable gets updated in the CG correction
+        # code if the CG correction is applied.
+        ram_mask = xr.zeros_like(self.data["spin_angle_bin"], dtype=bool)
         # ram only includes spin-phase interval [0, 0.5)
         # which is the first half of the spin_angle_bins
-        elif spin_phase == "ram":
-            self.data = self.data.isel(
-                spin_angle_bin=slice(0, self.data["spin_angle_bin"].data.size // 2)
-            )
-        # anti-ram includes spin-phase interval [0.5, 1)
-        # which is the second half of the spin_angle_bins
-        elif spin_phase == "anti":
-            self.data = self.data.isel(
-                spin_angle_bin=slice(self.data["spin_angle_bin"].data.size // 2, None)
-            )
+        ram_mask[slice(0, self.data["spin_angle_bin"].data.size // 2)] = True
+        self.data["ram_mask"] = ram_mask
 
         # Rename some PSET vars to match L2 variables
         self.data = self.data.rename(
@@ -684,8 +677,6 @@ class HiPointingSet(LoHiBasePointingSet):
             self.data["exposure_factor"], self.data["epoch"].values[0]
         )
 
-        self.spatial_coords = ("spin_angle_bin",)
-
         # Update az_el_points using the base class method
         self.update_az_el_points()
 
@@ -810,12 +801,12 @@ class AbstractSkyMap(ABC):
         """
         return self.az_el_points.shape[0]
 
-    def project_pset_values_to_map(
+    def project_pset_values_to_map(  # noqa: PLR0912
         self,
         pointing_set: PointingSet,
         value_keys: list[str] | None = None,
         index_match_method: IndexMatchMethod = IndexMatchMethod.PUSH,
-        pset_valid_mask: NDArray | None = None,
+        pset_valid_mask: NDArray | xr.DataArray | None = None,
     ) -> None:
         """
         Project a pointing set's values to the map grid.
@@ -837,7 +828,7 @@ class AbstractSkyMap(ABC):
         index_match_method : IndexMatchMethod, optional
             The method of index matching to use for all values.
             Default is IndexMatchMethod.PUSH.
-        pset_valid_mask : NDArray, optional
+        pset_valid_mask : xarray.DataArray or NDArray, optional
             A boolean mask of shape (number of pointing set pixels,) indicating
             which pixels in the pointing set should be considered valid for projection.
             If None, all pixels are considered valid. Default is None.
@@ -849,11 +840,12 @@ class AbstractSkyMap(ABC):
         """
         if value_keys is None:
             value_keys = list(pointing_set.data.data_vars.keys())
-        for value_key in value_keys:
-            if value_key not in pointing_set.data.data_vars:
-                raise ValueError(f"Value key {value_key} not found in pointing set.")
+
+        if missing_keys := set(value_keys) - set(pointing_set.data.data_vars):
+            raise KeyError(f"Value keys not found in pointing set: {missing_keys}")
 
         if pset_valid_mask is None:
+            logger.debug("No pset_valid_mask provided, using all pixels as valid.")
             pset_valid_mask = np.ones(pointing_set.num_points, dtype=bool)
 
         if index_match_method is IndexMatchMethod.PUSH:
@@ -876,9 +868,12 @@ class AbstractSkyMap(ABC):
             )
 
         for value_key in value_keys:
+            if value_key not in pointing_set.data.data_vars:
+                raise ValueError(f"Value key {value_key} not found in pointing set.")
+
             # If multiple spatial axes present
             # (i.e (az, el) for rectangular coordinate PSET),
-            # flatten them in the values array to match the raveled indices
+            # stack them into a single coordinate to match the raveled indices
             raveled_pset_data = pointing_set.data[value_key].stack(
                 {CoordNames.GENERIC_PIXEL.value: pointing_set.spatial_coords}
             )
@@ -907,13 +902,22 @@ class AbstractSkyMap(ABC):
                 data_bc, indices_bc = xr.broadcast(
                     raveled_pset_data, matched_indices_push
                 )
+                # If the valid mask is a xr.DataArray, broadcast it to the same shape
+                if isinstance(pset_valid_mask, xr.DataArray):
+                    stacked_valid_mask = pset_valid_mask.stack(
+                        {CoordNames.GENERIC_PIXEL.value: pointing_set.spatial_coords}
+                    )
+                    _, pset_valid_mask_bc = xr.broadcast(data_bc, stacked_valid_mask)
+                    pset_valid_mask_values = pset_valid_mask_bc.values
+                else:
+                    pset_valid_mask_values = pset_valid_mask
 
                 # Extract numpy arrays for bincount operation
                 pointing_projected_values = map_utils.bin_single_array_at_indices(
                     value_array=data_bc.values,
                     projection_grid_shape=self.binning_grid_shape,
                     projection_indices=indices_bc.values,
-                    input_valid_mask=pset_valid_mask,
+                    input_valid_mask=pset_valid_mask_values,
                 )
                 # TODO: we may need to allow for unweighted/weighted means here by
                 # dividing pointing_projected_values by some binned weights.
@@ -934,10 +938,6 @@ class AbstractSkyMap(ABC):
                 self.data_1d[value_key].values[..., valid_map_mask] += (
                     pointing_projected_values
                 )
-            else:
-                raise NotImplementedError(
-                    "Only PUSH and PULL index matching methods are supported."
-                )
 
         # TODO: The max epoch needs to include the pset duration. Right now it
         #     is just capturing the start epoch. See issue #1747
@@ -1266,12 +1266,13 @@ class RectangularSkyMap(AbstractSkyMap):
             coords={**self.non_spatial_coords, **self.spatial_coords},
         )
 
-    def build_cdf_dataset(
+    def build_cdf_dataset(  # noqa: PLR0912
         self,
         instrument: str,
         level: str,
         descriptor: str,
         sensor: str | None = None,
+        drop_vars_with_no_attributes: bool = True,
     ) -> xr.Dataset:
         """
         Format the data into a xarray.Dataset and add required CDF variables.
@@ -1286,6 +1287,12 @@ class RectangularSkyMap(AbstractSkyMap):
             Descriptor for filename.
         sensor : str, optional
             Sensor number "45" or "90".
+        drop_vars_with_no_attributes : bool, optional
+            Default behavior is to drop any dataset variables that don't have
+            attributes defined in the CDF attribute manager. This ensures that
+            the output CDF doesn't have any of the intermedeiate variables left
+            over from computations. Sometimes, it is useful to output the
+            intermedeiate variables. To do so, set this to False.
 
         Returns
         -------
@@ -1389,13 +1396,18 @@ class RectangularSkyMap(AbstractSkyMap):
                     variable_name=name,
                     check_schema=check_schema,
                 )
-            except KeyError as e:
-                raise KeyError(
-                    f"Attributes for variable {name} not found in "
-                    f"loaded variable attributes."
-                ) from e
-
-            cdf_ds[name].attrs.update(var_attrs)
+                cdf_ds[name].attrs.update(var_attrs)
+            except KeyError:
+                if drop_vars_with_no_attributes:
+                    logger.debug(
+                        f"Dropping variable '{name}' that has no attributes defined."
+                    )
+                    cdf_ds = cdf_ds.drop_vars(name)
+                else:
+                    logger.debug(
+                        f"Variable '{name}' has no attributes defined. It will "
+                        f"be included in the output dataset with no attributes."
+                    )
 
         # Manually adjust epoch attributes
         cdf_ds["epoch"].attrs.update(