imap-processing 0.18.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

imap_processing/hi/utils.py

@@ -1,12 +1,12 @@
 """IMAP-Hi utils functions."""
 
 import re
-from collections.abc import Sequence
+from collections.abc import Iterable, Sequence
 from dataclasses import dataclass
 from enum import IntEnum
-from typing import Optional, Union
 
 import numpy as np
+import pandas as pd
 import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
@@ -15,11 +15,13 @@ from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 class HIAPID(IntEnum):
     """Create ENUM for apid."""
 
+    H45_MEMDMP = 740
     H45_APP_NHK = 754
     H45_SCI_CNT = 769
     H45_SCI_DE = 770
     H45_DIAG_FEE = 772
 
+    H90_MEMDMP = 804
     H90_APP_NHK = 818
     H90_SCI_CNT = 833
     H90_SCI_DE = 834
@@ -100,9 +102,9 @@ def parse_sensor_number(full_string: str) -> int:
 def full_dataarray(
     name: str,
     attrs: dict,
-    coords: Optional[dict[str, xr.DataArray]] = None,
-    shape: Optional[Union[int, Sequence[int]]] = None,
-    fill_value: Optional[float] = None,
+    coords: dict[str, xr.DataArray] | None = None,
+    shape: int | Sequence[int] | None = None,
+    fill_value: float | None = None,
 ) -> xr.DataArray:
     """
     Generate an empty xarray.DataArray with appropriate attributes.
@@ -158,9 +160,9 @@ def full_dataarray(
 
 def create_dataset_variables(
     variable_names: list[str],
-    variable_shape: Optional[Union[int, Sequence[int]]] = None,
-    coords: Optional[dict[str, xr.DataArray]] = None,
-    fill_value: Optional[float] = None,
+    variable_shape: int | Sequence[int] | None = None,
+    coords: dict[str, xr.DataArray] | None = None,
+    fill_value: float | None = None,
     att_manager_lookup_str: str = "{0}",
 ) -> dict[str, xr.DataArray]:
     """
@@ -247,3 +249,146 @@ class CoincidenceBitmap(IntEnum):
         matches = re.findall(pattern, detector_hit_str)
         # Sum the integer value assigned to the detector name for each match
         return sum(CoincidenceBitmap[m] for m in matches)
+
+
+class EsaEnergyStepLookupTable:
+    """Class for holding a esa_step to esa_energy lookup table."""
+
+    def __init__(self) -> None:
+        self.df = pd.DataFrame(
+            columns=["start_met", "end_met", "esa_step", "esa_energy_step"]
+        )
+        self._indexed = False
+
+        # Get the FILLVAL from the CDF attribute manager that will be returned
+        # for queries without matches
+        attr_mgr = ImapCdfAttributes()
+        attr_mgr.add_instrument_global_attrs("hi")
+        attr_mgr.add_instrument_variable_attrs(instrument="hi", level=None)
+        var_attrs = attr_mgr.get_variable_attributes(
+            "hi_de_esa_energy_step", check_schema=False
+        )
+        self._fillval = var_attrs["FILLVAL"]
+        self._esa_energy_step_dtype = var_attrs["dtype"]
+
+    def add_entry(
+        self, start_met: float, end_met: float, esa_step: int, esa_energy_step: int
+    ) -> None:
+        """
+        Add a single entry to the lookup table.
+
+        Parameters
+        ----------
+        start_met : float
+            Start mission elapsed time of the time range.
+        end_met : float
+            End mission elapsed time of the time range.
+        esa_step : int
+            ESA step value.
+        esa_energy_step : int
+            ESA energy step value to be stored.
+        """
+        new_row = pd.DataFrame(
+            {
+                "start_met": [start_met],
+                "end_met": [end_met],
+                "esa_step": [esa_step],
+                "esa_energy_step": [esa_energy_step],
+            }
+        )
+        self.df = pd.concat([self.df, new_row], ignore_index=True)
+        self._indexed = False
+
+    def _ensure_indexed(self) -> None:
+        """
+        Create index for faster queries if not already done.
+
+        Notes
+        -----
+        This method sorts the internal DataFrame by start_met and esa_step
+        for improved query performance.
+        """
+        if not self._indexed:
+            # Sort by start_met and esa_step for better query performance
+            self.df = self.df.sort_values(["start_met", "esa_step"]).reset_index(
+                drop=True
+            )
+            self._indexed = True
+
+    def query(
+        self,
+        query_met: float | Iterable[float],
+        esa_step: int | Iterable[float],
+    ) -> float | np.ndarray:
+        """
+        Query MET(s) and esa_step(s) to retrieve esa_energy_step(s).
+
+        Parameters
+        ----------
+        query_met : float or array_like
+            Mission elapsed time value(s) to query.
+            Can be a single float or array-like of floats.
+        esa_step : int or array_like
+            ESA step value(s) to match. Can be a single int or array-like of ints.
+            Must be same type (scalar or array-like) as query_met.
+
+        Returns
+        -------
+        float or numpy.ndarray
+            - If inputs are scalars: returns float (esa_energy_step)
+            - If inputs are array-like: returns numpy array of esa_energy_steps
+              with same length as inputs.
+              Contains FILLVAL for queries with no matches.
+
+        Raises
+        ------
+        ValueError
+            If one input is scalar and the other is array-like, or if both are
+            array-like but have different lengths.
+
+        Notes
+        -----
+        If multiple entries match a query, returns the first match found.
+        """
+        self._ensure_indexed()
+
+        # Check if inputs are scalars
+        is_scalar_met = np.isscalar(query_met)
+        is_scalar_step = np.isscalar(esa_step)
+
+        # Check for mismatched input types
+        if is_scalar_met != is_scalar_step:
+            raise ValueError(
+                "query_met and esa_step must both be scalars or both be array-like"
+            )
+
+        # Convert to arrays for uniform processing
+        query_mets = np.atleast_1d(query_met)
+        esa_steps = np.atleast_1d(esa_step)
+
+        # Ensure both arrays have the same shape
+        if query_mets.shape != esa_steps.shape:
+            raise ValueError(
+                "query_met and esa_step must have the same "
+                "length when both are array-like"
+            )
+
+        results = np.full_like(query_mets, self._fillval)
+
+        # Lookup esa_energy_steps for queries
+        for i, (qm, es) in enumerate(zip(query_mets, esa_steps, strict=False)):
+            mask = (
+                (self.df["start_met"] <= qm)
+                & (self.df["end_met"] >= qm)
+                & (self.df["esa_step"] == es)
+            )
+
+            matches = self.df[mask]
+            if not matches.empty:
+                results[i] = matches["esa_energy_step"].iloc[0]
+
+        # Return scalar for scalar inputs, array for array inputs
+        if is_scalar_met and is_scalar_step:
+            return results.astype(self._esa_energy_step_dtype)[0]
+        else:
+            return results.astype(self._esa_energy_step_dtype)

imap_processing/hit/l0/constants.py

@@ -114,6 +114,9 @@ FLAG_PATTERN = np.array([1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
 # Define size of science frame (num of packets)
 FRAME_SIZE = len(FLAG_PATTERN)
 
+# Mod 10 pattern
+MOD_10_PATTERN = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
+
 # Define the number of bits in the mantissa and exponent for
 # decompressing data
 MANTISSA_BITS = 12

imap_processing/hit/l0/decom_hit.py

@@ -260,12 +260,9 @@ def assemble_science_frames(sci_dataset: xr.Dataset) -> xr.Dataset:
         height event data per valid science frame added as new
         data variables.
     """
-    # TODO: Figure out how to handle partial science frames at the
-    #  beginning and end of CCSDS files. These science frames are split
-    #  across CCSDS files and still need to be processed with packets
-    #  from the previous file. Only discard incomplete science frames
-    #  in the middle of the CCSDS file. The code currently skips all
-    #  incomplete science frames.
+    # TODO: The code currently skips all incomplete science frames.
+    #  Only discard incomplete science frames in the middle of the CCSDS file or
+    #  use fill values?
 
     # Convert sequence flags and counters to NumPy arrays for vectorized operations
     seq_flgs = sci_dataset.seq_flgs.values

imap_processing/hit/l1a/hit_l1a.py

@@ -1,6 +1,8 @@
 """Decommutate HIT CCSDS data and create L1a data products."""
 
 import logging
+from datetime import datetime
+from pathlib import Path
 
 import numpy as np
 import xarray as xr
@@ -19,6 +21,11 @@ from imap_processing.hit.l0.constants import (
     ZENITH_ANGLES,
 )
 from imap_processing.hit.l0.decom_hit import decom_hit
+from imap_processing.spice.time import (
+    et_to_datetime64,
+    met_to_datetime64,
+    ttj2000ns_to_et,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -28,7 +35,7 @@ logger = logging.getLogger(__name__)
 fillval = -9223372036854775808
 
 
-def hit_l1a(packet_file: str) -> list[xr.Dataset]:
+def hit_l1a(packet_file: Path, packet_date: str | None) -> list[xr.Dataset]:
     """
     Will process HIT L0 data into L1A data products.
 
@@ -36,31 +43,42 @@ def hit_l1a(packet_file: str) -> list[xr.Dataset]:
     ----------
     packet_file : str
         Path to the CCSDS data packet file.
+    packet_date : str
+        The date of the packet data in 'YYYYMMDD' format. This is used to filter
+        data to the correct processing day since L0 will have a 20-minute
+        buffer before and after the processing day.
 
     Returns
     -------
     processed_data : list[xarray.Dataset]
         List of Datasets of L1A processed data.
     """
+    if not packet_date:
+        raise ValueError("Packet date is required for processing L1A data.")
+
     # Unpack ccsds file to xarray datasets
-    datasets_by_apid = get_datasets_by_apid(packet_file)
+    datasets_by_apid = get_datasets_by_apid(str(packet_file))
 
     # Create the attribute manager for this data level
     attr_mgr = get_attribute_manager("l1a")
 
-    l1a_datasets = []
-
     # Process l1a data products
+    l1a_datasets = []
     if HitAPID.HIT_HSKP in datasets_by_apid:
         logger.info("Creating HIT L1A housekeeping dataset")
-        l1a_datasets.append(
-            process_housekeeping_data(
-                datasets_by_apid[HitAPID.HIT_HSKP], attr_mgr, "imap_hit_l1a_hk"
-            )
+        hk_dataset = process_housekeeping_data(
+            datasets_by_apid[HitAPID.HIT_HSKP], attr_mgr, "imap_hit_l1a_hk"
+        )
+        # filter the housekeeping dataset to the processing day
+        hk_dataset = filter_dataset_to_processing_day(
+            hk_dataset, str(packet_date), epoch_vals=hk_dataset["epoch"].values
         )
+        l1a_datasets.append(hk_dataset)
     if HitAPID.HIT_SCIENCE in datasets_by_apid:
         l1a_datasets.extend(
-            process_science(datasets_by_apid[HitAPID.HIT_SCIENCE], attr_mgr)
+            process_science(
+                datasets_by_apid[HitAPID.HIT_SCIENCE], attr_mgr, str(packet_date)
+            )
         )
     return l1a_datasets
 
@@ -102,7 +120,15 @@ def subcom_sectorates(sci_dataset: xr.Dataset) -> xr.Dataset:
     sci_dataset : xarray.Dataset
         Xarray dataset with sectored rates data organized by species.
     """
-    updated_dataset = sci_dataset.copy()
+    # Initialize the dataset with the required variables
+    updated_dataset = sci_dataset[
+        [
+            "sectorates",
+            "hdr_minute_cnt",
+            "livetime_counter",
+            "hdr_dynamic_threshold_state",
+        ]
+    ].copy(deep=True)
 
     # Calculate mod 10 values
     hdr_min_count_mod_10 = updated_dataset.hdr_minute_cnt.values % 10
@@ -308,25 +334,272 @@ def add_cdf_attributes(
     return dataset
 
 
+def find_complete_mod10_sets(mod_vals: np.ndarray) -> np.ndarray:
+    """
+    Find start indices where mod values match [0,1,2,3,4,5,6,7,8,9] pattern.
+
+    Parameters
+    ----------
+    mod_vals : np.ndarray
+        1D array of mod 10 values from the hdr_minute_cnt field in the L1A counts data.
+
+    Returns
+    -------
+    np.ndarray
+        Indices in mod_vals where the complete pattern [0, 1, ..., 9] starts.
+    """
+    # The pattern to match is an array from 0-9
+    window_size = 10
+
+    if mod_vals.size < window_size:
+        logger.warning(
+            "Mod 10 array is smaller than the required window size for "
+            "pattern matching."
+        )
+        return np.array([], dtype=int)
+    # Use sliding windows to find pattern matches
+    sw_view = np.lib.stride_tricks.sliding_window_view(mod_vals, window_size)
+    matches = np.all(sw_view == np.arange(window_size), axis=1)
+    return np.where(matches)[0]
+
+
+def subset_sectored_counts(
+    sectored_counts_dataset: xr.Dataset, packet_date: str
+) -> xr.Dataset:
+    """
+    Subset data for complete sets of sectored counts and corresponding livetime values.
+
+    A set of sectored data starts with hydrogen and ends with iron and correspond to
+    the mod 10 values 0-9. The livetime values from the previous 10 minutes are used
+    to calculate the rates for each set since those counts are transmitted 10 minutes
+    after they were collected. Therefore, only complete sets of sectored counts where
+    livetime from the previous 10 minutes are available are included in the output.
+
+    Parameters
+    ----------
+    sectored_counts_dataset : xarray.Dataset
+        The sectored counts dataset.
+
+    packet_date : str
+        The date of the packet data in 'YYYYMMDD' format, used for filtering.
+
+    Returns
+    -------
+    xarray.Dataset
+        A dataset of complete sectored counts and corresponding livetime values
+        for the processing day.
+    """
+    # TODO: Update to use fill values for partial frames rather than drop them
+
+    # Modify livetime_counter to use a new epoch coordinate
+    # that is aligned with the original epoch dimension. This
+    # ensures that livetime doesn't get filtered when the original
+    # epoch dimension is filtered for complete sets.
+    sectored_counts_dataset = update_livetime_coord(sectored_counts_dataset)
+
+    # Identify 10-minute intervals of complete sectored counts
+    # by using the mod 10 values of the header minute counts.
+    # Mod 10 determines the species and energy bin the data belongs
+    # to. A mapping of mod 10 values to species and energy bins is
+    # provided in l0/constants.py for reference.
+    bin_size = 10
+    mod_10: np.ndarray = sectored_counts_dataset.hdr_minute_cnt.values % 10
+    start_indices = find_complete_mod10_sets(mod_10)
+
+    # Filter out start indices that are less than or equal to the bin size
+    # since the previous 10 minutes are needed for calculating rates
+    if start_indices.size == 0:
+        raise ValueError(
+            "No data to process - valid start indices not found for "
+            "complete sectored counts."
+        )
+    else:
+        start_indices = start_indices[start_indices >= bin_size]
+
+    # Subset data for complete sets of sectored counts.
+    # Each set of sectored counts is 10 minutes long, so we take the indices
+    # starting from the start indices and extending to the bin size of 10.
+    # This creates a 1D array of indices that correspond to the complete
+    # sets of sectored counts which is used to filter the L1A dataset and
+    # create the L1B sectored rates dataset.
+    data_indices = np.concatenate(
+        [np.arange(idx, idx + bin_size) for idx in start_indices]
+    )
+    complete_sectored_counts_dataset = sectored_counts_dataset.isel(epoch=data_indices)
+
+    epoch_per_complete_set = np.repeat(
+        [
+            complete_sectored_counts_dataset.epoch[idx : idx + bin_size].mean().item()
+            for idx in range(0, len(complete_sectored_counts_dataset.epoch), 10)
+        ],
+        bin_size,
+    )
+
+    # Filter dataset for data in the processing day
+
+    # Trim the sectored data to epoch_per_complete_set values in the processing day
+    filtered_dataset = filter_dataset_to_processing_day(
+        complete_sectored_counts_dataset, packet_date, epoch_vals=epoch_per_complete_set
+    )
+
+    # Trim livetime to the size of the sectored data but shifted 10 minutes earlier.
+    filtered_dataset = subset_livetime(filtered_dataset)
+
+    return filtered_dataset
+
+
+def update_livetime_coord(sectored_dataset: xr.Dataset) -> xr.Dataset:
+    """
+    Update livetime_counter to use a new epoch coordinate.
+
+    Assign a new epoch coordinate to the `livetime_counter` variable.
+    This new coordinate is aligned with the original `epoch` dimension,
+    ensuring that `livetime_counter` remains unaffected when the original
+    `epoch` dimension is filtered for complete sets in `subset_sectored_counts`
+    function.
+
+    Parameters
+    ----------
+    sectored_dataset : xarray.Dataset
+        The dataset containing sectored counts and livetime_counter data.
+
+    Returns
+    -------
+    xarray.Dataset
+        The updated dataset with modified livetime_counter.
+    """
+    epoch_values = sectored_dataset.epoch.values
+    sectored_dataset = sectored_dataset.assign_coords(
+        {
+            "epoch_livetime": ("epoch", epoch_values),
+        }
+    )
+    da = sectored_dataset["livetime_counter"]
+    da = da.assign_coords(epoch_livetime=("epoch", epoch_values))
+
+    # Swap the dimension from 'epoch' to 'epoch_livetime'
+    da = da.swap_dims({"epoch": "epoch_livetime"})
+
+    # Update the dataset with the modified variable
+    sectored_dataset["livetime_counter"] = da
+
+    return sectored_dataset
+
+
+def subset_livetime(dataset: xr.Dataset) -> xr.Dataset:
+    """
+    Trim livetime to values shifted 10 minutes earlier than sectored data.
+
+    This ensures that the livetime values correspond to the sectored counts correctly
+    since sectored data is collected 10 minutes before it's transmitted.
+
+    Parameters
+    ----------
+    dataset : xarray.Dataset
+        The dataset containing sectored counts and livetime data.
+
+    Returns
+    -------
+    xarray.Dataset
+        The updated dataset with trimmed livetime data.
+    """
+    # epoch values are per science frame which is 1 minute
+    epoch_vals = dataset["epoch"].values
+    epoch_livetime_vals = dataset["epoch_livetime"].values
+
+    if not epoch_vals.size:
+        raise ValueError(
+            "Epoch values are empty. Cannot proceed with livetime subsetting."
+        )
+
+    # Get index positions of epoch[0] and epoch[-1] in epoch_livetime
+    start_idx = np.where(epoch_livetime_vals == epoch_vals[0])[0][0]
+    end_idx = np.where(epoch_livetime_vals == epoch_vals[-1])[0][0]
+
+    if start_idx < 10:
+        raise ValueError(
+            "Start index for livetime is less than 10. This indicates that the "
+            "dataset is too small to shift livetime correctly."
+        )
+
+    # Compute shifted indices by 10 minutes
+    start_trimmed = max(start_idx - 10, 0)
+    end_trimmed = max(end_idx - 10, 0)
+
+    return dataset.isel(epoch_livetime=slice(start_trimmed, end_trimmed + 1))
+
+
+def filter_dataset_to_processing_day(
+    dataset: xr.Dataset,
+    packet_date: str,
+    epoch_vals: np.ndarray,
+    sc_tick: bool = False,
+) -> xr.Dataset:
+    """
+    Filter the dataset for data within the processing day.
+
+    Parameters
+    ----------
+    dataset : xarray.Dataset
+        The dataset to filter.
+    packet_date : str
+        The date of the packet data in 'YYYYMMDD' format.
+    epoch_vals : np.ndarray
+        An array of epoch values. Used to identify indices of data that
+        belong in the processing day. For sectored counts data, an
+        array of mean epoch values for major frames (10 min. intervals)
+        is used to filter the dataset to ensure that major frames that span
+        midnight, but belong to the processing day, are included. For other
+        datasets, the dataset's epoch coordinate values will be used.
+    sc_tick : bool
+        If true, the dataset's sc_tick will be used to filter data as well.
+        This ensures that the ccsds headers that use sc_tick as a coordinate,
+        instead of epoch, also corresponds to the processing day.
+
+    Returns
+    -------
+    xarray.Dataset
+        The filtered dataset containing data within the processing day.
+    """
+    processing_day = datetime.strptime(packet_date, "%Y%m%d").strftime("%Y-%m-%d")
+
+    # Filter dataset by epoch indices in processing day
+    epoch_dt = et_to_datetime64(ttj2000ns_to_et(epoch_vals))
+    epoch_indices_in_processing_day = np.where(
+        epoch_dt.astype("datetime64[D]") == np.datetime64(processing_day)
+    )[0]
+    dataset = dataset.isel(epoch=epoch_indices_in_processing_day)
+
+    # If sc_tick is provided (coord for ccsds headers), filter by sc_tick too
+    if sc_tick:
+        sc_tick_dt = met_to_datetime64(dataset["sc_tick"].values)
+        indices_in_processing_day = np.where(
+            sc_tick_dt.astype("datetime64[D]") == np.datetime64(processing_day)
+        )[0]
+        dataset = dataset.isel(sc_tick=indices_in_processing_day)
+    return dataset
+
+
 def process_science(
-    dataset: xr.Dataset, attr_mgr: ImapCdfAttributes
+    dataset: xr.Dataset, attr_mgr: ImapCdfAttributes, packet_date: str
 ) -> list[xr.Dataset]:
     """
     Will process science datasets for CDF products.
 
-    Process binary science data for CDF creation. The data is
-    grouped into science frames, decommutated and decompressed,
-    and split into count rates and event datasets. Updates the
-    dataset attributes and coordinates and data variable
-    dimensions according to specifications in a cdf yaml file.
+    The function processes binary science data for CDF creation.
+    The data is decommutated, decompressed, grouped into science frames,
+    and split into count rates, sectored count rates, and event datasets.
+    It also updates the dataset attributes according to specifications
+    in a cdf yaml file.
 
     Parameters
     ----------
     dataset : xarray.Dataset
         A dataset containing HIT science data.
-
     attr_mgr : ImapCdfAttributes
         Attribute manager used to get the data product field's attributes.
+    packet_date : str
+        The date of the packet data, used for processing.
 
     Returns
     -------
@@ -338,20 +611,37 @@ def process_science(
     # Decommutate and decompress the science data
     sci_dataset = decom_hit(dataset)
 
-    # Organize sectored rates by species type
-    sci_dataset = subcom_sectorates(sci_dataset)
+    # Create dataset for sectored data organized by species type
+    sectored_dataset = subcom_sectorates(sci_dataset)
+
+    # Subset sectored data for complete sets (10 min intervals covering all species)
+    sectored_dataset = subset_sectored_counts(sectored_dataset, packet_date)
+
+    # TODO:
+    #  - headers are values per packet rather than per frame. Do these need to align
+    #    with the science frames?
+    #    For instance, the mean epoch for a frame that spans midnight might contain
+    #    packets from the previous day but filtering sc_tick by processing day will
+    #    exclude those packets. Is this an issue?
+
+    # Filter the science dataset to only include data from the processing day
+    sci_dataset = filter_dataset_to_processing_day(
+        sci_dataset, packet_date, epoch_vals=sci_dataset["epoch"].values, sc_tick=True
+    )
 
     # Split the science data into count rates and event datasets
     pha_raw_dataset = xr.Dataset(
         {"pha_raw": sci_dataset["pha_raw"]}, coords={"epoch": sci_dataset["epoch"]}
     )
-    count_rates_dataset = sci_dataset.drop_vars("pha_raw")
+    count_rates_dataset = sci_dataset.drop_vars(["pha_raw", "sectorates"])
 
     # Calculate uncertainties for count rates
     count_rates_dataset = calculate_uncertainties(count_rates_dataset)
+    sectored_count_rates_dataset = calculate_uncertainties(sectored_dataset)
 
     l1a_datasets: dict = {
-        "imap_hit_l1a_counts": count_rates_dataset,
+        "imap_hit_l1a_counts-standard": count_rates_dataset,
+        "imap_hit_l1a_counts-sectored": sectored_count_rates_dataset,
         "imap_hit_l1a_direct-events": pha_raw_dataset,
     }