imap-processing 0.19.0__py3-none-any.whl → 0.19.2__py3-none-any.whl

imap_processing/lo/l1b/lo_l1b.py

@@ -9,6 +9,7 @@ import numpy as np
 import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.lo import lo_ancillary
 from imap_processing.lo.l1b.tof_conversions import (
     TOF0_CONV,
     TOF1_CONV,
@@ -16,20 +17,24 @@ from imap_processing.lo.l1b.tof_conversions import (
     TOF3_CONV,
 )
 from imap_processing.spice.geometry import SpiceFrame, instrument_pointing
+from imap_processing.spice.repoint import get_pointing_times
+from imap_processing.spice.spin import get_spin_number
 from imap_processing.spice.time import met_to_ttj2000ns, ttj2000ns_to_et
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
 
-def lo_l1b(dependencies: dict) -> list[Path]:
+def lo_l1b(sci_dependencies: dict, anc_dependencies: list) -> list[Path]:
     """
     Will process IMAP-Lo L1A data into L1B CDF data products.
 
     Parameters
     ----------
-    dependencies : dict
+    sci_dependencies : dict
         Dictionary of datasets needed for L1B data product creation in xarray Datasets.
+    anc_dependencies : list
+        List of ancillary file paths needed for L1B data product creation.
 
     Returns
     -------
@@ -43,17 +48,20 @@ def lo_l1b(dependencies: dict) -> list[Path]:
     # create the attribute manager to access L1A fillval attributes
     attr_mgr_l1a = ImapCdfAttributes()
     attr_mgr_l1a.add_instrument_variable_attrs(instrument="lo", level="l1a")
-    logger.info(f"\n Dependencies: {list(dependencies.keys())}\n")
+    logger.info(f"\n Dependencies: {list(sci_dependencies.keys())}\n")
     # if the dependencies are used to create Annotated Direct Events
-    if "imap_lo_l1a_de" in dependencies and "imap_lo_l1a_spin" in dependencies:
+    if "imap_lo_l1a_de" in sci_dependencies and "imap_lo_l1a_spin" in sci_dependencies:
         logger.info("\nProcessing IMAP-Lo L1B Direct Events...")
         logical_source = "imap_lo_l1b_de"
         # get the dependency dataset for l1b direct events
-        l1a_de = dependencies["imap_lo_l1a_de"]
-        spin_data = dependencies["imap_lo_l1a_spin"]
+        l1a_de = sci_dependencies["imap_lo_l1a_de"]
+        spin_data = sci_dependencies["imap_lo_l1a_spin"]
 
         # Initialize the L1B DE dataset
         l1b_de = initialize_l1b_de(l1a_de, attr_mgr_l1b, logical_source)
+        pointing_start_met, pointing_end_met = get_pointing_times(
+            l1a_de["met"].values[0].item()
+        )
         # Get the start and end times for each spin epoch
         acq_start, acq_end = convert_start_end_acq_times(spin_data)
         # Get the average spin durations for each epoch
@@ -66,7 +74,7 @@ def lo_l1b(dependencies: dict) -> list[Path]:
         # spin bins are 0 - 60 bins
         l1b_de = set_spin_bin(l1b_de, spin_angle)
         # set the spin cycle for each direct event
-        l1b_de = set_spin_cycle(l1a_de, l1b_de)
+        l1b_de = set_spin_cycle(pointing_start_met, l1a_de, l1b_de)
         # get spin start times for each event
         spin_start_time = get_spin_start_times(l1a_de, l1b_de, spin_data, acq_end)
         # get the absolute met for each event
@@ -75,6 +83,10 @@ def lo_l1b(dependencies: dict) -> list[Path]:
         )
         # set the epoch for each event
         l1b_de = set_each_event_epoch(l1b_de)
+        # Set the ESA mode for each direct event
+        l1b_de = set_esa_mode(
+            pointing_start_met, pointing_end_met, anc_dependencies, l1b_de
+        )
         # Set the average spin duration for each direct event
         l1b_de = set_avg_spin_durations_per_event(
             l1a_de, l1b_de, avg_spin_durations_per_cycle
@@ -133,7 +145,7 @@ def initialize_l1b_de(
         # TODO: Add pos to YAML file
         # attrs=attr_mgr.get_variable_attributes("pos"),
     )
-    l1b_de["mode"] = xr.DataArray(
+    l1b_de["mode_bit"] = xr.DataArray(
         l1a_de["mode"].values,
         dims=["epoch"],
         # TODO: Add mode to YAML file
@@ -155,6 +167,65 @@ def initialize_l1b_de(
     return l1b_de
 
 
+def set_esa_mode(
+    pointing_start_met: float,
+    pointing_end_met: float,
+    anc_dependencies: list,
+    l1b_de: xr.Dataset,
+) -> xr.Dataset:
+    """
+    Set the ESA mode for each direct event.
+
+    The ESA mode is determined from the sweep table for the time period of the pointing.
+
+    Parameters
+    ----------
+    pointing_start_met : float
+        Start time for the pointing in MET seconds.
+    pointing_end_met : float
+        End time for the pointing in MET seconds.
+    anc_dependencies : list
+        List of ancillary file paths.
+    l1b_de : xarray.Dataset
+        The L1B DE dataset.
+
+    Returns
+    -------
+    l1b_de : xr.Dataset
+        The L1B DE dataset with the ESA mode added.
+    """
+    # Read the sweep table from the ancillary files
+    sweep_df = lo_ancillary.read_ancillary_file(
+        next(str(s) for s in anc_dependencies if "sweep-table" in str(s))
+    )
+
+    # Get the sweep table rows that correspond to the time period of the pointing
+    pointing_sweep_df = sweep_df[
+        (sweep_df["GoodTime_start"] >= pointing_start_met)
+        & (sweep_df["GoodTime_start"] <= pointing_end_met)
+    ]
+
+    # Check that there is only one ESA mode in the sweep table for the pointing
+    if len(pointing_sweep_df["ESA_Mode"].unique()) == 1:
+        # Update the ESA mode strings to be 0 for HiRes and 1 for HiThr
+        sweep_df["esa_mode"] = sweep_df["ESA_Mode"].map({"HiRes": 0, "HiThr": 1})
+        # Get the ESA mode for the pointing
+        esa_mode = sweep_df["esa_mode"].values[0]
+        # Repeat the ESA mode for each direct event in the pointing
+        esa_mode_array = np.repeat(esa_mode, len(l1b_de["epoch"]))
+    else:
+        raise ValueError("Multiple ESA modes found in sweep table for pointing.")
+
+    l1b_de["esa_mode"] = xr.DataArray(
+        esa_mode_array,
+        dims=["epoch"],
+        # TODO: Add esa_mode to YAML file
+        # attrs=attr_mgr.get_variable_attributes("esa_mode"),
+    )
+
+    return l1b_de
+
+
 def convert_start_end_acq_times(
     spin_data: xr.Dataset,
 ) -> tuple[xr.DataArray, xr.DataArray]:
@@ -252,7 +323,9 @@ def set_spin_bin(l1b_de: xr.Dataset, spin_angle: np.ndarray) -> xr.Dataset:
     return l1b_de
 
 
-def set_spin_cycle(l1a_de: xr.Dataset, l1b_de: xr.Dataset) -> xr.Dataset:
+def set_spin_cycle(
+    pointing_start_met: float, l1a_de: xr.Dataset, l1b_de: xr.Dataset
+) -> xr.Dataset:
     """
     Set the spin cycle for each direct event.
 
@@ -265,6 +338,8 @@ def set_spin_cycle(l1a_de: xr.Dataset, l1b_de: xr.Dataset) -> xr.Dataset:
 
     Parameters
     ----------
+    pointing_start_met : float
+        The start time of the pointing in MET seconds.
     l1a_de : xarray.Dataset
         The L1A DE dataset.
     l1b_de : xarray.Dataset
@@ -275,19 +350,18 @@ def set_spin_cycle(l1a_de: xr.Dataset, l1b_de: xr.Dataset) -> xr.Dataset:
     l1b_de : xarray.Dataset
         The L1B DE dataset with the spin cycle added for each direct event.
     """
+    spin_start_num = get_spin_number(pointing_start_met)
     counts = l1a_de["de_count"].values
     # split the esa_steps into ASC groups
     de_asc_groups = np.split(l1a_de["esa_step"].values, np.cumsum(counts)[:-1])
     spin_cycle = []
-    for i, esa_asc_group in enumerate(de_asc_groups):
-        # TODO: Spin Number does not reset for each pointing. Need to figure out
-        #  how to retain this information across days
-        # increment the spin_start by 28 after each aggregated science cycle
-        spin_start = i * 28
+    for esa_asc_group in de_asc_groups:
         # calculate the spin cycle for each DE in the ASC group
         # TODO: Add equation number in algorithm document when new version is
-        # available. Add to docstring as well
-        spin_cycle.extend(spin_start + 7 + (esa_asc_group - 1) * 2)
+        #  available. Add to docstring as well
+        spin_cycle.extend(spin_start_num + 7 + (esa_asc_group - 1) * 2)
+        # increment the spin start number by 28 for the next ASC
+        spin_start_num += 28
 
     l1b_de["spin_cycle"] = xr.DataArray(
         spin_cycle,

imap_processing/lo/l1c/lo_l1c.py

@@ -1,5 +1,6 @@
 """IMAP-Lo L1C Data Processing."""
 
+import logging
 from dataclasses import Field
 from enum import Enum
 
@@ -67,14 +68,36 @@ def lo_l1c(sci_dependencies: dict, anc_dependencies: list) -> list[xr.Dataset]:
         logical_source = "imap_lo_l1c_pset"
         l1b_de = sci_dependencies["imap_lo_l1b_de"]
         l1b_goodtimes_only = filter_goodtimes(l1b_de, anc_dependencies)
-        pset = initialize_pset(l1b_goodtimes_only, attr_mgr, logical_source)
-        full_counts = create_pset_counts(l1b_goodtimes_only)
 
         # Set the pointing start and end times based on the first epoch
         pointing_start_met, pointing_end_met = get_pointing_times(
             ttj2000ns_to_met(l1b_goodtimes_only["epoch"][0].item())
         )
 
+        pset = xr.Dataset(
+            coords={"epoch": np.array([met_to_ttj2000ns(pointing_start_met)])},
+            attrs=attr_mgr.get_global_attributes(logical_source),
+        )
+
+        # ESA mode needs to be added to L1B DE. Adding try statement
+        # to avoid error until it's available in the dataset
+        if "esa_mode" not in l1b_de:
+            logging.debug(
+                "ESA mode not found in L1B DE dataset. \
+                Setting to default value of 0 for Hi-Res."
+            )
+            pset["esa_mode"] = xr.DataArray(
+                np.array([0]),
+                dims=["epoch"],
+                attrs=attr_mgr.get_variable_attributes("esa_mode"),
+            )
+        else:
+            pset["esa_mode"] = xr.DataArray(
+                np.array([l1b_de["esa_mode"].values[0]]),
+                dims=["epoch"],
+                attrs=attr_mgr.get_variable_attributes("esa_mode"),
+            )
+
         pset["pointing_start_met"] = xr.DataArray(
             np.array([pointing_start_met]),
             dims="epoch",
@@ -86,12 +109,6 @@ def lo_l1c(sci_dependencies: dict, anc_dependencies: list) -> list[xr.Dataset]:
             attrs=attr_mgr.get_variable_attributes("pointing_end_met"),
         )
 
-        # Set the epoch to the start of the pointing
-        pset["epoch"] = xr.DataArray(
-            met_to_ttj2000ns(pset["pointing_start_met"].values),
-            attrs=attr_mgr.get_variable_attributes("epoch"),
-        )
-
         # Get the start and end spin numbers based on the pointing start and end MET
         pset["start_spin_number"] = xr.DataArray(
             [get_spin_number(pset["pointing_start_met"].item())],
@@ -104,6 +121,8 @@ def lo_l1c(sci_dependencies: dict, anc_dependencies: list) -> list[xr.Dataset]:
             attrs=attr_mgr.get_variable_attributes("end_spin_number"),
         )
 
+        full_counts = create_pset_counts(l1b_de, FilterType.NONE)
+
         # Set the counts
         pset["triples_counts"] = create_pset_counts(
             l1b_goodtimes_only, FilterType.TRIPLES
@@ -118,6 +137,32 @@ def lo_l1c(sci_dependencies: dict, anc_dependencies: list) -> list[xr.Dataset]:
         pset["exposure_time"] = calculate_exposure_times(
             full_counts, l1b_goodtimes_only
         )
+
+        # Set backgrounds
+        (
+            pset["h_background_rates"],
+            pset["h_background_rates_stat_uncert"],
+            pset["h_background_rates_sys_err"],
+        ) = set_background_rates(
+            pset["pointing_start_met"].item(),
+            pset["pointing_end_met"].item(),
+            FilterType.HYDROGEN,
+            anc_dependencies,
+            attr_mgr,
+        )
+
+        (
+            pset["o_background_rates"],
+            pset["o_background_rates_stat_uncert"],
+            pset["o_background_rates_sys_err"],
+        ) = set_background_rates(
+            pset["pointing_start_met"].item(),
+            pset["pointing_end_met"].item(),
+            FilterType.OXYGEN,
+            anc_dependencies,
+            attr_mgr,
+        )
+
     pset.attrs = attr_mgr.get_global_attributes(logical_source)
 
     pset = pset.assign_coords(
@@ -131,44 +176,6 @@ def lo_l1c(sci_dependencies: dict, anc_dependencies: list) -> list[xr.Dataset]:
     return [pset]
 
 
-def initialize_pset(
-    l1b_de: xr.Dataset, attr_mgr: ImapCdfAttributes, logical_source: str
-) -> xr.Dataset:
-    """
-    Initialize the PSET dataset and set the Epoch.
-
-    The Epoch time is set to the first of the L1B
-    Direct Event times. There is one Epoch per PSET file.
-
-    Parameters
-    ----------
-    l1b_de : xarray.Dataset
-        L1B Direct Event dataset.
-    attr_mgr : ImapCdfAttributes
-        Attribute manager used to get the L1C attributes.
-    logical_source : str
-        The logical source of the pset.
-
-    Returns
-    -------
-    pset : xarray.Dataset
-        Initialized PSET dataset.
-    """
-    pset = xr.Dataset(
-        attrs=attr_mgr.get_global_attributes(logical_source),
-    )
-    # TODO: Need to create utility to get start of repointing to use
-    #  for the pset epoch time. Setting to first DE for now
-    pset_epoch = l1b_de["epoch"][0].item()
-    pset["epoch"] = xr.DataArray(
-        np.array([pset_epoch]),
-        dims=["epoch"],
-        attrs=attr_mgr.get_variable_attributes("epoch"),
-    )
-
-    return pset
-
-
 def filter_goodtimes(l1b_de: xr.Dataset, anc_dependencies: list) -> xr.Dataset:
     """
     Filter the L1B Direct Event dataset to only include good times.
@@ -189,10 +196,12 @@ def filter_goodtimes(l1b_de: xr.Dataset, anc_dependencies: list) -> xr.Dataset:
         Filtered L1B Direct Event dataset.
     """
     # the goodtimes are currently the only ancillary file needed for L1C processing
-    goodtimes_table_df = lo_ancillary.read_ancillary_file(anc_dependencies[0])
+    goodtimes_table_df = lo_ancillary.read_ancillary_file(
+        next(str(s) for s in anc_dependencies if "good-times" in str(s))
+    )
 
     # convert goodtimes from MET to TTJ2000
-    goodtimes_start = met_to_ttj2000ns(goodtimes_table_df["GoodTime_strt"])
+    goodtimes_start = met_to_ttj2000ns(goodtimes_table_df["GoodTime_start"])
     goodtimes_end = met_to_ttj2000ns(goodtimes_table_df["GoodTime_end"])
 
     # Create a mask for epochs within any of the start/end time ranges
@@ -254,9 +263,9 @@ def create_pset_counts(
             "001000",
         ],
         # hydrogen species identifier
-        FilterType.HYDROGEN: "h",
+        FilterType.HYDROGEN: "H",
         # oxygen species identifier
-        FilterType.OXYGEN: "o",
+        FilterType.OXYGEN: "O",
     }
 
     # if the filter string is triples or doubles, filter using the coincidence type
@@ -484,3 +493,108 @@ def create_datasets(
             )
 
     return dataset
+
+
+def set_background_rates(
+    pointing_start_met: float,
+    pointing_end_met: float,
+    species: FilterType,
+    anc_dependencies: list,
+    attr_mgr: ImapCdfAttributes,
+) -> tuple[xr.DataArray, xr.DataArray, xr.DataArray]:
+    """
+    Set the background rates for the specified species.
+
+    The background rates are set to a constant value of 0.01 counts/s for all bins.
+
+    Parameters
+    ----------
+    pointing_start_met : float
+        The start MET time of the pointing.
+    pointing_end_met : float
+        The end MET time of the pointing.
+    species : FilterType
+        The species to set the background rates for. Can be "h" or "o".
+    anc_dependencies : list
+        Ancillary files needed for L1C data product creation.
+    attr_mgr : ImapCdfAttributes
+        Attribute manager used to get the L1C attributes.
+
+    Returns
+    -------
+    background_rates : tuple[xr.DataArray, xr.DataArray, xr.DataArray]
+        Tuple containing:
+        - The background rates for the specified species.
+        - The statistical uncertainties for the background rates.
+        - The systematic errors for the background rates.
+    """
+    if species not in {FilterType.HYDROGEN, FilterType.OXYGEN}:
+        raise ValueError(f"Species must be 'h' or 'o', but got {species.value}.")
+
+    bg_rates = np.zeros(
+        (N_ESA_ENERGY_STEPS, N_SPIN_ANGLE_BINS, N_OFF_ANGLE_BINS), dtype=np.float16
+    )
+    bg_stat_uncert = np.zeros(
+        (N_ESA_ENERGY_STEPS, N_SPIN_ANGLE_BINS, N_OFF_ANGLE_BINS), dtype=np.float16
+    )
+    bg_sys_err = np.zeros(
+        (N_ESA_ENERGY_STEPS, N_SPIN_ANGLE_BINS, N_OFF_ANGLE_BINS), dtype=np.float16
+    )
+
+    # read in the background rates from ancillary file
+    if species == FilterType.HYDROGEN:
+        background_df = lo_ancillary.read_ancillary_file(
+            next(str(s) for s in anc_dependencies if "hydrogen-background" in str(s))
+        )
+    else:
+        background_df = lo_ancillary.read_ancillary_file(
+            next(str(s) for s in anc_dependencies if "oxygen-background" in str(s))
+        )
+
+    # find to the rows for the current pointing
+    pointing_bg_df = background_df[
+        (background_df["GoodTime_strt"] >= pointing_start_met)
+        & (background_df["GoodTime_end"] <= pointing_end_met)
+    ]
+
+    # convert the bin start and end resolution from 6 degrees to .1 degrees
+    pointing_bg_df["bin_strt"] = pointing_bg_df["bin_strt"] * 60
+    # The last bin end in the file is 0, which means 60 degrees. This is
+    # converted to 0.1 degree resolution of 3600
+    pointing_bg_df["bin_end"] = pointing_bg_df["bin_end"] * 60
+    pointing_bg_df.loc[pointing_bg_df["bin_end"] == 0, "bin_end"] = 3600
+    # for each row in the bg ancillary file for this pointing
+    for _, row in pointing_bg_df.iterrows():
+        bin_start = int(row["bin_strt"])
+        bin_end = int(row["bin_end"])
+        # for each energy step, set the background rate and uncertainty
+        for esa_step in range(0, 7):
+            value = row[f"E-Step{esa_step + 1}"]
+            if row["type"] == "rate":
+                bg_rates[esa_step, bin_start:bin_end, :] = value
+            elif row["type"] == "sigma":
+                bg_stat_uncert[esa_step, bin_start:bin_end, :] = value
+            else:
+                raise ValueError("Unknown background type in ancillary file.")
+    # set the background rates, uncertainties, and systematic errors
+    bg_rates_data = xr.DataArray(
+        data=bg_rates,
+        dims=["esa_energy_step", "spin_angle", "off_angle"],
+        attrs=attr_mgr.get_variable_attributes(f"{species.value}_background_rates"),
+    )
+    bg_stat_uncert_data = xr.DataArray(
+        data=bg_stat_uncert,
+        dims=["esa_energy_step", "spin_angle", "off_angle"],
+        attrs=attr_mgr.get_variable_attributes(
+            f"{species.value}_background_rates_stat_uncert"
+        ),
+    )
+    bg_sys_err_data = xr.DataArray(
+        data=bg_sys_err,
+        dims=["esa_energy_step", "spin_angle", "off_angle"],
+        attrs=attr_mgr.get_variable_attributes(
+            f"{species.value}_background_rates_sys_err"
+        ),
+    )
+
+    return bg_rates_data, bg_stat_uncert_data, bg_sys_err_data