imap-processing 1.0.1__py3-none-any.whl → 1.0.2__py3-none-any.whl

imap_processing/ialirt/process_ephemeris.py

@@ -72,6 +72,7 @@ def calculate_azimuth_and_elevation(
     altitude: float,
     observation_time: float | np.ndarray,
     target: str = SpiceBody.IMAP.name,
+    obsref: str = "ITRF93",
 ) -> tuple:
     """
     Calculate azimuth and elevation.
@@ -91,6 +92,9 @@ def calculate_azimuth_and_elevation(
         is to be computed. Expressed as ephemeris time, seconds past J2000 TDB.
     target : str (Optional)
         The target body. Default is "IMAP".
+    obsref : str (Optional)
+        Body-fixed, body-centered reference frame wrt
+        observer's center.
 
     Returns
     -------
@@ -120,7 +124,7 @@ def calculate_azimuth_and_elevation(
             elplsz=True,  # Elevation increases from the XY plane toward +Z
             obspos=ground_station_position_ecef,  # observer pos. to center of motion
             obsctr="EARTH",  # Name of the center of motion
-            obsref="IAU_EARTH",  # Body-fixed, body-centered reference frame wrt
+            obsref=obsref,  # Body-fixed, body-centered reference frame wrt
             # observer's center
         )
         azimuth.append(np.rad2deg(azel_results[0][1]))
@@ -223,7 +227,7 @@ def build_output(
 
     # For now, assume that kernel management will be handled by ensure_spice
     azimuth, elevation = calculate_azimuth_and_elevation(
-        longitude, latitude, altitude, time_range
+        longitude, latitude, altitude, time_range, obsref="ITRF93"
     )
 
     output_dict["time"] = et_to_utc(time_range, format_str="ISOC")

imap_processing/ialirt/utils/create_xarray.py

@@ -52,7 +52,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset:  # noqa: PLR0
         ["radial", "tangential", "normal"],
         name="RTN_component",
         dims=["RTN_component"],
-        attrs=cdf_manager.get_variable_attributes("RTN_componentt", check_schema=False),
+        attrs=cdf_manager.get_variable_attributes("RTN_component", check_schema=False),
     )
 
     esa_step = xr.DataArray(
@@ -85,7 +85,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset:  # noqa: PLR0
         name="codice_hi_h_spin_angle",
         dims=["codice_hi_h_spin_angle"],
         attrs=cdf_manager.get_variable_attributes(
-            "codice_hi_h_spin_anglen", check_schema=False
+            "codice_hi_h_spin_angle", check_schema=False
         ),
     )
 
@@ -156,6 +156,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset:  # noqa: PLR0
                 "sc_velocity_GSE",
                 "mag_hk_status",
                 "spice_kernels",
+                "instrument",
             ]:
                 continue
             elif key in ["mag_B_GSE", "mag_B_GSM", "mag_B_RTN"]:

imap_processing/lo/l1c/lo_l1c.py

@@ -302,7 +302,7 @@ def create_pset_counts(
     # Create the histogram with 3600 longitude bins, 40 latitude bins, and 7 energy bins
     lon_edges = np.arange(3601)
     lat_edges = np.arange(41)
-    energy_edges = np.arange(8)
+    energy_edges = np.arange(1, 9)
 
     hist, _edges = np.histogramdd(
         data,

imap_processing/lo/l2/lo_l2.py

@@ -938,8 +938,10 @@ def calculate_bootstrap_corrections(dataset: xr.Dataset) -> xr.Dataset:
         bootstrap_factor_array,
         dims=["energy_i", "energy_k"],
         coords={
-            "energy_i": list(range(7)),
-            "energy_k": list(range(8)),  # Include virtual channel 7 (index 7)
+            "energy_i": dataset["energy"].values,
+            # Add an extra coordinate for the virtual E8 channel, unused
+            # in the broadcasting calculations
+            "energy_k": np.concatenate([dataset["energy"].values, [np.nan]]),
         },
     )
 
@@ -1001,7 +1003,7 @@ def calculate_bootstrap_corrections(dataset: xr.Dataset) -> xr.Dataset:
         # NOTE: The paper uses 1-based indexing and we use 0-based indexing
         #       so there is an off-by-one difference in the indices.
         bootstrap_intensity_i[:] = (
-            j_c_prime_i - bootstrap_factor.sel(energy_i=i, energy_k=7) * j_8_b[0, ...]
+            j_c_prime_i - bootstrap_factor.isel(energy_i=i, energy_k=7) * j_8_b[0, ...]
         )
         # NOTE: We will square root at the end to get the uncertainty, but
         #       all equations are with variances
@@ -1013,7 +1015,7 @@ def calculate_bootstrap_corrections(dataset: xr.Dataset) -> xr.Dataset:
 
         # Get bootstrap factors for this i and the relevant k values
         # Rename energy_k dimension to energy for alignment with intensity
-        bootstrap_factors_k = bootstrap_factor.sel(
+        bootstrap_factors_k = bootstrap_factor.isel(
             energy_i=i, energy_k=k_indices
         ).rename({"energy_k": "energy"})
 

imap_processing/quality_flags.py

@@ -64,6 +64,7 @@ class ImapAttitudeUltraFlags(FlagNameMixin):
     AUXMISMATCH = 2**1  # bit 1 # aux packet does not match Universal Spin Table
     SPINPHASE = 2**2  # bit 2 # spin phase flagged by Universal Spin Table
     SPINPERIOD = 2**3  # bit 3 # spin period flagged by Universal Spin Table
+    DURINGREPOINT = 2**4  # bit 4 # spin during a repointing
 
 
 class ImapRatesUltraFlags(FlagNameMixin):

imap_processing/swapi/constants.py

@@ -0,0 +1,4 @@
+"""Constants for SWAPI processing."""
+
+NUM_PACKETS_PER_SWEEP = 12
+NUM_ENERGY_STEPS = 72

imap_processing/swapi/l1/swapi_l1.py

@@ -12,6 +12,8 @@ from imap_processing import imap_module_directory
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.cdf.utils import load_cdf
 from imap_processing.quality_flags import SWAPIFlags
+from imap_processing.spice.time import met_to_utc, ttj2000ns_to_met
+from imap_processing.swapi.constants import NUM_ENERGY_STEPS, NUM_PACKETS_PER_SWEEP
 from imap_processing.swapi.swapi_utils import SWAPIAPID, SWAPIMODE
 from imap_processing.utils import packet_file_to_datasets
 
@@ -41,10 +43,10 @@ def filter_good_data(full_sweep_sci: xr.Dataset) -> npt.NDArray:
     """
     # PLAN_ID for current sweep should all be one value and
     # SWEEP_TABLE should all be one value.
-    plan_id = full_sweep_sci["plan_id"].data.reshape(-1, 12)
-    sweep_table = full_sweep_sci["sweep_table"].data.reshape(-1, 12)
+    plan_id = full_sweep_sci["plan_id"].data.reshape(-1, NUM_PACKETS_PER_SWEEP)
+    sweep_table = full_sweep_sci["sweep_table"].data.reshape(-1, NUM_PACKETS_PER_SWEEP)
 
-    mode = full_sweep_sci["mode"].data.reshape(-1, 12)
+    mode = full_sweep_sci["mode"].data.reshape(-1, NUM_PACKETS_PER_SWEEP)
 
     sweep_indices = (sweep_table == sweep_table[:, 0, None]).all(axis=1)
     plan_id_indices = (plan_id == plan_id[:, 0, None]).all(axis=1)
@@ -62,10 +64,10 @@ def filter_good_data(full_sweep_sci: xr.Dataset) -> npt.NDArray:
     # From this: [0 24]
     # To this: [[ 0  1  2  3  4  5  6  7  8  9 10 11]
     # [24 25 26 27 28 29 30 31 32 33 34 35]]
-    cycle_start_indices = np.where(bad_data_indices == 0)[0] * 12
-    bad_cycle_indices = cycle_start_indices[..., None] + np.arange(12)[
-        None, ...
-    ].reshape(-1)
+    cycle_start_indices = np.where(bad_data_indices == 0)[0] * NUM_PACKETS_PER_SWEEP
+    bad_cycle_indices = cycle_start_indices[..., None] + np.arange(
+        NUM_PACKETS_PER_SWEEP
+    )[None, ...].reshape(-1)
 
     logger.debug("Cycle data was bad due to one of below reasons:")
     logger.debug(
@@ -162,7 +164,7 @@ def find_sweep_starts(packets: xr.Dataset) -> npt.NDArray:
     indices_start : numpy.ndarray
         Array of indices of start cycle.
     """
-    if packets["shcoarse"].size < 12:
+    if packets["shcoarse"].size < NUM_PACKETS_PER_SWEEP:
         return np.array([], np.int64)
 
     # calculate time difference between consecutive sweep
@@ -387,7 +389,7 @@ def process_sweep_data(full_sweep_sci: xr.Dataset, cem_prefix: str) -> xr.Datase
     # [ 2  3  4  5  6  7  8  9  10  11  12  13]]]
     # In other word, we grouped each cem's
     # data by full sweep.
-    current_cem_counts = current_cem_counts.reshape(6, -1, 12)
+    current_cem_counts = current_cem_counts.reshape(6, -1, NUM_PACKETS_PER_SWEEP)
 
     # Then, we go from above to
     # to this final output:
@@ -421,7 +423,7 @@ def process_sweep_data(full_sweep_sci: xr.Dataset, cem_prefix: str) -> xr.Datase
     all_cem_data = np.stack(current_cem_counts, axis=-1)
     # This line just flatten the inner most array to
     # (total_full_sweeps x 72)
-    all_cem_data = all_cem_data.reshape(-1, 72)
+    all_cem_data = all_cem_data.reshape(-1, NUM_ENERGY_STEPS)
     return all_cem_data
 
 
@@ -490,7 +492,9 @@ def process_swapi_science(
     # ===================================================================
     # Quality flags
     # ===================================================================
-    quality_flags_data = np.zeros((total_full_sweeps, 72), dtype=np.uint16)
+    quality_flags_data = np.zeros(
+        (total_full_sweeps, NUM_ENERGY_STEPS), dtype=np.uint16
+    )
 
     # Add science data quality flags
     # Have to match datatype to bitwise OR
@@ -547,7 +551,7 @@ def process_swapi_science(
 
     for flag_name in hk_flags_name:
         current_flag = np.repeat(good_sweep_hk_data[flag_name.lower()].data, 6).reshape(
-            -1, 72
+            -1, NUM_ENERGY_STEPS
         )
         # Use getattr to dynamically access the flag in SWAPIFlags class
         flag_to_set = getattr(SWAPIFlags, flag_name)
@@ -568,7 +572,9 @@ def process_swapi_science(
     # Use center time for epoch to line up with mission requests. Center time
     # of SWAPI is time of 7th packet(aka SEQ_NUMBER == 6) creation time at the
     # beginning of 7th packet.
-    epoch_values = good_sweep_sci["epoch"].data.reshape(total_full_sweeps, 12)[:, 6]
+    epoch_values = good_sweep_sci["epoch"].data.reshape(
+        total_full_sweeps, NUM_PACKETS_PER_SWEEP
+    )[:, 6]
 
     epoch_time = xr.DataArray(
         epoch_values,
@@ -626,20 +632,33 @@ def process_swapi_science(
 
     # Add other support data
     dataset["sweep_table"] = xr.DataArray(
-        good_sweep_sci["sweep_table"].data.reshape(total_full_sweeps, 12)[:, 0],
+        good_sweep_sci["sweep_table"].data.reshape(
+            total_full_sweeps, NUM_PACKETS_PER_SWEEP
+        )[:, 0],
         name="sweep_table",
         dims=["epoch"],
         attrs=cdf_manager.get_variable_attributes("sweep_table"),
     )
     dataset["plan_id"] = xr.DataArray(
-        good_sweep_sci["plan_id"].data.reshape(total_full_sweeps, 12)[:, 0],
+        good_sweep_sci["plan_id"].data.reshape(
+            total_full_sweeps, NUM_PACKETS_PER_SWEEP
+        )[:, 0],
         name="plan_id",
         dims=["epoch"],
         attrs=cdf_manager.get_variable_attributes("plan_id"),
     )
     # Store start time for L3 purposes per SWAPI requests
+    # Per SWAPI request, convert start time of sweep to UTC time.
+    sci_start_time = met_to_utc(
+        ttj2000ns_to_met(
+            good_sweep_sci["epoch"].data.reshape(
+                total_full_sweeps, NUM_PACKETS_PER_SWEEP
+            )[:, 0]
+        ),
+        precision=0,
+    )
     dataset["sci_start_time"] = xr.DataArray(
-        good_sweep_sci["epoch"].data.reshape(total_full_sweeps, 12)[:, 0],
+        sci_start_time,
         name="sci_start_time",
         dims=["epoch"],
         attrs=cdf_manager.get_variable_attributes("sci_start_time"),
@@ -650,7 +669,9 @@ def process_swapi_science(
     # updated every 6th step. This is used in L2 to calculate last 9 fine
     # energy steps.
     dataset["esa_lvl5"] = xr.DataArray(
-        good_sweep_sci["esa_lvl5"].data.reshape(total_full_sweeps, 12)[:, 11],
+        good_sweep_sci["esa_lvl5"].data.reshape(
+            total_full_sweeps, NUM_PACKETS_PER_SWEEP
+        )[:, 11],
         name="esa_lvl5",
         dims=["epoch"],
         attrs=cdf_manager.get_variable_attributes("esa_lvl5"),
@@ -661,19 +682,25 @@ def process_swapi_science(
     #   SWP_HK.FPGA_TYPE - Type number of the FPGA
     #   SWP_HK.FPGA_REV - Revision number of the FPGA
     dataset["lut_choice"] = xr.DataArray(
-        good_sweep_hk_data["lut_choice"].data.reshape(total_full_sweeps, 12)[:, 0],
+        good_sweep_hk_data["lut_choice"].data.reshape(
+            total_full_sweeps, NUM_PACKETS_PER_SWEEP
+        )[:, 0],
         name="lut_choice",
         dims=["epoch"],
         attrs=cdf_manager.get_variable_attributes("lut_choice"),
     )
     dataset["fpga_type"] = xr.DataArray(
-        good_sweep_hk_data["fpga_type"].data.reshape(total_full_sweeps, 12)[:, 0],
+        good_sweep_hk_data["fpga_type"].data.reshape(
+            total_full_sweeps, NUM_PACKETS_PER_SWEEP
+        )[:, 0],
         name="fpga_type",
         dims=["epoch"],
         attrs=cdf_manager.get_variable_attributes("fpga_type"),
     )
     dataset["fpga_rev"] = xr.DataArray(
-        good_sweep_hk_data["fpga_rev"].data.reshape(total_full_sweeps, 12)[:, 0],
+        good_sweep_hk_data["fpga_rev"].data.reshape(
+            total_full_sweeps, NUM_PACKETS_PER_SWEEP
+        )[:, 0],
         name="fpga_rev",
         dims=["epoch"],
         attrs=cdf_manager.get_variable_attributes("fpga_rev"),

imap_processing/swapi/l2/swapi_l2.py

@@ -8,6 +8,7 @@ import pandas as pd
 import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.swapi.constants import NUM_ENERGY_STEPS
 
 logger = logging.getLogger(__name__)
 
@@ -72,15 +73,28 @@ def solve_full_sweep_energy(
             (esa_table_df["timestamp"] <= time) & (esa_table_df["Sweep #"] == sweep_id)
         ]
         if subset.empty:
-            first_63_energies.append(np.full(63, np.nan, dtype=np.float64))
-            continue
+            # Get the earliest timestamp available
+            earliest_time = esa_table_df["timestamp"].min()
+
+            # Find the sweep's ESA data for the earliest time and sweep_id
+            earliest_subset = esa_table_df[
+                (esa_table_df["timestamp"] == earliest_time)
+                & (esa_table_df["Sweep #"] == sweep_id)
+            ]
+            if earliest_subset.empty:
+                raise ValueError(
+                    f"No matching ESA table entry found for sweep ID {sweep_id} "
+                    f"at time {time}, and no entries found for earliest time "
+                    f"{earliest_time}."
+                )
+            subset = earliest_subset
 
         # Subset data can contain multiple 72 energy values with last 9 fine energies
         # with 'Solve' value. We need to sort by time and ESA step to maintain correct
         # order. Then take the last group of 72 steps values and select first 63
         # values only.
         subset = subset.sort_values(["timestamp", "ESA Step #"])
-        grouped = subset["Energy"].values.reshape(-1, 72)
+        grouped = subset["Energy"].values.reshape(-1, NUM_ENERGY_STEPS)
         first_63 = grouped[-1, :63]
         first_63_energies.append(first_63)
 

imap_processing/ultra/l1a/ultra_l1a.py

@@ -43,7 +43,7 @@ logger = logging.getLogger(__name__)
 
 
 def ultra_l1a(  # noqa: PLR0912
-    packet_file: str, apid_input: int | None = None
+    packet_file: str, apid_input: int | None = None, create_derived_l1b: bool = False
 ) -> list[xr.Dataset]:
     """
     Will process ULTRA L0 data into L1A CDF files at output_filepath.
@@ -54,6 +54,8 @@ def ultra_l1a(  # noqa: PLR0912
         Path to the CCSDS data packet file.
     apid_input : Optional[int]
         Optional apid.
+    create_derived_l1b : bool
+        Whether to create the l1b datasets with derived values.
 
     Returns
     -------
@@ -64,7 +66,17 @@ def ultra_l1a(  # noqa: PLR0912
         f"{imap_module_directory}/ultra/packet_definitions/ULTRA_SCI_COMBINED.xml"
     )
 
+    # Keep a list to track the two versions, l1a and l1b with the derived values.
+    decommutated_packet_datasets = []
     datasets_by_apid = packet_file_to_datasets(packet_file, xtce)
+    decommutated_packet_datasets.append(datasets_by_apid)
+    if create_derived_l1b:
+        # For the housekeeping products, we can create the l1b at the same time
+        # as the l1a since there is no additional processing needed.
+        datasets_by_apid = packet_file_to_datasets(
+            packet_file, xtce, use_derived_value=True
+        )
+        decommutated_packet_datasets.append(datasets_by_apid)
 
     output_datasets = []
 
@@ -109,77 +121,114 @@ def ultra_l1a(  # noqa: PLR0912
     attr_mgr.add_instrument_global_attrs("ultra")
     attr_mgr.add_instrument_variable_attrs("ultra", "l1a")
 
-    for apid in apids:
-        if apid in ULTRA_AUX.apid:
-            decom_ultra_dataset = datasets_by_apid[apid]
-            gattr_key = ULTRA_AUX.logical_source[ULTRA_AUX.apid.index(apid)]
-        elif apid in all_l1a_image_apids:
-            packet_props = all_l1a_image_apids[apid]
-            decom_ultra_dataset = process_ultra_tof(
-                datasets_by_apid[apid], packet_props
-            )
-            gattr_key = packet_props.logical_source[packet_props.apid.index(apid)]
-        elif apid in ULTRA_RATES.apid:
-            decom_ultra_dataset = process_ultra_rates(datasets_by_apid[apid])
-            decom_ultra_dataset = decom_ultra_dataset.drop_vars("fastdata_00")
-            gattr_key = ULTRA_RATES.logical_source[ULTRA_RATES.apid.index(apid)]
-        elif apid in ULTRA_ENERGY_RATES.apid:
-            decom_ultra_dataset = process_ultra_energy_rates(datasets_by_apid[apid])
-            decom_ultra_dataset = decom_ultra_dataset.drop_vars("ratedata")
-            gattr_key = ULTRA_ENERGY_RATES.logical_source[
-                ULTRA_ENERGY_RATES.apid.index(apid)
-            ]
-        elif apid in all_event_apids:
-            decom_ultra_dataset = process_ultra_events(datasets_by_apid[apid], apid)
-            gattr_key = all_event_apids[apid]
+    for i, datasets_by_apid in enumerate(decommutated_packet_datasets):
+        for apid in apids:
+            if apid in ULTRA_AUX.apid:
+                decom_ultra_dataset = datasets_by_apid[apid]
+                gattr_key = ULTRA_AUX.logical_source[ULTRA_AUX.apid.index(apid)]
+            elif apid in all_l1a_image_apids:
+                packet_props = all_l1a_image_apids[apid]
+                decom_ultra_dataset = process_ultra_tof(
+                    datasets_by_apid[apid], packet_props
+                )
+                gattr_key = packet_props.logical_source[packet_props.apid.index(apid)]
+            elif apid in ULTRA_RATES.apid:
+                decom_ultra_dataset = process_ultra_rates(datasets_by_apid[apid])
+                decom_ultra_dataset = decom_ultra_dataset.drop_vars("fastdata_00")
+                gattr_key = ULTRA_RATES.logical_source[ULTRA_RATES.apid.index(apid)]
+            elif apid in ULTRA_ENERGY_RATES.apid:
+                decom_ultra_dataset = process_ultra_energy_rates(datasets_by_apid[apid])
+                decom_ultra_dataset = decom_ultra_dataset.drop_vars("ratedata")
+                gattr_key = ULTRA_ENERGY_RATES.logical_source[
+                    ULTRA_ENERGY_RATES.apid.index(apid)
+                ]
+            elif apid in all_event_apids:
+                # We don't want to process the event l1b datasets since those l1b
+                # products need more information
+                if i == 1:
+                    continue
+                decom_ultra_dataset = process_ultra_events(datasets_by_apid[apid], apid)
+                gattr_key = all_event_apids[apid]
+                # Add coordinate attributes
+                attrs = attr_mgr.get_variable_attributes("event_id")
+                decom_ultra_dataset.coords["event_id"].attrs.update(attrs)
+            elif apid in ULTRA_ENERGY_SPECTRA.apid:
+                decom_ultra_dataset = process_ultra_energy_spectra(
+                    datasets_by_apid[apid]
+                )
+                decom_ultra_dataset = decom_ultra_dataset.drop_vars("compdata")
+                gattr_key = ULTRA_ENERGY_SPECTRA.logical_source[
+                    ULTRA_ENERGY_SPECTRA.apid.index(apid)
+                ]
+            elif apid in ULTRA_MACROS_CHECKSUM.apid:
+                decom_ultra_dataset = process_ultra_macros_checksum(
+                    datasets_by_apid[apid]
+                )
+                gattr_key = ULTRA_MACROS_CHECKSUM.logical_source[
+                    ULTRA_MACROS_CHECKSUM.apid.index(apid)
+                ]
+            elif apid in ULTRA_HK.apid:
+                decom_ultra_dataset = datasets_by_apid[apid]
+                gattr_key = ULTRA_HK.logical_source[ULTRA_HK.apid.index(apid)]
+            elif apid in ULTRA_CMD_TEXT.apid:
+                decom_ultra_dataset = datasets_by_apid[apid]
+                decoded_strings = [
+                    s.decode("ascii").rstrip("\x00")
+                    for s in decom_ultra_dataset["text"].values
+                ]
+                decom_ultra_dataset = decom_ultra_dataset.drop_vars("text")
+                decom_ultra_dataset["text"] = xr.DataArray(
+                    decoded_strings,
+                    dims=["epoch"],
+                    coords={"epoch": decom_ultra_dataset["epoch"]},
+                )
+                gattr_key = ULTRA_CMD_TEXT.logical_source[
+                    ULTRA_CMD_TEXT.apid.index(apid)
+                ]
+            elif apid in ULTRA_CMD_ECHO.apid:
+                decom_ultra_dataset = process_ultra_cmd_echo(datasets_by_apid[apid])
+                gattr_key = ULTRA_CMD_ECHO.logical_source[
+                    ULTRA_CMD_ECHO.apid.index(apid)
+                ]
+            else:
+                logger.error(f"APID {apid} not recognized.")
+                continue
+
+            decom_ultra_dataset.attrs.update(attr_mgr.get_global_attributes(gattr_key))
+
+            if i == 1:
+                # Derived values dataset at l1b
+                # We already have the l1a attributes, just update the l1a -> l1b
+                # in the metadata.
+                decom_ultra_dataset.attrs["Data_type"] = decom_ultra_dataset.attrs[
+                    "Data_type"
+                ].replace("1A", "1B")
+                decom_ultra_dataset.attrs["Logical_source"] = decom_ultra_dataset.attrs[
+                    "Logical_source"
+                ].replace("l1a", "l1b")
+                decom_ultra_dataset.attrs["Logical_source_description"] = (
+                    decom_ultra_dataset.attrs["Logical_source_description"].replace(
+                        "1A", "1B"
+                    )
+                )
+
+            # Add data variable attributes
+            for key in decom_ultra_dataset.data_vars:
+                attrs = attr_mgr.get_variable_attributes(key.lower())
+                decom_ultra_dataset.data_vars[key].attrs.update(attrs)
+                if i == 1:
+                    # For l1b datasets, the FILLVAL and VALIDMIN/MAX may be
+                    # different datatypes, so we can't use them directly from l1a.
+                    # just remove them for now since we don't really have a need for
+                    # for them currently.
+                    for attr_key in ["FILLVAL", "VALIDMIN", "VALIDMAX"]:
+                        if attr_key in decom_ultra_dataset.data_vars[key].attrs:
+                            decom_ultra_dataset.data_vars[key].attrs.pop(attr_key)
+
             # Add coordinate attributes
-            attrs = attr_mgr.get_variable_attributes("event_id")
-            decom_ultra_dataset.coords["event_id"].attrs.update(attrs)
-        elif apid in ULTRA_ENERGY_SPECTRA.apid:
-            decom_ultra_dataset = process_ultra_energy_spectra(datasets_by_apid[apid])
-            decom_ultra_dataset = decom_ultra_dataset.drop_vars("compdata")
-            gattr_key = ULTRA_ENERGY_SPECTRA.logical_source[
-                ULTRA_ENERGY_SPECTRA.apid.index(apid)
-            ]
-        elif apid in ULTRA_MACROS_CHECKSUM.apid:
-            decom_ultra_dataset = process_ultra_macros_checksum(datasets_by_apid[apid])
-            gattr_key = ULTRA_MACROS_CHECKSUM.logical_source[
-                ULTRA_MACROS_CHECKSUM.apid.index(apid)
-            ]
-        elif apid in ULTRA_HK.apid:
-            decom_ultra_dataset = datasets_by_apid[apid]
-            gattr_key = ULTRA_HK.logical_source[ULTRA_HK.apid.index(apid)]
-        elif apid in ULTRA_CMD_TEXT.apid:
-            decom_ultra_dataset = datasets_by_apid[apid]
-            decoded_strings = [
-                s.decode("ascii").rstrip("\x00")
-                for s in decom_ultra_dataset["text"].values
-            ]
-            decom_ultra_dataset = decom_ultra_dataset.drop_vars("text")
-            decom_ultra_dataset["text"] = xr.DataArray(
-                decoded_strings,
-                dims=["epoch"],
-                coords={"epoch": decom_ultra_dataset["epoch"]},
-            )
-            gattr_key = ULTRA_CMD_TEXT.logical_source[ULTRA_CMD_TEXT.apid.index(apid)]
-        elif apid in ULTRA_CMD_ECHO.apid:
-            decom_ultra_dataset = process_ultra_cmd_echo(datasets_by_apid[apid])
-            gattr_key = ULTRA_CMD_ECHO.logical_source[ULTRA_CMD_ECHO.apid.index(apid)]
-        else:
-            logger.error(f"APID {apid} not recognized.")
-            continue
-
-        decom_ultra_dataset.attrs.update(attr_mgr.get_global_attributes(gattr_key))
-
-        # Add data variable attributes
-        for key in decom_ultra_dataset.data_vars:
-            attrs = attr_mgr.get_variable_attributes(key.lower())
-            decom_ultra_dataset.data_vars[key].attrs.update(attrs)
-
-        # Add coordinate attributes
-        attrs = attr_mgr.get_variable_attributes("epoch", check_schema=False)
-        decom_ultra_dataset.coords["epoch"].attrs.update(attrs)
-
-        output_datasets.append(decom_ultra_dataset)
+            attrs = attr_mgr.get_variable_attributes("epoch", check_schema=False)
+            decom_ultra_dataset.coords["epoch"].attrs.update(attrs)
+
+            output_datasets.append(decom_ultra_dataset)
 
     return output_datasets

imap_processing/ultra/l1b/de.py

@@ -5,10 +5,13 @@ import xarray as xr
 
 from imap_processing.cdf.utils import parse_filename_like
 from imap_processing.quality_flags import (
+    ImapAttitudeUltraFlags,
     ImapDEOutliersUltraFlags,
     ImapDEScatteringUltraFlags,
 )
 from imap_processing.spice.geometry import SpiceFrame
+from imap_processing.spice.repoint import get_repoint_data
+from imap_processing.spice.time import et_to_met
 from imap_processing.ultra.l1b.lookup_utils import get_geometric_factor
 from imap_processing.ultra.l1b.ultra_l1b_annotated import (
     get_annotated_particle_velocity,
@@ -74,6 +77,10 @@ def calculate_de(
     spin_number = get_spin_number(
         de_dataset["shcoarse"].values, de_dataset["spin"].values
     )
+    repoint_id = de_dataset.attrs.get("Repointing", None)
+    if repoint_id is not None:
+        repoint_id = int(repoint_id.replace("repoint", ""))
+
     de_dict["spin"] = spin_number
 
     # Add already populated fields.
@@ -311,7 +318,19 @@ def calculate_de(
     ultra_frame = getattr(SpiceFrame, f"IMAP_ULTRA_{sensor}")
 
     # Account for counts=0 (event times have FILL value)
-    valid_events = event_times != FILLVAL_FLOAT32
+    valid_events = (event_times != FILLVAL_FLOAT32).copy()
+    # TODO - find a better solution than filtering out data from repointings?
+    if repoint_id is not None:
+        in_pointing = calculate_events_in_pointing(
+            repoint_id, event_times[valid_events]
+        )
+        # Update quality flags for valid events that are not in the pointing
+        quality_flags[valid_events][~in_pointing] |= (
+            ImapAttitudeUltraFlags.DURINGREPOINT.value
+        )
+        # Update valid_events to only include times within a pointing
+        valid_events[valid_events] &= in_pointing
+
     if np.any(valid_events):
         (
             sc_velocity[valid_events],
@@ -371,3 +390,40 @@ def calculate_de(
     dataset = create_dataset(de_dict, name, "l1b")
 
     return dataset
+
+
+def calculate_events_in_pointing(
+    repoint_id: int,
+    event_times: np.ndarray,
+) -> np.ndarray:
+    """
+    Calculate boolean array of events within a pointing.
+
+    Parameters
+    ----------
+    repoint_id : int
+        The repointing ID.
+    event_times : np.ndarray
+        Array of event times in ET.
+
+    Returns
+    -------
+    in_pointing : np.ndarray
+        Boolean array indicating whether each event is within the pointing period
+        combined with the valid_events mask.
+    """
+    # TODO add this as a helper function in repoint.py
+    repoint_data = get_repoint_data()
+    # To find the pointing start and stop, get the end of the current repointing
+    # and the start of the next repointing
+    repoint_row = repoint_data[repoint_data["repoint_id"] == repoint_id]
+    next_repoint_row = repoint_data[repoint_data["repoint_id"] == repoint_id + 1]
+    pointing_start_met = repoint_row["repoint_end_met"].values[0]
+    pointing_end_met = next_repoint_row["repoint_start_met"].values[0]
+
+    # Check which events are within the pointing
+    in_pointing = (et_to_met(event_times) >= pointing_start_met) & (
+        et_to_met(event_times) <= pointing_end_met
+    )
+
+    return in_pointing

imap_processing/ultra/l1b/ultra_l1b_annotated.py

@@ -52,7 +52,6 @@ def get_annotated_particle_velocity(
         from_frame=instrument_frame,
         to_frame=spacecraft_frame,
     )
-
     # Particle velocity in the pointing (DPS) frame wrt spacecraft.
     particle_velocity_dps_spacecraft = frame_transform(
         et=time,