imap-processing 0.11.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

imap_processing/hi/l1a/hi_l1a.py

@@ -7,8 +7,8 @@ from typing import Union
 import xarray as xr
 
 from imap_processing import imap_module_directory
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.hi.l1a.histogram import create_dataset as hist_create_dataset
-from imap_processing.hi.l1a.housekeeping import process_housekeeping
 from imap_processing.hi.l1a.science_direct_event import science_direct_event
 from imap_processing.hi.utils import HIAPID
 from imap_processing.utils import packet_file_to_datasets
@@ -16,7 +16,7 @@ from imap_processing.utils import packet_file_to_datasets
 logger = logging.getLogger(__name__)
 
 
-def hi_l1a(packet_file_path: Union[str, Path], data_version: str) -> list[xr.Dataset]:
+def hi_l1a(packet_file_path: Union[str, Path]) -> list[xr.Dataset]:
     """
     Will process IMAP raw data to l1a.
 
@@ -24,50 +24,75 @@ def hi_l1a(packet_file_path: Union[str, Path], data_version: str) -> list[xr.Dat
     ----------
     packet_file_path : str
         Data packet file path.
-    data_version : str
-        Version of the data product being created.
 
     Returns
     -------
     processed_data : list[xarray.Dataset]
         List of processed xarray dataset.
     """
-    packet_def_file = (
-        imap_module_directory / "hi/packet_definitions/TLM_HI_COMBINED_SCI.xml"
-    )
-    datasets_by_apid = packet_file_to_datasets(
-        packet_file=packet_file_path, xtce_packet_definition=packet_def_file
-    )
+    datasets_by_apid = hi_packet_file_to_datasets(packet_file_path)
 
     # Process science to l1a.
     processed_data = []
     for apid in datasets_by_apid:
-        if apid in [HIAPID.H45_SCI_CNT, HIAPID.H90_SCI_CNT]:
-            logger.info(
-                "Processing histogram data for [%s] packets", HIAPID.H45_SCI_CNT.name
-            )
-            data = hist_create_dataset(datasets_by_apid[apid])
-        elif apid in [HIAPID.H45_SCI_DE, HIAPID.H90_SCI_DE]:
-            logger.info(
-                "Processing direct event data for [%s] packets", HIAPID.H45_SCI_DE.name
-            )
+        try:
+            apid_enum = HIAPID(apid)
+        except ValueError as err:
+            raise RuntimeError(f"Encountered unexpected APID [{apid}]") from err
 
+        logger.info(f"Processing IMAP-Hi data for {apid_enum.name} packets")
+
+        if apid_enum in [HIAPID.H45_SCI_CNT, HIAPID.H90_SCI_CNT]:
+            data = hist_create_dataset(datasets_by_apid[apid])
+            gattr_key = "imap_hi_l1a_hist_attrs"
+        elif apid_enum in [HIAPID.H45_SCI_DE, HIAPID.H90_SCI_DE]:
             data = science_direct_event(datasets_by_apid[apid])
-        elif apid in [HIAPID.H45_APP_NHK, HIAPID.H90_APP_NHK]:
-            logger.info(
-                "Processing housekeeping data for [%s] packets", HIAPID.H45_APP_NHK.name
-            )
-            data = process_housekeeping(datasets_by_apid[apid])
-        else:
-            raise RuntimeError(f"Encountered unexpected APID [{apid}]")
+            gattr_key = "imap_hi_l1a_de_attrs"
+        elif apid_enum in [HIAPID.H45_APP_NHK, HIAPID.H90_APP_NHK]:
+            data = datasets_by_apid[apid]
+            gattr_key = "imap_hi_l1a_hk_attrs"
+        elif apid_enum in [HIAPID.H45_DIAG_FEE, HIAPID.H90_DIAG_FEE]:
+            data = datasets_by_apid[apid]
+            gattr_key = "imap_hi_l1a_diagfee_attrs"
 
-        # TODO: revisit this
-        data.attrs["Data_version"] = data_version
+        # Update dataset global attributes
+        attr_mgr = ImapCdfAttributes()
+        attr_mgr.add_instrument_global_attrs("hi")
+        data.attrs.update(attr_mgr.get_global_attributes(gattr_key))
 
         # set the sensor string in Logical_source
-        sensor_str = HIAPID(apid).sensor
+        sensor_str = apid_enum.sensor
         data.attrs["Logical_source"] = data.attrs["Logical_source"].format(
             sensor=sensor_str
         )
         processed_data.append(data)
     return processed_data
+
+
+def hi_packet_file_to_datasets(
+    packet_file_path: Union[str, Path], use_derived_value: bool = False
+) -> dict[int, xr.Dataset]:
+    """
+    Extract hi datasets from packet file.
+
+    Parameters
+    ----------
+    packet_file_path : str
+        L0 packet file path.
+    use_derived_value : bool
+        Whether to use the derived value from the XTCE definition. Default is False.
+
+    Returns
+    -------
+    datasets : dict[int, xarray.Dataset]
+        Dictionary of xarray datasets keyed by APID.
+    """
+    packet_def_file = (
+        imap_module_directory / "hi/packet_definitions/TLM_HI_COMBINED_SCI.xml"
+    )
+    datasets_by_apid = packet_file_to_datasets(
+        packet_file=packet_file_path,
+        xtce_packet_definition=packet_def_file,
+        use_derived_value=use_derived_value,
+    )
+    return datasets_by_apid

imap_processing/hi/l1a/histogram.py

@@ -116,7 +116,6 @@ def create_dataset(input_ds: xr.Dataset) -> xr.Dataset:
     )
 
     dataset.update(new_vars)
-    dataset.attrs.update(attr_mgr.get_global_attributes("imap_hi_l1a_hist_attrs"))
 
     return dataset
 

imap_processing/hi/l1a/science_direct_event.py

@@ -31,11 +31,11 @@ def parse_direct_events(de_data: bytes) -> dict[str, npt.ArrayLike]:
     IMAP-Hi direct event data information is stored in
     48-bits as follows:
 
-    |        Read 48-bits into 2, 16, 10, 10, 10, bits. Each of these breaks
+    |        Read 48-bits into 16, 2, 10, 10, 10, bits. Each of these breaks
     |        down as:
     |
-    |            start_bitmask_data - 2 bits (tA=1, tB=2, tC1=3, META=0)
     |            de_tag - 16 bits
+    |            start_bitmask_data - 2 bits (tA=1, tB=2, tC1=3)
     |            tof_1 - 10 bit counter
     |            tof_2 - 10 bit counter
     |            tof_3 - 10 bit counter
@@ -70,16 +70,16 @@ def parse_direct_events(de_data: bytes) -> dict[str, npt.ArrayLike]:
     # direct events.
     # Considering the 6-bytes of data for each DE as 3 2-byte words,
     # each word contains the following:
-    # word_0: 2-bits of Trigger ID, upper 14-bits of de_tag
-    # word_1: lower 2-bits of de_tag, 10-bits tof_1, upper 4-bits of tof_2
+    # word_0: full 16-bits is the de_tag
+    # word_1: 2-bits of Trigger ID, 10-bits tof_1, upper 4-bits of tof_2
     # word_2: lower 6-bits of tof_2, 10-bits of tof_3
     data_uint16 = np.reshape(
         np.frombuffer(de_data, dtype=">u2"), (3, -1), order="F"
     ).astype(np.uint16)
 
     de_dict = dict()
-    de_dict["trigger_id"] = (data_uint16[0] >> 14).astype(np.uint8)
-    de_dict["de_tag"] = (data_uint16[0] << 2) + (data_uint16[1] >> 14)
+    de_dict["de_tag"] = data_uint16[0]
+    de_dict["trigger_id"] = (data_uint16[1] >> 14).astype(np.uint8)
     de_dict["tof_1"] = (data_uint16[1] & int(b"00111111_11110000", 2)) >> 4
     de_dict["tof_2"] = ((data_uint16[1] & int(b"00000000_00001111", 2)) << 6) + (
         data_uint16[2] >> 10
@@ -147,10 +147,8 @@ def create_dataset(de_data_dict: dict[str, npt.ArrayLike]) -> xr.Dataset:
         attrs=event_met_attrs,
     )
 
-    de_global_attrs = attr_mgr.get_global_attributes("imap_hi_l1a_de_attrs")
     dataset = xr.Dataset(
         coords={"epoch": epoch, "event_met": event_met},
-        attrs=de_global_attrs,
     )
 
     for var_name, data in de_data_dict.items():

imap_processing/hi/l1b/hi_l1b.py

@@ -2,6 +2,8 @@
 
 import logging
 from enum import IntEnum
+from pathlib import Path
+from typing import Union
 
 import numpy as np
 import xarray as xr
@@ -12,6 +14,7 @@ from imap_processing.cdf.utils import parse_filename_like
 from imap_processing.hi.l1a.science_direct_event import HALF_CLOCK_TICK_S
 from imap_processing.hi.utils import (
     HIAPID,
+    CoincidenceBitmap,
     HiConstants,
     create_dataset_variables,
     parse_sensor_number,
@@ -25,7 +28,7 @@ from imap_processing.spice.spin import (
     get_spacecraft_spin_phase,
 )
 from imap_processing.spice.time import met_to_sclkticks, sct_to_et
-from imap_processing.utils import convert_raw_to_eu
+from imap_processing.utils import packet_file_to_datasets
 
 
 class TriggerId(IntEnum):
@@ -36,76 +39,102 @@ class TriggerId(IntEnum):
     C = 3
 
 
-class CoincidenceBitmap(IntEnum):
-    """IntEnum class for coincidence type bitmap values."""
-
-    A = 2**3
-    B = 2**2
-    C1 = 2**1
-    C2 = 2**0
-
-
 logger = logging.getLogger(__name__)
 ATTR_MGR = ImapCdfAttributes()
 ATTR_MGR.add_instrument_global_attrs("hi")
 ATTR_MGR.add_instrument_variable_attrs(instrument="hi", level=None)
 
 
-def hi_l1b(l1a_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
+def hi_l1b(dependency: Union[str, Path, xr.Dataset]) -> list[xr.Dataset]:
     """
     High level IMAP-HI L1B processing function.
 
     Parameters
     ----------
-    l1a_dataset : xarray.Dataset
-        L1A dataset to process.
-    data_version : str
-        Version of the data product being created.
+    dependency : str or xarray.Dataset
+        Path to L0 file or L1A dataset to process.
 
     Returns
     -------
-    l1b_dataset : xarray.Dataset
-        Processed xarray dataset.
+    l1b_dataset : list[xarray.Dataset]
+        Processed xarray datasets.
     """
-    logger.info(
-        f"Running Hi L1B processing on dataset: {l1a_dataset.attrs['Logical_source']}"
-    )
-    logical_source_parts = parse_filename_like(l1a_dataset.attrs["Logical_source"])
-    # TODO: apid is not currently stored in all L1A data but should be.
-    #    Use apid to determine what L1B processing function to call
-
     # Housekeeping processing
-    if logical_source_parts["descriptor"].endswith("hk"):
-        # if packet_enum in (HIAPID.H45_APP_NHK, HIAPID.H90_APP_NHK):
-        packet_enum = HIAPID(l1a_dataset["pkt_apid"].data[0])
-        conversion_table_path = str(
-            imap_module_directory / "hi" / "l1b" / "hi_eng_unit_convert_table.csv"
-        )
-        l1b_dataset = convert_raw_to_eu(
-            l1a_dataset,
-            conversion_table_path=conversion_table_path,
-            packet_name=packet_enum.name,
-            comment="#",  # type: ignore[arg-type]
-            # Todo error, Argument "comment" to "convert_raw_to_eu" has incompatible
-            # type "str"; expected "dict[Any, Any]"
-            converters={"mnemonic": str.lower},
-        )
-
-        l1b_dataset.attrs.update(ATTR_MGR.get_global_attributes("imap_hi_l1b_hk_attrs"))
-    elif logical_source_parts["descriptor"].endswith("de"):
-        l1b_dataset = annotate_direct_events(l1a_dataset)
-    else:
-        raise NotImplementedError(
-            f"No Hi L1B processing defined for file type: "
+    if isinstance(dependency, (Path, str)):
+        logger.info(f"Running Hi L1B processing on file: {dependency}")
+        l1b_datasets = housekeeping(dependency)
+    elif isinstance(dependency, xr.Dataset):
+        l1a_dataset = dependency
+        logger.info(
+            f"Running Hi L1B processing on dataset: "
             f"{l1a_dataset.attrs['Logical_source']}"
         )
-    # Update global attributes
-    l1b_dataset.attrs["Logical_source"] = l1b_dataset.attrs["Logical_source"].format(
-        sensor=logical_source_parts["sensor"]
+        logical_source_parts = parse_filename_like(l1a_dataset.attrs["Logical_source"])
+        # TODO: apid is not currently stored in all L1A data but should be.
+        #    Use apid to determine what L1B processing function to call
+
+        # DE processing
+        if logical_source_parts["descriptor"].endswith("de"):
+            l1b_datasets = [annotate_direct_events(l1a_dataset)]
+            l1b_datasets[0].attrs["Logical_source"] = (
+                l1b_datasets[0]
+                .attrs["Logical_source"]
+                .format(sensor=logical_source_parts["sensor"])
+            )
+        else:
+            raise NotImplementedError(
+                f"No Hi L1B processing defined for file type: "
+                f"{l1a_dataset.attrs['Logical_source']}"
+            )
+
+    return l1b_datasets
+
+
+def housekeeping(packet_file_path: Union[str, Path]) -> list[xr.Dataset]:
+    """
+    Will process IMAP raw data to l1b housekeeping dataset.
+
+    In order to use `space_packet_parser` and the xtce which contains the
+    DN to EU conversion factors, the L0 packet file is used to go straight to
+    L1B.
+
+    Parameters
+    ----------
+    packet_file_path : str
+        Packet file path.
+
+    Returns
+    -------
+    processed_data : list[xarray.Dataset]
+        Housekeeping datasets with engineering units.
+    """
+    packet_def_file = (
+        imap_module_directory / "hi/packet_definitions/TLM_HI_COMBINED_SCI.xml"
     )
-    # TODO: revisit this
-    l1b_dataset.attrs["Data_version"] = data_version
-    return l1b_dataset
+    # TODO: If raw and derived values can be gotten from one call to
+    #    packet_file_to_datasets, the L1A and L1B could be generated
+    #    in a single L1A/B function.
+    datasets_by_apid = packet_file_to_datasets(
+        packet_file=packet_file_path,
+        xtce_packet_definition=packet_def_file,
+        use_derived_value=True,
+    )
+
+    # Extract only the HK datasets
+    attr_mgr = ImapCdfAttributes()
+    attr_mgr.add_instrument_global_attrs("hi")
+    datasets = list()
+    for apid in [HIAPID.H45_APP_NHK, HIAPID.H90_APP_NHK]:
+        if apid in datasets_by_apid:
+            datasets.append(datasets_by_apid[apid])
+            # Update the dataset global attributes
+            datasets[-1].attrs.update(
+                ATTR_MGR.get_global_attributes("imap_hi_l1b_hk_attrs")
+            )
+            datasets[-1].attrs["Logical_source"] = (
+                datasets[-1].attrs["Logical_source"].format(sensor=apid.sensor)
+            )
+    return datasets
 
 
 def annotate_direct_events(l1a_dataset: xr.Dataset) -> xr.Dataset:
@@ -124,7 +153,7 @@ def annotate_direct_events(l1a_dataset: xr.Dataset) -> xr.Dataset:
     """
     l1b_dataset = l1a_dataset.copy()
     l1b_dataset.update(de_esa_energy_step(l1b_dataset))
-    l1b_dataset.update(compute_coincidence_type_and_time_deltas(l1b_dataset))
+    l1b_dataset.update(compute_coincidence_type_and_tofs(l1b_dataset))
     l1b_dataset.update(de_nominal_bin_and_spin_phase(l1b_dataset))
     l1b_dataset.update(compute_hae_coordinates(l1b_dataset))
     l1b_dataset.update(
@@ -154,14 +183,14 @@ def annotate_direct_events(l1a_dataset: xr.Dataset) -> xr.Dataset:
     return l1b_dataset
 
 
-def compute_coincidence_type_and_time_deltas(
+def compute_coincidence_type_and_tofs(
     dataset: xr.Dataset,
 ) -> dict[str, xr.DataArray]:
     """
-    Compute coincidence type and time deltas.
+    Compute coincidence type and time of flights.
 
-    Generates the new variables "coincidence_type", "delta_t_ab", "delta_t_ac1",
-    "delta_t_bc1", and "delta_t_c1c2" and returns a dictionary with the new
+    Generates the new variables "coincidence_type", "tof_ab", "tof_ac1",
+    "tof_bc1", and "tof_c1c2" and returns a dictionary with the new
     variables that can be added to the input dataset by calling the
     xarray.Dataset.update method.
 
@@ -178,16 +207,16 @@ def compute_coincidence_type_and_time_deltas(
     new_vars = create_dataset_variables(
         [
             "coincidence_type",
-            "delta_t_ab",
-            "delta_t_ac1",
-            "delta_t_bc1",
-            "delta_t_c1c2",
+            "tof_ab",
+            "tof_ac1",
+            "tof_bc1",
+            "tof_c1c2",
         ],
         len(dataset.event_met),
         att_manager_lookup_str="hi_de_{0}",
     )
 
-    # compute masks needed for coincidence type and delta t calculations
+    # compute masks needed for coincidence type and ToF calculations
     a_first = dataset.trigger_id.values == TriggerId.A
     b_first = dataset.trigger_id.values == TriggerId.B
     c_first = dataset.trigger_id.values == TriggerId.C
@@ -221,56 +250,56 @@ def compute_coincidence_type_and_time_deltas(
     # |      2      |      B      |  t_a - t_b  | t_c1 - t_b  | t_c2 - t_c1 |
     # |      3      |      C      |  t_a - t_c1 | t_b  - t_c1 | t_c2 - t_c1 |
 
-    # Prepare for delta_t calculations by converting TOF values to nanoseconds
+    # Prepare for L1B ToF calculations by converting L1A TOF values to nanoseconds
     tof_1_ns = (dataset.tof_1.values * HiConstants.TOF1_TICK_DUR).astype(np.int32)
     tof_2_ns = (dataset.tof_2.values * HiConstants.TOF2_TICK_DUR).astype(np.int32)
     tof_3_ns = (dataset.tof_3.values * HiConstants.TOF3_TICK_DUR).astype(np.int32)
 
-    # # ********** delta_t_ab = (t_b - t_a) **********
+    # # ********** tof_ab = (t_b - t_a) **********
     # Table: row 1, column 1
     a_and_tof1 = a_first & tof1_valid
-    new_vars["delta_t_ab"].values[a_and_tof1] = tof_1_ns[a_and_tof1]
+    new_vars["tof_ab"].values[a_and_tof1] = tof_1_ns[a_and_tof1]
     # Table: row 2, column 1
     b_and_tof1 = b_first & tof1_valid
-    new_vars["delta_t_ab"].values[b_and_tof1] = -1 * tof_1_ns[b_and_tof1]
+    new_vars["tof_ab"].values[b_and_tof1] = -1 * tof_1_ns[b_and_tof1]
     # Table: row 3, column 1 and 2
-    # delta_t_ab = (t_b - t_c1) - (t_a - t_c1) = (t_b - t_a)
+    # tof_ab = (t_b - t_c1) - (t_a - t_c1) = (t_b - t_a)
     c_and_tof1and2 = c_first & tof1and2_valid
-    new_vars["delta_t_ab"].values[c_and_tof1and2] = (
+    new_vars["tof_ab"].values[c_and_tof1and2] = (
         tof_2_ns[c_and_tof1and2] - tof_1_ns[c_and_tof1and2]
     )
 
-    # ********** delta_t_ac1 = (t_c1 - t_a) **********
+    # ********** tof_ac1 = (t_c1 - t_a) **********
     # Table: row 1, column 2
     a_and_tof2 = a_first & tof2_valid
-    new_vars["delta_t_ac1"].values[a_and_tof2] = tof_2_ns[a_and_tof2]
+    new_vars["tof_ac1"].values[a_and_tof2] = tof_2_ns[a_and_tof2]
     # Table: row 2, column 1 and 2
-    # delta_t_ac1 = (t_c1 - t_b) - (t_a - t_b) = (t_c1 - t_a)
+    # tof_ac1 = (t_c1 - t_b) - (t_a - t_b) = (t_c1 - t_a)
     b_and_tof1and2 = b_first & tof1and2_valid
-    new_vars["delta_t_ac1"].values[b_and_tof1and2] = (
+    new_vars["tof_ac1"].values[b_and_tof1and2] = (
         tof_2_ns[b_and_tof1and2] - tof_1_ns[b_and_tof1and2]
     )
     # Table: row 3, column 1
     c_and_tof1 = c_first & tof1_valid
-    new_vars["delta_t_ac1"].values[c_and_tof1] = -1 * tof_1_ns[c_and_tof1]
+    new_vars["tof_ac1"].values[c_and_tof1] = -1 * tof_1_ns[c_and_tof1]
 
-    # ********** delta_t_bc1 = (t_c1 - t_b) **********
+    # ********** tof_bc1 = (t_c1 - t_b) **********
     # Table: row 1, column 1 and 2
-    # delta_t_bc1 = (t_c1 - t_a) - (t_b - t_a) => (t_c1 - t_b)
+    # tof_bc1 = (t_c1 - t_a) - (t_b - t_a) => (t_c1 - t_b)
     a_and_tof1and2 = a_first & tof1and2_valid
-    new_vars["delta_t_bc1"].values[a_and_tof1and2] = (
+    new_vars["tof_bc1"].values[a_and_tof1and2] = (
         tof_2_ns[a_and_tof1and2] - tof_1_ns[a_and_tof1and2]
     )
     # Table: row 2, column 2
     b_and_tof2 = b_first & tof2_valid
-    new_vars["delta_t_bc1"].values[b_and_tof2] = tof_2_ns[b_and_tof2]
+    new_vars["tof_bc1"].values[b_and_tof2] = tof_2_ns[b_and_tof2]
     # Table: row 3, column 2
     c_and_tof2 = c_first & tof2_valid
-    new_vars["delta_t_bc1"].values[c_and_tof2] = -1 * tof_2_ns[c_and_tof2]
+    new_vars["tof_bc1"].values[c_and_tof2] = -1 * tof_2_ns[c_and_tof2]
 
-    # ********** delta_t_c1c2 = (t_c2 - t_c1) **********
+    # ********** tof_c1c2 = (t_c2 - t_c1) **********
     # Table: all rows, column 3
-    new_vars["delta_t_c1c2"].values[tof3_valid] = tof_3_ns[tof3_valid]
+    new_vars["tof_c1c2"].values[tof3_valid] = tof_3_ns[tof3_valid]
 
     return new_vars
 
@@ -325,8 +354,8 @@ def compute_hae_coordinates(dataset: xr.Dataset) -> dict[str, xr.DataArray]:
     Parameters
     ----------
     dataset : xarray.Dataset
-        The partial L1B dataset that has had coincidence type, time deltas, and
-        spin phase computed and added to the L1A data.
+        The partial L1B dataset that has had coincidence type, times of flight,
+        and spin phase computed and added to the L1A data.
 
     Returns
     -------