imap-processing 0.7.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

imap_processing/hit/l1a/hit_l1a.py

@@ -1,39 +1,23 @@
 """Decommutate HIT CCSDS data and create L1a data products."""
 
 import logging
-from enum import IntEnum
 
-import numpy as np
 import xarray as xr
 
-from imap_processing import imap_module_directory
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
-from imap_processing.utils import packet_file_to_datasets
+from imap_processing.hit.hit_utils import (
+    HitAPID,
+    get_attribute_manager,
+    get_datasets_by_apid,
+    process_housekeeping_data,
+)
+from imap_processing.hit.l0.decom_hit import decom_hit
 
 logger = logging.getLogger(__name__)
 
 # TODO review logging levels to use (debug vs. info)
 
 
-class HitAPID(IntEnum):
-    """
-    HIT APID Mappings.
-
-    Attributes
-    ----------
-    HIT_HSKP: int
-        Housekeeping
-    HIT_SCIENCE : int
-        Science
-    HIT_IALRT : int
-        I-ALiRT
-    """
-
-    HIT_HSKP = 1251
-    HIT_SCIENCE = 1252
-    HIT_IALRT = 1253
-
-
 def hit_l1a(packet_file: str, data_version: str) -> list[xr.Dataset]:
     """
     Will process HIT L0 data into L1A data products.
@@ -50,29 +34,19 @@ def hit_l1a(packet_file: str, data_version: str) -> list[xr.Dataset]:
     processed_data : list[xarray.Dataset]
         List of Datasets of L1A processed data.
     """
-    # TODO add logging
-
-    # Unpack ccsds file
-    packet_definition = (
-        imap_module_directory / "hit/packet_definitions/hit_packet_definitions.xml"
-    )
-    datasets_by_apid = packet_file_to_datasets(
-        packet_file=packet_file,
-        xtce_packet_definition=packet_definition,
-        use_derived_value=False,
-    )
+    # Unpack ccsds file to xarray datasets
+    datasets_by_apid = get_datasets_by_apid(packet_file)
 
     # Create the attribute manager for this data level
-    attr_mgr = ImapCdfAttributes()
-    attr_mgr.add_instrument_global_attrs(instrument="hit")
-    attr_mgr.add_instrument_variable_attrs(instrument="hit", level="l1a")
-    attr_mgr.add_global_attribute("Data_version", data_version)
+    attr_mgr = get_attribute_manager(data_version, "l1a")
 
-    # Process science to l1a.
+    # Process l1a data products
     if HitAPID.HIT_HSKP in datasets_by_apid:
-        datasets_by_apid[HitAPID.HIT_HSKP] = process_housekeeping(
-            datasets_by_apid[HitAPID.HIT_HSKP], attr_mgr
+        logger.info("Creating HIT L1A housekeeping dataset")
+        datasets_by_apid[HitAPID.HIT_HSKP] = process_housekeeping_data(
+            datasets_by_apid[HitAPID.HIT_HSKP], attr_mgr, "imap_hit_l1a_hk"
         )
+
     if HitAPID.HIT_SCIENCE in datasets_by_apid:
         # TODO complete science data processing
         print("Skipping science data for now")
@@ -83,44 +57,6 @@ def hit_l1a(packet_file: str, data_version: str) -> list[xr.Dataset]:
     return list(datasets_by_apid.values())
 
 
-def concatenate_leak_variables(
-    dataset: xr.Dataset, adc_channels: xr.DataArray
-) -> xr.Dataset:
-    """
-    Concatenate leak variables in the dataset.
-
-    Updates the housekeeping dataset to replace the individual
-    leak_i_00, leak_i_01, ..., leak_i_63 variables with a single
-    leak_i variable as a 2D array. "i" here represents current
-    in the leakage current [Voltage] data.
-
-    Parameters
-    ----------
-    dataset : xarray.Dataset
-        Dataset containing 64 leak variables.
-    adc_channels : xarray.DataArray
-        DataArray to be used as a dimension for the concatenated leak variables.
-
-    Returns
-    -------
-    dataset : xarray.Dataset
-        Updated dataset with concatenated leak variables.
-    """
-    # Stack 64 leak variables (leak_00, leak_01, ..., leak_63)
-    leak_vars = [dataset[f"leak_i_{i:02d}"] for i in range(64)]
-
-    # Concatenate along 'adc_channels' and reorder dimensions
-    stacked_leaks = xr.concat(leak_vars, dim=adc_channels).transpose(
-        "epoch", "adc_channels"
-    )
-    dataset["leak_i"] = stacked_leaks
-
-    # Drop the individual leak variables
-    updated_dataset = dataset.drop_vars([f"leak_i_{i:02d}" for i in range(64)])
-
-    return updated_dataset
-
-
 def process_science(dataset: xr.Dataset, attr_mgr: ImapCdfAttributes) -> xr.Dataset:
     """
     Will process science dataset for CDF product.
@@ -147,10 +83,12 @@ def process_science(dataset: xr.Dataset, attr_mgr: ImapCdfAttributes) -> xr.Data
     logger.info("Creating HIT L1A science datasets")
 
     # Logical sources for the two products.
-    # logical_sources = ["imap_hit_l1a_sci-counts", "imap_hit_l1a_pulse-height-event"]
+    # logical_sources = ["imap_hit_l1a_count-rates", "imap_hit_l1a_pulse-height-event"]
+
+    # Decommutate and decompress the science data
+    sci_dataset = decom_hit(dataset)
 
     # TODO: Complete this function
-    #  - call decom_hit.py to decommutate the science data
     #  - split the science data into count rates and event datasets
     #  - update dimensions and add attributes to the dataset and data arrays
     #  - return list of two datasets (count rates and events)?
@@ -158,97 +96,4 @@ def process_science(dataset: xr.Dataset, attr_mgr: ImapCdfAttributes) -> xr.Data
     # logger.info("HIT L1A event dataset created")
     # logger.info("HIT L1A count rates dataset created")
 
-    return dataset
-
-
-def process_housekeeping(
-    dataset: xr.Dataset, attr_mgr: ImapCdfAttributes
-) -> xr.Dataset:
-    """
-    Will process housekeeping dataset for CDF product.
-
-    Updates the housekeeping dataset to replace with a single
-    leak_i variable as a 2D array. Also updates the dataset
-    attributes and coordinates and data variable dimensions
-    according to specifications in a cdf yaml file.
-
-    Parameters
-    ----------
-    dataset : xarray.Dataset
-        Dataset containing HIT housekeeping data.
-
-    attr_mgr : ImapCdfAttributes
-        Attribute manager used to get the data product field's attributes.
-
-    Returns
-    -------
-    dataset : xarray.Dataset
-        An updated dataset ready for CDF conversion.
-    """
-    logger.info("Creating HIT L1A housekeeping dataset")
-
-    logical_source = "imap_hit_l1a_hk"
-
-    # Drop keys that are not CDF data variables
-    drop_keys = [
-        "pkt_apid",
-        "sc_tick",
-        "version",
-        "type",
-        "sec_hdr_flg",
-        "seq_flgs",
-        "src_seq_ctr",
-        "pkt_len",
-        "hskp_spare1",
-        "hskp_spare2",
-        "hskp_spare3",
-        "hskp_spare4",
-        "hskp_spare5",
-    ]
-
-    # Drop variables not needed for CDF
-    dataset = dataset.drop_vars(drop_keys)
-
-    # Create data arrays for dependencies
-    adc_channels = xr.DataArray(
-        np.arange(64, dtype=np.uint8),
-        name="adc_channels",
-        dims=["adc_channels"],
-        attrs=attr_mgr.get_variable_attributes("adc_channels"),
-    )
-
-    # NOTE: LABL_PTR_1 should be CDF_CHAR.
-    adc_channels_label = xr.DataArray(
-        adc_channels.values.astype(str),
-        name="adc_channels_label",
-        dims=["adc_channels_label"],
-        attrs=attr_mgr.get_variable_attributes("adc_channels_label"),
-    )
-
-    # Update dataset coordinates and attributes
-    dataset = dataset.assign_coords(
-        {
-            "adc_channels": adc_channels,
-            "adc_channels_label": adc_channels_label,
-        }
-    )
-    dataset.attrs = attr_mgr.get_global_attributes(logical_source)
-
-    # Stack 64 leak variables (leak_00, leak_01, ..., leak_63)
-    dataset = concatenate_leak_variables(dataset, adc_channels)
-
-    # Assign attributes and dimensions to each data array in the Dataset
-    for field in dataset.data_vars.keys():
-        # Create a dict of dimensions using the DEPEND_I keys in the
-        # attributes
-        dims = {
-            key: value
-            for key, value in attr_mgr.get_variable_attributes(field).items()
-            if "DEPEND" in key
-        }
-        dataset[field].attrs = attr_mgr.get_variable_attributes(field)
-        dataset[field].assign_coords(dims)
-
-    dataset.epoch.attrs = attr_mgr.get_variable_attributes("epoch")
-
-    return dataset
+    return sci_dataset

imap_processing/hit/l1b/hit_l1b.py

@@ -1,179 +1,59 @@
 """IMAP-HIT L1B data processing."""
 
 import logging
-from dataclasses import fields
 
-import numpy as np
 import xarray as xr
 
-from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
-from imap_processing.hit.l0.data_classes.housekeeping import Housekeeping
-from imap_processing.spice.time import met_to_j2000ns
+from imap_processing.hit.hit_utils import (
+    HitAPID,
+    get_attribute_manager,
+    get_datasets_by_apid,
+    process_housekeeping_data,
+)
 
 logger = logging.getLogger(__name__)
 
 # TODO review logging levels to use (debug vs. info)
 
 
-def hit_l1b(l1a_dataset: xr.Dataset, data_version: str) -> list[xr.Dataset]:
+def hit_l1b(dependencies: dict, data_version: str) -> list[xr.Dataset]:
     """
     Will process HIT data to L1B.
 
+    Processes dependencies needed to create L1B data products.
+
     Parameters
     ----------
-    l1a_dataset : xarray.Dataset
-        L1A data.
+    dependencies : dict
+        Dictionary of dependencies that are L1A xarray datasets
+        for science data and a file path string to a CCSDS file
+        for housekeeping data.
     data_version : str
         Version of the data product being created.
 
     Returns
     -------
-    cdf_filepaths : xarray.Dataset
-        L1B processed data.
+    processed_data : list[xarray.Dataset]
+        List of L1B datasets.
     """
-    # create the attribute manager for this data level
-    attr_mgr = ImapCdfAttributes()
-    attr_mgr.add_instrument_global_attrs(instrument="hit")
-    attr_mgr.add_instrument_variable_attrs(instrument="hit", level="l1b")
-    attr_mgr.add_global_attribute("Data_version", data_version)
-
-    # TODO: Check for type of L1A dataset and determine what L1B products to make
-    #   Need more info from instrument teams. Work with housekeeping data for now
-    logical_source = "imap_hit_l1b_hk"
-
-    # Create datasets
-    datasets = []
-    if "_hk" in logical_source:
-        dataset = create_hk_dataset(attr_mgr)
-        datasets.append(dataset)
-    elif "_sci" in logical_source:
-        # process science data. placeholder for future code
+    # Create the attribute manager for this data level
+    attr_mgr = get_attribute_manager(data_version, "l1b")
+
+    # Create L1B datasets
+    datasets: list = []
+    if "imap_hit_l0_raw" in dependencies:
+        # Unpack ccsds file to xarray datasets
+        packet_file = dependencies["imap_hit_l0_raw"]
+        datasets_by_apid = get_datasets_by_apid(packet_file, derived=True)
+        # Process housekeeping to l1b.
+        datasets.append(
+            process_housekeeping_data(
+                datasets_by_apid[HitAPID.HIT_HSKP], attr_mgr, "imap_hit_l1b_hk"
+            )
+        )
+        logger.info("HIT L1B housekeeping dataset created")
+    if "imap_hit_l1a_countrates" in dependencies:
+        # TODO: process science data. placeholder for future code
         pass
 
     return datasets
-
-
-# TODO: This is going to work differently when we have sample data
-def create_hk_dataset(attr_mgr: ImapCdfAttributes) -> xr.Dataset:
-    """
-    Create a housekeeping dataset.
-
-    Parameters
-    ----------
-    attr_mgr : ImapCdfAttributes
-        Attribute manager used to get the data product field's attributes.
-
-    Returns
-    -------
-    hk_dataset : xarray.dataset
-        Dataset with all data product fields in xarray.DataArray.
-    """
-    logger.info("Creating datasets for HIT L1B data")
-
-    # TODO: TEMPORARY. Need to update to use the L1B data class once that exists.
-    #  Using l1a housekeeping data class for now since l1b housekeeping has the
-    #  same data fields
-    data_fields = fields(Housekeeping)
-
-    # TODO define keys to skip. This will change later.
-    skip_keys = [
-        "shcoarse",
-        "ground_sw_version",
-        "packet_file_name",
-        "ccsds_header",
-        "leak_i_raw",
-    ]
-
-    logical_source = "imap_hit_l1b_hk"
-
-    # Create fake data for now
-
-    # Convert integers into datetime64[s]
-    epoch_converted_time = met_to_j2000ns([0, 1, 2])
-
-    # Shape for dims
-    n_epoch = 3
-    n_channels = 64
-
-    # Create xarray data arrays for dependencies
-    epoch_time = xr.DataArray(
-        data=epoch_converted_time,
-        name="epoch",
-        dims=["epoch"],
-        attrs=attr_mgr.get_variable_attributes("epoch"),
-    )
-
-    adc_channels = xr.DataArray(
-        np.arange(n_channels, dtype=np.uint16),
-        name="adc_channels",
-        dims=["adc_channels"],
-        attrs=attr_mgr.get_variable_attributes("adc_channels"),
-    )
-
-    # Create xarray dataset
-    hk_dataset = xr.Dataset(
-        coords={"epoch": epoch_time, "adc_channels": adc_channels},
-        attrs=attr_mgr.get_global_attributes(logical_source),
-    )
-
-    # Create xarray data array for each data field
-    for data_field in data_fields:
-        field = data_field.name.lower()
-        if field not in skip_keys:
-            # Create a list of all the dimensions using the DEPEND_I keys in the
-            # attributes
-            dims = [
-                value
-                for key, value in attr_mgr.get_variable_attributes(field).items()
-                if "DEPEND" in key
-            ]
-
-            # TODO: This is temporary.
-            #  The data will be set in the data class when that's created
-            if field == "leak_i":
-                # 2D array - needs two dims
-                hk_dataset[field] = xr.DataArray(
-                    np.ones((n_epoch, n_channels), dtype=np.uint16),
-                    dims=dims,
-                    attrs=attr_mgr.get_variable_attributes(field),
-                )
-            elif field in [
-                "preamp_l234a",
-                "preamp_l1a",
-                "preamp_l1b",
-                "preamp_l234b",
-                "temp0",
-                "temp1",
-                "temp2",
-                "temp3",
-                "analog_temp",
-                "hvps_temp",
-                "idpu_temp",
-                "lvps_temp",
-                "ebox_3d4vd",
-                "ebox_5d1vd",
-                "ebox_p12va",
-                "ebox_m12va",
-                "ebox_p5d7va",
-                "ebox_m5d7va",
-                "ref_p5v",
-                "l1ab_bias",
-                "l2ab_bias",
-                "l34a_bias",
-                "l34b_bias",
-                "ebox_p2d0vd",
-            ]:
-                hk_dataset[field] = xr.DataArray(
-                    np.ones(3, dtype=np.float16),
-                    dims=dims,
-                    attrs=attr_mgr.get_variable_attributes(field),
-                )
-            else:
-                hk_dataset[field] = xr.DataArray(
-                    [1, 1, 1],
-                    dims=dims,
-                    attrs=attr_mgr.get_variable_attributes(field),
-                )
-
-    logger.info("HIT L1B datasets created")
-    return hk_dataset

imap_processing/idex/idex_l1a.py

@@ -2,6 +2,16 @@
 Perform IDEX L1a Processing.
 
 This module processes decommutated IDEX packets and creates L1a data products.
+
+Examples
+--------
+.. code-block:: python
+
+    from imap_processing.idex.idex_l1a import PacketParser
+
+    l0_file = "imap_processing/tests/idex/imap_idex_l0_sci_20231214_v001.pkts"
+    l1a_data = PacketParser(l0_file, data_version)
+    l1a_data.write_l1a_cdf()
 """
 
 import logging
@@ -102,15 +112,6 @@ class PacketParser:
         The path and filename to the L0 file to read.
     data_version : str
         The version of the data product being created.
-
-    Examples
-    --------
-    .. code-block:: python
-
-        from imap_processing.idex.idex_l1a import PacketParser
-        l0_file = "imap_processing/tests/idex/imap_idex_l0_sci_20230725_v001.pkts"
-        l1a_data = PacketParser(l0_file, data_version)
-        l1a_data.write_l1a_cdf()
     """
 
     def __init__(self, packet_file: Union[str, Path], data_version: str) -> None:

imap_processing/lo/l0/decompression_tables/decompression_tables.py

@@ -63,7 +63,7 @@ CASE_DECODER = {
     (1, 0): VariableFields(True, True, True, False, False, False),
     (2, 0): VariableFields(True, True, False, False, False, True),
     (3, 0): VariableFields(True, False, False, False, False, False),
-    (4, 1): VariableFields(True, False, False, False, False, True),
+    (4, 1): VariableFields(True, False, True, False, False, True),
     (4, 0): VariableFields(True, False, False, True, False, False),
     (5, 0): VariableFields(True, False, True, False, False, False),
     (6, 1): VariableFields(True, False, False, False, False, True),

imap_processing/lo/l0/lo_science.py

@@ -367,7 +367,7 @@ def combine_segmented_packets(dataset: xr.Dataset) -> xr.Dataset:
     # Mark the segment splits with comma to identify padding bits
     # when parsing the binary
     dataset["events"] = [
-        ",".join(dataset["data"].values[start : end + 1])
+        "".join(dataset["data"].values[start : end + 1])
         for start, end in zip(seg_starts, seg_ends)
     ]
     # drop any group of segmented packets that aren't sequential