imap-processing 0.6.0__py3-none-any.whl

imap_processing/hit/l1a/hit_l1a.py

@@ -0,0 +1,254 @@
+"""Decommutate HIT CCSDS data and create L1a data products."""
+
+import logging
+from enum import IntEnum
+
+import numpy as np
+import xarray as xr
+
+from imap_processing import imap_module_directory
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.utils import packet_file_to_datasets
+
+logger = logging.getLogger(__name__)
+
+# TODO review logging levels to use (debug vs. info)
+
+
+class HitAPID(IntEnum):
+    """
+    HIT APID Mappings.
+
+    Attributes
+    ----------
+    HIT_HSKP: int
+        Housekeeping
+    HIT_SCIENCE : int
+        Science
+    HIT_IALRT : int
+        I-ALiRT
+    """
+
+    HIT_HSKP = 1251
+    HIT_SCIENCE = 1252
+    HIT_IALRT = 1253
+
+
+def hit_l1a(packet_file: str, data_version: str) -> list[xr.Dataset]:
+    """
+    Will process HIT L0 data into L1A data products.
+
+    Parameters
+    ----------
+    packet_file : str
+        Path to the CCSDS data packet file.
+    data_version : str
+        Version of the data product being created.
+
+    Returns
+    -------
+    processed_data : list[xarray.Dataset]
+        List of Datasets of L1A processed data.
+    """
+    # TODO add logging
+
+    # Unpack ccsds file
+    packet_definition = (
+        imap_module_directory / "hit/packet_definitions/hit_packet_definitions.xml"
+    )
+    datasets_by_apid = packet_file_to_datasets(
+        packet_file=packet_file,
+        xtce_packet_definition=packet_definition,
+        use_derived_value=False,
+    )
+
+    # Create the attribute manager for this data level
+    attr_mgr = ImapCdfAttributes()
+    attr_mgr.add_instrument_global_attrs(instrument="hit")
+    attr_mgr.add_instrument_variable_attrs(instrument="hit", level="l1a")
+    attr_mgr.add_global_attribute("Data_version", data_version)
+
+    # Process science to l1a.
+    if HitAPID.HIT_HSKP in datasets_by_apid:
+        datasets_by_apid[HitAPID.HIT_HSKP] = process_housekeeping(
+            datasets_by_apid[HitAPID.HIT_HSKP], attr_mgr
+        )
+    if HitAPID.HIT_SCIENCE in datasets_by_apid:
+        # TODO complete science data processing
+        print("Skipping science data for now")
+        datasets_by_apid[HitAPID.HIT_SCIENCE] = process_science(
+            datasets_by_apid[HitAPID.HIT_SCIENCE], attr_mgr
+        )
+
+    return list(datasets_by_apid.values())
+
+
+def concatenate_leak_variables(
+    dataset: xr.Dataset, adc_channels: xr.DataArray
+) -> xr.Dataset:
+    """
+    Concatenate leak variables in the dataset.
+
+    Updates the housekeeping dataset to replace the individual
+    leak_i_00, leak_i_01, ..., leak_i_63 variables with a single
+    leak_i variable as a 2D array. "i" here represents current
+    in the leakage current [Voltage] data.
+
+    Parameters
+    ----------
+    dataset : xarray.Dataset
+        Dataset containing 64 leak variables.
+    adc_channels : xarray.DataArray
+        DataArray to be used as a dimension for the concatenated leak variables.
+
+    Returns
+    -------
+    dataset : xarray.Dataset
+        Updated dataset with concatenated leak variables.
+    """
+    # Stack 64 leak variables (leak_00, leak_01, ..., leak_63)
+    leak_vars = [dataset[f"leak_i_{i:02d}"] for i in range(64)]
+
+    # Concatenate along 'adc_channels' and reorder dimensions
+    stacked_leaks = xr.concat(leak_vars, dim=adc_channels).transpose(
+        "epoch", "adc_channels"
+    )
+    dataset["leak_i"] = stacked_leaks
+
+    # Drop the individual leak variables
+    updated_dataset = dataset.drop_vars([f"leak_i_{i:02d}" for i in range(64)])
+
+    return updated_dataset
+
+
+def process_science(dataset: xr.Dataset, attr_mgr: ImapCdfAttributes) -> xr.Dataset:
+    """
+    Will process science dataset for CDF product.
+
+    Process binary science data for CDF creation. The data is
+    grouped into science frames, decommutated and decompressed,
+    and split into count rates and event datasets. Updates the
+    dataset attributes and coordinates and data variable
+    dimensions according to specifications in a cdf yaml file.
+
+    Parameters
+    ----------
+    dataset : xarray.Dataset
+        Dataset containing HIT science data.
+
+    attr_mgr : ImapCdfAttributes
+        Attribute manager used to get the data product field's attributes.
+
+    Returns
+    -------
+    dataset : xarray.Dataset
+        An updated dataset ready for CDF conversion.
+    """
+    logger.info("Creating HIT L1A science datasets")
+
+    # Logical sources for the two products.
+    # logical_sources = ["imap_hit_l1a_sci-counts", "imap_hit_l1a_pulse-height-event"]
+
+    # TODO: Complete this function
+    #  - call decom_hit.py to decommutate the science data
+    #  - split the science data into count rates and event datasets
+    #  - update dimensions and add attributes to the dataset and data arrays
+    #  - return list of two datasets (count rates and events)?
+
+    # logger.info("HIT L1A event dataset created")
+    # logger.info("HIT L1A count rates dataset created")
+
+    return dataset
+
+
+def process_housekeeping(
+    dataset: xr.Dataset, attr_mgr: ImapCdfAttributes
+) -> xr.Dataset:
+    """
+    Will process housekeeping dataset for CDF product.
+
+    Updates the housekeeping dataset to replace with a single
+    leak_i variable as a 2D array. Also updates the dataset
+    attributes and coordinates and data variable dimensions
+    according to specifications in a cdf yaml file.
+
+    Parameters
+    ----------
+    dataset : xarray.Dataset
+        Dataset containing HIT housekeeping data.
+
+    attr_mgr : ImapCdfAttributes
+        Attribute manager used to get the data product field's attributes.
+
+    Returns
+    -------
+    dataset : xarray.Dataset
+        An updated dataset ready for CDF conversion.
+    """
+    logger.info("Creating HIT L1A housekeeping dataset")
+
+    logical_source = "imap_hit_l1a_hk"
+
+    # Drop keys that are not CDF data variables
+    drop_keys = [
+        "pkt_apid",
+        "sc_tick",
+        "version",
+        "type",
+        "sec_hdr_flg",
+        "seq_flgs",
+        "src_seq_ctr",
+        "pkt_len",
+        "hskp_spare1",
+        "hskp_spare2",
+        "hskp_spare3",
+        "hskp_spare4",
+        "hskp_spare5",
+    ]
+
+    # Drop variables not needed for CDF
+    dataset = dataset.drop_vars(drop_keys)
+
+    # Create data arrays for dependencies
+    adc_channels = xr.DataArray(
+        np.arange(64, dtype=np.uint8),
+        name="adc_channels",
+        dims=["adc_channels"],
+        attrs=attr_mgr.get_variable_attributes("adc_channels"),
+    )
+
+    # NOTE: LABL_PTR_1 should be CDF_CHAR.
+    adc_channels_label = xr.DataArray(
+        adc_channels.values.astype(str),
+        name="adc_channels_label",
+        dims=["adc_channels_label"],
+        attrs=attr_mgr.get_variable_attributes("adc_channels_label"),
+    )
+
+    # Update dataset coordinates and attributes
+    dataset = dataset.assign_coords(
+        {
+            "adc_channels": adc_channels,
+            "adc_channels_label": adc_channels_label,
+        }
+    )
+    dataset.attrs = attr_mgr.get_global_attributes(logical_source)
+
+    # Stack 64 leak variables (leak_00, leak_01, ..., leak_63)
+    dataset = concatenate_leak_variables(dataset, adc_channels)
+
+    # Assign attributes and dimensions to each data array in the Dataset
+    for field in dataset.data_vars.keys():
+        # Create a dict of dimensions using the DEPEND_I keys in the
+        # attributes
+        dims = {
+            key: value
+            for key, value in attr_mgr.get_variable_attributes(field).items()
+            if "DEPEND" in key
+        }
+        dataset[field].attrs = attr_mgr.get_variable_attributes(field)
+        dataset[field].assign_coords(dims)
+
+    dataset.epoch.attrs = attr_mgr.get_variable_attributes("epoch")
+
+    return dataset

imap_processing/hit/l1b/hit_l1b.py

@@ -0,0 +1,179 @@
+"""IMAP-HIT L1B data processing."""
+
+import logging
+from dataclasses import fields
+
+import numpy as np
+import xarray as xr
+
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.hit.l0.data_classes.housekeeping import Housekeeping
+from imap_processing.spice.time import met_to_j2000ns
+
+logger = logging.getLogger(__name__)
+
+# TODO review logging levels to use (debug vs. info)
+
+
+def hit_l1b(l1a_dataset: xr.Dataset, data_version: str) -> list[xr.Dataset]:
+    """
+    Will process HIT data to L1B.
+
+    Parameters
+    ----------
+    l1a_dataset : xarray.Dataset
+        L1A data.
+    data_version : str
+        Version of the data product being created.
+
+    Returns
+    -------
+    cdf_filepaths : xarray.Dataset
+        L1B processed data.
+    """
+    # create the attribute manager for this data level
+    attr_mgr = ImapCdfAttributes()
+    attr_mgr.add_instrument_global_attrs(instrument="hit")
+    attr_mgr.add_instrument_variable_attrs(instrument="hit", level="l1b")
+    attr_mgr.add_global_attribute("Data_version", data_version)
+
+    # TODO: Check for type of L1A dataset and determine what L1B products to make
+    #   Need more info from instrument teams. Work with housekeeping data for now
+    logical_source = "imap_hit_l1b_hk"
+
+    # Create datasets
+    datasets = []
+    if "_hk" in logical_source:
+        dataset = create_hk_dataset(attr_mgr)
+        datasets.append(dataset)
+    elif "_sci" in logical_source:
+        # process science data. placeholder for future code
+        pass
+
+    return datasets
+
+
+# TODO: This is going to work differently when we have sample data
+def create_hk_dataset(attr_mgr: ImapCdfAttributes) -> xr.Dataset:
+    """
+    Create a housekeeping dataset.
+
+    Parameters
+    ----------
+    attr_mgr : ImapCdfAttributes
+        Attribute manager used to get the data product field's attributes.
+
+    Returns
+    -------
+    hk_dataset : xarray.dataset
+        Dataset with all data product fields in xarray.DataArray.
+    """
+    logger.info("Creating datasets for HIT L1B data")
+
+    # TODO: TEMPORARY. Need to update to use the L1B data class once that exists.
+    #  Using l1a housekeeping data class for now since l1b housekeeping has the
+    #  same data fields
+    data_fields = fields(Housekeeping)
+
+    # TODO define keys to skip. This will change later.
+    skip_keys = [
+        "shcoarse",
+        "ground_sw_version",
+        "packet_file_name",
+        "ccsds_header",
+        "leak_i_raw",
+    ]
+
+    logical_source = "imap_hit_l1b_hk"
+
+    # Create fake data for now
+
+    # Convert integers into datetime64[s]
+    epoch_converted_time = met_to_j2000ns([0, 1, 2])
+
+    # Shape for dims
+    n_epoch = 3
+    n_channels = 64
+
+    # Create xarray data arrays for dependencies
+    epoch_time = xr.DataArray(
+        data=epoch_converted_time,
+        name="epoch",
+        dims=["epoch"],
+        attrs=attr_mgr.get_variable_attributes("epoch"),
+    )
+
+    adc_channels = xr.DataArray(
+        np.arange(n_channels, dtype=np.uint16),
+        name="adc_channels",
+        dims=["adc_channels"],
+        attrs=attr_mgr.get_variable_attributes("adc_channels"),
+    )
+
+    # Create xarray dataset
+    hk_dataset = xr.Dataset(
+        coords={"epoch": epoch_time, "adc_channels": adc_channels},
+        attrs=attr_mgr.get_global_attributes(logical_source),
+    )
+
+    # Create xarray data array for each data field
+    for data_field in data_fields:
+        field = data_field.name.lower()
+        if field not in skip_keys:
+            # Create a list of all the dimensions using the DEPEND_I keys in the
+            # attributes
+            dims = [
+                value
+                for key, value in attr_mgr.get_variable_attributes(field).items()
+                if "DEPEND" in key
+            ]
+
+            # TODO: This is temporary.
+            #  The data will be set in the data class when that's created
+            if field == "leak_i":
+                # 2D array - needs two dims
+                hk_dataset[field] = xr.DataArray(
+                    np.ones((n_epoch, n_channels), dtype=np.uint16),
+                    dims=dims,
+                    attrs=attr_mgr.get_variable_attributes(field),
+                )
+            elif field in [
+                "preamp_l234a",
+                "preamp_l1a",
+                "preamp_l1b",
+                "preamp_l234b",
+                "temp0",
+                "temp1",
+                "temp2",
+                "temp3",
+                "analog_temp",
+                "hvps_temp",
+                "idpu_temp",
+                "lvps_temp",
+                "ebox_3d4vd",
+                "ebox_5d1vd",
+                "ebox_p12va",
+                "ebox_m12va",
+                "ebox_p5d7va",
+                "ebox_m5d7va",
+                "ref_p5v",
+                "l1ab_bias",
+                "l2ab_bias",
+                "l34a_bias",
+                "l34b_bias",
+                "ebox_p2d0vd",
+            ]:
+                hk_dataset[field] = xr.DataArray(
+                    np.ones(3, dtype=np.float16),
+                    dims=dims,
+                    attrs=attr_mgr.get_variable_attributes(field),
+                )
+            else:
+                hk_dataset[field] = xr.DataArray(
+                    [1, 1, 1],
+                    dims=dims,
+                    attrs=attr_mgr.get_variable_attributes(field),
+                )
+
+    logger.info("HIT L1B datasets created")
+    return hk_dataset