imap-processing 0.6.0__py3-none-any.whl

imap_processing/lo/l0/lo_science.py

@@ -0,0 +1,150 @@
+"""Processing function for Lo Science Data."""
+
+from collections import namedtuple
+
+import numpy as np
+import xarray as xr
+
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.lo.l0.utils.bit_decompression import (
+    DECOMPRESSION_TABLES,
+    Decompress,
+    decompress_int,
+)
+
+HistPacking = namedtuple(
+    "HistPacking",
+    [
+        "bit_length",
+        "section_length",
+        "shape",  # (azimuth, esa_step)
+    ],
+)
+
+HIST_DATA_META = {
+    # field: bit_length, section_length, shape
+    "start_a": HistPacking(12, 504, (6, 7)),
+    "start_c": HistPacking(12, 504, (6, 7)),
+    "stop_b0": HistPacking(12, 504, (6, 7)),
+    "stop_b3": HistPacking(12, 504, (6, 7)),
+    "tof0_count": HistPacking(8, 336, (6, 7)),
+    "tof1_count": HistPacking(8, 336, (6, 7)),
+    "tof2_count": HistPacking(8, 336, (6, 7)),
+    "tof3_count": HistPacking(8, 336, (6, 7)),
+    "tof0_tof1": HistPacking(8, 3360, (60, 7)),
+    "tof0_tof2": HistPacking(8, 3360, (60, 7)),
+    "tof1_tof2": HistPacking(8, 3360, (60, 7)),
+    "silver": HistPacking(8, 3360, (60, 7)),
+    "disc_tof0": HistPacking(8, 336, (6, 7)),
+    "disc_tof1": HistPacking(8, 336, (6, 7)),
+    "disc_tof2": HistPacking(8, 336, (6, 7)),
+    "disc_tof3": HistPacking(8, 336, (6, 7)),
+    "pos0": HistPacking(12, 504, (6, 7)),
+    "pos1": HistPacking(12, 504, (6, 7)),
+    "pos2": HistPacking(12, 504, (6, 7)),
+    "pos3": HistPacking(12, 504, (6, 7)),
+    "hydrogen": HistPacking(8, 3360, (60, 7)),
+    "oxygen": HistPacking(8, 3360, (60, 7)),
+}
+
+
+def parse_histogram(dataset: xr.Dataset, attr_mgr: ImapCdfAttributes) -> xr.Dataset:
+    """
+    Parse and decompress binary histogram data for Lo.
+
+    Parameters
+    ----------
+    dataset : xr.Dataset
+        Lo science counts from packets_to_dataset function.
+    attr_mgr : ImapCdfAttributes
+        CDF attribute manager for Lo L1A.
+
+    Returns
+    -------
+    dataset : xr.Dataset
+        Parsed and decompressed histogram data.
+    """
+    hist_bin = dataset.sci_cnt
+
+    # initialize the starting bit for the sections of data
+    section_start = 0
+    # for each field type in the histogram data
+    for field in HIST_DATA_META:
+        data_meta = HIST_DATA_META[field]
+        # for each histogram binary string decompress
+        # the data
+        decompressed_data = [
+            decompress(
+                bin_str, data_meta.bit_length, section_start, data_meta.section_length
+            )
+            for bin_str in hist_bin.values
+        ]
+
+        # add on the epoch length (equal to number of packets) to the
+        # field shape
+        data_shape = (len(hist_bin), data_meta.shape[0], data_meta.shape[1])
+
+        # get the dimension names from the CDF attr manager
+        dims = [
+            value
+            for key, value in attr_mgr.get_variable_attributes(field).items()
+            if "DEPEND" in key
+        ]
+        # reshape the decompressed data
+        shaped_data = np.array(decompressed_data, dtype=np.uint32).reshape(data_shape)
+        # add the data to the dataset
+        dataset[field] = xr.DataArray(
+            shaped_data, dims=dims, attrs=attr_mgr.get_variable_attributes(field)
+        )
+
+        # increment for the start of the next section
+        section_start += data_meta.section_length
+
+    return dataset
+
+
+def decompress(
+    bin_str: str, bits_per_index: int, section_start: int, section_length: int
+) -> list[int]:
+    """
+    Parse and decompress binary histogram data for Lo.
+
+    Parameters
+    ----------
+    bin_str : str
+        Binary string to decompress.
+    bits_per_index : int
+        Number of bits per index of the data section.
+    section_start : int
+        The start bit for the section of data.
+    section_length : int
+        The length of the section of data.
+
+    Returns
+    -------
+    decompressed_ints : list[int]
+        Decompressed integers for the data section.
+    """
+    # select the decompression method based on the bit length
+    # of the compressed data
+    if bits_per_index == 8:
+        decompress = Decompress.DECOMPRESS8TO16
+    elif bits_per_index == 12:
+        decompress = Decompress.DECOMPRESS12TO16
+    else:
+        raise ValueError(f"Invalid bits_per_index: {bits_per_index}")
+
+    # parse the binary and convert to integers
+    raw_ints = [
+        int(bin_str[i : i + bits_per_index], 2)
+        for i in range(section_start, section_start + section_length, bits_per_index)
+    ]
+
+    # decompress raw integers
+    decompressed_ints: list[int] = decompress_int(
+        raw_ints,
+        decompress,
+        DECOMPRESSION_TABLES,
+    )
+
+    return decompressed_ints

imap_processing/lo/l0/utils/binary_string.py

@@ -0,0 +1,59 @@
+"""Class for binary data strings."""
+
+
+class BinaryString:
+    """
+    Iterate through a binary string.
+
+    The binary string class retains the current
+    binary string bit position and updates it
+    each time the string is accessed.
+
+    Parameters
+    ----------
+    bin : str
+        Binary string.
+
+    Attributes
+    ----------
+    bin : str
+        Binary string.
+    bit_pos : int
+        Current position in the binary string.
+
+    Methods
+    -------
+    next_bits(num_bits)
+        Returns the next x bits in the binary string
+        using the current bit position as the reference point.
+        The bit position is updated to be the bit immediately
+        following the last returned bit.
+    """
+
+    bin: str
+    bit_pos: int
+
+    def __init__(self, bin: str) -> None:
+        self.bin = bin
+        self.bit_pos = 0
+
+    def next_bits(self, num_bits: int) -> str:
+        """
+        Return the next x bits in the binary string.
+
+        The bit position updated to be the bit immediately
+        following the last returned bit.
+
+        Parameters
+        ----------
+        num_bits : int
+            The number of bits you want to read off the binary string.
+
+        Returns
+        -------
+        bits : str
+            A binary string containing the next x bits that were requested.
+        """
+        bits = self.bin[self.bit_pos : self.bit_pos + num_bits]
+        self.bit_pos += num_bits
+        return bits

imap_processing/lo/l0/utils/bit_decompression.py

@@ -0,0 +1,62 @@
+"""Decompression for 8 to 12, 8 to 16, and 12 to 16 bits."""
+
+from enum import Enum
+from pathlib import Path
+
+import numpy as np
+
+
+class Decompress(Enum):
+    """The decompression options."""
+
+    DECOMPRESS8TO12 = "8_to_12"
+    DECOMPRESS8TO16 = "8_to_16"
+    DECOMPRESS12TO16 = "12_to_16"
+
+
+# Load all decompression tables into a dictionary.
+DECOMPRESSION_TABLES = {
+    enum: np.loadtxt(
+        Path(__file__).parent.parent / f"decompression_tables/{enum.value}_bit.csv",
+        delimiter=",",
+        skiprows=1,
+    )
+    for enum in Decompress
+}
+
+
+def decompress_int(
+    compressed_values: list, decompression: Decompress, decompression_lookup: dict
+) -> list[int]:
+    # No idea what the correct type is for the return. Mypy says it is Any
+    """
+    Will decompress a data field using a specified bit conversion.
+
+    Parameters
+    ----------
+    compressed_values : list
+        Compressed integers.
+    decompression : Decompress
+        The decompression to use.
+    decompression_lookup : dict
+        Dictionary containing all the decompression tables.
+
+    Returns
+    -------
+    decompressed : list[int]
+        The decompressed integer.
+    """
+    valid_decompression = [
+        Decompress.DECOMPRESS8TO12,
+        Decompress.DECOMPRESS8TO16,
+        Decompress.DECOMPRESS12TO16,
+    ]
+    if decompression not in valid_decompression:
+        raise ValueError(
+            "Invalid decompression method. Must be one of the following Enums: "
+            + "Decompress.DECOMPRESS8TO12, Decompress.DECOMPRESS8TO12, "
+            + "Decompress.DECOMPRESS8TO12"
+        )
+    data = decompression_lookup[decompression]
+    decompressed: list[int] = data[compressed_values, 1]
+    return decompressed

imap_processing/lo/l0/utils/lo_base.py

@@ -0,0 +1,57 @@
+"""General Lo L0 data class used for parsing data and setting attributes."""
+
+from dataclasses import dataclass, fields
+
+from space_packet_parser.parser import Packet
+
+from imap_processing.ccsds.ccsds_data import CcsdsData
+
+
+@dataclass
+class LoBase:
+    """
+    Data structure for common values across histogram and direct events data.
+
+    Attributes
+    ----------
+    ground_sw_version : str
+        Ground software version.
+    packet_file_name : str
+        File name of the source packet.
+    ccsds_header : CcsdsData
+        CCSDS header data.
+
+    Methods
+    -------
+    parse_data(packet):
+        Parse the packet and assign to class variable using the xtce defined named.
+    """
+
+    ground_sw_version: str
+    packet_file_name: str
+    ccsds_header: CcsdsData
+
+    def set_attributes(self, packet: Packet) -> None:
+        """
+        Set dataclass attributes with packet data.
+
+        Parameters
+        ----------
+        packet : space_packet_parser.parser.Packet
+            A single Lo L0 packet from space packet parser.
+        """
+        attributes = [field.name for field in fields(self)]
+
+        # For each item in packet, assign it to the matching attribute in the class.
+        for key, item in packet.data.items():
+            value = (
+                item.derived_value if item.derived_value is not None else item.raw_value
+            )
+            if "SPARE" in key or "CHKSUM" in key:
+                continue
+            if key not in attributes:
+                raise KeyError(
+                    f"Did not find matching attribute in {self.__class__} data class"
+                    f"for {key}"
+                )
+            setattr(self, key, value)

imap_processing/lo/l1a/lo_l1a.py

@@ -0,0 +1,157 @@
+"""IMAP-Lo L1A Data Processing."""
+
+import logging
+from pathlib import Path
+
+import numpy as np
+import xarray as xr
+
+from imap_processing import imap_module_directory
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.lo.l0.lo_apid import LoAPID
+from imap_processing.lo.l0.lo_science import parse_histogram
+from imap_processing.utils import packet_file_to_datasets
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+def lo_l1a(dependency: Path, data_version: str) -> list[xr.Dataset]:
+    """
+    Will process IMAP-Lo L0 data into L1A CDF data products.
+
+    Parameters
+    ----------
+    dependency : Path
+        Dependency file needed for data product creation.
+        Should always be only one for L1A.
+    data_version : str
+        Version of the data product being created.
+
+    Returns
+    -------
+    created_file_paths : list[Path]
+        Location of created CDF files.
+    """
+    xtce_file = imap_module_directory / "lo/packet_definitions/lo_xtce.xml"
+
+    logger.info("\nDecommutating packets and converting to dataset")
+    datasets_by_apid = packet_file_to_datasets(
+        packet_file=dependency.resolve(),
+        xtce_packet_definition=xtce_file.resolve(),
+        use_derived_value=False,
+    )
+
+    # create the attribute manager for this data level
+    attr_mgr = ImapCdfAttributes()
+    attr_mgr.add_instrument_global_attrs(instrument="lo")
+    attr_mgr.add_instrument_variable_attrs(instrument="lo", level="l1a")
+    attr_mgr.add_global_attribute("Data_version", data_version)
+
+    logger.info(
+        f"\nProcessing {LoAPID(LoAPID.ILO_SCI_CNT).name} "
+        f"packet (APID: {LoAPID.ILO_SCI_CNT.value})"
+    )
+    if LoAPID.ILO_SCI_CNT in datasets_by_apid:
+        logical_source = "imap_lo_l1a_histogram"
+        datasets_by_apid[LoAPID.ILO_SCI_CNT] = parse_histogram(
+            datasets_by_apid[LoAPID.ILO_SCI_CNT], attr_mgr
+        )
+        datasets_by_apid[LoAPID.ILO_SCI_CNT] = add_dataset_attrs(
+            datasets_by_apid[LoAPID.ILO_SCI_CNT], attr_mgr, logical_source
+        )
+
+    good_apids = [LoAPID.ILO_SCI_CNT]
+    logger.info(f"\nReturning datasets: {[LoAPID(apid) for apid in good_apids]}")
+    return [datasets_by_apid[good_apid] for good_apid in good_apids]
+
+
+def add_dataset_attrs(
+    dataset: xr.Dataset, attr_mgr: ImapCdfAttributes, logical_source: str
+) -> xr.Dataset:
+    """
+    Add Attributes to the dataset.
+
+    Parameters
+    ----------
+    dataset : xr.Dataset
+        Lo dataset from packets_to_dataset function.
+    attr_mgr : ImapCdfAttributes
+        CDF attribute manager for Lo L1A.
+    logical_source : str
+        Logical source for the data.
+
+    Returns
+    -------
+    dataset : xr.Dataset
+        Data with attributes added.
+    """
+    # TODO: may want up split up these if statements into their
+    # own functions
+    if logical_source == "imap_lo_l1a_histogram":
+        azimuth_60 = xr.DataArray(
+            data=np.arange(0, 6, dtype=np.uint8),
+            name="azimuth_60",
+            dims=["azimuth_60"],
+            attrs=attr_mgr.get_variable_attributes("azimuth_60"),
+        )
+        azimuth_60_label = xr.DataArray(
+            data=azimuth_60.values.astype(str),
+            name="azimuth_60_label",
+            dims=["azimuth_60_label"],
+            attrs=attr_mgr.get_variable_attributes("azimuth_60_label"),
+        )
+        azimuth_6 = xr.DataArray(
+            data=np.arange(0, 60, dtype=np.uint8),
+            name="azimuth_6",
+            dims=["azimuth_6"],
+            attrs=attr_mgr.get_variable_attributes("azimuth_6"),
+        )
+        azimuth_6_label = xr.DataArray(
+            data=azimuth_6.values.astype(str),
+            name="azimuth_6_label",
+            dims=["azimuth_6_label"],
+            attrs=attr_mgr.get_variable_attributes("azimuth_6_label"),
+        )
+
+        esa_step = xr.DataArray(
+            data=np.arange(1, 8, dtype=np.uint8),
+            name="esa_step",
+            dims=["esa_step"],
+            attrs=attr_mgr.get_variable_attributes("esa_step"),
+        )
+        esa_step_label = xr.DataArray(
+            esa_step.values.astype(str),
+            name="esa_step_label",
+            dims=["esa_step_label"],
+            attrs=attr_mgr.get_variable_attributes("esa_step_label"),
+        )
+
+        dataset.shcoarse.attrs.update(attr_mgr.get_variable_attributes("shcoarse"))
+        dataset.epoch.attrs.update(attr_mgr.get_variable_attributes("epoch"))
+
+        dataset = dataset.assign_coords(
+            azimuth_60=azimuth_60,
+            azimuth_60_label=azimuth_60_label,
+            azimuth_6=azimuth_6,
+            azimuth_6_label=azimuth_6_label,
+            esa_step=esa_step,
+            esa_step_label=esa_step_label,
+        )
+        dataset.attrs.update(attr_mgr.get_global_attributes(logical_source))
+        # remove the binary field and CCSDS header from the dataset
+        dataset = dataset.drop_vars(
+            [
+                "sci_cnt",
+                "chksum",
+                "version",
+                "type",
+                "sec_hdr_flg",
+                "pkt_apid",
+                "seq_flgs",
+                "src_seq_ctr",
+                "pkt_len",
+            ]
+        )
+
+    return dataset

imap_processing/lo/l1b/lo_l1b.py

@@ -0,0 +1,160 @@
+"""IMAP-Lo L1B Data Processing."""
+
+from collections import namedtuple
+from dataclasses import Field
+from pathlib import Path
+
+import numpy as np
+import xarray as xr
+
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.spice.time import met_to_j2000ns
+
+
+def lo_l1b(dependencies: dict, data_version: str) -> list[Path]:
+    """
+    Will process IMAP-Lo L1A data into L1B CDF data products.
+
+    Parameters
+    ----------
+    dependencies : dict
+        Dictionary of datasets needed for L1B data product creation in xarray Datasets.
+    data_version : str
+        Version of the data product being created.
+
+    Returns
+    -------
+    created_file_paths : list[pathlib.Path]
+        Location of created CDF files.
+    """
+    # create the attribute manager for this data level
+    attr_mgr = ImapCdfAttributes()
+    attr_mgr.add_instrument_global_attrs(instrument="lo")
+    attr_mgr.add_instrument_variable_attrs(instrument="lo", level="l1b")
+    attr_mgr.add_global_attribute("Data_version", data_version)
+
+    # if the dependencies are used to create Annotated Direct Events
+
+    if "imap_lo_l1a_de" in dependencies and "imap_lo_l1a_spin" in dependencies:
+        logical_source = "imap_lo_l1b_de"
+        # TODO: TEMPORARY. Need to update to use the L1B data class once that exists
+        #  and I have sample data.
+        data_field_tup = namedtuple("data_field_tup", ["name"])
+        data_fields = [
+            data_field_tup("ESA_STEP"),
+            data_field_tup("MODE"),
+            data_field_tup("TOF0"),
+            data_field_tup("TOF1"),
+            data_field_tup("TOF2"),
+            data_field_tup("TOF3"),
+            data_field_tup("COINCIDENCE_TYPE"),
+            data_field_tup("POS"),
+            data_field_tup("COINCIDENCE"),
+            data_field_tup("BADTIME"),
+            data_field_tup("DIRECTION"),
+        ]
+
+    dataset: list[Path] = create_datasets(attr_mgr, logical_source, data_fields)  # type: ignore[arg-type]
+    # TODO Remove once data_fields is removed from create_datasets
+    return dataset
+
+
+# TODO: This is going to work differently when I sample data.
+#  The data_fields input is temporary.
+def create_datasets(
+    attr_mgr: ImapCdfAttributes,
+    logical_source: str,
+    data_fields: list[Field],
+) -> xr.Dataset:
+    """
+    Create a dataset using the populated data classes.
+
+    Parameters
+    ----------
+    attr_mgr : ImapCdfAttributes
+        Attribute manager used to get the data product field's attributes.
+    logical_source : str
+        The logical source of the data product that's being created.
+    data_fields : list[dataclasses.Field]
+        List of Fields for data classes.
+
+    Returns
+    -------
+    dataset : xarray.Dataset
+        Dataset with all data product fields in xr.DataArray.
+    """
+    # TODO: Once L1B DE processing is implemented using the spin packet
+    #  and relative L1A DE time to calculate the absolute DE time,
+    #  this epoch conversion will go away and the time in the DE dataclass
+    #  can be used direction
+    epoch_converted_time = met_to_j2000ns([0, 1, 2])
+
+    # Create a data array for the epoch time
+    # TODO: might need to update the attrs to use new YAML file
+    epoch_time = xr.DataArray(
+        data=epoch_converted_time,
+        name="epoch",
+        dims=["epoch"],
+        attrs=attr_mgr.get_variable_attributes("epoch"),
+    )
+
+    if logical_source == "imap_lo_l1b_de":
+        direction_vec = xr.DataArray(
+            data=[0, 1, 2],
+            name="direction_vec",
+            dims=["direction_vec"],
+            attrs=attr_mgr.get_variable_attributes("direction_vec"),
+        )
+
+        direction_vec_label = xr.DataArray(
+            data=direction_vec.values.astype(str),
+            name="direction_vec_label",
+            dims=["direction_vec_label"],
+            attrs=attr_mgr.get_variable_attributes("direction_vec_label"),
+        )
+
+        dataset = xr.Dataset(
+            coords={
+                "epoch": epoch_time,
+                "direction_vec": direction_vec,
+                "direction_vec_label": direction_vec_label,
+            },
+            attrs=attr_mgr.get_global_attributes(logical_source),
+        )
+
+    # Loop through the data fields that were pulled from the
+    # data class. These should match the field names given
+    # to each field in the YAML attribute file
+    for data_field in data_fields:
+        field = data_field.name.lower()
+        # Create a list of all the dimensions using the DEPEND_I keys in the
+        # YAML attributes
+        dims = [
+            value
+            for key, value in attr_mgr.get_variable_attributes(field).items()
+            if "DEPEND" in key
+        ]
+
+        # Create a data array for the current field and add it to the dataset
+        # TODO: TEMPORARY. need to update to use l1a data once that's available.
+        #  Won't need to check for the direction field when I have sample data either.
+        if field == "direction":
+            dataset[field] = xr.DataArray(
+                [[0, 0, 1], [0, 1, 0], [0, 0, 1]],
+                dims=dims,
+                attrs=attr_mgr.get_variable_attributes(field),
+            )
+        # TODO: This is temporary.
+        #  The data type will be set in the data class when that's created
+        elif field in ["tof0", "tof1", "tof2", "tof3"]:
+            dataset[field] = xr.DataArray(
+                [np.float16(1), np.float16(1), np.float16(1)],
+                dims=dims,
+                attrs=attr_mgr.get_variable_attributes(field),
+            )
+        else:
+            dataset[field] = xr.DataArray(
+                [1, 1, 1], dims=dims, attrs=attr_mgr.get_variable_attributes(field)
+            )
+
+    return dataset