imap-processing 0.8.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

imap_processing/codice/constants.py

@@ -184,7 +184,7 @@ DATA_PRODUCT_CONFIGURATIONS = {
             "energy_label",
         ],  # TODO: These will likely change
         "dataset_name": "imap_codice_l1a_lo-counters-aggregated",
-        "dims": ["epoch", "inst_az", "spin_sector", "esa_step"],
+        "dims": ["epoch", "esa_step", "inst_az", "spin_sector"],
         "instrument": "lo",
         "num_counters": 1,
         "num_energy_steps": 128,
@@ -205,7 +205,7 @@ DATA_PRODUCT_CONFIGURATIONS = {
             "energy_label",
         ],  # TODO: These will likely change
         "dataset_name": "imap_codice_l1a_lo-counters-singles",
-        "dims": ["epoch", "inst_az", "spin_sector", "esa_step"],
+        "dims": ["epoch", "esa_step", "inst_az", "spin_sector"],
         "instrument": "lo",
         "num_counters": 1,
         "num_energy_steps": 128,
@@ -225,9 +225,9 @@ DATA_PRODUCT_CONFIGURATIONS = {
         "variable_names": LO_INST_COUNTS_SINGLES_VARIABLE_NAMES,
     },
     CODICEAPID.COD_LO_SW_ANGULAR_COUNTS: {
-        "coords": ["epoch", "inst_az", "spin_sector", "esa_step", "energy_label"],
+        "coords": ["epoch", "energy_label", "esa_step", "inst_az", "spin_sector"],
         "dataset_name": "imap_codice_l1a_lo-sw-angular",
-        "dims": ["epoch", "inst_az", "spin_sector", "esa_step"],
+        "dims": ["epoch", "esa_step", "inst_az", "spin_sector"],
         "instrument": "lo",
         "num_counters": 4,
         "num_energy_steps": 128,
@@ -248,7 +248,7 @@ DATA_PRODUCT_CONFIGURATIONS = {
     CODICEAPID.COD_LO_NSW_ANGULAR_COUNTS: {
         "coords": ["epoch", "inst_az", "spin_sector", "esa_step", "energy_label"],
         "dataset_name": "imap_codice_l1a_lo-nsw-angular",
-        "dims": ["epoch", "inst_az", "spin_sector", "esa_step"],
+        "dims": ["epoch", "esa_step", "inst_az", "spin_sector"],
         "instrument": "lo",
         "num_counters": 1,
         "num_energy_steps": 128,
@@ -269,7 +269,7 @@ DATA_PRODUCT_CONFIGURATIONS = {
     CODICEAPID.COD_LO_SW_PRIORITY_COUNTS: {
         "coords": ["epoch", "inst_az", "spin_sector", "esa_step", "energy_label"],
         "dataset_name": "imap_codice_l1a_lo-sw-priority",
-        "dims": ["epoch", "inst_az", "spin_sector", "esa_step"],
+        "dims": ["epoch", "esa_step", "inst_az", "spin_sector"],
         "instrument": "lo",
         "num_counters": 5,
         "num_energy_steps": 128,
@@ -290,7 +290,7 @@ DATA_PRODUCT_CONFIGURATIONS = {
     CODICEAPID.COD_LO_NSW_PRIORITY_COUNTS: {
         "coords": ["epoch", "inst_az", "spin_sector", "esa_step", "energy_label"],
         "dataset_name": "imap_codice_l1a_lo-nsw-priority",
-        "dims": ["epoch", "inst_az", "spin_sector", "esa_step"],
+        "dims": ["epoch", "esa_step", "inst_az", "spin_sector"],
         "instrument": "lo",
         "num_counters": 2,
         "num_energy_steps": 128,
@@ -311,7 +311,7 @@ DATA_PRODUCT_CONFIGURATIONS = {
     CODICEAPID.COD_LO_SW_SPECIES_COUNTS: {
         "coords": ["epoch", "inst_az", "spin_sector", "esa_step", "energy_label"],
         "dataset_name": "imap_codice_l1a_lo-sw-species",
-        "dims": ["epoch", "inst_az", "spin_sector", "esa_step"],
+        "dims": ["epoch", "esa_step", "inst_az", "spin_sector"],
         "instrument": "lo",
         "num_counters": 16,
         "num_energy_steps": 128,
@@ -332,7 +332,7 @@ DATA_PRODUCT_CONFIGURATIONS = {
     CODICEAPID.COD_LO_NSW_SPECIES_COUNTS: {
         "coords": ["epoch", "inst_az", "spin_sector", "esa_step", "energy_label"],
         "dataset_name": "imap_codice_l1a_lo-nsw-species",
-        "dims": ["epoch", "inst_az", "spin_sector", "esa_step"],
+        "dims": ["epoch", "esa_step", "inst_az", "spin_sector"],
         "instrument": "lo",
         "num_counters": 8,
         "num_energy_steps": 128,

imap_processing/codice/decompress.py

@@ -50,7 +50,9 @@ def _apply_lossy_a(compressed_bytes: bytes) -> list[int]:
         The 24- or 32-bit decompressed values.
     """
     compressed_values = list(compressed_bytes)
-    decompressed_values = [LOSSY_A_TABLE[item] for item in compressed_values]
+    decompressed_values = [
+        LOSSY_A_TABLE[item - 1] if item > 0 else 0 for item in compressed_values
+    ]
     return decompressed_values
 
 
@@ -71,7 +73,9 @@ def _apply_lossy_b(compressed_bytes: bytes) -> list[int]:
         The 24- or 32-bit decompressed values.
     """
     compressed_values = list(compressed_bytes)
-    decompressed_values = [LOSSY_B_TABLE[item] for item in compressed_values]
+    decompressed_values = [
+        LOSSY_B_TABLE[item - 1] if item > 0 else 0 for item in compressed_values
+    ]
     return decompressed_values
 
 

imap_processing/glows/l1a/glows_l1a.py

@@ -7,11 +7,10 @@ import numpy as np
 import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
-from imap_processing.cdf.utils import J2000_EPOCH
 from imap_processing.glows.l0.decom_glows import decom_packets
 from imap_processing.glows.l0.glows_l0_data import DirectEventL0
 from imap_processing.glows.l1a.glows_l1a_data import DirectEventL1A, HistogramL1A
-from imap_processing.spice.time import met_to_j2000ns
+from imap_processing.spice.time import J2000_EPOCH, met_to_j2000ns
 
 
 def create_glows_attr_obj(data_version: str) -> ImapCdfAttributes:

imap_processing/hi/l1a/hi_l1a.py

@@ -33,7 +33,7 @@ def hi_l1a(packet_file_path: Union[str, Path], data_version: str) -> list[xr.Dat
         List of processed xarray dataset.
     """
     packet_def_file = (
-        imap_module_directory / "hi/packet_definitions/hi_packet_definition.xml"
+        imap_module_directory / "hi/packet_definitions/TLM_HI_COMBINED_SCI.xml"
     )
     datasets_by_apid = packet_file_to_datasets(
         packet_file=packet_file_path, xtce_packet_definition=packet_def_file
@@ -42,18 +42,18 @@ def hi_l1a(packet_file_path: Union[str, Path], data_version: str) -> list[xr.Dat
     # Process science to l1a.
     processed_data = []
     for apid in datasets_by_apid:
-        if apid == HIAPID.H45_SCI_CNT:
+        if apid in [HIAPID.H45_SCI_CNT, HIAPID.H90_SCI_CNT]:
             logger.info(
                 "Processing histogram data for [%s] packets", HIAPID.H45_SCI_CNT.name
             )
             data = hist_create_dataset(datasets_by_apid[apid])
-        elif apid == HIAPID.H45_SCI_DE:
+        elif apid in [HIAPID.H45_SCI_DE, HIAPID.H90_SCI_DE]:
             logger.info(
                 "Processing direct event data for [%s] packets", HIAPID.H45_SCI_DE.name
             )
 
             data = science_direct_event(datasets_by_apid[apid])
-        elif apid == HIAPID.H45_APP_NHK:
+        elif apid in [HIAPID.H45_APP_NHK, HIAPID.H90_APP_NHK]:
             logger.info(
                 "Processing housekeeping data for [%s] packets", HIAPID.H45_APP_NHK.name
             )

imap_processing/hi/l1a/histogram.py

@@ -2,36 +2,36 @@
 
 import numpy as np
 import xarray as xr
+from numpy._typing import NDArray
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
-from imap_processing.utils import convert_to_binary_string
 
 # define the names of the 24 counter arrays
 # contained in the histogram packet
 QUALIFIED_COUNTERS = (
-    "qual_ab",
-    "qual_c1c2",
-    "qual_ac1",
-    "qual_bc1",
-    "qual_abc1",
-    "qual_ac1c2",
-    "qual_bc1c2",
-    "qual_abc1c2",
+    "ab_qualified",
+    "c1c2_qualified",
+    "ac1_qualified",
+    "bc1_qualified",
+    "abc1_qualified",
+    "ac1c2_qualified",
+    "bc1c2_qualified",
+    "abc1c2_qualified",
 )
 LONG_COUNTERS = (
-    "long_a",
-    "long_b",
-    "long_c",
-    "long_ab",
-    "long_c1c2",
-    "long_ac1",
-    "long_bc1",
-    "long_abc1",
-    "long_ac1c2",
-    "long_bc1c2",
-    "long_abc1c2",
+    "a_first_only",
+    "b_first_only",
+    "c_first_only",
+    "ab_long",
+    "c1c2_long",
+    "ac1_long",
+    "bc1_long",
+    "abc1_long",
+    "ac1c2_long",
+    "bc1c2_long",
+    "abc1c2_long",
 )
-TOTAL_COUNTERS = ("total_a", "total_b", "total_c", "fee_de_sent", "fee_de_recd")
+TOTAL_COUNTERS = ("a_total", "b_total", "c_total", "fee_de_recd", "fee_de_sent")
 
 
 def create_dataset(input_ds: xr.Dataset) -> xr.Dataset:
@@ -41,99 +41,56 @@ def create_dataset(input_ds: xr.Dataset) -> xr.Dataset:
     Parameters
     ----------
     input_ds : xarray.Dataset
-        Dataset of packets.
+        Dataset of packets generated using the
+        `imap_processing.utils.packet_file_to_datasets` function.
 
     Returns
     -------
     dataset : xarray.Dataset
         Dataset with all metadata field data in xr.DataArray.
     """
-    dataset = allocate_histogram_dataset(len(input_ds.epoch))
-
-    # TODO: Move into the allocate dataset function?
-    dataset["epoch"].data[:] = input_ds["epoch"].data
-    dataset["ccsds_met"].data = input_ds["ccsds_met"].data
-    dataset["esa_stepping_num"].data = input_ds["esa_step"].data
-
-    # unpack the packets data into the Dataset
-    # (npackets, 24 * 90 * 12)
-    # TODO: Look into avoiding the for-loops below
-    #       It seems like we could try to reshape the arrays and do some numpy
-    #       broadcasting rather than for-loops directly here
-    for i_epoch, counters_bytes_data in enumerate(input_ds["counters"].data):
-        binary_str_val = convert_to_binary_string(counters_bytes_data)
-        # unpack 24 arrays of 90 12-bit unsigned integers
-        counter_ints = [
-            int(binary_str_val[i * 12 : (i + 1) * 12], 2) for i in range(90 * 24)
-        ]
-        # populate the dataset with the unpacked integers
-        for i_counter, counter in enumerate(
-            (*QUALIFIED_COUNTERS, *LONG_COUNTERS, *TOTAL_COUNTERS)
-        ):
-            dataset[counter][i_epoch] = counter_ints[
-                i_counter * 90 : (i_counter + 1) * 90
-            ]
-
-    return dataset
-
-
-def allocate_histogram_dataset(num_packets: int) -> xr.Dataset:
-    """
-    Allocate empty xarray.Dataset for specified number of Hi Histogram packets.
-
-    Parameters
-    ----------
-    num_packets : int
-        The number of Hi Histogram packets to allocate space for
-        in the xarray.Dataset.
-
-    Returns
-    -------
-    dataset : xarray.Dataset
-        Empty xarray.Dataset ready to be filled with packet data.
-    """
     attr_mgr = ImapCdfAttributes()
     attr_mgr.add_instrument_global_attrs(instrument="hi")
     attr_mgr.add_instrument_variable_attrs(instrument="hi", level=None)
-    # preallocate the xr.DataArrays for all CDF attributes based on number of packets
-    coords = dict()
-    coords["epoch"] = xr.DataArray(
-        np.empty(num_packets, dtype="datetime64[ns]"),
-        name="epoch",
-        dims=["epoch"],
-        attrs=attr_mgr.get_variable_attributes("epoch"),
-    )
-    # Histogram data is binned in 90, 4-degree bins
-    coords["angle"] = xr.DataArray(
-        np.arange(2, 360, 4),
-        name="angle",
-        dims=["angle"],
-        attrs=attr_mgr.get_variable_attributes("hi_hist_angle"),
-    )
 
-    data_vars = dict()
-    # Generate label variables
-    data_vars["angle_label"] = xr.DataArray(
-        coords["angle"].values.astype(str),
-        name="angle_label",
-        dims=["angle"],
-        attrs=attr_mgr.get_variable_attributes(
-            "hi_hist_angle_label", check_schema=False
-        ),
-    )
-    # Other data variables
-    data_vars["ccsds_met"] = xr.DataArray(
-        np.empty(num_packets, dtype=np.uint32),
-        dims=["epoch"],
-        attrs=attr_mgr.get_variable_attributes("hi_hist_ccsds_met"),
+    # Rename shcoarse variable (do this first since it copies the input_ds)
+    dataset = input_ds.rename_vars({"shcoarse": "ccsds_met"})
+
+    dataset.epoch.attrs.update(
+        attr_mgr.get_variable_attributes("epoch"),
     )
-    data_vars["esa_stepping_num"] = xr.DataArray(
-        np.empty(num_packets, dtype=np.uint8),
-        dims=["epoch"],
-        attrs=attr_mgr.get_variable_attributes("hi_hist_esa_step"),
+    # Add the hist_angle coordinate
+    # Histogram data is binned in 90, 4-degree bins
+    attrs = attr_mgr.get_variable_attributes("hi_hist_angle")
+    dataset.coords.update(
+        {
+            "angle": xr.DataArray(
+                np.arange(2, 360, 4),
+                name="angle",
+                dims=["angle"],
+                attrs=attrs,
+            )
+        }
     )
-
-    # Allocate xarray.DataArray objects for the 24 90-element histogram counters
+    # Update existing variable attributes
+    for var_name in [
+        "version",
+        "type",
+        "sec_hdr_flg",
+        "pkt_apid",
+        "seq_flgs",
+        "src_seq_ctr",
+        "pkt_len",
+        "ccsds_met",
+        "esa_step",
+        "num_of_spins",
+        "cksum",
+    ]:
+        attrs = attr_mgr.get_variable_attributes(f"hi_hist_{var_name}")
+        dataset.data_vars[var_name].attrs.update(attrs)
+
+    new_vars = dict()
+    # Populate 90-element histogram counters
     default_counter_attrs = attr_mgr.get_variable_attributes("hi_hist_counters")
     for counter_name in (*QUALIFIED_COUNTERS, *LONG_COUNTERS, *TOTAL_COUNTERS):
         # Inject counter name into generic counter attributes
@@ -141,15 +98,56 @@ def allocate_histogram_dataset(num_packets: int) -> xr.Dataset:
         for key, val in counter_attrs.items():
             if isinstance(val, str) and "{counter_name}" in val:
                 counter_attrs[key] = val.format(counter_name=counter_name)
-        data_vars[counter_name] = xr.DataArray(
-            data=np.empty((num_packets, len(coords["angle"])), np.uint16),
+        # Instantiate the counter DataArray
+        new_vars[counter_name] = xr.DataArray(
+            data=unpack_hist_counter(input_ds[counter_name].data.sum()),
             dims=["epoch", "angle"],
             attrs=counter_attrs,
         )
 
-    dataset = xr.Dataset(
-        data_vars=data_vars,
-        coords=coords,
-        attrs=attr_mgr.get_global_attributes("imap_hi_l1a_hist_attrs"),
+    # Generate label variable for angle coordinate
+    new_vars["angle_label"] = xr.DataArray(
+        dataset.coords["angle"].values.astype(str),
+        name="angle_label",
+        dims=["angle"],
+        attrs=attr_mgr.get_variable_attributes(
+            "hi_hist_angle_label", check_schema=False
+        ),
     )
+
+    dataset.update(new_vars)
+    dataset.attrs.update(attr_mgr.get_global_attributes("imap_hi_l1a_hist_attrs"))
+
     return dataset
+
+
+def unpack_hist_counter(counter_bytes: bytes) -> NDArray[np.uint16]:
+    """
+    Unpack Hi SCI_CNT counter data for a single counter.
+
+    Parameters
+    ----------
+    counter_bytes : bytes
+        Sum individual bytes for all epochs of a Hi SCI_CNT counter.
+
+    Returns
+    -------
+    output_array : numpy.ndarray[numpy.uint16]
+        The unpacked 12-bit unsigned integers for the input bytes. The
+        output array has a shape of (n, 90) where n is the number of SCI_CNT
+        packets in the input dataset.
+    """
+    # Interpret bytes for all epochs of current counter as uint8 array
+    counter_uint8 = np.frombuffer(counter_bytes, dtype=np.uint8)
+    # Split into triplets of upper-byte, split-byte and lower-byte arrays
+    upper_uint8, split_unit8, lower_uint8 = np.reshape(
+        counter_uint8, (3, -1), order="F"
+    ).astype(np.uint16)
+    # Compute even indexed uint12 values from upper-byte and first 4-bits of
+    # split-byte
+    even_uint12 = (upper_uint8 << 4) + (split_unit8 >> 4)
+    # Compute odd indexed uint12 values from lower 4-bits of split-byte and
+    # lower-byte
+    odd_uint12 = ((split_unit8 & (2**4 - 1)) << 8) + lower_uint8
+    output_array = np.column_stack((even_uint12, odd_uint12)).reshape(-1, 90)
+    return output_array