imap-processing 0.6.0__py3-none-any.whl

imap_processing/swapi/swapi_utils.py

@@ -0,0 +1,25 @@
+"""
+Various classes and functions used throughout SWAPI processing.
+
+This module contains utility classes and functions that are used by various
+other SWAPI processing modules.
+"""
+
+from enum import IntEnum
+
+
+class SWAPIAPID(IntEnum):
+    """Create ENUM for apid."""
+
+    SWP_HK = 1184
+    SWP_SCI = 1188
+    SWP_AUT = 1192
+
+
+class SWAPIMODE(IntEnum):
+    """Create ENUM for MODE."""
+
+    LVENG = 0
+    LVSCI = 1
+    HVENG = 2
+    HVSCI = 3

imap_processing/swe/__init__.py

@@ -0,0 +1 @@
+__version__ = "001"

imap_processing/swe/l1a/swe_l1a.py

@@ -0,0 +1,48 @@
+"""Contains code to perform SWE L1a processing."""
+
+import logging
+
+import xarray as xr
+
+from imap_processing import imap_module_directory
+from imap_processing.swe.l1a.swe_science import swe_science
+from imap_processing.swe.utils.swe_utils import (
+    SWEAPID,
+)
+from imap_processing.utils import packet_file_to_datasets
+
+logger = logging.getLogger(__name__)
+
+
+def swe_l1a(packet_file: str, data_version: str) -> xr.Dataset:
+    """
+    Will process SWE l0 data into l1a data.
+
+    Receive all L0 data file. Based on appId, it
+    call its function to process. If appId is science, it requires more work
+    than other appId such as appId of housekeeping.
+
+    Parameters
+    ----------
+    packet_file : str
+        Path where the raw packet file is stored.
+    data_version : str
+        Data version to write to CDF files and the Data_version CDF attribute.
+        Should be in the format Vxxx.
+
+    Returns
+    -------
+    List
+        List of xarray.Dataset.
+    """
+    xtce_document = (
+        f"{imap_module_directory}/swe/packet_definitions/swe_packet_definition.xml"
+    )
+    datasets_by_apid = packet_file_to_datasets(
+        packet_file, xtce_document, use_derived_value=False
+    )
+
+    # TODO: figure out how to handle non-science data
+    return swe_science(
+        l0_dataset=datasets_by_apid[SWEAPID.SWE_SCIENCE], data_version=data_version
+    )

imap_processing/swe/l1a/swe_science.py

@@ -0,0 +1,223 @@
+"""Contains code to perform SWE L1a science processing."""
+
+import logging
+
+import numpy as np
+import xarray as xr
+
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.swe.utils.swe_utils import SWEAPID
+
+logger = logging.getLogger(__name__)
+
+
+def decompressed_counts(cem_count: int) -> int:
+    """
+    Decompressed counts from the CEMs.
+
+    Parameters
+    ----------
+    cem_count : int
+        CEM counts. Eg. 243.
+
+    Returns
+    -------
+    int
+        Decompressed count. Eg. 40959.
+    """
+    # index is the first four bits of input data
+    # multi is the last four bits of input data
+    index = cem_count // 16
+    multi = cem_count % 16
+
+    # This is look up table for the index to get
+    # base and step_size to calculate the decompressed count.
+    decompress_table = {
+        0: {"base": 0, "step_size": 1},
+        1: {"base": 16, "step_size": 1},
+        2: {"base": 32, "step_size": 2},
+        3: {"base": 64, "step_size": 4},
+        4: {"base": 128, "step_size": 8},
+        5: {"base": 256, "step_size": 16},
+        6: {"base": 512, "step_size": 16},
+        7: {"base": 768, "step_size": 16},
+        8: {"base": 1024, "step_size": 32},
+        9: {"base": 1536, "step_size": 32},
+        10: {"base": 2048, "step_size": 64},
+        11: {"base": 3072, "step_size": 128},
+        12: {"base": 5120, "step_size": 256},
+        13: {"base": 9216, "step_size": 512},
+        14: {"base": 17408, "step_size": 1024},
+        15: {"base": 33792, "step_size": 2048},
+    }
+
+    # decompression formula from SWE algorithm document CN102D-D0001 and page 16.
+    # N = base[index] + multi * step_size[index] + (step_size[index] - 1) / 2
+    # NOTE: for (step_size[index] - 1) / 2, we only keep the whole number part of
+    # the quotient
+
+    return (
+        decompress_table[index]["base"]
+        + (multi * decompress_table[index]["step_size"])
+        + ((decompress_table[index]["step_size"] - 1) // 2)
+    )
+
+
+def swe_science(l0_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
+    """
+    SWE L1a science processing.
+
+    SWE L1A algorithm steps:
+        - Read data from each SWE packet file
+        - Uncompress counts data
+        - Store metadata fields and data in DataArray of xarray
+        - Save data to dataset.
+
+    In each packet, SWE collects data for 15 seconds. In each second,
+    it collect data for 12 energy steps and at each energy step,
+    it collects 7 data from each 7 CEMs.
+
+    Each L1A data from each packet will have this shape: 15 rows, 12 columns,
+    and each cell in 15 x 12 table contains 7 element array.
+    These dimension maps to this:
+
+    |     15 rows --> 15 seconds
+    |     12 column --> 12 energy steps in each second
+    |     7 element --> 7 CEMs counts
+
+    In L1A, we don't do anything besides read raw data, uncompress counts data and
+    store data in 15 x 12 x 7 array.
+
+    SWE want to keep all value as it is in L1A. Post L1A, we group data into full cycle
+    and convert raw data to engineering data as needed.
+
+    Parameters
+    ----------
+    l0_dataset : xarray.Dataset
+        Raw packet data from SWE stored as an xarray dataset.
+
+    data_version : str
+        Data version for the 'Data_version' CDF attribute. This is the version of the
+        output file.
+
+    Returns
+    -------
+    dataset : xarray.Dataset
+        The xarray dataset with data.
+    """
+    # We know we can only have 8 bit numbers input, so iterate over all
+    # possibilities once up front
+    decompression_table = np.array([decompressed_counts(i) for i in range(256)])
+
+    # Loop through each packet individually with a list comprehension and
+    # perform the following steps:
+    # 1. Turn the binary string  of 0s and 1s to an int
+    # 2. Convert the int into a bytes object of length 1260 (10080 / 8)
+    #    Eg. "0000000011110011" --> b'\x00\xf3'
+    #    1260 = 15 seconds x 12 energy steps x 7 CEMs
+    # 3. Read that bytes data to a numpy array of uint8 through the buffer protocol
+    # 4. Reshape the data to 180 x 7
+    raw_science_array = np.array(
+        [
+            np.frombuffer(
+                int(binary_string, 2).to_bytes(1260, byteorder="big"), dtype=np.uint8
+            ).reshape(180, 7)
+            for binary_string in l0_dataset["science_data"].values
+        ]
+    )
+
+    # Decompress the raw science data using numpy broadcasting logic
+    # science_array will be the same shape as raw_science_array (npackets, 180, 7)
+    science_array = decompression_table[raw_science_array]
+
+    # Load CDF attrs
+    cdf_attrs = ImapCdfAttributes()
+    cdf_attrs.add_instrument_global_attrs("swe")
+    cdf_attrs.add_instrument_variable_attrs("swe", "l1a")
+    cdf_attrs.add_global_attribute("Data_version", data_version)
+
+    epoch_time = xr.DataArray(
+        l0_dataset["epoch"],
+        name="epoch",
+        dims=["epoch"],
+        attrs=cdf_attrs.get_variable_attributes("epoch"),
+    )
+
+    spin_angle = xr.DataArray(
+        np.arange(180),
+        name="spin_angle",
+        dims=["spin_angle"],
+        attrs=cdf_attrs.get_variable_attributes("spin_angle"),
+    )
+
+    # NOTE: LABL_PTR_1 should be CDF_CHAR.
+    spin_angle_label = xr.DataArray(
+        spin_angle.values.astype(str),
+        name="spin_angle_label",
+        dims=["spin_angle_label"],
+        attrs=cdf_attrs.get_variable_attributes("spin_angle_label"),
+    )
+
+    polar_angle = xr.DataArray(
+        np.arange(7),
+        name="polar_angle",
+        dims=["polar_angle"],
+        attrs=cdf_attrs.get_variable_attributes("polar_angle"),
+    )
+
+    # NOTE: LABL_PTR_2 should be CDF_CHAR.
+    polar_angle_label = xr.DataArray(
+        polar_angle.values.astype(str),
+        name="polar_angle_label",
+        dims=["polar_angle_label"],
+        attrs=cdf_attrs.get_variable_attributes("polar_angle_label"),
+    )
+
+    science_xarray = xr.DataArray(
+        science_array,
+        dims=["epoch", "spin_angle", "polar_angle"],
+        attrs=cdf_attrs.get_variable_attributes("science_data"),
+    )
+
+    raw_science_xarray = xr.DataArray(
+        raw_science_array,
+        dims=["epoch", "spin_angle", "polar_angle"],
+        attrs=cdf_attrs.get_variable_attributes("raw_counts"),
+    )
+
+    # Add APID to global attrs for following processing steps
+    l1a_global_attrs = cdf_attrs.get_global_attributes("imap_swe_l1a_sci")
+    # Formatting to string to be complaint with ISTP
+    l1a_global_attrs["packet_apid"] = SWEAPID.SWE_SCIENCE.value
+    dataset = xr.Dataset(
+        coords={
+            "epoch": epoch_time,
+            "spin_angle": spin_angle,
+            "polar_angle": polar_angle,
+            "spin_angle_label": spin_angle_label,
+            "polar_angle_label": polar_angle_label,
+        },
+        attrs=l1a_global_attrs,
+    )
+    dataset["science_data"] = science_xarray
+    dataset["raw_science_data"] = raw_science_xarray
+    # TODO: Remove the header in packet_file_to_datasets
+    #       The science_data variable is also in the l1 dataset with different values
+    l0_dataset = l0_dataset.drop_vars(
+        [
+            "science_data",
+            "version",
+            "type",
+            "sec_hdr_flg",
+            "pkt_apid",
+            "seq_flgs",
+            "src_seq_ctr",
+            "pkt_len",
+        ]
+    )
+    for var_name, arr in l0_dataset.variables.items():
+        arr.attrs = cdf_attrs.get_variable_attributes(var_name)
+    dataset = dataset.merge(l0_dataset)
+
+    logger.info("SWE L1A science data processing completed.")
+    return dataset

imap_processing/swe/l1b/engineering_unit_convert_table.csv

@@ -0,0 +1,65 @@
+index,packetName,mnemonic,convertAs,segNumber,lowValue,highValue,c0,c1,c2,c3,c4,c5,c6,c7
+0,SWE_APP_HK,HVPS_CEM_DAC,UNSEGMENTED_POLY,1,0,0,0,1.025641,0,0,0,0,0,0
+1,SWE_APP_HK,THRESHHOLD,UNSEGMENTED_POLY,1,0,0,0,0.001221,0,0,0,0,0,0
+2,SWE_APP_HK,HVPS_VBULK,UNSEGMENTED_POLY,1,0,0,0,0.51282,0,0,0,0,0,0
+3,SWE_APP_HK,HVPS_VCEM,UNSEGMENTED_POLY,1,0,0,0,1.34616,0,0,0,0,0,0
+4,SWE_APP_HK,HVPS_VESA,UNSEGMENTED_POLY,1,0,0,0,0.38462,0,0,0,0,0,0
+5,SWE_APP_HK,HVPS_VESA_LOW_RANGE,UNSEGMENTED_POLY,1,0,0,0,0.0078526,0,0,0,0,0,0
+6,SWE_APP_HK,HVPS_ICEM,UNSEGMENTED_POLY,1,0,0,0,0.000064103,0,0,0,0,0,0
+7,SWE_APP_HK,FEE_TEMP,UNSEGMENTED_POLY,1,0,0,-450.3382536,2.1927679,-0.004465576,4.89959E-06,-2.96758E-09,9.35868E-13,-1.1994E-16,0
+8,SWE_APP_HK,SENSOR_TEMP,UNSEGMENTED_POLY,1,0,0,-450.3382536,2.1927679,-0.004465576,4.89959E-06,-2.96758E-09,9.35868E-13,-1.1994E-16,0
+9,SWE_APP_HK,HVPS_TEMP,UNSEGMENTED_POLY,1,0,0,-450.3382536,2.1927679,-0.004465576,4.89959E-06,-2.96758E-09,9.35868E-13,-1.1994E-16,0
+10,SWE_APP_HK,LVPS_BACK_BOARD_TEMP,UNSEGMENTED_POLY,1,0,0,-273.2,0.144461908,0,0,0,0,0,0
+11,SWE_APP_HK,LVPS_FRONT_BOARD_TEMP,UNSEGMENTED_POLY,1,0,0,-273.2,0.144461908,0,0,0,0,0,0
+12,SWE_APP_HK,LVPS_MID_BOARD_TEMP,UNSEGMENTED_POLY,1,0,0,-273.2,0.144461908,0,0,0,0,0,0
+13,SWE_APP_HK,LVPS_3_3V_VMON,UNSEGMENTED_POLY,1,0,0,0,0.001611,0,0,0,0,0,0
+14,SWE_APP_HK,LVPS_5V_P_VMON,UNSEGMENTED_POLY,1,0,0,0,0.002441,0,0,0,0,0,0
+15,SWE_APP_HK,LVPS_5V_N_VMON,UNSEGMENTED_POLY,1,0,0,0,-0.002441,0,0,0,0,0,0
+16,SWE_APP_HK,LVPS_12V_P_VMON,UNSEGMENTED_POLY,1,0,0,0,0.005859,0,0,0,0,0,0
+17,SWE_APP_HK,LVPS_12V_N_VMON,UNSEGMENTED_POLY,1,0,0,0,-0.005859,0,0,0,0,0,0
+18,SWE_APP_HK,LVPS_3_3V_IMON,UNSEGMENTED_POLY,1,0,0,0,0.000916,0,0,0,0,0,0
+19,SWE_APP_HK,LVPS_5V_P_IMON,UNSEGMENTED_POLY,1,0,0,0,0.000461,0,0,0,0,0,0
+20,SWE_APP_HK,LVPS_5V_N_IMON,UNSEGMENTED_POLY,1,0,0,0,-0.000461,0,0,0,0,0,0
+21,SWE_APP_HK,LVPS_12V_P_IMON,UNSEGMENTED_POLY,1,0,0,0,0.000704,0,0,0,0,0,0
+22,SWE_APP_HK,LVPS_12V_N_IMON,UNSEGMENTED_POLY,1,0,0,0,-0.000704,0,0,0,0,0,0
+23,SWE_APP_HK,CDH_PROCESSOR_TEMP,UNSEGMENTED_POLY,1,0,0,-450.3382536,2.1927679,-0.004465576,4.89959E-06,-2.96758E-09,9.35868E-13,-1.1994E-16,0
+24,SWE_APP_HK,CDH_1_8V_LDO_TEMP,UNSEGMENTED_POLY,1,0,0,-450.3382536,2.1927679,-0.004465576,4.89959E-06,-2.96758E-09,9.35868E-13,-1.1994E-16,0
+25,SWE_APP_HK,CDH_1_5V_LDO_TEMP,UNSEGMENTED_POLY,1,0,0,-450.3382536,2.1927679,-0.004465576,4.89959E-06,-2.96758E-09,9.35868E-13,-1.1994E-16,0
+26,SWE_APP_HK,CDH_SDRAM_TEMP,UNSEGMENTED_POLY,1,0,0,-450.3382536,2.1927679,-0.004465576,4.89959E-06,-2.96758E-09,9.35868E-13,-1.1994E-16,0
+27,SWE_APP_HK,CDH_1_5V_VMON,UNSEGMENTED_POLY,1,0,0,0,0.001221,0,0,0,0,0,0
+28,SWE_APP_HK,CDH_1_8V_VMON,UNSEGMENTED_POLY,1,0,0,0,0.001221,0,0,0,0,0,0
+29,SWE_APP_HK,CDH_3_3V_VMON,UNSEGMENTED_POLY,1,0,0,0,0.001221,0,0,0,0,0,0
+30,SWE_APP_HK,CDH_12V_P_VMON,UNSEGMENTED_POLY,1,0,0,0,0.005897,0,0,0,0,0,0
+31,SWE_APP_HK,CDH_12V_N_VMON,UNSEGMENTED_POLY,1,0,0,0,-0.005841,0,0,0,0,0,0
+32,SWE_APP_HK,CDH_5V_VMON,UNSEGMENTED_POLY,1,0,0,0,0.002441,0,0,0,0,0,0
+33,SWE_APP_HK,CDH_ANALOG_REF_VMON,UNSEGMENTED_POLY,1,0,0,0,0.002441,0,0,0,0,0,0
+34,SWE_SCIENCE,SPIN_PHASE,UNSEGMENTED_POLY,1,0,0,0,0.005493,0,0,0,0,0,0
+35,SWE_SCIENCE,SPIN_PERIOD,UNSEGMENTED_POLY,1,0,0,0,0.00032,0,0,0,0,0,0
+36,SWE_SCIENCE,THRESHOLD_DAC,UNSEGMENTED_POLY,1,0,0,0,0.001221,0,0,0,0,0,0
+37,SWE_BOOT_HK,MDM25P_14_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0
+38,SWE_BOOT_HK,MDM25P_15_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0
+39,SWE_BOOT_HK,MDM25P_16_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0
+40,SWE_BOOT_HK,LVPS_12V_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.144461908,0,0,0,0,0,0
+41,SWE_BOOT_HK,LVPS_5V_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.144461908,0,0,0,0,0,0
+42,SWE_BOOT_HK,LVPS_3p3V_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.144461908,0,0,0,0,0,0
+43,SWE_BOOT_HK,LVPS_3p3V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0
+44,SWE_BOOT_HK,LVPS_5V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0
+45,SWE_BOOT_HK,LVPS_n5V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0
+46,SWE_BOOT_HK,LVPS_12V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0
+47,SWE_BOOT_HK,LVPS_n12V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0
+48,SWE_BOOT_HK,LVPS_3p3V_I,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0
+49,SWE_BOOT_HK,LVPS_5V_I,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0
+50,SWE_BOOT_HK,LVPS_n5V_I,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0
+51,SWE_BOOT_HK,LVPS_12V_I,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0
+52,SWE_BOOT_HK,LVPS_n12V_I,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0
+53,SWE_BOOT_HK,CDH_1p5V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0
+54,SWE_BOOT_HK,CDH_1p8V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0
+55,SWE_BOOT_HK,CDH_3p3V,UNSEGMENTED_POLY,1,0,0,0,0.001220703,0,0,0,0,0,0
+56,SWE_BOOT_HK,CDH_12V,UNSEGMENTED_POLY,1,0,0,0,0.005897117,0,0,0,0,0,0
+57,SWE_BOOT_HK,CDH_n12V,UNSEGMENTED_POLY,1,0,0,0,-0.005840685,0,0,0,0,0,0
+58,SWE_BOOT_HK,CDH_5V,UNSEGMENTED_POLY,1,0,0,0,0.002412457,0,0,0,0,0,0
+59,SWE_BOOT_HK,CDH_5V_ADC,UNSEGMENTED_POLY,1,0,0,0,0.002412457,0,0,0,0,0,0
+60,SWE_BOOT_HK,CDH_PROCESSOR_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0
+61,SWE_BOOT_HK,CDH_1P8V_LDO_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0
+62,SWE_BOOT_HK,CDH_1P5V_LDO_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0
+63,SWE_BOOT_HK,CDH_SDRAM_T,UNSEGMENTED_POLY,1,0,0,-273.2,0.122070313,0,0,0,0,0,0