imap-processing 0.11.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

imap_processing/swapi/l2/swapi_l2.py

@@ -2,6 +2,9 @@
 
 import logging
 
+import numpy as np
+import numpy.typing as npt
+import pandas as pd
 import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
@@ -12,7 +15,96 @@ logger = logging.getLogger(__name__)
 TIME_PER_BIN = 0.167  # seconds
 
 
-def swapi_l2(l1_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
+def solve_full_sweep_energy(
+    esa_lvl5_data: np.ndarray, esa_table_df: pd.DataFrame, lut_notes_df: pd.DataFrame
+) -> npt.NDArray:
+    """
+    Calculate the energy of each full sweep data.
+
+    Parameters
+    ----------
+    esa_lvl5_data : numpy.ndarray
+        The L1 data input.
+    esa_table_df : pandas.DataFrame
+        The ESA unit conversion table that contains first 63 energies.
+    lut_notes_df : pandas.DataFrame
+        The LUT notes table that contains the last 9 fine energies.
+
+    Returns
+    -------
+    energy : numpy.ndarray
+        The energy of each full sweep data.
+    """
+    # Read 0 - 62 energy steps' fixed energy value
+    fixed_energy_values = esa_table_df["Energy"].values[:63]
+
+    # Find last 9 fine energy values of all sweeps data
+    # -------------------------------------------------
+    # First, verify that all values in the LUT-notes table's 'ESA DAC (Hex)' column
+    # exactly matches a value in the esa_lvl5_data.
+    has_exact_match = np.isin(esa_lvl5_data, lut_notes_df["ESA DAC (Hex)"].values)
+    if not np.all(has_exact_match):
+        raise ValueError(
+            "These ESA_LVL5 values not found in lut-notes table: "
+            f"{esa_lvl5_data[np.where(~has_exact_match)[0]]} "
+        )
+
+    # Find index of 71st energy step for all sweeps data in lut-notes table.
+    # Tried using np.where(np.isin(...)) or df.index[np.isin(...)] to find the index
+    # of each value in esa_lvl5_data within the LUT table. However, these methods
+    # return only the unique matching indices — not one index per input value.
+    # For example, given the input:
+    #   ['12F1', '12F1', '12F1', '12F1']
+    # np.where(np.isin(...)) would return:
+    #   [336]
+    # because it finds that '12F1' exists in the LUT and only returns its position once.
+    # What we actually need is:
+    #   [336, 336, 336, 336]
+    # — one index for *each* occurrence in the input, preserving its shape and order.
+    # Therefore, instead of relying on np.isin or similar, we explicitly use
+    # np.where in a loop to find the index of each value in esa_lvl5_data individually,
+    # ensuring the output array has the same shape as the input.
+
+    last_energy_step_indices = np.array(
+        [
+            np.where(lut_notes_df["ESA DAC (Hex)"].values == val)[0][0]
+            for val in esa_lvl5_data
+        ]
+    )
+    # Use back tracking steps to find all 9 fine energy value indices
+    # Eg. [0, -4, -8, ..., -28, -32]
+    steps = np.arange(9) * -4
+
+    # Find indices of last 9 fine energy values of all sweeps data
+    fine_energy_indices = last_energy_step_indices[:, None] + steps
+
+    # NOTE: Per SWAPI instruction, set every index that result in negative
+    # indices during back tracking to zero index. SWAPI calls this
+    # "flooring" the index. For example, if the 71st energy step index results
+    # in less than 32, then it would result in some negative indices. Eg.
+    #    71st index = 31
+    #    nine fine energy indices = [31, 27, 23, 19, 15, 11, 7, 3, -1]
+    #    flooring = [31, 27, 23, 19, 15, 11, 7, 3, 0]
+    fine_energy_indices[fine_energy_indices < 0] = 0
+
+    energy_values = lut_notes_df["Energy"].values[fine_energy_indices]
+
+    # Expand to match the number of rows in energy_values
+    first_63_values = np.tile(
+        fixed_energy_values, (energy_values.shape[0], 1)
+    )  # (epoch, 63)
+
+    # Append the first_63_values in front of energy_values
+    sweeps_energy_value = np.hstack((first_63_values, energy_values))
+
+    return sweeps_energy_value
+
+
+def swapi_l2(
+    l1_dataset: xr.Dataset,
+    esa_table_df: pd.DataFrame,
+    lut_notes_df: pd.DataFrame,
+) -> xr.Dataset:
     """
     Produce science data to L2.
 
@@ -32,8 +124,10 @@ def swapi_l2(l1_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
     ----------
     l1_dataset : xarray.Dataset
         The L1 data input.
-    data_version : str
-        Version of the data product being created.
+    esa_table_df : pandas.DataFrame
+        The ESA unit conversion table that contains first 63 energies.
+    lut_notes_df : pandas.DataFrame
+        The LUT notes table that contains the last 9 fine energies.
 
     Returns
     -------
@@ -56,13 +150,30 @@ def swapi_l2(l1_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
         "lut_choice",
         "fpga_type",
         "fpga_rev",
+        "esa_lvl5",
     ]
     l2_dataset = l1_dataset[l1_data_keys]
 
+    # Find energy of each full sweep data
+    # -----------------------------------
+    # Convert unpacked ESA_LVL5 values to hex to match the LUT table
+    # value
+    esa_lvl5_hex = np.vectorize(lambda x: format(x, "X"))(l1_dataset["esa_lvl5"].values)
+    esa_energy = solve_full_sweep_energy(
+        esa_lvl5_hex,
+        esa_table_df=esa_table_df,
+        lut_notes_df=lut_notes_df,
+    )
+
+    l2_dataset["swp_esa_energy"] = xr.DataArray(
+        esa_energy,
+        name="esa_energy",
+        dims=["epoch", "energy"],
+        attrs=cdf_manager.get_variable_attributes("esa_energy"),
+    )
+
     # Update L2 specific attributes
-    l2_dataset.attrs["Data_version"] = data_version
     l2_global_attrs = cdf_manager.get_global_attributes("imap_swapi_l2_sci")
-    l2_dataset.attrs["Data_level"] = l2_global_attrs["Data_level"]
     l2_dataset.attrs["Data_type"] = l2_global_attrs["Data_type"]
     l2_dataset.attrs["Logical_source"] = l2_global_attrs["Logical_source"]
     l2_dataset.attrs["Logical_source_description"] = l2_global_attrs[

imap_processing/swapi/swapi_utils.py

@@ -6,6 +6,10 @@ other SWAPI processing modules.
 """
 
 from enum import IntEnum
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
 
 
 class SWAPIAPID(IntEnum):
@@ -23,3 +27,31 @@ class SWAPIMODE(IntEnum):
     LVSCI = 1
     HVENG = 2
     HVSCI = 3
+
+
+def read_swapi_lut_table(file_path: Path) -> pd.DataFrame:
+    """
+    Read the LUT table from a CSV file.
+
+    Parameters
+    ----------
+    file_path : pathlib.Path
+        The path to the LUT table CSV file.
+
+    Returns
+    -------
+    pandas.DataFrame
+        The LUT table as a DataFrame.
+    """
+    df = pd.read_csv(file_path)
+
+    # Clean and convert 'Energy' column from comma-separated strings to integers
+    df["Energy"] = (
+        df["Energy"]
+        .astype(str)
+        .str.replace(",", "", regex=False)
+        .replace("Solve", -1)
+        .astype(np.int64)
+    )
+
+    return df

imap_processing/swe/l1a/swe_l1a.py

@@ -5,6 +5,7 @@ import logging
 import xarray as xr
 
 from imap_processing import imap_module_directory
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.swe.l1a.swe_science import swe_science
 from imap_processing.swe.utils.swe_utils import (
     SWEAPID,
@@ -14,7 +15,7 @@ from imap_processing.utils import packet_file_to_datasets
 logger = logging.getLogger(__name__)
 
 
-def swe_l1a(packet_file: str, data_version: str) -> xr.Dataset:
+def swe_l1a(packet_file: str) -> xr.Dataset:
     """
     Will process SWE l0 data into l1a data.
 
@@ -26,9 +27,6 @@ def swe_l1a(packet_file: str, data_version: str) -> xr.Dataset:
     ----------
     packet_file : str
         Path where the raw packet file is stored.
-    data_version : str
-        Data version to write to CDF files and the Data_version CDF attribute.
-        Should be in the format Vxxx.
 
     Returns
     -------
@@ -42,12 +40,46 @@ def swe_l1a(packet_file: str, data_version: str) -> xr.Dataset:
         packet_file, xtce_document, use_derived_value=False
     )
 
-    if SWEAPID.SWE_SCIENCE not in datasets_by_apid:
-        logger.info("No science data found in packet file.")
-        return []
-    # TODO: figure out how to handle non-science data
-    return [
-        swe_science(
-            l0_dataset=datasets_by_apid[SWEAPID.SWE_SCIENCE], data_version=data_version
+    processed_data = []
+
+    if SWEAPID.SWE_SCIENCE in datasets_by_apid:
+        logger.info("Processing SWE science data.")
+        processed_data.append(
+            swe_science(l0_dataset=datasets_by_apid[SWEAPID.SWE_SCIENCE])
+        )
+
+    # Process non-science data
+    # Define minimal CDF attrs for the non science dataset
+    imap_attrs = ImapCdfAttributes()
+    imap_attrs.add_instrument_global_attrs("swe")
+    imap_attrs.add_instrument_variable_attrs("swe", "l1a")
+    non_science_attrs = imap_attrs.get_variable_attributes("non_science_attrs")
+    epoch_attrs = imap_attrs.get_variable_attributes("epoch", check_schema=False)
+
+    if SWEAPID.SWE_APP_HK in datasets_by_apid:
+        logger.info("Processing SWE housekeeping data.")
+        hk_ds = datasets_by_apid[SWEAPID.SWE_APP_HK]
+        hk_ds.attrs.update(imap_attrs.get_global_attributes("imap_swe_l1a_hk"))
+        hk_ds["epoch"].attrs.update(epoch_attrs)
+        # Add attrs to HK data variables
+        for var_name in hk_ds.data_vars:
+            hk_ds[var_name].attrs.update(non_science_attrs)
+        processed_data.append(hk_ds)
+
+    if SWEAPID.SWE_CEM_RAW in datasets_by_apid:
+        logger.info("Processing SWE CEM raw data.")
+        cem_raw_ds = datasets_by_apid[SWEAPID.SWE_CEM_RAW]
+        cem_raw_ds.attrs.update(
+            imap_attrs.get_global_attributes("imap_swe_l1a_cem-raw")
         )
-    ]
+        cem_raw_ds["epoch"].attrs.update(epoch_attrs)
+
+        # Add attrs to CEM raw data variables
+        for var_name in cem_raw_ds.data_vars:
+            cem_raw_ds[var_name].attrs.update(non_science_attrs)
+        processed_data.append(cem_raw_ds)
+
+    if len(processed_data) == 0:
+        logger.info("Data contains unknown APID.")
+
+    return processed_data

imap_processing/swe/l1a/swe_science.py

@@ -6,6 +6,7 @@ import numpy as np
 import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.swe.utils import swe_constants
 from imap_processing.swe.utils.swe_utils import SWEAPID
 
 logger = logging.getLogger(__name__)
@@ -63,7 +64,7 @@ def decompressed_counts(cem_count: int) -> int:
     )
 
 
-def swe_science(l0_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
+def swe_science(l0_dataset: xr.Dataset) -> xr.Dataset:
     """
     SWE L1a science processing.
 
@@ -96,10 +97,6 @@ def swe_science(l0_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
     l0_dataset : xarray.Dataset
         Raw packet data from SWE stored as an xarray dataset.
 
-    data_version : str
-        Data version for the 'Data_version' CDF attribute. This is the version of the
-        output file.
-
     Returns
     -------
     dataset : xarray.Dataset
@@ -119,7 +116,9 @@ def swe_science(l0_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
     # 4. Reshape the data to 180 x 7
     raw_science_array = np.array(
         [
-            np.frombuffer(binary_string, dtype=np.uint8).reshape(180, 7)
+            np.frombuffer(binary_string, dtype=np.uint8).reshape(
+                180, swe_constants.N_CEMS
+            )
             for binary_string in l0_dataset["science_data"].values
         ]
     )
@@ -132,20 +131,19 @@ def swe_science(l0_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
     cdf_attrs = ImapCdfAttributes()
     cdf_attrs.add_instrument_global_attrs("swe")
     cdf_attrs.add_instrument_variable_attrs("swe", "l1a")
-    cdf_attrs.add_global_attribute("Data_version", data_version)
 
     epoch_time = xr.DataArray(
         l0_dataset["epoch"],
         name="epoch",
         dims=["epoch"],
-        attrs=cdf_attrs.get_variable_attributes("epoch"),
+        attrs=cdf_attrs.get_variable_attributes("epoch", check_schema=False),
     )
 
     spin_sector = xr.DataArray(
         np.arange(180),
         name="spin_sector",
         dims=["spin_sector"],
-        attrs=cdf_attrs.get_variable_attributes("spin_sector"),
+        attrs=cdf_attrs.get_variable_attributes("spin_sector", check_schema=False),
     )
 
     # NOTE: LABL_PTR_1 should be CDF_CHAR.
@@ -153,14 +151,16 @@ def swe_science(l0_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
         spin_sector.values.astype(str),
         name="spin_sector_label",
         dims=["spin_sector"],
-        attrs=cdf_attrs.get_variable_attributes("spin_sector_label"),
+        attrs=cdf_attrs.get_variable_attributes(
+            "spin_sector_label", check_schema=False
+        ),
     )
 
     cem_id = xr.DataArray(
-        np.arange(7),
+        np.arange(swe_constants.N_CEMS),
         name="cem_id",
         dims=["cem_id"],
-        attrs=cdf_attrs.get_variable_attributes("cem_id"),
+        attrs=cdf_attrs.get_variable_attributes("cem_id", check_schema=False),
     )
 
     # NOTE: LABL_PTR_2 should be CDF_CHAR.
@@ -168,7 +168,7 @@ def swe_science(l0_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
         cem_id.values.astype(str),
         name="cem_id_label",
         dims=["cem_id"],
-        attrs=cdf_attrs.get_variable_attributes("cem_id_label"),
+        attrs=cdf_attrs.get_variable_attributes("cem_id_label", check_schema=False),
     )
 
     science_xarray = xr.DataArray(