imap-processing 0.12.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

imap_processing/swapi/l2/swapi_l2.py

@@ -2,6 +2,9 @@
 
 import logging
 
+import numpy as np
+import numpy.typing as npt
+import pandas as pd
 import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
@@ -12,7 +15,96 @@ logger = logging.getLogger(__name__)
 TIME_PER_BIN = 0.167  # seconds
 
 
-def swapi_l2(l1_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
+def solve_full_sweep_energy(
+    esa_lvl5_data: np.ndarray, esa_table_df: pd.DataFrame, lut_notes_df: pd.DataFrame
+) -> npt.NDArray:
+    """
+    Calculate the energy of each full sweep data.
+
+    Parameters
+    ----------
+    esa_lvl5_data : numpy.ndarray
+        The L1 data input.
+    esa_table_df : pandas.DataFrame
+        The ESA unit conversion table that contains first 63 energies.
+    lut_notes_df : pandas.DataFrame
+        The LUT notes table that contains the last 9 fine energies.
+
+    Returns
+    -------
+    energy : numpy.ndarray
+        The energy of each full sweep data.
+    """
+    # Read 0 - 62 energy steps' fixed energy value
+    fixed_energy_values = esa_table_df["Energy"].values[:63]
+
+    # Find last 9 fine energy values of all sweeps data
+    # -------------------------------------------------
+    # First, verify that all values in the LUT-notes table's 'ESA DAC (Hex)' column
+    # exactly matches a value in the esa_lvl5_data.
+    has_exact_match = np.isin(esa_lvl5_data, lut_notes_df["ESA DAC (Hex)"].values)
+    if not np.all(has_exact_match):
+        raise ValueError(
+            "These ESA_LVL5 values not found in lut-notes table: "
+            f"{esa_lvl5_data[np.where(~has_exact_match)[0]]} "
+        )
+
+    # Find index of 71st energy step for all sweeps data in lut-notes table.
+    # Tried using np.where(np.isin(...)) or df.index[np.isin(...)] to find the index
+    # of each value in esa_lvl5_data within the LUT table. However, these methods
+    # return only the unique matching indices — not one index per input value.
+    # For example, given the input:
+    #   ['12F1', '12F1', '12F1', '12F1']
+    # np.where(np.isin(...)) would return:
+    #   [336]
+    # because it finds that '12F1' exists in the LUT and only returns its position once.
+    # What we actually need is:
+    #   [336, 336, 336, 336]
+    # — one index for *each* occurrence in the input, preserving its shape and order.
+    # Therefore, instead of relying on np.isin or similar, we explicitly use
+    # np.where in a loop to find the index of each value in esa_lvl5_data individually,
+    # ensuring the output array has the same shape as the input.
+
+    last_energy_step_indices = np.array(
+        [
+            np.where(lut_notes_df["ESA DAC (Hex)"].values == val)[0][0]
+            for val in esa_lvl5_data
+        ]
+    )
+    # Use back tracking steps to find all 9 fine energy value indices
+    # Eg. [0, -4, -8, ..., -28, -32]
+    steps = np.arange(9) * -4
+
+    # Find indices of last 9 fine energy values of all sweeps data
+    fine_energy_indices = last_energy_step_indices[:, None] + steps
+
+    # NOTE: Per SWAPI instruction, set every index that result in negative
+    # indices during back tracking to zero index. SWAPI calls this
+    # "flooring" the index. For example, if the 71st energy step index results
+    # in less than 32, then it would result in some negative indices. Eg.
+    #    71st index = 31
+    #    nine fine energy indices = [31, 27, 23, 19, 15, 11, 7, 3, -1]
+    #    flooring = [31, 27, 23, 19, 15, 11, 7, 3, 0]
+    fine_energy_indices[fine_energy_indices < 0] = 0
+
+    energy_values = lut_notes_df["Energy"].values[fine_energy_indices]
+
+    # Expand to match the number of rows in energy_values
+    first_63_values = np.tile(
+        fixed_energy_values, (energy_values.shape[0], 1)
+    )  # (epoch, 63)
+
+    # Append the first_63_values in front of energy_values
+    sweeps_energy_value = np.hstack((first_63_values, energy_values))
+
+    return sweeps_energy_value
+
+
+def swapi_l2(
+    l1_dataset: xr.Dataset,
+    esa_table_df: pd.DataFrame,
+    lut_notes_df: pd.DataFrame,
+) -> xr.Dataset:
     """
     Produce science data to L2.
 
@@ -32,8 +124,10 @@ def swapi_l2(l1_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
     ----------
     l1_dataset : xarray.Dataset
         The L1 data input.
-    data_version : str
-        Version of the data product being created.
+    esa_table_df : pandas.DataFrame
+        The ESA unit conversion table that contains first 63 energies.
+    lut_notes_df : pandas.DataFrame
+        The LUT notes table that contains the last 9 fine energies.
 
     Returns
     -------
@@ -56,11 +150,29 @@ def swapi_l2(l1_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
         "lut_choice",
         "fpga_type",
         "fpga_rev",
+        "esa_lvl5",
     ]
     l2_dataset = l1_dataset[l1_data_keys]
 
+    # Find energy of each full sweep data
+    # -----------------------------------
+    # Convert unpacked ESA_LVL5 values to hex to match the LUT table
+    # value
+    esa_lvl5_hex = np.vectorize(lambda x: format(x, "X"))(l1_dataset["esa_lvl5"].values)
+    esa_energy = solve_full_sweep_energy(
+        esa_lvl5_hex,
+        esa_table_df=esa_table_df,
+        lut_notes_df=lut_notes_df,
+    )
+
+    l2_dataset["swp_esa_energy"] = xr.DataArray(
+        esa_energy,
+        name="esa_energy",
+        dims=["epoch", "energy"],
+        attrs=cdf_manager.get_variable_attributes("esa_energy"),
+    )
+
     # Update L2 specific attributes
-    l2_dataset.attrs["Data_version"] = data_version
     l2_global_attrs = cdf_manager.get_global_attributes("imap_swapi_l2_sci")
     l2_dataset.attrs["Data_type"] = l2_global_attrs["Data_type"]
     l2_dataset.attrs["Logical_source"] = l2_global_attrs["Logical_source"]

imap_processing/swapi/swapi_utils.py

@@ -6,6 +6,10 @@ other SWAPI processing modules.
 """
 
 from enum import IntEnum
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
 
 
 class SWAPIAPID(IntEnum):
@@ -23,3 +27,31 @@ class SWAPIMODE(IntEnum):
     LVSCI = 1
     HVENG = 2
     HVSCI = 3
+
+
+def read_swapi_lut_table(file_path: Path) -> pd.DataFrame:
+    """
+    Read the LUT table from a CSV file.
+
+    Parameters
+    ----------
+    file_path : pathlib.Path
+        The path to the LUT table CSV file.
+
+    Returns
+    -------
+    pandas.DataFrame
+        The LUT table as a DataFrame.
+    """
+    df = pd.read_csv(file_path)
+
+    # Clean and convert 'Energy' column from comma-separated strings to integers
+    df["Energy"] = (
+        df["Energy"]
+        .astype(str)
+        .str.replace(",", "", regex=False)
+        .replace("Solve", -1)
+        .astype(np.int64)
+    )
+
+    return df

imap_processing/swe/l1a/swe_l1a.py

@@ -15,7 +15,7 @@ from imap_processing.utils import packet_file_to_datasets
 logger = logging.getLogger(__name__)
 
 
-def swe_l1a(packet_file: str, data_version: str) -> xr.Dataset:
+def swe_l1a(packet_file: str) -> xr.Dataset:
     """
     Will process SWE l0 data into l1a data.
 
@@ -27,9 +27,6 @@ def swe_l1a(packet_file: str, data_version: str) -> xr.Dataset:
     ----------
     packet_file : str
         Path where the raw packet file is stored.
-    data_version : str
-        Data version to write to CDF files and the Data_version CDF attribute.
-        Should be in the format Vxxx.
 
     Returns
     -------
@@ -48,17 +45,13 @@ def swe_l1a(packet_file: str, data_version: str) -> xr.Dataset:
     if SWEAPID.SWE_SCIENCE in datasets_by_apid:
         logger.info("Processing SWE science data.")
         processed_data.append(
-            swe_science(
-                l0_dataset=datasets_by_apid[SWEAPID.SWE_SCIENCE],
-                data_version=data_version,
-            )
+            swe_science(l0_dataset=datasets_by_apid[SWEAPID.SWE_SCIENCE])
         )
 
     # Process non-science data
     # Define minimal CDF attrs for the non science dataset
     imap_attrs = ImapCdfAttributes()
     imap_attrs.add_instrument_global_attrs("swe")
-    imap_attrs.add_global_attribute("Data_version", data_version)
     imap_attrs.add_instrument_variable_attrs("swe", "l1a")
     non_science_attrs = imap_attrs.get_variable_attributes("non_science_attrs")
     epoch_attrs = imap_attrs.get_variable_attributes("epoch", check_schema=False)

imap_processing/swe/l1a/swe_science.py

@@ -64,7 +64,7 @@ def decompressed_counts(cem_count: int) -> int:
     )
 
 
-def swe_science(l0_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
+def swe_science(l0_dataset: xr.Dataset) -> xr.Dataset:
     """
     SWE L1a science processing.
 
@@ -97,10 +97,6 @@ def swe_science(l0_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
     l0_dataset : xarray.Dataset
         Raw packet data from SWE stored as an xarray dataset.
 
-    data_version : str
-        Data version for the 'Data_version' CDF attribute. This is the version of the
-        output file.
-
     Returns
     -------
     dataset : xarray.Dataset
@@ -135,20 +131,19 @@ def swe_science(l0_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
     cdf_attrs = ImapCdfAttributes()
     cdf_attrs.add_instrument_global_attrs("swe")
     cdf_attrs.add_instrument_variable_attrs("swe", "l1a")
-    cdf_attrs.add_global_attribute("Data_version", data_version)
 
     epoch_time = xr.DataArray(
         l0_dataset["epoch"],
         name="epoch",
         dims=["epoch"],
-        attrs=cdf_attrs.get_variable_attributes("epoch"),
+        attrs=cdf_attrs.get_variable_attributes("epoch", check_schema=False),
     )
 
     spin_sector = xr.DataArray(
         np.arange(180),
         name="spin_sector",
         dims=["spin_sector"],
-        attrs=cdf_attrs.get_variable_attributes("spin_sector"),
+        attrs=cdf_attrs.get_variable_attributes("spin_sector", check_schema=False),
     )
 
     # NOTE: LABL_PTR_1 should be CDF_CHAR.
@@ -156,14 +151,16 @@ def swe_science(l0_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
         spin_sector.values.astype(str),
         name="spin_sector_label",
         dims=["spin_sector"],
-        attrs=cdf_attrs.get_variable_attributes("spin_sector_label"),
+        attrs=cdf_attrs.get_variable_attributes(
+            "spin_sector_label", check_schema=False
+        ),
     )
 
     cem_id = xr.DataArray(
         np.arange(swe_constants.N_CEMS),
         name="cem_id",
         dims=["cem_id"],
-        attrs=cdf_attrs.get_variable_attributes("cem_id"),
+        attrs=cdf_attrs.get_variable_attributes("cem_id", check_schema=False),
     )
 
     # NOTE: LABL_PTR_2 should be CDF_CHAR.
@@ -171,7 +168,7 @@ def swe_science(l0_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
         cem_id.values.astype(str),
         name="cem_id_label",
         dims=["cem_id"],
-        attrs=cdf_attrs.get_variable_attributes("cem_id_label"),
+        attrs=cdf_attrs.get_variable_attributes("cem_id_label", check_schema=False),
     )
 
     science_xarray = xr.DataArray(