imap-processing 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

imap_processing/tests/swapi/test_swapi_decom.py

@@ -2,36 +2,34 @@ import pandas as pd
 import pytest
 
 from imap_processing import imap_module_directory
-from imap_processing.decom import decom_packets
 from imap_processing.swapi.l1.swapi_l1 import (
     SWAPIAPID,
 )
-from imap_processing.utils import group_by_apid
+from imap_processing.utils import packet_file_to_datasets
 
 
 @pytest.fixture(scope="session")
 def decom_test_data(swapi_l0_test_data_path):
-    """Read test data from file"""
+    """Read test data from file with derived values"""
     test_file = "imap_swapi_l0_raw_20240924_v001.pkts"
     packet_file = imap_module_directory / swapi_l0_test_data_path / test_file
     packet_definition = (
         f"{imap_module_directory}/swapi/packet_definitions/swapi_packet_definition.xml"
     )
-    data_list = []
-    data_list.extend(decom_packets(packet_file, packet_definition))
-    return data_list
+    return packet_file_to_datasets(
+        packet_file, packet_definition, use_derived_value=False
+    )
 
 
 def test_number_of_packets(decom_test_data):
     """This test and validate number of packets."""
-    grouped_data = group_by_apid(decom_test_data)
-    sci_packets = grouped_data[SWAPIAPID.SWP_SCI]
+    sci_packets = decom_test_data[SWAPIAPID.SWP_SCI]
     expected_sci_packets = 153
-    assert len(sci_packets) == expected_sci_packets
+    assert len(sci_packets["epoch"]) == expected_sci_packets
 
-    hk_packets = grouped_data[SWAPIAPID.SWP_HK]
+    hk_packets = decom_test_data[SWAPIAPID.SWP_HK]
     expected_hk_packets = 17
-    assert len(hk_packets) == expected_hk_packets
+    assert len(hk_packets["epoch"]) == expected_hk_packets
 
 
 def test_swapi_sci_data(decom_test_data, swapi_l0_validation_data_path):
@@ -42,32 +40,30 @@ def test_swapi_sci_data(decom_test_data, swapi_l0_validation_data_path):
         index_col="SHCOARSE",
     )
 
-    grouped_data = group_by_apid(decom_test_data)
-    sci_packets = grouped_data[SWAPIAPID.SWP_SCI]
-    first_data = sci_packets[0]
-    validation_data = raw_validation_data.loc[first_data["SHCOARSE"]]
+    sci_packets = decom_test_data[SWAPIAPID.SWP_SCI]
+    first_data = sci_packets.isel(epoch=0)
+    validation_data = raw_validation_data.loc[first_data["shcoarse"].values]
 
     # compare raw values of validation data
-    for key, value in first_data.items():
-        # check if the data is the same
-        if key == "PLAN_ID_SCIENCE":
-            # We had to work around this because HK and SCI packet uses
-            # PLAN_ID but they uses different length of bits.
-            assert value == validation_data["PLAN_ID"]
-        elif key == "SPARE_2_SCIENCE":
-            # Same for this SPARE_2 as above case
-            assert value == validation_data["SPARE_2"]
-        elif key == "MODE":
-            assert value.raw_value == validation_data[key]
-        elif "RNG" in key:
-            assert value.raw_value == validation_data[key]
-        else:
-            # for SHCOARSE we need the name of the column.
-            # This is done because pandas removed it from the
-            # main columns to make it the index.
-            assert value.raw_value == (
-                validation_data[key] if key != "SHCOARSE" else validation_data.name
-            )
+    for key in raw_validation_data.columns:
+        if key in [
+            "PHAPID",
+            "timestamp",
+            "PHGROUPF",
+            "PHSHF",
+            "PHVERNO",
+            "PHSEQCNT",
+            "PHDLEN",
+            "PHTYPE",
+        ]:
+            continue
+
+        # for SHCOARSE we need the name of the column.
+        # This is done because pandas removed it from the
+        # main columns to make it the index.
+        assert first_data[key.lower()].values == (
+            validation_data[key] if key != "SHCOARSE" else validation_data.name
+        )
 
 
 def test_swapi_hk_data(decom_test_data, swapi_l0_validation_data_path):
@@ -78,39 +74,49 @@ def test_swapi_hk_data(decom_test_data, swapi_l0_validation_data_path):
         index_col="SHCOARSE",
     )
 
-    grouped_data = group_by_apid(decom_test_data)
-    hk_packets = grouped_data[SWAPIAPID.SWP_HK]
-    first_data = hk_packets[0]
-    validation_data = raw_validation_data.loc[first_data["SHCOARSE"]]
-    bad_keys = [
-        "N5_V",
-        "SCEM_I",
-        "P5_I",
-        "PHD_LLD1_V",
-        "SPARE_4",
-        "P_CEM_CMD_LVL_MON",
-        "S_CEM_CMD_LVL_MON",
-        "ESA_CMD_LVL_MON",
-        "PHD_LLD2_V",
-        "CHKSUM",
-    ]
+    hk_packets = decom_test_data[SWAPIAPID.SWP_HK]
+    first_data = hk_packets.isel(epoch=0)
+    validation_data = raw_validation_data.loc[first_data["shcoarse"].values]
+
     # compare raw values of validation data
-    for key, value in first_data.items():
-        if key == "PLAN_ID_HK":
-            # We had to work around this because HK and SCI packet uses
-            # PLAN_ID but they uses different length of bits.
-            assert value == validation_data["PLAN_ID"]
-        elif key == "SPARE_2_HK":
-            # Same for this SPARE_2 as PLAN_ID
-            assert value == validation_data["SPARE_2"]
-        elif key == "SHCOARSE":
-            # for SHCOARSE we need the name of the column.
-            # This is done because pandas removed it from the main columns
-            # to make it the index.
-            assert value == validation_data.name
-        elif key in bad_keys:
-            # TODO: remove this elif after getting good validation data
-            # Validation data has wrong value for N5_V
+    for key in raw_validation_data.columns:
+        if key in [
+            "PHAPID",
+            "timestamp",
+            "PHGROUPF",
+            "PHSHF",
+            "PHVERNO",
+            "PHSEQCNT",
+            "PHDLEN",
+            "PHTYPE",
+        ]:
+            continue
+
+        value_mismatching_keys = [
+            "SCEM_I",
+            "N5_V",
+            "P5_I",
+            "PHD_LLD1_V",
+            "P_CEM_CMD_LVL_MON",
+            "S_CEM_CMD_LVL_MON",
+            "ESA_CMD_LVL_MON",
+            "PHD_LLD2_V",
+            "CHKSUM",
+        ]
+
+        extra_keys_val_data = [
+            "ESA_GATE_SET",
+            "P5V_ESA_V_MON",
+            "M5V_ESA_V_MON",
+            "P5V_ESA_I_MON",
+            "M5V_ESA_I_MON",
+        ]
+
+        if key in extra_keys_val_data or key in value_mismatching_keys:
             continue
-        else:
-            assert value.raw_value == validation_data[key]
+        # for SHCOARSE we need the name of the column.
+        # This is done because pandas removed it from the
+        # main columns to make it the index.
+        assert first_data[key.lower()].values == (
+            validation_data[key] if key != "SHCOARSE" else validation_data.name
+        )

imap_processing/tests/swapi/test_swapi_l1.py

@@ -39,7 +39,7 @@ def test_filter_good_data():
     total_sweeps = 3
     ds = xr.Dataset(
         {
-            "plan_id_science": xr.DataArray(np.full((total_sweeps * 12), 1)),
+            "plan_id": xr.DataArray(np.full((total_sweeps * 12), 1)),
             "sweep_table": xr.DataArray(np.repeat(np.arange(total_sweeps), 12)),
             "mode": xr.DataArray(np.full((total_sweeps * 12), SWAPIMODE.HVSCI.value)),
         },
@@ -69,9 +69,9 @@ def test_filter_good_data():
     expected = [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]
     np.testing.assert_array_equal(filter_good_data(ds), expected)
 
-    # Check for bad plan_id_science data.
+    # Check for bad plan_id data.
     ds["sweep_table"] = xr.DataArray(np.repeat(np.arange(total_sweeps), 12))
-    ds["plan_id_science"][24 : total_sweeps * 12] = np.arange(0, 12)
+    ds["plan_id"][24 : total_sweeps * 12] = np.arange(0, 12)
     np.testing.assert_array_equal(filter_good_data(ds), np.arange(0, 24))
 
 
@@ -157,7 +157,7 @@ def test_process_swapi_science(decom_test_data):
     )
 
     # make PLAN_ID data incorrect. Now processed data should have less sweeps
-    ds_data["plan_id_science"].data[:24] = np.arange(24)
+    ds_data["plan_id"].data[:24] = np.arange(24)
     processed_data = process_swapi_science(
         ds_data, decom_test_data[SWAPIAPID.SWP_HK], data_version="001"
     )

imap_processing/tests/swe/conftest.py

@@ -74,3 +74,36 @@ def l1a_validation_df():
     # Fill NaNs with the previous value
     df["shcoarse"] = df["shcoarse"].ffill()
     return df
+
+
+@pytest.fixture(scope="session")
+def l1b_validation_df():
+    """Read validation data from file"""
+    l1_val_path = imap_module_directory / "tests/swe/l1_validation"
+    filename = "swe_l0_unpacked-data_20240510_v001_VALIDATION_L1B_v3.dat"
+
+    # Define column names for validation data
+    column_names = [
+        "shcoarse",
+        "cem_1",
+        "cem_2",
+        "cem_3",
+        "cem_4",
+        "cem_5",
+        "cem_6",
+        "cem_7",
+    ]
+
+    # Read the data, specifying na_values and delimiter
+    df = pd.read_csv(
+        l1_val_path / filename,
+        skiprows=12,  # Skip the first 10 rows of comments
+        sep=r"\s*,\s*",  # Regex to handle spaces and commas as delimiters
+        names=column_names,
+        na_values=["", " "],  # Treat empty strings or spaces as NaN
+        engine="python",
+    )
+
+    # Fill NaNs with the previous value
+    df["shcoarse"] = df["shcoarse"].ffill()
+    return df