imap-processing 0.12.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

imap_processing/tests/idex/conftest.py

@@ -1,14 +1,22 @@
+from unittest import mock
+
 import numpy as np
 import pytest
 import xarray as xr
 
 from imap_processing import imap_module_directory
 from imap_processing.idex.idex_l1a import PacketParser
+from imap_processing.idex.idex_l1b import idex_l1b
+from imap_processing.idex.idex_l2a import idex_l2a
 
 TEST_DATA_PATH = imap_module_directory / "tests" / "idex" / "test_data"
 
-TEST_L0_FILE = TEST_DATA_PATH / "imap_idex_l0_raw_20231218_v001.pkts"
+TEST_L0_FILE_SCI = TEST_DATA_PATH / "imap_idex_l0_raw_20231218_v001.pkts"
+TEST_L0_FILE_EVT = TEST_DATA_PATH / "imap_idex_l0_raw_20250108_v001.pkts"  # 1418
+TEST_L0_FILE_CATLST = TEST_DATA_PATH / "imap_idex_l0_raw_20241206_v001.pkts"  # 1419
+
 L1A_EXAMPLE_FILE = TEST_DATA_PATH / "idex_l1a_validation_file.h5"
+L1B_EXAMPLE_FILE = TEST_DATA_PATH / "idex_l1b_validation_file.h5"
 
 pytestmark = pytest.mark.external_test_data
 
@@ -26,19 +34,43 @@ SPICE_ARRAYS = [
 ]
 
 
-@pytest.fixture(scope="module")
-def decom_test_data() -> xr.Dataset:
+@pytest.fixture
+def decom_test_data_sci() -> xr.Dataset:
     """Return a ``xarray`` dataset containing test data.
 
     Returns
     -------
     dataset : xarray.Dataset
-        A ``xarray`` dataset containing the test data
+        A ``xarray`` dataset containing the science test data
+    """
+    return PacketParser(TEST_L0_FILE_SCI).data[0]
+
+
+@pytest.fixture
+def decom_test_data_catlst() -> xr.Dataset:
+    """Return a ``xarray`` dataset containing the catalog list summary data.
+
+    Returns
+    -------
+    dataset : xarray.Dataset
+        A ``xarray`` dataset containing the catalog list summary data.
+    """
+    return PacketParser(TEST_L0_FILE_CATLST).data[0]
+
+
+@pytest.fixture
+def decom_test_data_evt() -> xr.Dataset:
+    """Return a ``xarray`` dataset containing the event log data.
+
+    Returns
+    -------
+    dataset : xarray.Dataset
+        A ``xarray`` dataset containing the event log data.
     """
-    return PacketParser(TEST_L0_FILE, "001").data
+    return PacketParser(TEST_L0_FILE_EVT).data[0]
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture
 def l1a_example_data(_download_test_data):
     """
     Pytest fixture to load example L1A data (produced by the IDEX team) for testing.
@@ -51,6 +83,39 @@ def l1a_example_data(_download_test_data):
     return load_hdf_file(L1A_EXAMPLE_FILE)
 
 
+@pytest.fixture
+def l2a_dataset(decom_test_data_sci: xr.Dataset) -> xr.Dataset:
+    """Return a ``xarray`` dataset containing test data.
+
+    Returns
+    -------
+    dataset : xr.Dataset
+        A ``xarray`` dataset containing the test data
+    """
+    spin_phase_angles = xr.DataArray(
+        np.random.randint(0, 360, len(decom_test_data_sci.epoch))
+    )
+    with mock.patch(
+        "imap_processing.idex.idex_l1b.get_spice_data",
+        return_value={"spin_phase": spin_phase_angles},
+    ):
+        dataset = idex_l2a(idex_l1b(decom_test_data_sci))
+    return dataset
+
+
+@pytest.fixture
+def l1b_example_data(_download_test_data):
+    """
+    Pytest fixture to load example L1B data (produced by the IDEX team) for testing.
+
+    Returns
+    -------
+    dict
+      A dictionary containing the 6 waveform and telemetry arrays
+    """
+    return load_hdf_file(L1B_EXAMPLE_FILE)
+
+
 def get_spice_data_side_effect_func(l1a_ds, idex_attrs):
     # Create a mock dictionary of spice arrays
 
@@ -95,7 +160,7 @@ def load_hdf_file(path: str) -> xr.Dataset:
         ds = tree.to_dataset()
         # Sort dimensions by shape. The high sampling time dimension is always less
         # than the low sampling time.
-        dims = [k for k, v in sorted(ds.dims.items(), key=lambda item: item[1])]
+        dims = [k for k, v in sorted(ds.sizes.items(), key=lambda item: item[1])]
         ds = ds.rename({dims[0]: "time_low", dims[1]: "time_high"}).expand_dims(
             {"event": [event_num]}
         )

imap_processing/tests/idex/test_idex_l0.py

@@ -6,38 +6,38 @@ import xarray as xr
 from imap_processing import imap_module_directory
 
 
-def test_idex_decom_length(decom_test_data: xr.Dataset):
+def test_idex_decom_length(decom_test_data_sci: xr.Dataset):
     """Verify that the output data has the expected number of data variables.
 
     Parameters
     ----------
-    decom_test_data : xarray.Dataset
+    decom_test_data_sci : xarray.Dataset
         The dataset to test with
     """
-    assert len(decom_test_data) == 110
+    assert len(decom_test_data_sci) == 110
 
 
-def test_idex_decom_event_num(decom_test_data: xr.Dataset):
+def test_idex_decom_event_num(decom_test_data_sci: xr.Dataset):
     """Verify that 14 impacts were gathered by the test data.
 
     Parameters
     ----------
-    decom_test_data : xarray.Dataset
+    decom_test_data_sci : xarray.Dataset
         The dataset to test with
     """
-    for var in decom_test_data:
-        if "epoch" in decom_test_data[var].dims:
-            assert len(decom_test_data[var]) == 14
+    for var in decom_test_data_sci:
+        if "epoch" in decom_test_data_sci[var].dims:
+            assert len(decom_test_data_sci[var]) == 14
 
 
-def test_idex_tof_high_data(decom_test_data: xr.Dataset):
+def test_idex_tof_high_data(decom_test_data_sci: xr.Dataset):
     """Verify that a sample of the data is correct.
 
     ``impact_14_tof_high_data.txt`` has been verified correct by the IDEX team
 
     Parameters
     ----------
-    decom_test_data : xarray.Dataset
+    decom_test_data_sci : xarray.Dataset
         The dataset to test with
     """
 
@@ -45,4 +45,26 @@ def test_idex_tof_high_data(decom_test_data: xr.Dataset):
         f"{imap_module_directory}/tests/idex/test_data/impact_14_tof_high_data.txt"
     ) as f:
         data = np.array([int(line.rstrip("\n")) for line in f])
-    assert (decom_test_data["TOF_High"][13].data == data).all()
+    assert (decom_test_data_sci["TOF_High"][13].data == data).all()
+
+
+def test_catlst_event_num(decom_test_data_catlst: xr.Dataset):
+    """Verify that a sample of the data is correct.
+
+    Parameters
+    ----------
+    decom_test_data_catlst : xarray.Dataset
+        The dataset to test with
+    """
+    assert len(decom_test_data_catlst["epoch"]) == 1
+
+
+def test_evt_event_num(decom_test_data_evt: xr.Dataset):
+    """Verify that a sample of the data is correct.
+
+    Parameters
+    ----------
+    decom_test_data_evt : xarray.Dataset
+        The dataset to test with
+    """
+    assert len(decom_test_data_evt["epoch"]) == 28

imap_processing/tests/idex/test_idex_l1a.py

@@ -14,44 +14,44 @@ from imap_processing.idex.decode import _decode_sub_frame, read_bits, rice_decod
 from imap_processing.idex.idex_l1a import PacketParser
 
 
-def test_idex_cdf_file(decom_test_data: xr.Dataset):
+def test_idex_cdf_file(decom_test_data_sci: xr.Dataset):
     """Verify the CDF file can be created with no errors.
 
     Parameters
     ----------
-    decom_test_data : xarray.Dataset
+    decom_test_data_sci : xarray.Dataset
         The dataset to test with
     """
-    file_name = write_cdf(decom_test_data)
+    file_name = write_cdf(decom_test_data_sci)
 
     assert file_name.exists()
-    assert file_name.name == "imap_idex_l1a_sci-1week_20231218_v001.cdf"
+    assert file_name.name == "imap_idex_l1a_sci-1week_20231218_v999.cdf"
 
 
-def test_bad_cdf_attributes(decom_test_data: xr.Dataset):
+def test_bad_cdf_attributes(decom_test_data_sci: xr.Dataset):
     """Ensure an ``ISTPError`` is raised when using bad CDF attributes.
 
     Parameters
     ----------
-    decom_test_data : xarray.Dataset
+    decom_test_data_sci : xarray.Dataset
         The dataset to test with
     """
-    tof_catdesc = decom_test_data["TOF_High"].attrs["CATDESC"]
-    del decom_test_data["TOF_High"].attrs["CATDESC"]
+    tof_catdesc = decom_test_data_sci["TOF_High"].attrs["CATDESC"]
+    del decom_test_data_sci["TOF_High"].attrs["CATDESC"]
 
     with pytest.raises(ISTPError):
-        write_cdf(decom_test_data, istp=True, terminate_on_warning=True)
+        write_cdf(decom_test_data_sci, istp=True, terminate_on_warning=True)
 
     # Add attributes back so future tests do not fail
-    decom_test_data["TOF_High"].attrs["CATDESC"] = tof_catdesc
+    decom_test_data_sci["TOF_High"].attrs["CATDESC"] = tof_catdesc
 
 
-def test_bad_cdf_file_data(decom_test_data: xr.Dataset):
+def test_bad_cdf_file_data(decom_test_data_sci: xr.Dataset):
     """Ensure an ``ISTPError`` is raised when using bad data.
 
     Parameters
     ----------
-    decom_test_data : xarray.Dataset
+    decom_test_data_sci : xarray.Dataset
         The dataset to test with
     """
     bad_data_attrs = {
@@ -75,22 +75,22 @@ def test_bad_cdf_file_data(decom_test_data: xr.Dataset):
         dims=("bad_data"),
         attrs=bad_data_attrs,
     )
-    decom_test_data["Bad_data"] = bad_data_xr
+    decom_test_data_sci["Bad_data"] = bad_data_xr
 
     with pytest.raises(ISTPError):
-        write_cdf(decom_test_data, istp=True, terminate_on_warning=True)
+        write_cdf(decom_test_data_sci, istp=True, terminate_on_warning=True)
 
-    del decom_test_data["Bad_data"]
+    del decom_test_data_sci["Bad_data"]
 
 
-def test_idex_tof_high_data_from_cdf(decom_test_data: xr.Dataset):
+def test_idex_tof_high_data_from_cdf(decom_test_data_sci: xr.Dataset):
     """Verify that a sample of the data is correct inside the CDF file.
 
     ``impact_14_tof_high_data.txt`` has been verified correct by the IDEX team
 
     Parameters
     ----------
-    decom_test_data : xarray.Dataset
+    decom_test_data_sci : xarray.Dataset
         The dataset to test with
     """
     with open(
@@ -98,13 +98,14 @@ def test_idex_tof_high_data_from_cdf(decom_test_data: xr.Dataset):
     ) as f:
         data = np.array([int(line.rstrip()) for line in f])
 
-    file_name = write_cdf(decom_test_data)
+    file_name = write_cdf(decom_test_data_sci)
     l1_data = load_cdf(file_name)
     assert (l1_data["TOF_High"][13].data == data).all()
 
 
+@pytest.mark.external_test_data
 def test_validate_l1a_idex_data_variables(
-    decom_test_data: xr.Dataset, l1a_example_data: xr.Dataset
+    decom_test_data_sci: xr.Dataset, l1a_example_data: xr.Dataset
 ):
     """
     Verify that each of the 6 waveform and telemetry arrays are equal to the
@@ -113,7 +114,7 @@ def test_validate_l1a_idex_data_variables(
 
     Parameters
     ----------
-    decom_test_data : xarray.Dataset
+    decom_test_data_sci : xarray.Dataset
         The dataset to test with
     l1a_example_data: xarray.Dataset
         A dataset containing the 6 waveform and telemetry arrays
@@ -141,7 +142,7 @@ def test_validate_l1a_idex_data_variables(
             cdf_var = match_variables.get(var, var.lower())
 
             np.testing.assert_array_equal(
-                decom_test_data[cdf_var],
+                decom_test_data_sci[cdf_var],
                 l1a_example_data[var],
                 f"The array '{cdf_var}' does not equal the expected example "
                 f"array '{var}' produced by the IDEX team",
@@ -157,8 +158,8 @@ def test_compressed_packet():
     compressed = Path(f"{test_data_dir}/compressed_2023_102_14_24_55.pkts")
     non_compressed = Path(f"{test_data_dir}/non_compressed_2023_102_14_22_26.pkts")
 
-    decompressed = PacketParser(compressed, "001").data
-    expected = PacketParser(non_compressed, "001").data
+    decompressed = PacketParser(compressed).data[0]
+    expected = PacketParser(non_compressed).data[0]
 
     waveforms = [
         "TOF_High",
@@ -289,3 +290,29 @@ def test_decode_sub_frame_psel_3():
     bstring = warmup1 + warmup2 + residual_1 + residual_2 + residual_3
     ints, bp = _decode_sub_frame(bstring, bp=0, psel=psel, k=k, n_bits=10)
     assert ints == [1, 2, 4, 1, 5]
+
+
+def test_cdf_creation_catlst(decom_test_data_catlst: xr.Dataset):
+    """Verify that a sample of the data can be written to a cdf without errors.
+
+    Parameters
+    ----------
+    decom_test_data_catlst : xarray.Dataset
+        The dataset to test with
+    """
+    filename = write_cdf(decom_test_data_catlst)
+
+    assert filename.name == "imap_idex_l1a_catlst_20241206_v999.cdf"
+
+
+def test_cdf_creation_evt(decom_test_data_evt: xr.Dataset):
+    """Verify that a sample of the data can be written to a cdf without errors.
+
+    Parameters
+    ----------
+    decom_test_data_evt : xarray.Dataset
+        The dataset to test with
+    """
+    filename = write_cdf(decom_test_data_evt)
+
+    assert filename.name == "imap_idex_l1a_evt_20250108_v999.cdf"

imap_processing/tests/idex/test_idex_l1b.py

@@ -20,9 +20,9 @@ from imap_processing.tests.idex import conftest
 from imap_processing.tests.idex.conftest import get_spice_data_side_effect_func
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture
 @mock.patch("imap_processing.idex.idex_l1b.get_spice_data")
-def l1b_dataset(mock_get_spice_data, decom_test_data: xr.Dataset) -> xr.Dataset:
+def l1b_dataset(mock_get_spice_data, decom_test_data_sci: xr.Dataset) -> xr.Dataset:
     """Return a ``xarray`` dataset containing test data.
 
     Returns
@@ -32,12 +32,11 @@ def l1b_dataset(mock_get_spice_data, decom_test_data: xr.Dataset) -> xr.Dataset:
     """
 
     mock_get_spice_data.side_effect = get_spice_data_side_effect_func
-
-    dataset = idex_l1b(decom_test_data, data_version="001")
+    dataset = idex_l1b(decom_test_data_sci)
     return dataset
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_spice_functions():
     """Mock spice functions to avoid loading kernels."""
     with (
@@ -56,7 +55,7 @@ def mock_spice_functions():
         yield mock_state, mock_pointing, mock_lon
 
 
-def test_l1b_cdf_filenames(l1b_dataset: xr.Dataset):
+def test_l1b_logical_source(l1b_dataset: xr.Dataset):
     """Tests that the ``idex_l1b`` function generates datasets
     with the expected logical source.
 
@@ -81,7 +80,7 @@ def test_idex_cdf_file(l1b_dataset: xr.Dataset):
     file_name = write_cdf(l1b_dataset)
 
     assert file_name.exists()
-    assert file_name.name == "imap_idex_l1b_sci-1week_20231218_v001.cdf"
+    assert file_name.name == "imap_idex_l1b_sci-1week_20231218_v999.cdf"
 
 
 def test_idex_waveform_units(l1b_dataset: xr.Dataset):
@@ -156,27 +155,27 @@ def test_unpack_instrument_settings():
     assert np.all(unpacked_dict["test_var1"] == 4)
 
 
-def test_get_trigger_settings_success(decom_test_data):
+def test_get_trigger_settings_success(decom_test_data_sci):
     """
     Check that the output to 'get_trigger_mode_and_level' matches expected arrays.
 
     Parameters
     ----------
-    decom_test_data : xarray.Dataset
+    decom_test_data_sci : xarray.Dataset
         L1a dataset
     """
     # Change the trigger mode and level for the first event to check that output is
     # correct when the modes and levels vary from event to event
-    decom_test_data["idx__txhdrmgtrigmode"][0] = 1
-    decom_test_data["idx__txhdrhgtrigmode"][0] = 0
+    decom_test_data_sci["idx__txhdrmgtrigmode"][0] = 1
+    decom_test_data_sci["idx__txhdrhgtrigmode"][0] = 0
 
-    n_epochs = len(decom_test_data["epoch"])
-    trigger_settings = get_trigger_mode_and_level(decom_test_data)
+    n_epochs = len(decom_test_data_sci["epoch"])
+    trigger_settings = get_trigger_mode_and_level(decom_test_data_sci)
 
     expected_modes = np.full(n_epochs, "HGThreshold")
     expected_modes[0] = "MGThreshold"
     expected_levels = np.full(n_epochs, 0.16762)
-    expected_levels[0] = 1023.0
+    expected_levels[0] = 1023.0 * 1.13e-2
 
     assert (trigger_settings["triggermode"].data == expected_modes).all(), (
         f"The dict entry 'triggermode' values did not match the expected values: "
@@ -185,22 +184,22 @@ def test_get_trigger_settings_success(decom_test_data):
 
     assert (trigger_settings["triggerlevel"].data == expected_levels).all(), (
         f"The dict entry 'triggerlevel' values did not match the expected values: "
-        f"{expected_levels}. Found: {trigger_settings['triggerlevels'].data}"
+        f"{expected_levels}. Found: {trigger_settings['triggerlevel'].data}"
     )
 
 
-def test_get_trigger_settings_failure(decom_test_data):
+def test_get_trigger_settings_failure(decom_test_data_sci):
     """
     Check that an error is thrown when there are more than one valid trigger for an
     event
 
     Parameters
     ----------
-    decom_test_data : xarray.Dataset
+    decom_test_data_sci : xarray.Dataset
         L1a dataset
     """
-    decom_test_data["idx__txhdrhgtrigmode"][0] = 1
-    decom_test_data["idx__txhdrmgtrigmode"][0] = 2
+    decom_test_data_sci["idx__txhdrhgtrigmode"][0] = 1
+    decom_test_data_sci["idx__txhdrmgtrigmode"][0] = 2
 
     error_ms = (
         "Only one channel can trigger a dust event. Please make sure there is "
@@ -210,14 +209,14 @@ def test_get_trigger_settings_failure(decom_test_data):
     )
 
     with pytest.raises(ValueError, match=error_ms):
-        get_trigger_mode_and_level(decom_test_data)
+        get_trigger_mode_and_level(decom_test_data_sci)
 
 
 @pytest.mark.usefixtures("use_fake_spin_data_for_time")
 def test_get_spice_data(
     mock_spice_functions,
     use_fake_spin_data_for_time,
-    decom_test_data,
+    decom_test_data_sci,
     furnish_kernels,
 ):
     """
@@ -225,11 +224,11 @@ def test_get_spice_data(
 
     Parameters
     ----------
-    decom_test_data : xarray.Dataset
+    decom_test_data_sci : xarray.Dataset
         L1a dataset
     """
     kernels = ["naif0012.tls"]
-    times = decom_test_data["shcoarse"].data
+    times = decom_test_data_sci["shcoarse"].data
     use_fake_spin_data_for_time(np.min(times), np.max(times))
 
     # Mock attribute manager variable attrs
@@ -241,8 +240,88 @@ def test_get_spice_data(
     ):
         mock_attrs.return_value = {"CATDESC": "Test var"}
 
-        spice_data = get_spice_data(decom_test_data, idex_attrs)
+        spice_data = get_spice_data(decom_test_data_sci, idex_attrs)
 
     for array in conftest.SPICE_ARRAYS:
         assert array in spice_data
-        assert len(spice_data[array]) == len(decom_test_data["epoch"])
+        assert len(spice_data[array]) == len(decom_test_data_sci["epoch"])
+
+
+@pytest.mark.external_test_data
+def test_validate_l1b_idex_data_variables(
+    l1b_dataset: xr.Dataset, l1b_example_data: xr.Dataset
+):
+    """
+    Verify that each of the 6 waveform and telemetry arrays are equal to the
+    corresponding array produced by the IDEX team using the same l0 file.
+
+    Parameters
+    ----------
+    l1b_dataset : xarray.Dataset
+        The dataset to test with
+    l1b_example_data: xr.Dataset
+        A dataset containing the 6 waveform and telemetry arrays
+    """
+    # Lookup table to match the SDC array names to the Idex Team array names
+
+    match_variables = {
+        "TOF L": "TOF_Low",
+        "TOF H": "TOF_High",
+        "TOF M": "TOF_Mid",
+        "Target H": "Target_High",
+        "Target L": "Target_Low",
+        "Ion Grid": "Ion_Grid",
+        "Time (high sampling)": "time_high_sample_rate",
+        "Time (low sampling)": "time_low_sample_rate",
+        "current_2V5_bus": "current_2p5v_bus",
+        "current_3V3_bus": "current_3p3v_bus",
+        "current_neg2V5_bus": "current_neg2p5v_bus",
+        "voltage_3V3_op_ref": "voltage_3p3_op_ref",
+        "voltage_3V3_ref": "voltage_3p3_ref",
+        "voltage_pos3V3_bus": "voltage_pos3p3v_bus",
+    }
+
+    # The Engineering data is converting to UTC, and the SDC is converting to J2000,
+    # for 'epoch' and 'Timestamp' so this test is using the raw time value 'SCHOARSE' to
+    # validate time
+    # SPICE data is mocked.
+    arrays_to_skip = [
+        "Timestamp",
+        "Epoch",
+        "Pitch",
+        "Roll",
+        "Yaw",
+        "Declination",
+        "PositionX",
+        "PositionY",
+        "PositionZ",
+        "VelocityX",
+        "VelocityY",
+        "VelocityZ",
+        "RightAscension",
+    ]
+    # Compare each corresponding variable
+    for var in l1b_example_data.data_vars:
+        if var not in arrays_to_skip:
+            # Get the corresponding array name
+            cdf_var = match_variables.get(var, var.lower().replace(".", "p"))
+
+            warning = (
+                f"The array '{cdf_var}' does not equal the expected example array "
+            )
+            f"'{var}' produced by the IDEX team"
+
+            if l1b_dataset[cdf_var].dtype == object:
+                assert (l1b_dataset[cdf_var].data == l1b_example_data[var]).all(), (
+                    warning
+                )
+
+            else:
+                (
+                    np.testing.assert_array_almost_equal(
+                        l1b_dataset[cdf_var].data,
+                        l1b_example_data[var],
+                        decimal=1e-04,
+                    ),
+                    warning,
+                )