imap-processing 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

imap_processing/tests/idex/test_idex_l1b.py

@@ -10,11 +10,19 @@ import xarray as xr
 from imap_processing import imap_module_directory
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.cdf.utils import write_cdf
-from imap_processing.idex.idex_l1b import idex_l1b, unpack_instrument_settings
+from imap_processing.idex.idex_l1b import (
+    get_spice_data,
+    get_trigger_mode_and_level,
+    idex_l1b,
+    unpack_instrument_settings,
+)
+from imap_processing.tests.idex import conftest
+from imap_processing.tests.idex.conftest import get_spice_data_side_effect_func
 
 
 @pytest.fixture(scope="module")
-def l1b_dataset(decom_test_data: xr.Dataset) -> xr.Dataset:
+@mock.patch("imap_processing.idex.idex_l1b.get_spice_data")
+def l1b_dataset(mock_get_spice_data, decom_test_data: xr.Dataset) -> xr.Dataset:
     """Return a ``xarray`` dataset containing test data.
 
     Returns
@@ -22,10 +30,32 @@ def l1b_dataset(decom_test_data: xr.Dataset) -> xr.Dataset:
     dataset : xr.Dataset
         A ``xarray`` dataset containing the test data
     """
+
+    mock_get_spice_data.side_effect = get_spice_data_side_effect_func
+
     dataset = idex_l1b(decom_test_data, data_version="001")
     return dataset
 
 
+@pytest.fixture()
+def mock_spice_functions():
+    """Mock spice functions to avoid loading kernels."""
+    with (
+        mock.patch("imap_processing.idex.idex_l1b.imap_state") as mock_state,
+        mock.patch(
+            "imap_processing.idex.idex_l1b.instrument_pointing"
+        ) as mock_pointing,
+        mock.patch("imap_processing.idex.idex_l1b.solar_longitude") as mock_lon,
+    ):
+        mock_state.side_effect = lambda t, observer: np.ones((len(t), 6))
+        mock_pointing.side_effect = lambda t, instrument, to_frame, cartesian: np.ones(
+            (len(t), 3)
+        )
+        mock_lon.side_effect = lambda t, degrees: np.ones(len(t))
+
+        yield mock_state, mock_pointing, mock_lon
+
+
 def test_l1b_cdf_filenames(l1b_dataset: xr.Dataset):
     """Tests that the ``idex_l1b`` function generates datasets
     with the expected logical source.
@@ -124,3 +154,95 @@ def test_unpack_instrument_settings():
 
     assert np.all(unpacked_dict["test_var0"] == 2)
     assert np.all(unpacked_dict["test_var1"] == 4)
+
+
+def test_get_trigger_settings_success(decom_test_data):
+    """
+    Check that the output to 'get_trigger_mode_and_level' matches expected arrays.
+
+    Parameters
+    ----------
+    decom_test_data : xarray.Dataset
+        L1a dataset
+    """
+    # Change the trigger mode and level for the first event to check that output is
+    # correct when the modes and levels vary from event to event
+    decom_test_data["idx__txhdrmgtrigmode"][0] = 1
+    decom_test_data["idx__txhdrhgtrigmode"][0] = 0
+
+    n_epochs = len(decom_test_data["epoch"])
+    trigger_settings = get_trigger_mode_and_level(decom_test_data)
+
+    expected_modes = np.full(n_epochs, "HGThreshold")
+    expected_modes[0] = "MGThreshold"
+    expected_levels = np.full(n_epochs, 0.16762)
+    expected_levels[0] = 1023.0
+
+    assert (trigger_settings["triggermode"].data == expected_modes).all(), (
+        f"The dict entry 'triggermode' values did not match the expected values: "
+        f"{expected_modes}. Found: {trigger_settings['triggermode'].data}"
+    )
+
+    assert (trigger_settings["triggerlevel"].data == expected_levels).all(), (
+        f"The dict entry 'triggerlevel' values did not match the expected values: "
+        f"{expected_levels}. Found: {trigger_settings['triggerlevels'].data}"
+    )
+
+
+def test_get_trigger_settings_failure(decom_test_data):
+    """
+    Check that an error is thrown when there are more than one valid trigger for an
+    event
+
+    Parameters
+    ----------
+    decom_test_data : xarray.Dataset
+        L1a dataset
+    """
+    decom_test_data["idx__txhdrhgtrigmode"][0] = 1
+    decom_test_data["idx__txhdrmgtrigmode"][0] = 2
+
+    error_ms = (
+        "Only one channel can trigger a dust event. Please make sure there is "
+        "only one valid trigger value per event. This caused Merge Error: "
+        "conflicting values for variable 'trigger_mode' on objects to be "
+        "combined. You can skip this check by specifying compat='override'."
+    )
+
+    with pytest.raises(ValueError, match=error_ms):
+        get_trigger_mode_and_level(decom_test_data)
+
+
+@pytest.mark.usefixtures("use_fake_spin_data_for_time")
+def test_get_spice_data(
+    mock_spice_functions,
+    use_fake_spin_data_for_time,
+    decom_test_data,
+    furnish_kernels,
+):
+    """
+    Test the get_spice_data() function.
+
+    Parameters
+    ----------
+    decom_test_data : xarray.Dataset
+        L1a dataset
+    """
+    kernels = ["naif0012.tls"]
+    times = decom_test_data["shcoarse"].data
+    use_fake_spin_data_for_time(np.min(times), np.max(times))
+
+    # Mock attribute manager variable attrs
+    idex_attrs = ImapCdfAttributes()
+
+    with (
+        furnish_kernels(kernels),
+        mock.patch.object(idex_attrs, "get_variable_attributes") as mock_attrs,
+    ):
+        mock_attrs.return_value = {"CATDESC": "Test var"}
+
+        spice_data = get_spice_data(decom_test_data, idex_attrs)
+
+    for array in conftest.SPICE_ARRAYS:
+        assert array in spice_data
+        assert len(spice_data[array]) == len(decom_test_data["epoch"])

imap_processing/tests/lo/test_lo_l1a.py

@@ -1,4 +1,5 @@
 import numpy as np
+import pandas as pd
 
 from imap_processing import imap_module_directory
 from imap_processing.lo.l1a.lo_l1a import lo_l1a
@@ -9,7 +10,11 @@ def test_lo_l1a():
     dependency = (
         imap_module_directory / "tests/lo/test_pkts/imap_lo_l0_raw_20240803_v002.pkts"
     )
-    expected_logical_source = ["imap_lo_l1a_histogram", "imap_lo_l1a_de"]
+    expected_logical_source = [
+        "imap_lo_l1a_spin",
+        "imap_lo_l1a_histogram",
+        "imap_lo_l1a_de",
+    ]
     output_dataset = lo_l1a(dependency, "001")
 
     # Assert
@@ -54,4 +59,59 @@ def test_lo_l1a_dataset():
     output_datasets = lo_l1a(dependency, "001")
 
     # Assert
-    np.testing.assert_array_equal(hist_fields_lower, output_datasets[0].data_vars)
+    np.testing.assert_array_equal(hist_fields_lower, output_datasets[1].data_vars)
+
+
+def test_validate_spin_data():
+    # Arrange
+    dependency = (
+        imap_module_directory / "tests/lo/test_pkts/imap_lo_l0_raw_20240803_v002.pkts"
+    )
+    validation_path = (
+        imap_module_directory / "tests/lo/validation_data/"
+        "Instrument_FM1_T104_R129_20240803_ILO_SPIN_EU.csv"
+    )
+    validation_data = pd.read_csv(validation_path)
+
+    spin_fields = [
+        "shcoarse",
+        "num_completed",
+        "acq_start_sec",
+        "acq_start_subsec",
+        "acq_end_sec",
+        "acq_end_subsec",
+        "start_sec_spin",
+        "start_subsec_spin",
+        "esa_neg_dac_spin",
+        "esa_pos_dac_spin",
+        "valid_period_spin",
+        "valid_phase_spin",
+        "period_source_spin",
+    ]
+    # The validation data contains a duplicate set of columns for the same values.
+    # They are formatted as column_prefix_<spin> and column_prefix[<spin>]
+    # adding a condition to remove the one with [ before combining those columns into
+    # a list
+    bad_fields = [col for col in validation_data.columns if "[" in col]
+    validation_data = validation_data.drop(bad_fields, axis=1)
+
+    # The validation contains columns for each of the 28 spins in the packet, so these
+    # need to be combined into a single list for the comparison
+    for field in spin_fields:
+        matching_columns = [
+            col for col in validation_data.columns if col.startswith(field.upper())
+        ]
+        if len(matching_columns) > 1:
+            validation_data[field.upper()] = validation_data[
+                matching_columns
+            ].values.tolist()
+            validation_data = validation_data.drop(matching_columns, axis=1)
+
+    # Act
+    output_dataset = lo_l1a(dependency, "001")
+
+    # Assert
+    for field in spin_fields:
+        np.testing.assert_array_equal(
+            output_dataset[0][field], validation_data[field.upper()].values.tolist()
+        )

imap_processing/tests/lo/test_lo_science.py

@@ -8,6 +8,7 @@ from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.lo.l0.lo_apid import LoAPID
 from imap_processing.lo.l0.lo_science import (
     combine_segmented_packets,
+    organize_spin_data,
     parse_de_bin,
     parse_events,
     parse_fixed_fields,
@@ -115,6 +116,38 @@ def segmented_pkts_fake_data():
     return dataset
 
 
+@pytest.fixture()
+def fake_spin_data():
+    dataset = xr.Dataset(
+        data_vars=dict(
+            num_completed=(["epoch"], np.array([0, 0])),
+            acq_start_sec=(["epoch"], np.array([1000000, 2000000])),
+            acq_start_subsec=(["epoch"], np.array([1000000, 2000000])),
+            acq_end_sec=(["epoch"], np.array([2000000, 3000000])),
+            acq_end_subsec=(["epoch"], np.array([2000000, 3000000])),
+        )
+    )
+    spin_fields = [
+        "start_sec_spin",
+        "start_subsec_spin",
+        "esa_neg_dac_spin",
+        "esa_pos_dac_spin",
+        "valid_period_spin",
+        "valid_phase_spin",
+        "period_source_spin",
+    ]
+
+    for spin_field in spin_fields:
+        packet_fields = [f"{spin_field}_{i}" for i in range(1, 29)]
+        fake_array_val = 0
+        for packet_field in packet_fields:
+            dataset[packet_field] = xr.DataArray(
+                np.array([fake_array_val, fake_array_val + 28]), dims="epoch"
+            )
+            fake_array_val += 1
+    return dataset
+
+
 @pytest.fixture()
 def attr_mgr():
     attr_mgr = ImapCdfAttributes()
@@ -266,3 +299,55 @@ def test_validate_parse_events(sample_data, attr_mgr):
 
     assert dataset["de_count"].values == 1998
     assert dataset["passes"].values == 8
+
+
+def test_organize_spin_data(fake_spin_data, attr_mgr):
+    # Arrange
+    data_by_epoch_spin = np.arange(0, 56).reshape(2, 28)
+    expected_dataset = xr.Dataset(
+        data_vars=dict(
+            num_completed=(["epoch"], np.array([0, 0])),
+            acq_start_sec=(["epoch"], np.array([1000000, 2000000])),
+            acq_start_subsec=(["epoch"], np.array([1000000, 2000000])),
+            acq_end_sec=(["epoch"], np.array([2000000, 3000000])),
+            acq_end_subsec=(["epoch"], np.array([2000000, 3000000])),
+            start_sec_spin=(
+                ["epoch", "spin"],
+                np.array(data_by_epoch_spin),
+            ),
+            start_subsec_spin=(
+                ["epoch", "spin"],
+                np.array(data_by_epoch_spin),
+            ),
+            esa_neg_dac_spin=(
+                ["epoch", "spin"],
+                np.array(data_by_epoch_spin),
+            ),
+            esa_pos_dac_spin=(
+                ["epoch", "spin"],
+                np.array(data_by_epoch_spin),
+            ),
+            valid_period_spin=(
+                ["epoch", "spin"],
+                np.array(data_by_epoch_spin),
+            ),
+            valid_phase_spin=(
+                ["epoch", "spin"],
+                np.array(data_by_epoch_spin),
+            ),
+            period_source_spin=(
+                ["epoch", "spin"],
+                np.array(data_by_epoch_spin),
+            ),
+        ),
+        coords=dict(
+            # acq_start + 1e6 * acq_start_subsec converted to J2000 epoch
+            epoch=(["epoch"], np.array([316576067184000000, 317576068184000000]))
+        ),
+    )
+
+    # Act
+    organized_data = organize_spin_data(fake_spin_data, attr_mgr)
+
+    # Assert
+    xr.testing.assert_equal(organized_data, expected_dataset)

imap_processing/tests/lo/validation_data/Instrument_FM1_T104_R129_20240803_ILO_SPIN_EU.csv

@@ -0,0 +1,2 @@
+SHCOARSE,NUM_COMPLETED,ACQ_START_SEC,ACQ_START_SUBSEC,ACQ_END_SEC,ACQ_END_SUBSEC,START_SEC_SPIN[0],START_SEC_SPIN[1],START_SEC_SPIN[2],START_SEC_SPIN[3],START_SEC_SPIN[4],START_SEC_SPIN[5],START_SEC_SPIN[6],START_SEC_SPIN[7],START_SEC_SPIN[8],START_SEC_SPIN[9],START_SEC_SPIN[10],START_SEC_SPIN[11],START_SEC_SPIN[12],START_SEC_SPIN[13],START_SEC_SPIN[14],START_SEC_SPIN[15],START_SEC_SPIN[16],START_SEC_SPIN[17],START_SEC_SPIN[18],START_SEC_SPIN[19],START_SEC_SPIN[20],START_SEC_SPIN[21],START_SEC_SPIN[22],START_SEC_SPIN[23],START_SEC_SPIN[24],START_SEC_SPIN[25],START_SEC_SPIN[26],START_SEC_SPIN[27],START_SEC_SPIN_1,START_SUBSEC_SPIN[0],START_SUBSEC_SPIN[1],START_SUBSEC_SPIN[2],START_SUBSEC_SPIN[3],START_SUBSEC_SPIN[4],START_SUBSEC_SPIN[5],START_SUBSEC_SPIN[6],START_SUBSEC_SPIN[7],START_SUBSEC_SPIN[8],START_SUBSEC_SPIN[9],START_SUBSEC_SPIN[10],START_SUBSEC_SPIN[11],START_SUBSEC_SPIN[12],START_SUBSEC_SPIN[13],START_SUBSEC_SPIN[14],START_SUBSEC_SPIN[15],START_SUBSEC_SPIN[16],START_SUBSEC_SPIN[17],START_SUBSEC_SPIN[18],START_SUBSEC_SPIN[19],START_SUBSEC_SPIN[20],START_SUBSEC_SPIN[21],START_SUBSEC_SPIN[22],START_SUBSEC_SPIN[23],START_SUBSEC_SPIN[24],START_SUBSEC_SPIN[25],START_SUBSEC_SPIN[26],START_SUBSEC_SPIN[27],START_SUBSEC_SPIN_1,ESA_NEG_DAC_SPIN[0],ESA_NEG_DAC_SPIN[1],ESA_NEG_DAC_SPIN[2],ESA_NEG_DAC_SPIN[3],ESA_NEG_DAC_SPIN[4],ESA_NEG_DAC_SPIN[5],ESA_NEG_DAC_SPIN[6],ESA_NEG_DAC_SPIN[7],ESA_NEG_DAC_SPIN[8],ESA_NEG_DAC_SPIN[9],ESA_NEG_DAC_SPIN[10],ESA_NEG_DAC_SPIN[11],ESA_NEG_DAC_SPIN[12],ESA_NEG_DAC_SPIN[13],ESA_NEG_DAC_SPIN[14],ESA_NEG_DAC_SPIN[15],ESA_NEG_DAC_SPIN[16],ESA_NEG_DAC_SPIN[17],ESA_NEG_DAC_SPIN[18],ESA_NEG_DAC_SPIN[19],ESA_NEG_DAC_SPIN[20],ESA_NEG_DAC_SPIN[21],ESA_NEG_DAC_SPIN[22],ESA_NEG_DAC_SPIN[23],ESA_NEG_DAC_SPIN[24],ESA_NEG_DAC_SPIN[25],ESA_NEG_DAC_SPIN[26],ESA_NEG_DAC_SPIN[27],ESA_NEG_DAC_SPIN_1,ESA_POS_DAC_SPIN[0],ESA_POS_DAC_SPIN[1],ESA_POS_DAC_SPIN[2],ESA_POS_DAC_SPIN[3],ESA_POS_DAC_SPIN[4],ESA_POS_DAC_SPIN[5],ESA_POS_DAC_SPIN[6],ESA_POS_DAC_SPIN[7],ESA_POS_DAC_SPIN[8],ESA_POS_DAC_SPIN[9],ESA_POS_DAC_SPIN[10],ESA_POS_DAC_SPIN[11],ESA_POS_DAC_SPIN[12],ESA_POS_DAC_SPIN[13],ESA_POS_DAC_SPIN[14],ESA_POS_DAC_SPIN[15],ESA_POS_DAC_SPIN[16],ESA_POS_DAC_SPIN[17],ESA_POS_DAC_SPIN[18],ESA_POS_DAC_SPIN[19],ESA_POS_DAC_SPIN[20],ESA_POS_DAC_SPIN[21],ESA_POS_DAC_SPIN[22],ESA_POS_DAC_SPIN[23],ESA_POS_DAC_SPIN[24],ESA_POS_DAC_SPIN[25],ESA_POS_DAC_SPIN[26],ESA_POS_DAC_SPIN[27],ESA_POS_DAC_SPIN_1,VALID_PERIOD_SPIN[0],VALID_PERIOD_SPIN[1],VALID_PERIOD_SPIN[2],VALID_PERIOD_SPIN[3],VALID_PERIOD_SPIN[4],VALID_PERIOD_SPIN[5],VALID_PERIOD_SPIN[6],VALID_PERIOD_SPIN[7],VALID_PERIOD_SPIN[8],VALID_PERIOD_SPIN[9],VALID_PERIOD_SPIN[10],VALID_PERIOD_SPIN[11],VALID_PERIOD_SPIN[12],VALID_PERIOD_SPIN[13],VALID_PERIOD_SPIN[14],VALID_PERIOD_SPIN[15],VALID_PERIOD_SPIN[16],VALID_PERIOD_SPIN[17],VALID_PERIOD_SPIN[18],VALID_PERIOD_SPIN[19],VALID_PERIOD_SPIN[20],VALID_PERIOD_SPIN[21],VALID_PERIOD_SPIN[22],VALID_PERIOD_SPIN[23],VALID_PERIOD_SPIN[24],VALID_PERIOD_SPIN[25],VALID_PERIOD_SPIN[26],VALID_PERIOD_SPIN[27],VALID_PERIOD_SPIN_1,VALID_PHASE_SPIN[0],VALID_PHASE_SPIN[1],VALID_PHASE_SPIN[2],VALID_PHASE_SPIN[3],VALID_PHASE_SPIN[4],VALID_PHASE_SPIN[5],VALID_PHASE_SPIN[6],VALID_PHASE_SPIN[7],VALID_PHASE_SPIN[8],VALID_PHASE_SPIN[9],VALID_PHASE_SPIN[10],VALID_PHASE_SPIN[11],VALID_PHASE_SPIN[12],VALID_PHASE_SPIN[13],VALID_PHASE_SPIN[14],VALID_PHASE_SPIN[15],VALID_PHASE_SPIN[16],VALID_PHASE_SPIN[17],VALID_PHASE_SPIN[18],VALID_PHASE_SPIN[19],VALID_PHASE_SPIN[20],VALID_PHASE_SPIN[21],VALID_PHASE_SPIN[22],VALID_PHASE_SPIN[23],VALID_PHASE_SPIN[24],VALID_PHASE_SPIN[25],VALID_PHASE_SPIN[26],VALID_PHASE_SPIN[27],VALID_PHASE_SPIN_1,PERIOD_SOURCE_SPIN[0],PERIOD_SOURCE_SPIN[1],PERIOD_SOURCE_SPIN[2],PERIOD_SOURCE_SPIN[3],PERIOD_SOURCE_SPIN[4],PERIOD_SOURCE_SPIN[5],PERIOD_SOURCE_SPIN[6],PERIOD_SOURCE_SPIN[7],PERIOD_SOURCE_SPIN[8],PERIOD_SOURCE_SPIN[9],PERIOD_SOURCE_SPIN[10],PERIOD_SOURCE_SPIN[11],PERIOD_SOURCE_SPIN[12],PERIOD_SOURCE_SPIN[13],PERIOD_SOURCE_SPIN[14],PERIOD_SOURCE_SPIN[15],PERIOD_SOURCE_SPIN[16],PERIOD_SOURCE_SPIN[17],PERIOD_SOURCE_SPIN[18],PERIOD_SOURCE_SPIN[19],PERIOD_SOURCE_SPIN[20],PERIOD_SOURCE_SPIN[21],PERIOD_SOURCE_SPIN[22],PERIOD_SOURCE_SPIN[23],PERIOD_SOURCE_SPIN[24],PERIOD_SOURCE_SPIN[25],PERIOD_SOURCE_SPIN[26],PERIOD_SOURCE_SPIN[27],PERIOD_SOURCE_SPIN_1,START_SEC_SPIN_2,START_SUBSEC_SPIN_2,ESA_NEG_DAC_SPIN_2,ESA_POS_DAC_SPIN_2,VALID_PERIOD_SPIN_2,VALID_PHASE_SPIN_2,PERIOD_SOURCE_SPIN_2,START_SEC_SPIN_3,START_SUBSEC_SPIN_3,ESA_NEG_DAC_SPIN_3,ESA_POS_DAC_SPIN_3,VALID_PERIOD_SPIN_3,VALID_PHASE_SPIN_3,PERIOD_SOURCE_SPIN_3,START_SEC_SPIN_4,START_SUBSEC_SPIN_4,ESA_NEG_DAC_SPIN_4,ESA_POS_DAC_SPIN_4,VALID_PERIOD_SPIN_4,VALID_PHASE_SPIN_4,PERIOD_SOURCE_SPIN_4,START_SEC_SPIN_5,START_SUBSEC_SPIN_5,ESA_NEG_DAC_SPIN_5,ESA_POS_DAC_SPIN_5,VALID_PERIOD_SPIN_5,VALID_PHASE_SPIN_5,PERIOD_SOURCE_SPIN_5,START_SEC_SPIN_6,START_SUBSEC_SPIN_6,ESA_NEG_DAC_SPIN_6,ESA_POS_DAC_SPIN_6,VALID_PERIOD_SPIN_6,VALID_PHASE_SPIN_6,PERIOD_SOURCE_SPIN_6,START_SEC_SPIN_7,START_SUBSEC_SPIN_7,ESA_NEG_DAC_SPIN_7,ESA_POS_DAC_SPIN_7,VALID_PERIOD_SPIN_7,VALID_PHASE_SPIN_7,PERIOD_SOURCE_SPIN_7,START_SEC_SPIN_8,START_SUBSEC_SPIN_8,ESA_NEG_DAC_SPIN_8,ESA_POS_DAC_SPIN_8,VALID_PERIOD_SPIN_8,VALID_PHASE_SPIN_8,PERIOD_SOURCE_SPIN_8,START_SEC_SPIN_9,START_SUBSEC_SPIN_9,ESA_NEG_DAC_SPIN_9,ESA_POS_DAC_SPIN_9,VALID_PERIOD_SPIN_9,VALID_PHASE_SPIN_9,PERIOD_SOURCE_SPIN_9,START_SEC_SPIN_10,START_SUBSEC_SPIN_10,ESA_NEG_DAC_SPIN_10,ESA_POS_DAC_SPIN_10,VALID_PERIOD_SPIN_10,VALID_PHASE_SPIN_10,PERIOD_SOURCE_SPIN_10,START_SEC_SPIN_11,START_SUBSEC_SPIN_11,ESA_NEG_DAC_SPIN_11,ESA_POS_DAC_SPIN_11,VALID_PERIOD_SPIN_11,VALID_PHASE_SPIN_11,PERIOD_SOURCE_SPIN_11,START_SEC_SPIN_12,START_SUBSEC_SPIN_12,ESA_NEG_DAC_SPIN_12,ESA_POS_DAC_SPIN_12,VALID_PERIOD_SPIN_12,VALID_PHASE_SPIN_12,PERIOD_SOURCE_SPIN_12,START_SEC_SPIN_13,START_SUBSEC_SPIN_13,ESA_NEG_DAC_SPIN_13,ESA_POS_DAC_SPIN_13,VALID_PERIOD_SPIN_13,VALID_PHASE_SPIN_13,PERIOD_SOURCE_SPIN_13,START_SEC_SPIN_14,START_SUBSEC_SPIN_14,ESA_NEG_DAC_SPIN_14,ESA_POS_DAC_SPIN_14,VALID_PERIOD_SPIN_14,VALID_PHASE_SPIN_14,PERIOD_SOURCE_SPIN_14,START_SEC_SPIN_15,START_SUBSEC_SPIN_15,ESA_NEG_DAC_SPIN_15,ESA_POS_DAC_SPIN_15,VALID_PERIOD_SPIN_15,VALID_PHASE_SPIN_15,PERIOD_SOURCE_SPIN_15,START_SEC_SPIN_16,START_SUBSEC_SPIN_16,ESA_NEG_DAC_SPIN_16,ESA_POS_DAC_SPIN_16,VALID_PERIOD_SPIN_16,VALID_PHASE_SPIN_16,PERIOD_SOURCE_SPIN_16,START_SEC_SPIN_17,START_SUBSEC_SPIN_17,ESA_NEG_DAC_SPIN_17,ESA_POS_DAC_SPIN_17,VALID_PERIOD_SPIN_17,VALID_PHASE_SPIN_17,PERIOD_SOURCE_SPIN_17,START_SEC_SPIN_18,START_SUBSEC_SPIN_18,ESA_NEG_DAC_SPIN_18,ESA_POS_DAC_SPIN_18,VALID_PERIOD_SPIN_18,VALID_PHASE_SPIN_18,PERIOD_SOURCE_SPIN_18,START_SEC_SPIN_19,START_SUBSEC_SPIN_19,ESA_NEG_DAC_SPIN_19,ESA_POS_DAC_SPIN_19,VALID_PERIOD_SPIN_19,VALID_PHASE_SPIN_19,PERIOD_SOURCE_SPIN_19,START_SEC_SPIN_20,START_SUBSEC_SPIN_20,ESA_NEG_DAC_SPIN_20,ESA_POS_DAC_SPIN_20,VALID_PERIOD_SPIN_20,VALID_PHASE_SPIN_20,PERIOD_SOURCE_SPIN_20,START_SEC_SPIN_21,START_SUBSEC_SPIN_21,ESA_NEG_DAC_SPIN_21,ESA_POS_DAC_SPIN_21,VALID_PERIOD_SPIN_21,VALID_PHASE_SPIN_21,PERIOD_SOURCE_SPIN_21,START_SEC_SPIN_22,START_SUBSEC_SPIN_22,ESA_NEG_DAC_SPIN_22,ESA_POS_DAC_SPIN_22,VALID_PERIOD_SPIN_22,VALID_PHASE_SPIN_22,PERIOD_SOURCE_SPIN_22,START_SEC_SPIN_23,START_SUBSEC_SPIN_23,ESA_NEG_DAC_SPIN_23,ESA_POS_DAC_SPIN_23,VALID_PERIOD_SPIN_23,VALID_PHASE_SPIN_23,PERIOD_SOURCE_SPIN_23,START_SEC_SPIN_24,START_SUBSEC_SPIN_24,ESA_NEG_DAC_SPIN_24,ESA_POS_DAC_SPIN_24,VALID_PERIOD_SPIN_24,VALID_PHASE_SPIN_24,PERIOD_SOURCE_SPIN_24,START_SEC_SPIN_25,START_SUBSEC_SPIN_25,ESA_NEG_DAC_SPIN_25,ESA_POS_DAC_SPIN_25,VALID_PERIOD_SPIN_25,VALID_PHASE_SPIN_25,PERIOD_SOURCE_SPIN_25,START_SEC_SPIN_26,START_SUBSEC_SPIN_26,ESA_NEG_DAC_SPIN_26,ESA_POS_DAC_SPIN_26,VALID_PERIOD_SPIN_26,VALID_PHASE_SPIN_26,PERIOD_SOURCE_SPIN_26,START_SEC_SPIN_27,START_SUBSEC_SPIN_27,ESA_NEG_DAC_SPIN_27,ESA_POS_DAC_SPIN_27,VALID_PERIOD_SPIN_27,VALID_PHASE_SPIN_27,PERIOD_SOURCE_SPIN_27,START_SEC_SPIN_28,START_SUBSEC_SPIN_28,ESA_NEG_DAC_SPIN_28,ESA_POS_DAC_SPIN_28,VALID_PERIOD_SPIN_28,VALID_PHASE_SPIN_28,PERIOD_SOURCE_SPIN_28,CHKSUM
+460400745,0,460400294,7833,460400745,2193,460400295,460400310,460400325,460400340,460400355,460400370,460400385,460400400,460400415,460400430,460400445,460400460,460400490,460400520,460400535,460400550,460400565,460400580,460400595,460400610,460400625,460400640,460400655,460400670,460400685,460400700,460400715,460400730,460400295,54,4485,4494,4469,4480,4459,4534,4444,4520,4497,4509,4486,46,53,4483,4446,4525,4432,4569,4484,4493,4474,4480,4460,4531,4442,4520,4495,54,0,73,73,142,142,272,272,535,535,1032,1032,2113,4095,4095,4095,4095,4095,142,142,272,272,535,535,1032,1032,2113,2113,4095,0,0,73,73,142,142,272,272,535,535,1032,1032,2113,4095,4095,4095,4095,4095,142,142,272,272,535,535,1032,1032,2113,2113,4095,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,460400310,4485,73,73,1,1,0,460400325,4494,73,73,1,1,0,460400340,4469,142,142,1,1,0,460400355,4480,142,142,1,1,0,460400370,4459,272,272,1,1,0,460400385,4534,272,272,1,1,0,460400400,4444,535,535,1,1,0,460400415,4520,535,535,1,1,0,460400430,4497,1032,1032,1,1,0,460400445,4509,1032,1032,1,1,0,460400460,4486,2113,2113,1,1,0,460400490,46,4095,4095,1,1,0,460400520,53,4095,4095,1,1,0,460400535,4483,4095,4095,1,1,0,460400550,4446,4095,4095,1,1,0,460400565,4525,4095,4095,1,1,0,460400580,4432,142,142,1,1,0,460400595,4569,142,142,1,1,0,460400610,4484,272,272,1,1,0,460400625,4493,272,272,1,1,0,460400640,4474,535,535,1,1,0,460400655,4480,535,535,1,1,0,460400670,4460,1032,1032,1,1,0,460400685,4531,1032,1032,1,1,0,460400700,4442,2113,2113,1,1,0,460400715,4520,2113,2113,1,1,0,460400730,4495,4095,4095,1,1,0,2422

imap_processing/tests/mag/conftest.py

@@ -0,0 +1,16 @@
+"""Shared modules for MAG tests"""
+
+from pathlib import Path
+
+import pytest
+
+from imap_processing.mag.l1a.mag_l1a import mag_l1a
+
+
+@pytest.fixture()
+def validation_l1a():
+    current_directory = Path(__file__).parent
+    test_file = current_directory / "validation" / "mag_l1_test_data.pkts"
+    # Test file contains only normal packets
+    l1a = mag_l1a(test_file, "v000")
+    return l1a

imap_processing/tests/mag/test_mag_decom.py

@@ -20,12 +20,14 @@ def cdf_attrs():
 
 def test_mag_decom():
     current_directory = Path(__file__).parent
-    burst_test_file = current_directory / "mag_l0_test_data.pkts"
+    burst_test_file = current_directory / "validation" / "mag_l0_test_data.pkts"
     packets = decom_packets(str(burst_test_file))
 
     l0 = packets["burst"] + packets["norm"]
 
-    expected_output = pd.read_csv(current_directory / "mag_l0_test_output.csv")
+    expected_output = pd.read_csv(
+        current_directory / "validation" / "mag_l0_test_output.csv"
+    )
     for index, test in enumerate(l0):
         assert test.ccsds_header.PKT_APID == expected_output["PHAPID"][index]
         assert test.ccsds_header.SRC_SEQ_CTR == expected_output["PHSEQCNT"][index]
@@ -59,7 +61,7 @@ def test_mag_decom():
 
 def test_mag_raw_xarray(cdf_attrs):
     current_directory = Path(__file__).parent
-    burst_test_file = current_directory / "mag_l0_test_data.pkts"
+    burst_test_file = current_directory / "validation" / "mag_l0_test_data.pkts"
     packets = decom_packets(str(burst_test_file))
     l0_norm = packets["norm"]
     l0_burst = packets["burst"]
@@ -94,7 +96,7 @@ def test_mag_raw_xarray(cdf_attrs):
 
 def test_mag_raw_cdf_generation(cdf_attrs):
     current_directory = Path(__file__).parent
-    test_file = current_directory / "mag_l0_test_data.pkts"
+    test_file = current_directory / "validation" / "mag_l0_test_data.pkts"
     packets = decom_packets(str(test_file))
     l0_norm = packets["norm"]
     l0_burst = packets["burst"]

imap_processing/tests/mag/test_mag_l1a.py

@@ -12,7 +12,7 @@ from imap_processing.mag.l1a.mag_l1a_data import (
     MagL1aPacketProperties,
     TimeTuple,
 )
-from imap_processing.spice.time import met_to_j2000ns
+from imap_processing.spice.time import met_to_ttj2000ns
 
 
 @pytest.fixture()
@@ -303,7 +303,7 @@ def test_different_vector_rates(
     uncompressed_vector_bytearray, expected_vectors, raw_compressed_vectors
 ):
     current_directory = Path(__file__).parent
-    test_file = current_directory / "mag_l1_test_data.pkts"
+    test_file = current_directory / "validation" / "mag_l1_test_data.pkts"
     # Test file contains only normal packets
     l0 = decom_packets(test_file)["norm"][0]
 
@@ -319,7 +319,6 @@ def test_different_vector_rates(
     )
     l1 = process_packets([l0])
     expected_day = np.datetime64("2023-11-30")
-
     assert len(l1["magi"][expected_day].vectors) == 16
     assert len(l1["mago"][expected_day].vectors) == 32
 
@@ -402,7 +401,7 @@ def test_padding_uncompressed(expected_vectors):
 
 def test_compare_validation_data():
     current_directory = Path(__file__).parent
-    test_file = current_directory / "mag_l1_test_data.pkts"
+    test_file = current_directory / "validation" / "mag_l1_test_data.pkts"
     # Test file contains only normal packets
     l0 = decom_packets(test_file)
     l1 = process_packets(l0["norm"])
@@ -414,7 +413,9 @@ def test_compare_validation_data():
     assert len(l1_mago.vectors) == 96
     assert len(l1_magi.vectors) == 96
 
-    validation_data = pd.read_csv(current_directory / "mag_l1a_test_output.csv")
+    validation_data = pd.read_csv(
+        current_directory / "validation" / "mag_l1a_test_output.csv"
+    )
 
     # Validation data does not have differing timestamps
     for index in validation_data.index:
@@ -482,6 +483,7 @@ def test_compressed_vector_data(expected_vectors, raw_compressed_vectors):
     expected_range_secondary = [3, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 1]
     # 16 bit width with range section
     headers = "01000010"
+    end_padding = "0000"
     input_data = np.array(
         [
             int(i)
@@ -491,6 +493,7 @@ def test_compressed_vector_data(expected_vectors, raw_compressed_vectors):
             + padding
             + range_primary
             + range_secondary
+            + end_padding
         ],
         dtype=np.uint8,
     )
@@ -509,7 +512,33 @@ def test_compressed_vector_data(expected_vectors, raw_compressed_vectors):
 
     assert primary_with_range.shape[0] == 16
     assert secondary_with_range.shape[0] == 16
+    assert np.array_equal(primary_with_range, primary_expected)
+    assert np.array_equal(secondary_with_range, secondary_expected)
 
+    # testing the case where a spare byte is included at the end.
+    end_padding = "000000000000"
+
+    input_data = np.array(
+        [
+            int(i)
+            for i in headers
+            + primary_compressed
+            + secondary_compressed
+            + padding
+            + range_primary
+            + range_secondary
+            + end_padding
+        ],
+        dtype=np.uint8,
+    )
+
+    input_data = np.packbits(input_data)
+    (primary_with_range, secondary_with_range) = MagL1a.process_compressed_vectors(
+        input_data, 16, 16
+    )
+
+    assert primary_with_range.shape[0] == 16
+    assert secondary_with_range.shape[0] == 16
     assert np.array_equal(primary_with_range, primary_expected)
     assert np.array_equal(secondary_with_range, secondary_expected)
 
@@ -775,7 +804,7 @@ def test_calculate_vector_time():
 
     test_data = MagL1a.calculate_vector_time(test_vectors, test_vecsec, start_time)
 
-    converted_start_time_ns = met_to_j2000ns(start_time.to_seconds())
+    converted_start_time_ns = met_to_ttj2000ns(start_time.to_seconds())
 
     skips_ns = np.timedelta64(int(1 / test_vecsec * 1e9), "ns")
     expected_data = np.array(
@@ -852,7 +881,7 @@ def test_mag_l1a_data():
 
 def test_mag_l1a():
     current_directory = Path(__file__).parent
-    test_file = current_directory / "mag_l1_test_data.pkts"
+    test_file = current_directory / "validation" / "mag_l1_test_data.pkts"
 
     output_data = mag_l1a(test_file, "v001")
 

imap_processing/tests/mag/test_mag_l1b.py

@@ -9,6 +9,7 @@ from imap_processing.mag.l1b.mag_l1b import (
     calibrate_vector,
     mag_l1b,
     mag_l1b_processing,
+    rescale_vector,
 )
 
 
@@ -38,8 +39,10 @@ def mag_l1a_dataset():
     compression_flags = xr.DataArray(
         np.zeros((20, 2), dtype=np.int8), dims=["epoch", "compression"]
     )
+    compression_flags[1, :] = np.array([1, 18], dtype=np.int8)
 
     vectors[0, :] = np.array([1, 1, 1, 0])
+    vectors[1, :] = np.array([7982, 48671, -68090, 0])
 
     output_dataset = xr.Dataset(
         coords={"epoch": epoch, "direction": direction, "compression": compression},
@@ -60,7 +63,12 @@ def test_mag_processing(mag_l1a_dataset):
     np.testing.assert_allclose(
         mag_l1b["vectors"][0].values, [2.2972, 2.2415, 2.2381, 0], atol=1e-4
     )
-    np.testing.assert_allclose(mag_l1b["vectors"][1].values, [0, 0, 0, 0])
+    np.testing.assert_allclose(
+        mag_l1b["vectors"][1].values,
+        [4584.1029091, 27238.73161294, -38405.22240195, 0.0],
+    )
+
+    # np.testing.assert_allclose(mag_l1b["vectors"][1].values, [0, 0, 0, 0])
 
     assert mag_l1b["vectors"].values.shape == mag_l1a_dataset["vectors"].values.shape
 
@@ -71,7 +79,6 @@ def test_mag_processing(mag_l1a_dataset):
     np.testing.assert_allclose(
         mag_l1b["vectors"][0].values, [2.27538, 2.23416, 2.23682, 0], atol=1e-5
     )
-    np.testing.assert_allclose(mag_l1b["vectors"][1].values, [0, 0, 0, 0])
 
     assert mag_l1b["vectors"].values.shape == mag_l1a_dataset["vectors"].values.shape
 
@@ -92,7 +99,9 @@ def test_mag_attributes(mag_l1a_dataset):
 
 def test_cdf_output():
     l1a_cdf = load_cdf(
-        Path(__file__).parent / "imap_mag_l1a_norm-magi_20251017_v001.cdf"
+        Path(__file__).parent
+        / "validation"
+        / "imap_mag_l1a_norm-magi_20251017_v001.cdf"
     )
     l1b_dataset = mag_l1b(l1a_cdf, "v001")
 
@@ -138,6 +147,19 @@ def test_mag_compression_scale(mag_l1a_dataset):
     assert np.allclose(output["vectors"].data[3][:3], scaled_vectors)
 
 
+def test_rescale_vector():
+    # From algo document examples
+    vector = np.array([10, -2000, 0])
+    expected_vector = np.array([2.5, -500, 0])
+    output = rescale_vector(vector, [1, 18])
+    assert np.allclose(output, expected_vector)
+
+    vector = np.array([32766, -2, 1])
+    expected_vector = np.array([65532, -4, 2])
+    output = rescale_vector(vector, [1, 15])
+    assert np.allclose(output, expected_vector)
+
+
 def test_calibrate_vector():
     # from MFOTOURFO
     cal_array = np.array(
@@ -161,8 +183,37 @@ def test_calibrate_vector():
     )
 
     calibration_matrix = xr.DataArray(cal_array)
+    # All cal vector comparisons were calculated by hand and confirmed by MAG team.
 
     cal_vector = calibrate_vector(np.array([1.0, 1.0, 1.0, 0]), calibration_matrix)
+
     expected_vector = np.array([2.2972, 2.2415, 2.2381, 0])
 
-    assert np.allclose(cal_vector, expected_vector, atol=1e-4)
+    assert np.allclose(cal_vector, expected_vector, atol=1e-9)
+
+    cal_vector = calibrate_vector(np.array([1.1, -2.0, 3.0, 1]), calibration_matrix)
+    expected_vector = np.array([(0.081202, -0.144636, 0.217628, 1)])
+    assert np.allclose(cal_vector, expected_vector, atol=1e-9)
+
+    cal_vector = calibrate_vector(
+        rescale_vector(np.array([7982, 48671, -68090, 0]), (1, 18)), calibration_matrix
+    )
+    expected_vector = [4584.1029091, 27238.73161294, -38405.22240195, 0.0]
+
+    assert np.allclose(cal_vector, expected_vector, atol=1e-9)
+
+
+def test_l1a_to_l1b(validation_l1a):
+    # Convert l1a input validation packet file to l1b
+    with pytest.raises(ValueError, match="Raw L1A"):
+        mag_l1b(validation_l1a[0], "v000")
+
+    l1b = [mag_l1b(i, "v000") for i in validation_l1a[1:]]
+
+    assert len(l1b) == len(validation_l1a) - 1
+
+    assert l1b[0].attrs["Logical_source"] == "imap_mag_l1b_norm-mago"
+    assert l1b[1].attrs["Logical_source"] == "imap_mag_l1b_norm-magi"
+
+    assert len(l1b[0]["vectors"].data) > 0
+    assert len(l1b[1]["vectors"].data) > 0