imap-processing 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

imap_processing/tests/idex/test_idex_l0.py

@@ -7,18 +7,18 @@ from imap_processing import imap_module_directory
 
 
 def test_idex_decom_length(decom_test_data: xr.Dataset):
-    """Verify that there are 6 data variables in the output.
+    """Verify that the output data has the expected number of data variables.
 
     Parameters
     ----------
     decom_test_data : xarray.Dataset
         The dataset to test with
     """
-    assert len(decom_test_data) == 42
+    assert len(decom_test_data) == 106
 
 
 def test_idex_decom_event_num(decom_test_data: xr.Dataset):
-    """Verify that 19 impacts were gathered by the test data.
+    """Verify that 14 impacts were gathered by the test data.
 
     Parameters
     ----------
@@ -26,7 +26,7 @@ def test_idex_decom_event_num(decom_test_data: xr.Dataset):
         The dataset to test with
     """
     for var in decom_test_data:
-        assert len(decom_test_data[var]) == 19
+        assert len(decom_test_data[var]) == 14
 
 
 def test_idex_tof_high_data(decom_test_data: xr.Dataset):

imap_processing/tests/idex/test_idex_l1a.py

@@ -21,7 +21,7 @@ def test_idex_cdf_file(decom_test_data: xr.Dataset):
     file_name = write_cdf(decom_test_data)
 
     assert file_name.exists()
-    assert file_name.name == "imap_idex_l1_sci_20230725_v001.cdf"
+    assert file_name.name == "imap_idex_l1a_sci_20231214_v001.cdf"
 
 
 def test_bad_cdf_attributes(decom_test_data: xr.Dataset):
@@ -32,11 +32,15 @@ def test_bad_cdf_attributes(decom_test_data: xr.Dataset):
     decom_test_data : xarray.Dataset
         The dataset to test with
     """
-    del decom_test_data["TOF_High"].attrs["DEPEND_1"]
+    tof_catdesc = decom_test_data["TOF_High"].attrs["CATDESC"]
+    del decom_test_data["TOF_High"].attrs["CATDESC"]
 
     with pytest.raises(ISTPError):
         write_cdf(decom_test_data)
 
+    # Add attributes back so future tests do not fail
+    decom_test_data["TOF_High"].attrs["CATDESC"] = tof_catdesc
+
 
 def test_bad_cdf_file_data(decom_test_data: xr.Dataset):
     """Ensure an ``ISTPError`` is raised when using bad data.
@@ -72,6 +76,8 @@ def test_bad_cdf_file_data(decom_test_data: xr.Dataset):
     with pytest.raises(ISTPError):
         write_cdf(decom_test_data)
 
+    del decom_test_data["Bad_data"]
+
 
 def test_idex_tof_high_data_from_cdf(decom_test_data: xr.Dataset):
     """Verify that a sample of the data is correct inside the CDF file.

imap_processing/tests/idex/test_idex_l1b.py

@@ -0,0 +1,126 @@
+"""Tests the L1b processing for IDEX data"""
+
+from unittest import mock
+
+import numpy as np
+import pandas as pd
+import pytest
+import xarray as xr
+
+from imap_processing import imap_module_directory
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.cdf.utils import write_cdf
+from imap_processing.idex.idex_l1b import idex_l1b, unpack_instrument_settings
+
+
+@pytest.fixture(scope="module")
+def l1b_dataset(decom_test_data: xr.Dataset) -> xr.Dataset:
+    """Return a ``xarray`` dataset containing test data.
+
+    Returns
+    -------
+    dataset : xr.Dataset
+        A ``xarray`` dataset containing the test data
+    """
+    dataset = idex_l1b(decom_test_data, data_version="001")
+    return dataset
+
+
+def test_l1b_cdf_filenames(l1b_dataset: xr.Dataset):
+    """Tests that the ``idex_l1b`` function generates datasets
+    with the expected logical source.
+
+    Parameters
+    ----------
+    l1b_dataset : xr.Dataset
+        A ``xarray`` dataset containing the test data
+    """
+    expected_src = "imap_idex_l1b_sci"
+    assert l1b_dataset.attrs["Logical_source"] == expected_src
+
+
+def test_idex_cdf_file(l1b_dataset: xr.Dataset):
+    """Verify the CDF file can be created with no errors.
+
+    Parameters
+    ----------
+    l1b_dataset : xarray.Dataset
+        The dataset to test with
+    """
+
+    file_name = write_cdf(l1b_dataset)
+
+    assert file_name.exists()
+    assert file_name.name == "imap_idex_l1b_sci_20231214_v001.cdf"
+
+
+def test_idex_waveform_units(l1b_dataset: xr.Dataset):
+    """Verify the CDF instrument settings and waveforms have the correct units.
+
+    Parameters
+    ----------
+    l1b_dataset : xarray.Dataset
+        The dataset to test with
+    """
+    cdf_var_defs_path = (
+        f"{imap_module_directory}/idex/idex_variable_unpacking_and_eu_conversion.csv"
+    )
+    cdf_var_defs = pd.read_csv(cdf_var_defs_path)
+
+    # Check instrument setting units
+    for _, row in cdf_var_defs.iterrows():
+        var_name = row["mnemonic"]
+        assert l1b_dataset[var_name].attrs["units"] == row["unit"]
+
+    # Check waveform units
+    waveform_var_names = [
+        "TOF_High",
+        "TOF_Low",
+        "TOF_Mid",
+        "Ion_Grid",
+        "Target_Low",
+        "Target_High",
+    ]
+
+    for var_name in waveform_var_names:
+        assert l1b_dataset[var_name].attrs["UNITS"] == "pC"
+
+
+def test_unpack_instrument_settings():
+    """
+    Check that the instrument setting variables are being unpacked correctly
+
+    Example
+    -------
+    In this example, we are using a test variable that has five bits
+    Idx__test_var01 = 0b10010
+
+    Int(0b10010) = 18
+
+    This should unpack into test_var0, and test_var1
+     - test_var0 is two bits long and starts at 0, and the unpacked value should be 2
+     - test_var1 is three bits long and starts at 3, and the unpacked value should be 4
+    """
+    # Create test dataset with an array shape = 5 all values = 18
+    test_ds = xr.Dataset({"idx__test_var01": xr.DataArray(np.full(5, 18))})
+
+    test_cdf_defs_df = pd.DataFrame(
+        {
+            "mnemonic": ["test_var0", "test_var1"],
+            "var_name": ["idx__test_var01", "idx__test_var01"],
+            "starting_bit": [0, 2],
+            "nbits_padding_before": [0, 0],
+            "unsigned_nbits": [2, 3],
+        }
+    )
+    idex_attrs = ImapCdfAttributes()
+    # Mock attribute manager variable attrs
+    with mock.patch.object(
+        idex_attrs, "get_variable_attributes", return_value={"CATDESC": "Test var"}
+    ):
+        unpacked_dict = unpack_instrument_settings(
+            test_ds, test_cdf_defs_df, idex_attrs
+        )
+
+    assert np.all(unpacked_dict["test_var0"] == 2)
+    assert np.all(unpacked_dict["test_var1"] == 4)

imap_processing/tests/lo/test_lo_l1a.py

@@ -1,29 +1,20 @@
-from pathlib import Path
-
 import numpy as np
-import pytest
 
 from imap_processing import imap_module_directory
 from imap_processing.lo.l1a.lo_l1a import lo_l1a
 
 
-@pytest.mark.skip(reason="not implemented")
-@pytest.mark.parametrize(
-    ("dependency", "expected_logical_source"),
-    [
-        (Path("imap_lo_l0_de_20100101_v001.pkts"), "imap_lo_l1a_de"),
-        (
-            Path("imap_lo_l0_spin_20100101_v001.pkt"),
-            "imap_lo_l1a_spin",
-        ),
-    ],
-)
-def test_lo_l1a(dependency, expected_logical_source):
+def test_lo_l1a():
     # Act
+    dependency = (
+        imap_module_directory / "tests/lo/test_pkts/imap_lo_l0_raw_20240803_v002.pkts"
+    )
+    expected_logical_source = ["imap_lo_l1a_histogram", "imap_lo_l1a_de"]
     output_dataset = lo_l1a(dependency, "001")
 
     # Assert
-    assert expected_logical_source == output_dataset.attrs["Logical_source"]
+    for dataset, logical_source in zip(output_dataset, expected_logical_source):
+        assert logical_source == dataset.attrs["Logical_source"]
 
 
 def test_lo_l1a_dataset():

imap_processing/tests/lo/test_lo_science.py

@@ -1,8 +1,11 @@
 import numpy as np
+import pandas as pd
 import pytest
 import xarray as xr
 
+from imap_processing import imap_module_directory
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.lo.l0.lo_apid import LoAPID
 from imap_processing.lo.l0.lo_science import (
     combine_segmented_packets,
     parse_de_bin,
@@ -10,10 +13,14 @@ from imap_processing.lo.l0.lo_science import (
     parse_fixed_fields,
     parse_variable_fields,
 )
+from imap_processing.utils import convert_to_binary_string, packet_file_to_datasets
 
 
 @pytest.fixture()
 def fake_de_dataset():
+    # binary packet fields
+    count = "0000000000000010"  # 2
+    passes = "00000000000000000000000000000001"  # 1
     # DE One
     absent_1 = "0000"  # case 0
     time_1 = "000001100100"  # 100
@@ -37,7 +44,9 @@ def fake_de_dataset():
     pos_2 = "00"  # 0
 
     de_data = (
-        absent_1
+        count
+        + passes
+        + absent_1
         + time_1
         + energy_1
         + mode_1
@@ -55,13 +64,27 @@ def fake_de_dataset():
     dataset = xr.Dataset(
         data_vars=dict(
             count=(["time"], np.array([2])),
-            data=(["time"], np.array([de_data])),
+            events=(["time"], np.array([de_data])),
         )
     )
 
     return dataset
 
 
+@pytest.fixture()
+def sample_data():
+    xtce_file = imap_module_directory / "lo/packet_definitions/lo_xtce.xml"
+    dependency = (
+        imap_module_directory / "tests/lo/test_pkts/imap_lo_l0_raw_20240803_v002.pkts"
+    )
+    datasets_by_apid = packet_file_to_datasets(
+        packet_file=dependency.resolve(),
+        xtce_packet_definition=xtce_file.resolve(),
+        use_derived_value=False,
+    )
+    return datasets_by_apid
+
+
 @pytest.fixture()
 def segmented_pkts_fake_data():
     dataset = xr.Dataset(
@@ -142,6 +165,9 @@ def test_parse_events(fake_de_dataset, attr_mgr):
 
 
 def test_parse_fixed_fields(initialized_dataset):
+    # Arrange
+    initialized_dataset.attrs["bit_pos"] = 48
+
     # Act
     dataset = parse_fixed_fields(initialized_dataset, 0, 0)
 
@@ -158,7 +184,7 @@ def test_parse_variable_fields(initialized_dataset):
     # Arrange
     initialized_dataset["coincidence_type"].values = np.array([0, 255])
     initialized_dataset["mode"].values = np.array([1, 255])
-    initialized_dataset.attrs["bit_pos"] = 20
+    initialized_dataset.attrs["bit_pos"] = 68
 
     # Act
     dataset = parse_variable_fields(initialized_dataset, 0, 0)
@@ -195,10 +221,48 @@ def test_combine_segmented_packets(segmented_pkts_fake_data):
         dataset["events"].values,
         np.array(
             [
-                "0000000001,0000000010,0000000100,0000001000",
+                "0000000001000000001000000001000000001000",
                 "0000010000",
-                "0100000000,1000000000",
+                "01000000001000000000",
             ]
         ),
     )
     np.testing.assert_array_equal(dataset["epoch"].values, np.array([0, 10, 30]))
+
+
+def test_validate_parse_events(sample_data, attr_mgr):
+    de_data = sample_data[LoAPID.ILO_SCI_DE]
+    validation_path = (
+        imap_module_directory / "tests/lo/validation_data/"
+        "Instrument_FM1_T104_R129_20240803_ILO_SCI_DE_dec_DN_with_fills.csv"
+    )
+
+    validation_data = pd.read_csv(validation_path)
+    de_fields = [
+        "coincidence_type",
+        "de_time",
+        "esa_step",
+        "mode",
+        "tof0",
+        "tof1",
+        "tof2",
+        "tof3",
+        "cksm",
+        "pos",
+    ]
+
+    de_data["data"] = xr.DataArray(
+        [convert_to_binary_string(data) for data in de_data["data"].values],
+        dims=de_data["data"].dims,
+        attrs=de_data["data"].attrs,
+    )
+    de_data = combine_segmented_packets(de_data)
+    dataset = parse_events(de_data, attr_mgr)
+
+    for field in de_fields:
+        np.testing.assert_array_equal(
+            dataset[field].values, validation_data[field.upper()].values
+        )
+
+    assert dataset["de_count"].values == 1998
+    assert dataset["passes"].values == 8