imap-processing 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

imap_processing/tests/ialirt/unit/test_process_codicehi.py

@@ -0,0 +1,106 @@
+import numpy as np
+import pytest
+
+from imap_processing import imap_module_directory
+from imap_processing.cdf.utils import load_cdf
+from imap_processing.ialirt.l0.process_codicehi import (
+    append_cod_hi_data,
+    find_groups,
+    process_codicehi,
+)
+from imap_processing.utils import packet_file_to_datasets
+
+
+@pytest.fixture(scope="session")
+def xtce_codicehi_path():
+    """Returns the xtce directory."""
+    return (
+        imap_module_directory / "ialirt" / "packet_definitions" / "ialirt_codicehi.xml"
+    )
+
+
+@pytest.fixture(scope="session")
+def binary_packet_path():
+    """Returns the xtce directory."""
+    return (
+        imap_module_directory
+        / "tests"
+        / "ialirt"
+        / "test_data"
+        / "l0"
+        / "hi_fsw_view_1_ccsds.bin"
+    )
+
+
+@pytest.fixture(scope="session")
+def codicehi_validation_data():
+    """Returns the test data directory."""
+    data_path = (
+        imap_module_directory
+        / "tests"
+        / "ialirt"
+        / "test_data"
+        / "l0"
+        / "imap_codice_l1a_hi-ialirt_20240523200000_v0.0.0.cdf"
+    )
+    data = load_cdf(data_path)
+
+    return data
+
+
+@pytest.fixture()
+def codicehi_test_data(binary_packet_path, xtce_codicehi_path):
+    """Create xarray data"""
+    apid = 1168
+    codicehi_test_data = packet_file_to_datasets(
+        binary_packet_path, xtce_codicehi_path
+    )[apid]
+
+    return codicehi_test_data
+
+
+def test_find_groups(codicehi_test_data):
+    """Tests find_groups"""
+
+    grouped_data = find_groups(codicehi_test_data)
+    unique_groups = np.unique(grouped_data["group"])
+    for group in unique_groups:
+        group_data = grouped_data["cod_hi_counter"].values[
+            grouped_data["group"] == group
+        ]
+        np.testing.assert_array_equal(group_data, np.arange(234))
+
+
+def test_append_cod_hi_data(codicehi_test_data):
+    """Tests append_cod_hi_data"""
+
+    grouped_data = find_groups(codicehi_test_data)
+    unique_groups = np.unique(grouped_data["group"])
+    for group in unique_groups:
+        mask = grouped_data["group"] == group
+        filtered_indices = np.where(mask)[0]
+        group_data = grouped_data.isel(epoch=filtered_indices)
+        expected_cod_hi_counter = np.repeat(group_data["cod_hi_counter"].values, 5)
+        appended_data = append_cod_hi_data(group_data)
+        assert np.array_equal(
+            appended_data["cod_hi_counter"].values, expected_cod_hi_counter.astype(int)
+        )
+
+
+def test_process_codicehi(codicehi_test_data, codicehi_validation_data, caplog):
+    """Tests process_codicehi."""
+    codicehi_product = process_codicehi(codicehi_test_data)
+    assert codicehi_product == [{}]
+
+    indices = (codicehi_test_data["cod_hi_acq"] != 0).values.nonzero()[0]
+    codicehi_test_data["cod_hi_counter"].values[indices[0] : indices[0] + 234] = (
+        np.random.permutation(234)
+    )
+
+    with caplog.at_level("WARNING"):
+        process_codicehi(codicehi_test_data)
+
+    assert any(
+        "does not contain all values from 0 to 233 without duplicates" in message
+        for message in caplog.text.splitlines()
+    )

imap_processing/tests/ialirt/unit/test_process_ephemeris.py

@@ -7,20 +7,46 @@ from imap_processing.ialirt import process_ephemeris
 from imap_processing.spice.time import str_to_et
 
 
-def test_calculate_doppler():
+@pytest.mark.external_kernel()
+def test_calculate_doppler(furnish_kernels):
     """
     Test the calculate_doppler() function.
     """
-    assert process_ephemeris.calculate_doppler(805794429.1837295) == 1
+    longitude = -71.41  # longitude in degrees
+    latitude = -33.94  # latitude in degrees
+    altitude = 0.157  # altitude in kilometers
+    # test single observation time
+    observation_time = 843307269.1823885  # "2026-09-22T00:00:00.000"
+
+    kernels = [
+        "pck00011.tpc",
+        "imap_spk_demo.bsp",
+        "de440s.bsp",
+        "earth_1962_240827_2124_combined.bpc",
+    ]
+    with furnish_kernels(kernels):
+        doppler_result = process_ephemeris.calculate_doppler(
+            longitude, latitude, altitude, observation_time
+        )
+    assert doppler_result is not None
+
+    # test array of observation times
+    time_endpoints = ("2026 SEP 22 00:00:00", "2026 SEP 22 23:59:59")
+    time_interval = int(1e3)  # seconds between data points
+    observation_time = np.arange(
+        str_to_et(time_endpoints[0]), str_to_et(time_endpoints[1]), time_interval
+    )
+    with furnish_kernels(kernels):
+        doppler_result = process_ephemeris.calculate_doppler(
+            longitude, latitude, altitude, observation_time
+        )
+    assert doppler_result is not None
 
 
 @pytest.mark.external_kernel()
 def test_latitude_longitude_to_ecef(furnish_kernels):
     """
     Test the latitude_longitude_to_ecef() function.
-
-    Test data is from https://naif.jpl.nasa.gov/pub/naif/toolkit_docs/C/cspice/
-    georec_c.html.
     """
     longitude = 118.0  # degrees
     latitude = 30.0  # degrees
@@ -92,6 +118,8 @@ def test_build_output(furnish_kernels):
         "naif0012.tls",
         "pck00011.tpc",
         "de440s.bsp",
+        "imap_spk_demo.bsp",
+        "earth_1962_240827_2124_combined.bpc",
     ]
     with furnish_kernels(kernels):
         output_dict = process_ephemeris.build_output(

imap_processing/tests/ialirt/unit/test_process_swapi.py

@@ -0,0 +1,85 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+from imap_processing import imap_module_directory
+from imap_processing.utils import packet_file_to_datasets
+
+
+@pytest.fixture(scope="session")
+def xtce_swapi_path():
+    """Returns the xtce auxiliary directory."""
+    return imap_module_directory / "ialirt" / "packet_definitions" / "ialirt_swapi.xml"
+
+
+@pytest.fixture(scope="session")
+def binary_packet_path():
+    """Returns the xtce directory."""
+    return (
+        imap_module_directory
+        / "tests"
+        / "ialirt"
+        / "test_data"
+        / "l0"
+        / "BinLog CCSDS_FRAG_TLM_20240826_152323Z_IALIRT_data_for_SDC.bin"
+    )
+
+
+@pytest.fixture(scope="session")
+def swapi_test_data():
+    """Returns the test data directory."""
+    data_path = (
+        imap_module_directory
+        / "tests"
+        / "ialirt"
+        / "test_data"
+        / "l0"
+        / "eu_SWP_IAL_20240826_152033.csv"
+    )
+    data = pd.read_csv(data_path)
+
+    return data
+
+
+@pytest.fixture()
+def xarray_data(binary_packet_path, xtce_swapi_path):
+    """Create xarray data"""
+    apid = 1187
+
+    xarray_data = packet_file_to_datasets(
+        binary_packet_path, xtce_swapi_path, use_derived_value=True
+    )[apid]
+    return xarray_data
+
+
+def test_decom_packets(xarray_data, swapi_test_data):
+    """This function checks that all instrument parameters are accounted for."""
+
+    # TODO: confirm w/ SWAPI team validity_enum flag can be
+    #  consistent with other instruments.
+    fields_to_test = {
+        "swapi_flag": "I_ALIRT_STATUS",
+        "swapi_reserved": "INST_RES_ST",
+        "swapi_seq": "SEQ_NUMBER",
+        "swapi_version": "SWEEP_TABLE",
+        "swapi_coin_1": "COIN_CNT0",
+        "swapi_coin_2": "COIN_CNT1",
+        "swapi_coin_3": "COIN_CNT2",
+        "swapi_coin_4": "COIN_CNT3",
+        "swapi_coin_5": "COIN_CNT4",
+        "swapi_coin_6": "COIN_CNT5",
+        "swapi_spare": "SPARE",
+    }
+    _, index, test_index = np.intersect1d(
+        xarray_data["swapi_acq"], swapi_test_data["ACQ_TIME"], return_indices=True
+    )
+
+    for xarray_field, test_field in fields_to_test.items():
+        actual_values = xarray_data[xarray_field].values[index]
+        expected_values = swapi_test_data[test_field].values[test_index]
+
+        # Assert that all values match
+        assert np.all(actual_values == expected_values), (
+            f"Mismatch found in {xarray_field}: "
+            f"actual {actual_values}, expected {expected_values}"
+        )

imap_processing/tests/ialirt/unit/test_process_swe.py

@@ -0,0 +1,106 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+from imap_processing import imap_module_directory
+from imap_processing.utils import packet_file_to_datasets
+
+
+@pytest.fixture(scope="session")
+def xtce_swe_path():
+    """Returns the xtce auxiliary directory."""
+    return imap_module_directory / "ialirt" / "packet_definitions" / "ialirt_swe.xml"
+
+
+@pytest.fixture(scope="session")
+def binary_packet_path():
+    """Returns the xtce directory."""
+    return (
+        imap_module_directory
+        / "tests"
+        / "ialirt"
+        / "test_data"
+        / "l0"
+        / "20240827095047_SWE_IALIRT_packet.bin"
+    )
+
+
+@pytest.fixture(scope="session")
+def swe_test_data():
+    """Returns the test data directory."""
+    data_path = (
+        imap_module_directory
+        / "tests"
+        / "ialirt"
+        / "test_data"
+        / "l0"
+        / "idle_export_eu.SWE_IALIRT_20240827_093852.csv"
+    )
+    data = pd.read_csv(data_path)
+
+    return data
+
+
+@pytest.fixture()
+def xarray_data(binary_packet_path, xtce_swe_path):
+    """Create xarray data"""
+    apid = 1360
+
+    xarray_data = packet_file_to_datasets(
+        binary_packet_path, xtce_swe_path, use_derived_value=True
+    )[apid]
+    return xarray_data
+
+
+def test_decom_packets(xarray_data, swe_test_data):
+    """This function checks that all instrument parameters are accounted for."""
+
+    fields_to_test = {
+        "swe_shcoarse": "SHCOARSE",
+        "swe_acq_sec": "ACQUISITION_TIME",
+        "swe_acq_sub": "ACQUISITION_TIME_SUBSECOND",
+        "swe_nom_flag": "INSTRUMENT_NOMINAL_FLAG",
+        "swe_ops_flag": "SCIENCE_OPERATION_FLAG",
+        "swe_seq": "SEQUENCE_NUMBER",
+        "swe_cem1_e1": "ELEC_COUNTS_SPIN_I_POL_0_E_0J",
+        "swe_cem1_e2": "ELEC_COUNTS_SPIN_I_POL_0_E_1J",
+        "swe_cem1_e3": "ELEC_COUNTS_SPIN_I_POL_0_E_2J",
+        "swe_cem1_e4": "ELEC_COUNTS_SPIN_I_POL_0_E_3J",
+        "swe_cem2_e1": "ELEC_COUNTS_SPIN_I_POL_1_E_0J",
+        "swe_cem2_e2": "ELEC_COUNTS_SPIN_I_POL_1_E_1J",
+        "swe_cem2_e3": "ELEC_COUNTS_SPIN_I_POL_1_E_2J",
+        "swe_cem2_e4": "ELEC_COUNTS_SPIN_I_POL_1_E_3J",
+        "swe_cem3_e1": "ELEC_COUNTS_SPIN_I_POL_2_E_0J",
+        "swe_cem3_e2": "ELEC_COUNTS_SPIN_I_POL_2_E_1J",
+        "swe_cem3_e3": "ELEC_COUNTS_SPIN_I_POL_2_E_2J",
+        "swe_cem3_e4": "ELEC_COUNTS_SPIN_I_POL_2_E_3J",
+        "swe_cem4_e1": "ELEC_COUNTS_SPIN_I_POL_3_E_0J",
+        "swe_cem4_e2": "ELEC_COUNTS_SPIN_I_POL_3_E_1J",
+        "swe_cem4_e3": "ELEC_COUNTS_SPIN_I_POL_3_E_2J",
+        "swe_cem4_e4": "ELEC_COUNTS_SPIN_I_POL_3_E_3J",
+        "swe_cem5_e1": "ELEC_COUNTS_SPIN_I_POL_4_E_0J",
+        "swe_cem5_e2": "ELEC_COUNTS_SPIN_I_POL_4_E_1J",
+        "swe_cem5_e3": "ELEC_COUNTS_SPIN_I_POL_4_E_2J",
+        "swe_cem5_e4": "ELEC_COUNTS_SPIN_I_POL_4_E_3J",
+        "swe_cem6_e1": "ELEC_COUNTS_SPIN_I_POL_5_E_0J",
+        "swe_cem6_e2": "ELEC_COUNTS_SPIN_I_POL_5_E_1J",
+        "swe_cem6_e3": "ELEC_COUNTS_SPIN_I_POL_5_E_2J",
+        "swe_cem6_e4": "ELEC_COUNTS_SPIN_I_POL_5_E_3J",
+        "swe_cem7_e1": "ELEC_COUNTS_SPIN_I_POL_6_E_0J",
+        "swe_cem7_e2": "ELEC_COUNTS_SPIN_I_POL_6_E_1J",
+        "swe_cem7_e3": "ELEC_COUNTS_SPIN_I_POL_6_E_2J",
+        "swe_cem7_e4": "ELEC_COUNTS_SPIN_I_POL_6_E_3J",
+    }
+    _, index, test_index = np.intersect1d(
+        xarray_data["swe_shcoarse"], swe_test_data["SHCOARSE"], return_indices=True
+    )
+
+    for xarray_field, test_field in fields_to_test.items():
+        actual_values = xarray_data[xarray_field].values[index]
+        expected_values = swe_test_data[test_field].values[test_index]
+
+        # Assert that all values match
+        assert np.all(actual_values == expected_values), (
+            f"Mismatch found in {xarray_field}: "
+            f"actual {actual_values}, expected {expected_values}"
+        )

imap_processing/tests/idex/conftest.py

@@ -1,11 +1,25 @@
 from pathlib import Path
 
+import numpy as np
 import pytest
 import xarray as xr
 
 from imap_processing import imap_module_directory
 from imap_processing.idex.idex_l1a import PacketParser
 
+SPICE_ARRAYS = [
+    "ephemeris_position_x",
+    "ephemeris_position_y",
+    "ephemeris_position_z",
+    "ephemeris_velocity_x",
+    "ephemeris_velocity_y",
+    "ephemeris_velocity_z",
+    "right_ascension",
+    "declination",
+    "solar_longitude",
+    "spin_phase",
+]
+
 
 @pytest.fixture(scope="module")
 def decom_test_data() -> xr.Dataset:
@@ -17,6 +31,20 @@ def decom_test_data() -> xr.Dataset:
         A ``xarray`` dataset containing the test data
     """
     test_file = Path(
-        f"{imap_module_directory}/tests/idex/imap_idex_l0_raw_20231214_v001.pkts"
+        f"{imap_module_directory}/tests/idex/test_data/imap_idex_l0_raw_20231214_v001.pkts"
     )
     return PacketParser(test_file, "001").data
+
+
+def get_spice_data_side_effect_func(l1a_ds, idex_attrs):
+    # Create a mock dictionary of spice arrays
+
+    return {
+        name: xr.DataArray(
+            name=name,
+            data=np.ones(len(l1a_ds["epoch"])),
+            dims="epoch",
+            attrs=idex_attrs.get_variable_attributes(name),
+        )
+        for name in SPICE_ARRAYS
+    }

imap_processing/tests/idex/test_idex_l0.py

@@ -14,7 +14,7 @@ def test_idex_decom_length(decom_test_data: xr.Dataset):
     decom_test_data : xarray.Dataset
         The dataset to test with
     """
-    assert len(decom_test_data) == 106
+    assert len(decom_test_data) == 110
 
 
 def test_idex_decom_event_num(decom_test_data: xr.Dataset):
@@ -26,7 +26,8 @@ def test_idex_decom_event_num(decom_test_data: xr.Dataset):
         The dataset to test with
     """
     for var in decom_test_data:
-        assert len(decom_test_data[var]) == 14
+        if "epoch" in decom_test_data[var].dims:
+            assert len(decom_test_data[var]) == 14
 
 
 def test_idex_tof_high_data(decom_test_data: xr.Dataset):
@@ -40,6 +41,8 @@ def test_idex_tof_high_data(decom_test_data: xr.Dataset):
         The dataset to test with
     """
 
-    with open(f"{imap_module_directory}/tests/idex/impact_14_tof_high_data.txt") as f:
+    with open(
+        f"{imap_module_directory}/tests/idex/test_data/impact_14_tof_high_data.txt"
+    ) as f:
         data = np.array([int(line.rstrip("\n")) for line in f])
     assert (decom_test_data["TOF_High"][13].data == data).all()

imap_processing/tests/idex/test_idex_l1a.py

@@ -1,5 +1,8 @@
 """Tests the L1 processing for decommutated IDEX data"""
 
+from pathlib import Path
+from unittest import mock
+
 import numpy as np
 import pytest
 import xarray as xr
@@ -7,6 +10,8 @@ from cdflib.xarray.xarray_to_cdf import ISTPError
 
 from imap_processing import imap_module_directory
 from imap_processing.cdf.utils import load_cdf, write_cdf
+from imap_processing.idex.decode import _decode_sub_frame, read_bits, rice_decode
+from imap_processing.idex.idex_l1a import PacketParser
 
 
 def test_idex_cdf_file(decom_test_data: xr.Dataset):
@@ -89,9 +94,154 @@ def test_idex_tof_high_data_from_cdf(decom_test_data: xr.Dataset):
     decom_test_data : xarray.Dataset
         The dataset to test with
     """
-    with open(f"{imap_module_directory}/tests/idex/impact_14_tof_high_data.txt") as f:
+    with open(
+        f"{imap_module_directory}/tests/idex/test_data/impact_14_tof_high_data.txt"
+    ) as f:
         data = np.array([int(line.rstrip()) for line in f])
 
     file_name = write_cdf(decom_test_data)
     l1_data = load_cdf(file_name)
     assert (l1_data["TOF_High"][13].data == data).all()
+
+
+def test_compressed_packet():
+    """
+    Test compressed data decompression against known non-compressed data.
+    """
+    test_data_dir = f"{imap_module_directory}/tests/idex/test_data"
+
+    compressed = Path(f"{test_data_dir}/compressed_2023_102_14_24_55.pkts")
+    non_compressed = Path(f"{test_data_dir}/non_compressed_2023_102_14_22_26.pkts")
+
+    decompressed = PacketParser(compressed, "001").data
+    expected = PacketParser(non_compressed, "001").data
+
+    waveforms = [
+        "TOF_High",
+        "TOF_Low",
+        "TOF_Mid",
+        "Ion_Grid",
+        "Target_High",
+        "Target_Low",
+    ]
+
+    # Compare each decompressed waveform with known non-compressed waveform.
+    for var in waveforms:
+        assert np.allclose(decompressed[var], expected[var]), (
+            f"Variable: {var} is different for the decompressed and non compressed "
+            f"datasets."
+        )
+
+
+def test_read_bits():
+    """
+    Test that read_bits() properly increments the pointer and reads the correct ints.
+    """
+
+    binary_str_6 = "110"
+    binary_str_neg_6 = "1010"
+    binary_str_1 = "0001"
+
+    full_string = binary_str_6 + binary_str_neg_6 + binary_str_1
+
+    pointer = 0
+    val, pointer = read_bits(full_string, pointer, 3)
+    assert val == 6
+    assert pointer == 3
+    # Test with a signed binary string
+    val, pointer = read_bits(full_string, pointer, 4, signed=True)
+    assert val == -6
+    assert pointer == 7
+
+    val, pointer = read_bits(full_string, pointer, 4)
+    assert val == 1
+    assert pointer == 11
+
+
+@mock.patch("imap_processing.idex.decode._decode_sub_frame")
+def test_rice_decode(mock_decode_sub_frame):
+    """
+    Verify that rice_decode() returns the expected list of integers.
+    """
+
+    mock_decode_sub_frame.return_value = [1 for _ in range(64)], 1
+    # mock binary string.
+    bstr = "0100000"
+    ints = rice_decode(bstr, True, 640)
+    # If there are 64 samples per block and the sample count is 640,
+    # There should be 10 calls to _decode_sub_frame()
+    assert mock_decode_sub_frame.call_count == 10
+    # There should be 640 samples
+    assert ints == [1 for _ in range(640)]
+
+
+def test_decode_sub_frame_psel_0():
+    """Test constant predictor (psel=0) where all samples equal first sample."""
+    psel = 0
+    bstring = "0" * 9 + "1"  # 1 in 10-bit binary
+    ints, bp = _decode_sub_frame(bstring, bp=0, psel=psel, k=0, n_bits=10)
+    assert ints == [1 for _ in range(64)]
+
+
+def test_decode_sub_frame_psel_1():
+    """Test verbatim predictor (psel=1) where each sample stored directly."""
+    # Create a binary string that is equal to 1, 2, 3, 4 to 64. Each int is represented
+    # in 10 bits to mock the expected high sample waveform data.
+    psel = 1
+    bstring = "".join([bin(i)[2:].zfill(10) for i in range(64)])
+    ints, bp = _decode_sub_frame(bstring, bp=0, psel=psel, k=0, n_bits=10)
+    assert ints == [i for i in range(64)]
+
+
+def test_decode_sub_frame_psel_2():
+    """Test linear predictor 1 (psel=2) with rice encoded residuals."""
+    psel = 2
+    k = 1
+    # This encoding takes the signed residual value (current value - last value)
+    # Example values 1, 2, 3, 8
+    # residual of 1
+    # quotient = 1 >> (k+1) = 0 (unary = "1")
+    # remainder = "01"
+    residual_1_and_2 = "101"
+    # residual of 5
+    # quotient = 4 >> (k+1) = 1 maps to 2 (unary = "001")
+    # remainder = "01"
+    residual_3 = "00101"
+    warmup = "0" * 9 + "1"  # equal to 1
+
+    bstring = warmup + residual_1_and_2 + residual_1_and_2 + residual_3
+    ints, bp = _decode_sub_frame(bstring, bp=0, psel=psel, k=k, n_bits=10)
+    assert ints == [1, 2, 3, 8]
+
+
+def test_decode_sub_frame_psel_3():
+    """Test linear predictor 2 (psel=3) with rice encoded residuals."""
+    # This predictor assumes that sample X(n) = 2X(n-1)-X(n-2). This predictor
+    # calculates the slope of the signal based on the two previous values
+    # (slope = X(n-1)-X(n-2)) and predicts the value of X(n) will follow on a straight
+    # line drawn between the two previous points.
+    # X(n) = X(n-1) + (X(n-1)-X(n-2))  = 2X(n-1)-X(n-2).
+    psel = 3
+    k = 1
+    # Example values: [1, 2, 4, 1, 5]
+    # For X(4)=1: predicted=3 (2*2-1), actual=4, residual=1
+    # For X(1)=-5: predicted=6 (2*4-2), actual=1, residual=-5
+    # For X(5)=7: predicted=-2 (2*1-4), actual=5, residual=7
+    # residual of 1
+    # quotient = 1 >> (K+1) = 0 (unary = "1")
+    # remainder = "01"
+    residual_1 = "101"
+    # residual of -5
+    # quotient = -5 >> (K+1) = -2 maps to 3 (unary = "0001")
+    # remainder = "11"
+    residual_2 = "000111"
+    # residual of 7
+    # quotient = 7 >> (K+1) = 1 maps to 2 (unary = "001")
+    # remainder = "11"
+    residual_3 = "00111"
+    warmup1 = "0" * 9 + "1"  # 1
+    warmup2 = "0" * 8 + "10"  # 2
+
+    bstring = warmup1 + warmup2 + residual_1 + residual_2 + residual_3
+    ints, bp = _decode_sub_frame(bstring, bp=0, psel=psel, k=k, n_bits=10)
+    assert ints == [1, 2, 4, 1, 5]