imap-processing 0.6.0__py3-none-any.whl

imap_processing/tests/swapi/test_swapi_l1.py

@@ -0,0 +1,354 @@
+import numpy as np
+import pytest
+import xarray as xr
+
+from imap_processing import imap_module_directory
+from imap_processing.cdf.utils import write_cdf
+from imap_processing.spice.time import met_to_j2000ns
+from imap_processing.swapi.l1.swapi_l1 import (
+    SWAPIAPID,
+    decompress_count,
+    filter_good_data,
+    find_sweep_starts,
+    get_indices_of_full_sweep,
+    process_swapi_science,
+    process_sweep_data,
+    swapi_l1,
+)
+from imap_processing.swapi.swapi_utils import SWAPIMODE
+from imap_processing.utils import packet_file_to_datasets
+
+
+@pytest.fixture(scope="session")
+def decom_test_data(swapi_l0_test_data_path):
+    """Read test data from file"""
+    test_file = "imap_swapi_l0_raw_20231012_v001.pkts"
+    packet_files = imap_module_directory / swapi_l0_test_data_path / test_file
+    packet_definition = (
+        f"{imap_module_directory}/swapi/packet_definitions/swapi_packet_definition.xml"
+    )
+    return packet_file_to_datasets(
+        packet_files, packet_definition, use_derived_value=False
+    )
+
+
+def test_filter_good_data():
+    """Test for bad data"""
+    # create test data for this test
+    total_sweeps = 3
+    ds = xr.Dataset(
+        {
+            "plan_id_science": xr.DataArray(np.full((total_sweeps * 12), 1)),
+            "sweep_table": xr.DataArray(np.repeat(np.arange(total_sweeps), 12)),
+            "mode": xr.DataArray(np.full((total_sweeps * 12), SWAPIMODE.HVENG.value)),
+        },
+        coords={"epoch": np.arange(total_sweeps * 12)},
+    )
+
+    # Check for no bad data
+    bad_data_indices = filter_good_data(ds)
+    assert len(bad_data_indices) == 36
+
+    # Check for bad MODE data, only HVENG is "good"
+    # TODO: update test when we update MODE from HVENG to HVSCI
+    ds["mode"] = xr.DataArray(
+        np.repeat(
+            [SWAPIMODE.LVENG.value, SWAPIMODE.LVSCI.value, SWAPIMODE.HVENG.value], 12
+        )
+    )
+    bad_data_indices = filter_good_data(ds)
+    np.testing.assert_array_equal(bad_data_indices, np.arange(24, 36))
+
+    # Check for bad sweep_table data.
+    # Reset MODE data and create first sweep to be mixed value
+    ds["mode"] = xr.DataArray(np.full((total_sweeps * 12), SWAPIMODE.HVENG.value))
+    ds["sweep_table"][:12] = np.arange(0, 12)
+    np.testing.assert_array_equal(filter_good_data(ds), np.arange(12, 36))
+
+    ds["sweep_table"][24:] = np.arange(0, 12)
+    expected = [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]
+    np.testing.assert_array_equal(filter_good_data(ds), expected)
+
+    # Check for bad plan_id_science data.
+    ds["sweep_table"] = xr.DataArray(np.repeat(np.arange(total_sweeps), 12))
+    ds["plan_id_science"][24 : total_sweeps * 12] = np.arange(0, 12)
+    np.testing.assert_array_equal(filter_good_data(ds), np.arange(0, 24))
+
+
+def test_decompress_count():
+    """Test for decompress count"""
+    # compressed + no-overflow, compressed + overflow, no compression
+    raw_values = np.array([[12, 0xFFFF, 12]])
+    compression_flag = np.array([[1, 1, 0]])
+    expected = np.array([[12 * 16, np.iinfo(np.int32).max, 12]], dtype=np.int32)
+    returned_value = decompress_count(raw_values, compression_flag)
+    np.testing.assert_array_equal(returned_value, expected)
+
+
+def test_find_sweep_starts():
+    """Test for find sweep starts"""
+    time = np.arange(26)
+    sequence_number = time % 12
+    ds = xr.Dataset(
+        {"seq_number": sequence_number, "shcoarse": np.arange(1, 27, 1)},
+        coords={"epoch": met_to_j2000ns(time)},
+    )
+
+    start_indices = find_sweep_starts(ds)
+    np.testing.assert_array_equal(start_indices, [0, 12])
+
+    ds["seq_number"].data[:12] = np.arange(3, 15)
+    start_indices = find_sweep_starts(ds)
+    np.testing.assert_array_equal(start_indices, [12])
+
+    # Creating test data that doesn't have start sequence.
+    # Sequence number range is 0-11.
+    ds["seq_number"] = np.arange(3, 29)
+    start_indices = find_sweep_starts(ds)
+    np.testing.assert_array_equal(start_indices, [])
+
+
+def test_get_full_indices():
+    """Test for correct full sweep indices"""
+    time = np.arange(26)
+    sequence_number = time % 12
+    ds = xr.Dataset(
+        {"seq_number": sequence_number, "shcoarse": np.arange(1, 27, 1)},
+        coords={"epoch": met_to_j2000ns(time)},
+    )
+
+    sweep_indices = get_indices_of_full_sweep(ds)
+    np.testing.assert_array_equal(sweep_indices, np.arange(0, 24))
+
+
+def test_swapi_algorithm(decom_test_data):
+    """Test SWAPI L1 algorithm"""
+    ds_data = decom_test_data[SWAPIAPID.SWP_SCI]
+    full_sweep_indices = get_indices_of_full_sweep(ds_data)
+    full_sweep_sci = ds_data.isel({"epoch": full_sweep_indices})
+    total_packets = len(full_sweep_sci["seq_number"].data)
+    # It takes 12 sequence data to make one full sweep
+    total_sequence = 12
+    total_full_sweeps = total_packets // total_sequence
+    pcem_counts = process_sweep_data(full_sweep_sci, "pcem_cnt")
+    # check that return value has correct shape
+    assert pcem_counts.shape == (total_full_sweeps, 72)
+    expected_count = [
+        0,
+        0,
+        0,
+        1,
+        2,
+        3,
+        4,
+        5,
+        6,
+        7,
+        8,
+        9,
+        10,
+        11,
+        12,
+        13,
+        14,
+        15,
+        16,
+        17,
+        18,
+        19,
+        20,
+        21,
+        22,
+        23,
+        24,
+        25,
+        26,
+        27,
+        28,
+        29,
+        30,
+        31,
+        32,
+        31,
+        30,
+        29,
+        28,
+        27,
+        26,
+        25,
+        24,
+        23,
+        22,
+        21,
+        20,
+        19,
+        18,
+        17,
+        16,
+        15,
+        14,
+        13,
+        12,
+        11,
+        10,
+        9,
+        8,
+        7,
+        6,
+        5,
+        4,
+        18,
+        20,
+        22,
+        24,
+        26,
+        28,
+        30,
+        32,
+        34,
+    ]
+
+    np.testing.assert_array_equal(pcem_counts[0], expected_count)
+
+
+def test_process_swapi_science(decom_test_data):
+    """Test process swapi science"""
+    ds_data = decom_test_data[SWAPIAPID.SWP_SCI]
+    processed_data = process_swapi_science(
+        ds_data, decom_test_data[SWAPIAPID.SWP_HK], data_version="001"
+    )
+
+    # Test dataset dimensions
+    assert processed_data.sizes == {
+        "epoch": 3,
+        "energy": 72,
+        "energy_label": 72,
+    }
+    # Test epoch data is correct
+    expected_epoch_datetime = met_to_j2000ns([48, 60, 72])
+    np.testing.assert_array_equal(processed_data["epoch"].data, expected_epoch_datetime)
+
+    expected_count = [
+        0,
+        0,
+        0,
+        1,
+        2,
+        3,
+        4,
+        5,
+        6,
+        7,
+        8,
+        9,
+        10,
+        11,
+        12,
+        13,
+        14,
+        15,
+        16,
+        17,
+        18,
+        19,
+        20,
+        21,
+        22,
+        23,
+        24,
+        25,
+        26,
+        27,
+        28,
+        29,
+        30,
+        31,
+        32,
+        31,
+        30,
+        29,
+        28,
+        27,
+        26,
+        25,
+        24,
+        23,
+        22,
+        21,
+        20,
+        19,
+        18,
+        17,
+        16,
+        15,
+        14,
+        13,
+        12,
+        11,
+        10,
+        9,
+        8,
+        7,
+        6,
+        5,
+        4,
+        18,
+        20,
+        22,
+        24,
+        26,
+        28,
+        30,
+        32,
+        34,
+    ]
+    # Test that we got expected counts and datashape
+    np.testing.assert_array_equal(processed_data["swp_pcem_counts"][0], expected_count)
+    assert processed_data["swp_pcem_counts"].shape == (3, 72)
+    # Test that we calculated uncertainty correctly
+    np.testing.assert_allclose(
+        np.sqrt(processed_data["swp_pcem_counts"][0]),
+        processed_data["swp_pcem_err_plus"][0],
+    )
+
+    # make PLAN_ID data incorrect
+    ds_data["plan_id_science"][:12] = np.arange(12)
+    processed_data = process_swapi_science(
+        ds_data, decom_test_data[SWAPIAPID.SWP_HK], data_version="001"
+    )
+
+    # Test dataset dimensions
+    assert processed_data.sizes == {
+        "epoch": 2,
+        "energy": 72,
+        "energy_label": 72,
+    }
+
+    # Test CDF File
+    # This time mismatch is because of sample data. Sample data has
+    # SHCOARSE time as 48, 60, 72. That's why time is different.
+    cdf_filename = "imap_swapi_l1_sci_20100101_v001.cdf"
+    cdf_path = write_cdf(processed_data)
+    assert cdf_path.name == cdf_filename
+
+
+def test_swapi_l1_cdf(swapi_l0_test_data_path):
+    """Test housekeeping processing and CDF file creation"""
+    test_packet_file = swapi_l0_test_data_path / "imap_swapi_l0_raw_20231012_v001.pkts"
+    processed_data = swapi_l1(test_packet_file, data_version="v001")
+
+    assert processed_data[0].attrs["Apid"] == f"{SWAPIAPID.SWP_SCI}"
+    assert processed_data[0].attrs["Plan_id"] == "0"
+    assert processed_data[0].attrs["Sweep_table"] == "0"
+
+    # Test CDF File
+    # sci cdf file
+    cdf_filename = "imap_swapi_l1_sci_20100101_v001.cdf"
+    cdf_path = write_cdf(processed_data[0])
+    assert cdf_path.name == cdf_filename
+
+    # hk cdf file
+    cdf_filename = "imap_swapi_l1_hk_20100101_v001.cdf"
+    # Ignore ISTP checks for HK data
+    cdf_path = write_cdf(processed_data[1], istp=False)
+    assert cdf_path.name == cdf_filename

imap_processing/tests/swapi/test_swapi_l2.py

@@ -0,0 +1,21 @@
+import numpy as np
+
+from imap_processing.cdf.utils import write_cdf
+from imap_processing.swapi.l1.swapi_l1 import swapi_l1
+from imap_processing.swapi.l2.swapi_l2 import TIME_PER_BIN, swapi_l2
+
+
+def test_swapi_l2_cdf(swapi_l0_test_data_path):
+    """Test housekeeping processing and CDF file creation"""
+    l0_data_path = swapi_l0_test_data_path / "imap_swapi_l0_raw_20231012_v001.pkts"
+    processed_data = swapi_l1(l0_data_path, data_version="v001")
+    l1_dataset = processed_data[0]
+
+    l2_dataset = swapi_l2(l1_dataset, data_version="v001")
+    l2_cdf = write_cdf(l2_dataset)
+    assert l2_cdf.name == "imap_swapi_l2_sci_20100101_v001.cdf"
+
+    # Test uncertainty variables are as expected
+    np.testing.assert_array_equal(
+        l2_dataset["swp_pcem_unc_plus"], l1_dataset["swp_pcem_err_plus"] / TIME_PER_BIN
+    )

imap_processing/tests/swe/conftest.py

@@ -0,0 +1,35 @@
+import pytest
+
+from imap_processing import imap_module_directory
+from imap_processing.swe.utils.swe_utils import SWEAPID
+from imap_processing.utils import packet_file_to_datasets
+
+
+@pytest.fixture(scope="session")
+def decom_test_data():
+    """Read test data from file"""
+    packet_file = (
+        imap_module_directory / "tests/swe/l0_data/2024051010_SWE_SCIENCE_packet.bin"
+    )
+    xtce_document = (
+        imap_module_directory / "swe/packet_definitions/swe_packet_definition.xml"
+    )
+    datasets_by_apid = packet_file_to_datasets(
+        packet_file, xtce_document, use_derived_value=False
+    )
+    return datasets_by_apid[SWEAPID.SWE_SCIENCE]
+
+
+@pytest.fixture(scope="session")
+def decom_test_data_derived():
+    """Read test data from file"""
+    packet_file = (
+        imap_module_directory / "tests/swe/l0_data/2024051010_SWE_SCIENCE_packet.bin"
+    )
+    xtce_document = (
+        imap_module_directory / "swe/packet_definitions/swe_packet_definition.xml"
+    )
+    datasets_by_apid = packet_file_to_datasets(
+        packet_file, xtce_document, use_derived_value=True
+    )
+    return datasets_by_apid[SWEAPID.SWE_SCIENCE]

imap_processing/tests/swe/decompressed/20230927173238_4th_quarter_decompressed.csv

@@ -0,0 +1,181 @@
+Index,CEM 1,CEM 2,CEM 3,CEM 4,CEM 5,CEM 6,CEM 7
+0,0,40959,0,40959,0,40959,935
+1,407,407,407,407,407,407,3519
+2,407,407,407,407,407,407,263
+3,407,407,407,407,407,407,1015
+4,407,407,407,407,407,407,3775
+5,407,407,407,407,407,407,1967
+6,407,407,407,407,407,407,519
+7,407,407,407,407,407,407,139
+8,407,407,407,407,407,407,1839
+9,407,407,407,407,407,407,487
+10,407,407,407,407,407,407,131
+11,407,407,407,407,407,407,251
+12,407,407,407,407,407,407,935
+13,407,407,407,407,407,407,3519
+14,407,407,407,407,407,407,263
+15,407,407,407,407,407,407,1015
+16,407,407,407,407,407,407,3775
+17,407,407,407,407,407,407,1967
+18,407,407,407,407,407,407,519
+19,407,407,407,407,407,407,139
+20,407,407,407,407,407,407,1839
+21,407,407,407,407,407,407,487
+22,407,407,407,407,407,407,131
+23,407,407,407,407,407,407,251
+24,407,407,407,407,407,407,935
+25,407,407,407,407,407,407,3519
+26,407,407,407,407,407,407,263
+27,407,407,407,407,407,407,1015
+28,407,407,407,407,407,407,3775
+29,407,407,407,407,407,407,1967
+30,407,407,407,407,407,407,519
+31,4031,4031,4031,4031,4031,4031,139
+32,4031,4031,4031,4031,4031,4031,1839
+33,4031,4031,4031,4031,4031,4031,487
+34,4031,4031,4031,4031,4031,4031,131
+35,4031,4031,4031,4031,4031,4031,251
+36,4031,4031,4031,4031,4031,4031,935
+37,4031,4031,4031,4031,4031,4031,3519
+38,4031,4031,4031,4031,4031,4031,263
+39,4031,4031,4031,4031,4031,4031,1015
+40,4031,4031,4031,4031,4031,4031,3775
+41,4031,4031,4031,4031,4031,4031,1967
+42,4031,4031,4031,4031,4031,4031,519
+43,4031,4031,4031,4031,4031,4031,139
+44,4031,4031,4031,4031,4031,4031,1839
+45,4031,4031,4031,4031,4031,4031,487
+46,4031,4031,4031,4031,4031,4031,131
+47,4031,4031,4031,4031,4031,4031,251
+48,4031,4031,4031,4031,4031,4031,935
+49,4031,4031,4031,4031,4031,4031,3519
+50,4031,4031,4031,4031,4031,4031,263
+51,4031,4031,4031,4031,4031,4031,1015
+52,4031,4031,4031,4031,4031,4031,3775
+53,4031,4031,4031,4031,4031,4031,1967
+54,4031,4031,4031,4031,4031,4031,519
+55,4031,4031,4031,4031,4031,4031,139
+56,4031,4031,4031,4031,4031,4031,1839
+57,4031,4031,4031,4031,4031,4031,487
+58,4031,4031,4031,4031,4031,4031,131
+59,4031,4031,4031,4031,4031,4031,251
+60,4031,4031,4031,4031,4031,4031,935
+61,8063,0,8063,0,8063,0,3519
+62,8063,0,8063,0,8063,0,263
+63,8063,0,8063,0,8063,0,1015
+64,8063,0,8063,0,8063,0,3775
+65,8063,0,8063,0,8063,0,1967
+66,8063,0,8063,0,8063,0,519
+67,8063,0,8063,0,8063,0,139
+68,8063,0,8063,0,8063,0,1839
+69,8063,0,8063,0,8063,0,487
+70,8063,0,8063,0,8063,0,131
+71,8063,0,8063,0,8063,0,251
+72,8063,0,8063,0,8063,0,935
+73,8063,0,8063,0,8063,0,3519
+74,8063,0,8063,0,8063,0,263
+75,8063,0,8063,0,8063,0,1015
+76,8063,0,8063,0,8063,0,3775
+77,8063,0,8063,0,8063,0,1967
+78,8063,0,8063,0,8063,0,519
+79,8063,0,8063,0,8063,0,139
+80,8063,0,8063,0,8063,0,1839
+81,8063,0,8063,0,8063,0,487
+82,8063,0,8063,0,8063,0,131
+83,8063,0,8063,0,8063,0,251
+84,8063,0,8063,0,8063,0,935
+85,8063,0,8063,0,8063,0,3519
+86,8063,0,8063,0,8063,0,263
+87,8063,0,8063,0,8063,0,1015
+88,8063,0,8063,0,8063,0,3775
+89,8063,0,8063,0,8063,0,1967
+90,8063,0,8063,0,8063,0,519
+91,407,0,407,0,407,0,139
+92,407,0,407,0,407,0,1839
+93,407,0,407,0,407,0,487
+94,407,0,407,0,407,0,131
+95,407,0,407,0,407,0,251
+96,407,0,407,0,407,0,935
+97,407,0,407,0,407,0,3519
+98,407,0,407,0,407,0,263
+99,407,0,407,0,407,0,1015
+100,407,0,407,0,407,0,3775
+101,407,0,407,0,407,0,1967
+102,407,0,407,0,407,0,519
+103,407,0,407,0,407,0,139
+104,407,0,407,0,407,0,1839
+105,407,0,407,0,407,0,487
+106,407,0,407,0,407,0,131
+107,407,0,407,0,407,0,251
+108,407,0,407,0,407,0,935
+109,407,0,407,0,407,0,3519
+110,407,0,407,0,407,0,263
+111,407,0,407,0,407,0,1015
+112,407,0,407,0,407,0,3775
+113,407,0,407,0,407,0,1967
+114,407,0,407,0,407,0,519
+115,407,0,407,0,407,0,139
+116,407,0,407,0,407,0,1839
+117,407,0,407,0,407,0,487
+118,407,0,407,0,407,0,131
+119,407,0,407,0,407,0,251
+120,407,0,407,0,407,0,935
+121,0,19967,0,19967,0,19967,3519
+122,0,19967,0,19967,0,19967,263
+123,0,19967,0,19967,0,19967,1015
+124,0,19967,0,19967,0,19967,3775
+125,0,19967,0,19967,0,19967,1967
+126,0,19967,0,19967,0,19967,519
+127,0,19967,0,19967,0,19967,139
+128,0,19967,0,19967,0,19967,1839
+129,0,19967,0,19967,0,19967,487
+130,0,19967,0,19967,0,19967,131
+131,0,19967,0,19967,0,19967,251
+132,0,19967,0,19967,0,19967,935
+133,0,19967,0,19967,0,19967,3519
+134,0,19967,0,19967,0,19967,263
+135,0,19967,0,19967,0,19967,1015
+136,0,19967,0,19967,0,19967,3775
+137,0,19967,0,19967,0,19967,1967
+138,0,19967,0,19967,0,19967,519
+139,0,19967,0,19967,0,19967,139
+140,0,19967,0,19967,0,19967,1839
+141,0,19967,0,19967,0,19967,487
+142,0,19967,0,19967,0,19967,131
+143,0,19967,0,19967,0,19967,251
+144,0,19967,0,19967,0,19967,935
+145,0,19967,0,19967,0,19967,3519
+146,0,19967,0,19967,0,19967,263
+147,0,19967,0,19967,0,19967,1015
+148,0,19967,0,19967,0,19967,3775
+149,0,19967,0,19967,0,19967,1967
+150,0,19967,0,19967,0,19967,519
+151,0,40959,0,40959,0,40959,139
+152,0,40959,0,40959,0,40959,1839
+153,0,40959,0,40959,0,40959,487
+154,0,40959,0,40959,0,40959,131
+155,0,40959,0,40959,0,40959,251
+156,0,40959,0,40959,0,40959,935
+157,0,40959,0,40959,0,40959,3519
+158,0,40959,0,40959,0,40959,263
+159,0,40959,0,40959,0,40959,1015
+160,0,40959,0,40959,0,40959,3775
+161,0,40959,0,40959,0,40959,1967
+162,0,40959,0,40959,0,40959,519
+163,0,40959,0,40959,0,40959,139
+164,0,40959,0,40959,0,40959,1839
+165,0,40959,0,40959,0,40959,487
+166,0,40959,0,40959,0,40959,131
+167,0,40959,0,40959,0,40959,251
+168,0,40959,0,40959,0,40959,935
+169,0,40959,0,40959,0,40959,3519
+170,0,40959,0,40959,0,40959,263
+171,0,40959,0,40959,0,40959,1015
+172,0,40959,0,40959,0,40959,3775
+173,0,40959,0,40959,0,40959,1967
+174,0,40959,0,40959,0,40959,519
+175,0,40959,0,40959,0,40959,139
+176,0,40959,0,40959,0,40959,1839
+177,0,40959,0,40959,0,40959,487
+178,0,40959,0,40959,0,40959,131
+179,0,40959,0,40959,0,40959,93