imap-processing 0.6.0__py3-none-any.whl

imap_processing/tests/ultra/unit/conftest.py

@@ -0,0 +1,210 @@
+"""Pytest plugin module for test data paths"""
+
+import sys
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+from imap_processing import decom
+from imap_processing.ultra.l0.decom_ultra import process_ultra_apids
+from imap_processing.ultra.l0.ultra_utils import (
+    ULTRA_AUX,
+    ULTRA_EVENTS,
+)
+from imap_processing.ultra.l1a import ultra_l1a
+from imap_processing.utils import group_by_apid
+
+
+@pytest.fixture()
+def ccsds_path():
+    """Returns the ccsds directory."""
+    return (
+        Path(sys.modules[__name__.split(".")[0]].__file__).parent
+        / "tests"
+        / "ultra"
+        / "test_data"
+        / "l0"
+        / "Ultra45_EM_SwRI_Cal_Run7_ThetaScan_20220530T225054.CCSDS"
+    )
+
+
+@pytest.fixture()
+def ccsds_path_events():
+    """Returns the ccsds directory."""
+    return (
+        Path(sys.modules[__name__.split(".")[0]].__file__).parent
+        / "tests"
+        / "ultra"
+        / "test_data"
+        / "l0"
+        / "FM45_7P_Phi0.0_BeamCal_LinearScan_phi0.04_theta-0.01_20230821T121304.CCSDS"
+    )
+
+
+@pytest.fixture()
+def ccsds_path_theta_0():
+    """Returns the ccsds directory."""
+    return (
+        Path(sys.modules[__name__.split(".")[0]].__file__).parent
+        / "tests"
+        / "ultra"
+        / "test_data"
+        / "l0"
+        / "FM45_40P_Phi28p5_BeamCal_LinearScan_phi28.50_theta-0.00"
+        "_20240207T102740.CCSDS"
+    )
+
+
+@pytest.fixture()
+def ccsds_path_tof():
+    """Returns the ccsds directory."""
+    return (
+        Path(sys.modules[__name__.split(".")[0]].__file__).parent
+        / "tests"
+        / "ultra"
+        / "test_data"
+        / "l0"
+        / "FM45_TV_Cycle6_Hot_Ops_Front212_20240124T063837.CCSDS"
+    )
+
+
+@pytest.fixture()
+def xtce_path():
+    """Returns the xtce image rates directory."""
+    return (
+        Path(sys.modules[__name__.split(".")[0]].__file__).parent
+        / "ultra"
+        / "packet_definitions"
+        / "ULTRA_SCI_COMBINED.xml"
+    )
+
+
+@pytest.fixture()
+def rates_test_path():
+    """Returns the xtce image rates test data directory."""
+    filename = (
+        "ultra45_raw_sc_ultraimgrates_Ultra45_EM_SwRI_Cal_Run7_ThetaScan_"
+        "20220530T225054.csv"
+    )
+    return (
+        Path(sys.modules[__name__.split(".")[0]].__file__).parent
+        / "tests"
+        / "ultra"
+        / "test_data"
+        / "l0"
+        / filename
+    )
+
+
+@pytest.fixture()
+def aux_test_path():
+    """Returns the xtce auxiliary test data directory."""
+    filename = (
+        "ultra45_raw_sc_auxdata_Ultra45_EM_SwRI_Cal_Run7_ThetaScan_"
+        "20220530T225054.csv"
+    )
+    return (
+        Path(sys.modules[__name__.split(".")[0]].__file__).parent
+        / "tests"
+        / "ultra"
+        / "test_data"
+        / "l0"
+        / filename
+    )
+
+
+@pytest.fixture()
+def events_test_path():
+    """Returns the xtce auxiliary test data directory."""
+    filename = (
+        "ultra45_raw_sc_ultrarawimgevent_FM45_7P_Phi00_BeamCal_"
+        "LinearScan_phi004_theta-001_20230821T121304.csv"
+    )
+    return (
+        Path(sys.modules[__name__.split(".")[0]].__file__).parent
+        / "tests"
+        / "ultra"
+        / "test_data"
+        / "l0"
+        / filename
+    )
+
+
+@pytest.fixture()
+def tof_test_path():
+    """Returns the xtce auxiliary test data directory."""
+    filename = (
+        "ultra45_raw_sc_enaphxtofhangimg_FM45_TV_Cycle6_Hot_Ops_"
+        "Front212_20240124T063837.csv"
+    )
+    return (
+        Path(sys.modules[__name__.split(".")[0]].__file__).parent
+        / "tests"
+        / "ultra"
+        / "test_data"
+        / "l0"
+        / filename
+    )
+
+
+@pytest.fixture()
+def decom_test_data(request, xtce_path):
+    """Read test data from file"""
+    apid = request.param["apid"]
+    filename = request.param["filename"]
+    ccsds_path = (
+        Path(sys.modules[__name__.split(".")[0]].__file__).parent
+        / "tests"
+        / "ultra"
+        / "test_data"
+        / "l0"
+        / filename
+    )
+
+    packets = decom.decom_packets(ccsds_path, xtce_path)
+    grouped_data = group_by_apid(packets)
+
+    data_packet_list = process_ultra_apids(grouped_data[apid], apid)
+    return data_packet_list, packets
+
+
+@pytest.fixture()
+def events_fsw_comparison_theta_0():
+    """FSW test data."""
+    filename = (
+        "ultra45_raw_sc_ultrarawimg_withFSWcalcs_FM45_40P_Phi28p5_"
+        "BeamCal_LinearScan_phi2850_theta-000_20240207T102740.csv"
+    )
+    return (
+        Path(sys.modules[__name__.split(".")[0]].__file__).parent
+        / "tests"
+        / "ultra"
+        / "test_data"
+        / "l0"
+        / filename
+    )
+
+
+@pytest.fixture()
+def de_dataset(ccsds_path_theta_0, xtce_path):
+    """L1A test data"""
+    packets = decom.decom_packets(ccsds_path_theta_0, xtce_path)
+    grouped_data = group_by_apid(packets)
+    decom_ultra_events = process_ultra_apids(
+        grouped_data[ULTRA_EVENTS.apid[0]], ULTRA_EVENTS.apid[0]
+    )
+    decom_ultra_aux = process_ultra_apids(
+        grouped_data[ULTRA_AUX.apid[0]], ULTRA_AUX.apid[0]
+    )
+    dataset = ultra_l1a.create_dataset(
+        {
+            ULTRA_EVENTS.apid[0]: decom_ultra_events,
+            ULTRA_AUX.apid[0]: decom_ultra_aux,
+        }
+    )
+    # Remove start_type with fill values
+    l1a_de_dataset = dataset.where(
+        dataset["START_TYPE"] != np.iinfo(np.int64).min, drop=True
+    )
+    return l1a_de_dataset

imap_processing/tests/ultra/unit/test_decom_apid_880.py

@@ -0,0 +1,98 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+from imap_processing.ultra.l0.ultra_utils import ULTRA_AUX
+from imap_processing.utils import group_by_apid
+
+
+@pytest.mark.parametrize(
+    "decom_test_data",
+    [
+        pytest.param(
+            {
+                "apid": ULTRA_AUX.apid[0],
+                "filename": "Ultra45_EM_SwRI_Cal_Run7_"
+                "ThetaScan_20220530T225054.CCSDS",
+            }
+        )
+    ],
+    indirect=True,
+)
+def test_aux_enumerated(decom_test_data):
+    """Test if enumerated values derived correctly"""
+
+    _, packets = decom_test_data
+
+    count = 0  # count number of packets with APID 880
+    total_packets = 23
+
+    grouped_data = group_by_apid(packets)
+    apid_data = grouped_data[880]
+
+    for packet in apid_data:
+        assert packet.data["SPINPERIODVALID"].derived_value == "INVALID"
+        assert packet.data["SPINPHASEVALID"].derived_value == "VALID"
+        assert packet.data["SPINPERIODSOURCE"].derived_value == "NOMINAL"
+        assert packet.data["CATBEDHEATERFLAG"].derived_value == "UNFLAGGED"
+        count += 1
+
+    assert count == total_packets
+
+
+@pytest.mark.parametrize(
+    "decom_test_data",
+    [
+        pytest.param(
+            {
+                "apid": ULTRA_AUX.apid[0],
+                "filename": "Ultra45_EM_SwRI_Cal_Run7_"
+                "ThetaScan_20220530T225054.CCSDS",
+            }
+        )
+    ],
+    indirect=True,
+)
+def test_aux_mode(decom_test_data):
+    """Test if enumerated values derived correctly"""
+
+    _, packets = decom_test_data
+
+    for packet in packets:
+        if packet.header["PKT_APID"].derived_value == 880:
+            assert packet.data["HWMODE"].derived_value == "MODE0"
+            assert packet.data["IMCENB"].derived_value == "MODE0"
+            assert packet.data["LEFTDEFLECTIONCHARGE"].derived_value == "MODE0"
+            assert packet.data["RIGHTDEFLECTIONCHARGE"].derived_value == "MODE0"
+
+
+@pytest.mark.parametrize(
+    "decom_test_data",
+    [
+        pytest.param(
+            {
+                "apid": ULTRA_AUX.apid[0],
+                "filename": "Ultra45_EM_SwRI_Cal_Run7_"
+                "ThetaScan_20220530T225054.CCSDS",
+            }
+        )
+    ],
+    indirect=True,
+)
+def test_aux_decom(decom_test_data, aux_test_path):
+    """This function reads validation data and checks that
+    decom data matches validation data for auxiliary packet"""
+
+    decom_ultra, _ = decom_test_data
+
+    df = pd.read_csv(aux_test_path, index_col="MET")
+
+    np.testing.assert_array_equal(df.SpinStartSeconds, decom_ultra["TIMESPINSTART"])
+    np.testing.assert_array_equal(
+        df.SpinStartSubseconds, decom_ultra["TIMESPINSTARTSUB"]
+    )
+    np.testing.assert_array_equal(df.SpinDuration, decom_ultra["DURATION"])
+    np.testing.assert_array_equal(df.SpinNumber, decom_ultra["SPINNUMBER"])
+    np.testing.assert_array_equal(df.SpinDataTime, decom_ultra["TIMESPINDATA"])
+    np.testing.assert_array_equal(df.SpinPeriod, decom_ultra["SPINPERIOD"])
+    np.testing.assert_array_equal(df.SpinPhase, decom_ultra["SPINPHASE"])

imap_processing/tests/ultra/unit/test_decom_apid_881.py

@@ -0,0 +1,50 @@
+import json
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from imap_processing.ultra.l0.ultra_utils import RATES_KEYS, ULTRA_RATES
+
+
+@pytest.mark.parametrize(
+    "decom_test_data",
+    [
+        pytest.param(
+            {
+                "apid": ULTRA_RATES.apid[0],
+                "filename": "Ultra45_EM_SwRI_Cal_Run7_"
+                "ThetaScan_20220530T225054.CCSDS",
+            }
+        )
+    ],
+    indirect=True,
+)
+def test_image_rate_decom(decom_test_data, rates_test_path):
+    """This function reads validation data and checks that decom data
+    matches validation data for image rate packet"""
+    decom_ultra, _ = decom_test_data
+
+    df = pd.read_csv(rates_test_path, index_col="MET")
+    total_packets = 23
+
+    np.testing.assert_array_equal(df.SID, decom_ultra["SID"])
+    np.testing.assert_array_equal(df.Spin, decom_ultra["SPIN"])
+    np.testing.assert_array_equal(df.AbortFlag, decom_ultra["ABORTFLAG"])
+    np.testing.assert_array_equal(df.StartDelay, decom_ultra["STARTDELAY"])
+
+    # Spot-check first packet
+    t0 = decom_ultra["SHCOARSE"][0]
+    expected_arr0 = json.loads(df.loc[t0].Counts)
+    arr = []
+    for name in RATES_KEYS:
+        arr.append(decom_ultra[name][0])
+    assert expected_arr0 == arr
+
+    # Spot-check last packet
+    tn = decom_ultra["SHCOARSE"][total_packets - 1]
+    expected_arrn = json.loads(df.loc[tn].Counts)
+    arr = []
+    for name in RATES_KEYS:
+        arr.append(decom_ultra[name][total_packets - 1])
+    assert expected_arrn == arr

imap_processing/tests/ultra/unit/test_decom_apid_883.py

@@ -0,0 +1,44 @@
+import json
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from imap_processing.ultra.l0.ultra_utils import ULTRA_TOF
+
+# TODO: discuss with instrument team incomplete set of SIDs
+
+
+@pytest.mark.parametrize(
+    "decom_test_data",
+    [
+        pytest.param(
+            {
+                "apid": ULTRA_TOF.apid[0],
+                "filename": "FM45_TV_Cycle6_Hot_Ops_" "Front212_20240124T063837.CCSDS",
+            }
+        )
+    ],
+    indirect=True,
+)
+def test_tof_decom(decom_test_data, tof_test_path):
+    """This function reads validation data and checks that decom data
+    matches validation data for image rate packet"""
+
+    decom_ultra, _ = decom_test_data
+    df = pd.read_csv(tof_test_path, index_col="SequenceCount")
+
+    np.testing.assert_array_equal(df.Spin, decom_ultra["SPIN"].flatten())
+    np.testing.assert_array_equal(df.AbortFlag, decom_ultra["ABORTFLAG"].flatten())
+    np.testing.assert_array_equal(df.StartDelay, decom_ultra["STARTDELAY"].flatten())
+    assert json.loads(df["P00s"].values[0])[0] == decom_ultra["P00"][0][0]
+
+    for count in df.index.get_level_values("SequenceCount").values:
+        df_data = df[df.index.get_level_values("SequenceCount") == count].Images.values[
+            0
+        ]
+        rows, cols = np.where(decom_ultra["SRC_SEQ_CTR"] == count)
+        decom_data = decom_ultra["PACKETDATA"][rows[0]][cols[0]]
+        df_data_array = np.array(json.loads(df_data)[0])
+
+        np.testing.assert_array_equal(df_data_array, decom_data)

imap_processing/tests/ultra/unit/test_decom_apid_896.py

@@ -0,0 +1,104 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+from imap_processing.ultra.l0.ultra_utils import ULTRA_EVENTS
+
+
+@pytest.mark.parametrize(
+    "decom_test_data",
+    [
+        pytest.param(
+            {
+                "apid": ULTRA_EVENTS.apid[0],
+                "filename": "FM45_7P_Phi0.0_BeamCal_LinearScan_phi0.04"
+                "_theta-0.01_20230821T121304.CCSDS",
+            }
+        )
+    ],
+    indirect=True,
+)
+def test_image_raw_events_decom(
+    decom_test_data, events_test_path, ccsds_path_events, xtce_path
+):
+    """This function reads validation data and checks that decom data
+    matches validation data for image rate packet"""
+    decom_ultra, _ = decom_test_data
+
+    df = pd.read_csv(events_test_path, index_col="MET")
+    df.replace(-1, np.iinfo(np.int64).min, inplace=True)
+
+    np.testing.assert_array_equal(df.SID, decom_ultra["SID"])
+    np.testing.assert_array_equal(df["Spin"], decom_ultra["SPIN"])
+    np.testing.assert_array_equal(df["AbortFlag"], decom_ultra["ABORTFLAG"])
+    np.testing.assert_array_equal(df["StartDelay"], decom_ultra["STARTDELAY"])
+    np.testing.assert_array_equal(df["Count"], decom_ultra["COUNT"])
+    np.testing.assert_array_equal(df["CoinType"], decom_ultra["COIN_TYPE"])
+    np.testing.assert_array_equal(df["StartType"], decom_ultra["START_TYPE"])
+    np.testing.assert_array_equal(df["StopType"], decom_ultra["STOP_TYPE"])
+    np.testing.assert_array_equal(df["StartPosTDC"], decom_ultra["START_POS_TDC"])
+    np.testing.assert_array_equal(df["StopNorthTDC"], decom_ultra["STOP_NORTH_TDC"])
+    np.testing.assert_array_equal(df["StopEastTDC"], decom_ultra["STOP_EAST_TDC"])
+    np.testing.assert_array_equal(df["StopSouthTDC"], decom_ultra["STOP_SOUTH_TDC"])
+    np.testing.assert_array_equal(df["StopWestTDC"], decom_ultra["STOP_WEST_TDC"])
+    np.testing.assert_array_equal(df["CoinNorthTDC"], decom_ultra["COIN_NORTH_TDC"])
+    np.testing.assert_array_equal(df["CoinSouthTDC"], decom_ultra["COIN_SOUTH_TDC"])
+    np.testing.assert_array_equal(
+        df["CoinDiscreteTDC"], decom_ultra["COIN_DISCRETE_TDC"]
+    )
+    np.testing.assert_array_equal(df["EnergyOrPH"], decom_ultra["ENERGY_PH"])
+    np.testing.assert_array_equal(df["PulseWidth"], decom_ultra["PULSE_WIDTH"])
+    np.testing.assert_array_equal(df["PhaseAngle"], decom_ultra["PHASE_ANGLE"])
+    np.testing.assert_array_equal(df["Bin"], decom_ultra["BIN"])
+
+
+@pytest.mark.parametrize(
+    "decom_test_data",
+    [
+        pytest.param(
+            {
+                "apid": ULTRA_EVENTS.apid[0],
+                "filename": "FM45_7P_Phi0.0_BeamCal_LinearScan_phi0.04"
+                "_theta-0.01_20230821T121304.CCSDS",
+            }
+        )
+    ],
+    indirect=True,
+)
+def test_image_raw_events_decom_flags(decom_test_data, events_test_path):
+    """This function reads validation data and checks that decom data
+    matches validation data for image rate packet"""
+
+    decom_ultra, _ = decom_test_data
+    df = pd.read_csv(events_test_path, index_col="MET")
+    df.replace(-1, np.iinfo(np.int64).min, inplace=True)
+
+    np.testing.assert_array_equal(df["CnT"], decom_ultra["EVENT_FLAG_CNT"])
+    np.testing.assert_array_equal(df["PHCmpSL"], decom_ultra["EVENT_FLAG_PHCMPSL"])
+    np.testing.assert_array_equal(df["PHCmpSR"], decom_ultra["EVENT_FLAG_PHCMPSR"])
+    np.testing.assert_array_equal(df["PHCmpCD"], decom_ultra["EVENT_FLAG_PHCMPCD"])
+    np.testing.assert_array_equal(df["SSDS7"], decom_ultra["SSD_FLAG_7"])
+    np.testing.assert_array_equal(df["SSDS6"], decom_ultra["SSD_FLAG_6"])
+    np.testing.assert_array_equal(df["SSDS5"], decom_ultra["SSD_FLAG_5"])
+    np.testing.assert_array_equal(df["SSDS4"], decom_ultra["SSD_FLAG_4"])
+    np.testing.assert_array_equal(df["SSDS3"], decom_ultra["SSD_FLAG_3"])
+    np.testing.assert_array_equal(df["SSDS2"], decom_ultra["SSD_FLAG_2"])
+    np.testing.assert_array_equal(df["SSDS1"], decom_ultra["SSD_FLAG_1"])
+    np.testing.assert_array_equal(df["SSDS0"], decom_ultra["SSD_FLAG_0"])
+    np.testing.assert_array_equal(df["CFDCoinTN"], decom_ultra["CFD_FLAG_COINTN"])
+    np.testing.assert_array_equal(df["CFDCoinBN"], decom_ultra["CFD_FLAG_COINBN"])
+    np.testing.assert_array_equal(df["CFDCoinTS"], decom_ultra["CFD_FLAG_COINTS"])
+    np.testing.assert_array_equal(df["CFDCoinBS"], decom_ultra["CFD_FLAG_COINBS"])
+    np.testing.assert_array_equal(df["CFDCoinD"], decom_ultra["CFD_FLAG_COIND"])
+    np.testing.assert_array_equal(df["CFDStartRF"], decom_ultra["CFD_FLAG_STARTRF"])
+    np.testing.assert_array_equal(df["CFDStartLF"], decom_ultra["CFD_FLAG_STARTLF"])
+    np.testing.assert_array_equal(df["CFDStartRP"], decom_ultra["CFD_FLAG_STARTRP"])
+    np.testing.assert_array_equal(df["CFDStartLP"], decom_ultra["CFD_FLAG_STARTLP"])
+    np.testing.assert_array_equal(df["CFDStopTN"], decom_ultra["CFD_FLAG_STOPTN"])
+    np.testing.assert_array_equal(df["CFDStopBN"], decom_ultra["CFD_FLAG_STOPBN"])
+    np.testing.assert_array_equal(df["CFDStopTE"], decom_ultra["CFD_FLAG_STOPTE"])
+    np.testing.assert_array_equal(df["CFDStopBE"], decom_ultra["CFD_FLAG_STOPBE"])
+    np.testing.assert_array_equal(df["CFDStopTS"], decom_ultra["CFD_FLAG_STOPTS"])
+    np.testing.assert_array_equal(df["CFDStopBS"], decom_ultra["CFD_FLAG_STOPBS"])
+    np.testing.assert_array_equal(df["CFDStopTW"], decom_ultra["CFD_FLAG_STOPTW"])
+    np.testing.assert_array_equal(df["CFDStopBW"], decom_ultra["CFD_FLAG_STOPBW"])

imap_processing/tests/ultra/unit/test_lookup_utils.py

@@ -0,0 +1,68 @@
+import numpy as np
+import pandas as pd
+
+from imap_processing import imap_module_directory
+from imap_processing.ultra.l1b.lookup_utils import (
+    get_back_position,
+    get_energy_norm,
+    get_image_params,
+    get_norm,
+    get_y_adjust,
+)
+
+BASE_PATH = imap_module_directory / "ultra" / "lookup_tables"
+
+
+def test_get_y_adjust():
+    """Tests function get_y_adjust."""
+
+    yadjust_path = BASE_PATH / "yadjust.csv"
+    yadjust_df = pd.read_csv(yadjust_path).set_index("dYLUT")
+
+    array = np.array([8])
+    res = get_y_adjust(array)
+
+    assert res == yadjust_df["dYAdj"][8]
+
+
+def test_get_stop_norm():
+    """Tests function get_stop_norm."""
+
+    tdc_norm_path = BASE_PATH / "ultra45_tdc_norm.csv"
+    tdc_norm_df = pd.read_csv(tdc_norm_path, header=1)
+
+    array = np.array([378])
+    stop_norm = get_norm(array, "SpE", "ultra45")
+
+    assert stop_norm == tdc_norm_df["SpE"][378]
+
+
+def test_get_back_position():
+    """Tests function get_back_position."""
+
+    back_pos_path = BASE_PATH / "ultra45_back-pos-luts.csv"
+    back_pos_df = pd.read_csv(back_pos_path, index_col="Index_offset")
+
+    array = np.array([-2000])
+    dn_converted = get_back_position(array, "XBkBt", "ultra45")
+
+    assert dn_converted == back_pos_df["XBkBt"].iloc[-2000]
+
+
+def test_get_egy_norm():
+    """Tests function get_egy_norm."""
+
+    egy_norm_path = BASE_PATH / "EgyNorm.mem.csv"
+    egy_norm_df = pd.read_csv(egy_norm_path)
+
+    norm_composite_energy = get_energy_norm(np.array([2]), np.array([2]))
+
+    assert int(norm_composite_energy) == egy_norm_df.iloc[2 * 4096 + 2]["NormEnergy"]
+
+
+def test_get_image_params():
+    """Tests function get_image_params."""
+
+    image_params = get_image_params("XFTLTOFF")
+
+    assert image_params == 49.3