imap-processing 0.6.0__py3-none-any.whl

imap_processing/tests/test_cli.py

@@ -0,0 +1,229 @@
+"""Tests coverage for imap_processing.cli."""
+
+import sys
+from unittest import mock
+
+import pytest
+import xarray as xr
+
+from imap_processing.cli import Codice, Hi, Hit, Ultra, _validate_args, main
+
+
+@pytest.fixture()
+def mock_instrument_dependencies():
+    with (
+        mock.patch("imap_processing.cli.imap_data_access.query") as mock_query,
+        mock.patch("imap_processing.cli.imap_data_access.download") as mock_download,
+        mock.patch("imap_processing.cli.imap_data_access.upload") as mock_upload,
+        mock.patch("imap_processing.cli.write_cdf") as mock_write_cdf,
+        mock.patch("imap_processing.cli.load_cdf") as mock_load_cdf,
+    ):
+        mocks = {
+            "mock_query": mock_query,
+            "mock_download": mock_download,
+            "mock_upload": mock_upload,
+            "mock_write_cdf": mock_write_cdf,
+            "mock_load_cdf": mock_load_cdf,
+        }
+        yield mocks
+
+
+@mock.patch("imap_processing.cli.Mag")
+def test_main(mock_instrument):
+    """Test imap_processing.cli.main()"""
+    test_args = [
+        "imap_cli",
+        "--instrument",
+        "mag",
+        "--dependency",
+        (
+            '[{"instrument": "mag", '
+            '"data_level": "l0", '
+            '"descriptor": "sci", '
+            '"version": "v001", '
+            '"start_date": "20240430"}]'
+        ),
+        "--data-level",
+        "l1a",
+        "--start-date",
+        "20240430",
+        "--end-date",
+        "20240501",
+        "--version",
+        "v001",
+        "--upload-to-sdc",
+    ]
+    with mock.patch.object(sys, "argv", test_args):
+        # Running without raising an exception is a pass.
+        # No asserts needed.
+        main()
+
+
+@pytest.mark.parametrize(
+    "instrument, data_level, raises_value_error",
+    [
+        ("mag", "l0", ""),
+        ("foo", "l0", "foo is not in the supported .*"),
+        ("codice", "l1z", "l1z is not a supported .*"),
+    ],
+)
+def test_validate_args(instrument, data_level, raises_value_error):
+    """Test coverage for imap_processing.cli._validate_args()"""
+    args = mock.Mock
+    args.instrument = instrument
+    args.data_level = data_level
+
+    if raises_value_error:
+        with pytest.raises(ValueError, match=raises_value_error):
+            _validate_args(args)
+    else:
+        _validate_args(args)
+
+
+@mock.patch("imap_processing.cli.codice_l1a.process_codice_l1a")
+def test_codice(mock_codice_l1a, mock_instrument_dependencies):
+    """Test coverage for cli.CoDICE class"""
+
+    test_dataset = xr.Dataset({}, attrs={"cdf_filename": "file0"})
+
+    mocks = mock_instrument_dependencies
+    mocks["mock_query"].return_value = [{"file_path": "/path/to/file0"}]
+    mocks["mock_download"].return_value = "file0"
+    mock_codice_l1a.return_value = test_dataset
+    mocks["mock_write_cdf"].side_effect = ["/path/to/file0"]
+
+    dependency_str = (
+        "[{"
+        "'instrument': 'codice',"
+        "'data_level': 'l0',"
+        "'descriptor': 'hskp',"
+        "'version': 'v001',"
+        "'start_date': '20230822'"
+        "}]"
+    )
+    instrument = Codice("l1a", dependency_str, "20230822", "20230822", "v001", True)
+    instrument.process()
+    assert mocks["mock_query"].call_count == 1
+    assert mocks["mock_download"].call_count == 1
+    assert mock_codice_l1a.call_count == 1
+    assert mocks["mock_upload"].call_count == 1
+
+
+@pytest.mark.parametrize("data_level, n_prods", [("l1a", 2), ("l1b", 1), ("l1c", 1)])
+def test_hi_l1(mock_instrument_dependencies, data_level, n_prods):
+    """Test coverage for cli.Hi class"""
+    mocks = mock_instrument_dependencies
+    mocks["mock_query"].return_value = [{"file_path": "/path/to/file0"}]
+    mocks["mock_download"].return_value = "file0"
+    mocks["mock_write_cdf"].side_effect = ["/path/to/file0", "/path/to/file1"]
+    mocks["mock_load_cdf"].return_value = xr.Dataset()
+
+    # patch autospec=True makes this test confirm that the function call in cli.py
+    # matches the mocked function signature.
+    with mock.patch(
+        f"imap_processing.cli.hi_{data_level}.hi_{data_level}", autospec=True
+    ) as mock_hi:
+        mock_hi.return_value = [f"{data_level}_file{n}" for n in range(n_prods)]
+        dependency_str = (
+            "[{"
+            "'instrument': 'lo',"
+            "'data_level': 'l0',"
+            "'descriptor': 'sci',"
+            "'version': 'v00-01',"
+            "'start_date': '20231212'"
+            "}]"
+        )
+        instrument = Hi(
+            data_level, dependency_str, "20231212", "20231213", "v005", True
+        )
+        instrument.process()
+        assert mocks["mock_query"].call_count == 1
+        assert mocks["mock_download"].call_count == 1
+        assert mock_hi.call_count == 1
+        assert mocks["mock_upload"].call_count == n_prods
+
+
+@mock.patch("imap_processing.cli.ultra_l1a.ultra_l1a")
+def test_ultra_l1a(mock_ultra_l1a, mock_instrument_dependencies):
+    """Test coverage for cli.Ultra class with l1a data level"""
+    mocks = mock_instrument_dependencies
+    mocks["mock_query"].return_value = [{"file_path": "/path/to/file0"}]
+    mocks["mock_download"].return_value = "dependency0"
+    mock_ultra_l1a.return_value = ["l1a_dataset0", "l1a_dataset1"]
+    mocks["mock_write_cdf"].side_effect = ["/path/to/product0", "/path/to/product1"]
+
+    dependency_str = (
+        "[{"
+        "'instrument': 'ultra',"
+        "'data_level': 'l0',"
+        "'descriptor': 'raw',"
+        "'version': 'v001',"
+        "'start_date': '20240207'"
+        "}]"
+    )
+    instrument = Ultra("l1a", dependency_str, "20240207", "20240208", "v001", True)
+    instrument.process()
+    assert mocks["mock_query"].call_count == 1
+    assert mocks["mock_download"].call_count == 1
+    assert mock_ultra_l1a.call_count == 1
+    assert mocks["mock_upload"].call_count == 2
+
+
+@mock.patch("imap_processing.cli.ultra_l1b.ultra_l1b")
+def test_ultra_l1b(mock_ultra_l1b, mock_instrument_dependencies):
+    """Test coverage for cli.Ultra class with l1b data level"""
+    mocks = mock_instrument_dependencies
+    mocks["mock_query"].return_value = [{"file_path": "/path/to/file0"}]
+    mocks["mock_download"].return_value = "dependency0"
+    mock_ultra_l1b.return_value = ["l1b_dataset0", "l1b_dataset1"]
+    mocks["mock_write_cdf"].side_effect = ["/path/to/product0", "/path/to/product1"]
+
+    instrument = Ultra("l1b", "[]", "20240207", "20240208", "v001", True)
+    instrument.process()
+    assert mocks["mock_query"].call_count == 0
+    assert mocks["mock_download"].call_count == 0
+    assert mock_ultra_l1b.call_count == 1
+    assert mocks["mock_upload"].call_count == 2
+
+
+@mock.patch("imap_processing.cli.ultra_l1c.ultra_l1c")
+def test_ultra_l1c(mock_ultra_l1c, mock_instrument_dependencies):
+    """Test coverage for cli.Ultra class with l1c data level"""
+    mocks = mock_instrument_dependencies
+    mocks["mock_query"].return_value = [{"file_path": "/path/to/file0"}]
+    mocks["mock_download"].return_value = "dependency0"
+    mock_ultra_l1c.return_value = ["l1c_dataset0", "l1c_dataset1"]
+    mocks["mock_write_cdf"].side_effect = ["/path/to/product0", "/path/to/product1"]
+
+    instrument = Ultra("l1c", "[]", "20240207", "20240208", "v001", True)
+    instrument.process()
+    assert mocks["mock_query"].call_count == 0
+    assert mocks["mock_download"].call_count == 0
+    assert mock_ultra_l1c.call_count == 1
+    assert mocks["mock_upload"].call_count == 2
+
+
+@mock.patch("imap_processing.cli.hit_l1a")
+def test_hit_l1a(mock_hit_l1a, mock_instrument_dependencies):
+    """Test coverage for cli.Hit class with l1a data level"""
+    mocks = mock_instrument_dependencies
+    mocks["mock_query"].return_value = [{"file_path": "/path/to/file0"}]
+    mocks["mock_download"].return_value = "dependency0"
+    mock_hit_l1a.return_value = ["l1a_dataset0", "l1a_dataset1"]
+    mocks["mock_write_cdf"].side_effect = ["/path/to/product0", "/path/to/product1"]
+
+    dependency_str = (
+        "[{"
+        "'instrument': 'hit',"
+        "'data_level': 'l0',"
+        "'descriptor': 'raw',"
+        "'version': 'v001',"
+        "'start_date': '20100105'"
+        "}]"
+    )
+    instrument = Hit("l1a", dependency_str, "20100105", "20100101", "v001", True)
+    instrument.process()
+    assert mocks["mock_query"].call_count == 1
+    assert mocks["mock_download"].call_count == 1
+    assert mock_hit_l1a.call_count == 1
+    assert mocks["mock_upload"].call_count == 2

imap_processing/tests/test_decom.py

@@ -0,0 +1,66 @@
+"""Tests aspects of the decommutation process that is generic to all instruments"""
+
+import pytest
+
+from imap_processing import imap_module_directory
+
+# Scrape the repo for all packet definition XML files
+xtce_document_list = imap_module_directory.glob("*/packet_definitions/*.xml")
+
+
+@pytest.mark.parametrize("xtce_document", xtce_document_list)
+def test_ccsds_header(xtce_document):
+    """Test if the XTCE document contains the proper CCSDS header information"""
+
+    # Define what is expected in the XTCE document
+    header_keys = [
+        ("VERSION", "3"),
+        ("TYPE", "1"),
+        ("SEC_HDR_FLG", "1"),
+        ("PKT_APID", "11"),
+        ("SEQ_FLGS", "2"),
+        ("SRC_SEQ_CTR", "14"),
+        ("PKT_LEN", "16"),
+    ]
+    header_entry_list = (
+        "<xtce:EntryList>"
+        '        <xtce:ParameterRefEntry parameterRef="VERSION" />'
+        '        <xtce:ParameterRefEntry parameterRef="TYPE" />'
+        '        <xtce:ParameterRefEntry parameterRef="SEC_HDR_FLG" />'
+        '        <xtce:ParameterRefEntry parameterRef="PKT_APID" />'
+        '        <xtce:ParameterRefEntry parameterRef="SEQ_FLGS" />'
+        '        <xtce:ParameterRefEntry parameterRef="SRC_SEQ_CTR" />'
+        '        <xtce:ParameterRefEntry parameterRef="PKT_LEN" />'
+        "</xtce:EntryList>"
+    )
+
+    # Read in the XTCE document
+    with open(xtce_document) as f:
+        document = f.read()
+
+    # Check that each header key is defined as a Parameter
+    for key in header_keys:
+        # Format for old XTCE generation format
+        old_xtce_gen = (
+            f'<xtce:Parameter name="{key[0]}" parameterTypeRef="UINT{key[1]}"'
+            in document
+            or f'<xtce:Parameter name="{key[0]}" parameterTypeRef="uint{key[1]}"'
+            in document
+        )
+        # Format for new XTCE generation format
+        new_xtce_gen = (
+            f'<xtce:IntegerParameterType name="{key[0]}" signed="false">'
+            f"\n\t\t\t\t"
+            f'<xtce:IntegerDataEncoding sizeInBits="{key[1]}" '
+            f'encoding="unsigned" />' in document
+        )
+
+        assert any((old_xtce_gen, new_xtce_gen))
+
+    # Check that the header is defined as a SequenceContainer in the XTCE file
+    # First remove discrepancies in whitespace, tabs, and newlines
+    header_entry_list = (
+        header_entry_list.replace(" ", "").replace("\t", "").replace("\n", "")
+    )
+    document = document.replace(" ", "").replace("\t", "").replace("\n", "")
+    assert header_entry_list in document

imap_processing/tests/test_quality_flags.py

@@ -0,0 +1,71 @@
+"""Test bitwise flagging."""
+
+import numpy as np
+
+from imap_processing.quality_flags import HitFlags, ImapLoFlags, ImapUltraFlags
+
+
+def test_quality_flags():
+    """Test the bitwise operations."""
+
+    # Test individual flags
+    assert HitFlags.NONE == 0x0
+    assert ImapUltraFlags.NONE == 0x0
+    assert ImapLoFlags.NONE == 0x0
+    assert HitFlags.INF == 2**0
+    assert ImapUltraFlags.INF == 2**0
+    assert ImapLoFlags.INF == 2**0
+    assert HitFlags.NEG == 2**1
+    assert ImapUltraFlags.NEG == 2**1
+    assert ImapLoFlags.NEG == 2**1
+    assert ImapUltraFlags.BADSPIN == 2**2
+    assert ImapLoFlags.BADSPIN == 2**2
+    assert ImapUltraFlags.FLAG1 == 2**3
+    assert ImapLoFlags.FLAG2 == 2**3
+    assert HitFlags.FLAG3 == 2**2
+
+    # Test combined flags for Ultra
+    flag = (
+        ImapUltraFlags.INF
+        | ImapUltraFlags.NEG
+        | ImapUltraFlags.BADSPIN
+        | ImapUltraFlags.FLAG1
+    )
+    assert flag & ImapUltraFlags.INF
+    assert flag & ImapUltraFlags.BADSPIN
+    assert flag & ImapUltraFlags.FLAG1
+    assert flag.name == "INF|NEG|BADSPIN|FLAG1"
+    assert flag.value == 15
+
+    # Test combined flags for Lo
+    flag = ImapLoFlags.INF | ImapLoFlags.NEG | ImapLoFlags.BADSPIN | ImapLoFlags.FLAG2
+    assert flag & ImapLoFlags.INF
+    assert flag & ImapLoFlags.BADSPIN
+    assert flag & ImapLoFlags.FLAG2
+    assert flag.name == "INF|NEG|BADSPIN|FLAG2"
+    assert flag.value == 15
+
+    # Test combined flags for HIT
+    flag = HitFlags.INF | HitFlags.NEG | HitFlags.FLAG3
+    assert flag & HitFlags.INF
+    assert flag & HitFlags.FLAG3
+    assert flag.name == "INF|NEG|FLAG3"
+    assert flag.value == 7
+
+    # Test use-case for Ultra
+    data = np.array([-6, np.inf, 2, 3])
+    quality = np.array(
+        [
+            ImapUltraFlags.INF | ImapUltraFlags.NEG,
+            ImapUltraFlags.INF,
+            ImapUltraFlags.NONE,
+            ImapUltraFlags.NONE,
+        ]
+    )
+    # Select data without INF flags
+    non_inf_mask = (quality & ImapUltraFlags.INF.value) == 0
+    np.array_equal(data[non_inf_mask], np.array([-6, 2, 3]))
+
+    # Select data without NEG or INF flags
+    non_neg_mask = (quality & ImapUltraFlags.NEG.value) == 0
+    np.array_equal(data[non_inf_mask & non_neg_mask], np.array([2, 3]))

imap_processing/tests/test_utils.py

@@ -0,0 +1,107 @@
+"""Tests coverage for imap_processing/utils.py"""
+
+import numpy as np
+import pandas as pd
+import pytest
+import xarray as xr
+
+from imap_processing import imap_module_directory, utils
+
+
+def test_convert_raw_to_eu(tmp_path):
+    """Test coverage for utils.convert_raw_to_eu()"""
+    # Generate a csv for testing
+    test_df = pd.DataFrame(
+        data={
+            "packetName": ["PACKET_0", "PACKET_0", "PACKET_1"],
+            "mnemonic": ["FIELD_0", "FIELD_1", "FIELD_0"],
+            "convertAs": ["UNSEGMENTED_POLY", "UNSEGMENTED_POLY", "FOO_METHOD"],
+            "c0": [0, 1, 2],
+            "c1": [0, 1, 2],
+            "c2": [0, 0, 2],
+            "c3": [0, 0, 2],
+            "c4": [0, 0, 2],
+            "c5": [0, 1, 2],
+            "c6": [0, 0, 2],
+            "c7": [0, 0, 2],
+            "unit": ["a", "b", "c"],
+        }
+    )
+    test_csv = tmp_path / "test_convert_table.csv"
+    with open(test_csv, "w") as f:
+        f.write("# Comment on first line of file\n")
+        test_df.to_csv(f, index=False)
+
+    # Generate a fake dataset for testing
+    n_packets = 3
+    field_0 = np.arange(n_packets)
+    field_1 = np.arange(n_packets) + 10
+    time = np.arange(n_packets) + 1000
+    dn_dataset = xr.Dataset(
+        data_vars=dict(
+            FIELD_0=(["time"], field_0),
+            FIELD_1=(["time"], field_1),
+        ),
+        coords=dict(
+            time=time,
+        ),
+    )
+
+    eu_dataset = utils.convert_raw_to_eu(
+        dn_dataset.copy(), test_csv.absolute(), "PACKET_0", comment="#"
+    )
+
+    # Check the converted values by manually doing the polynomial math
+    np.testing.assert_array_equal(eu_dataset["FIELD_0"].data, np.zeros(n_packets))
+    assert eu_dataset["FIELD_0"].attrs["units"] == test_df["unit"].iloc[0]
+
+    field_1_coeffs = (
+        test_df[test_df["mnemonic"] == "FIELD_1"].filter(regex=r"c\d").values[0]
+    )
+    field_1_compare = np.zeros(n_packets)
+    for p, coeff in enumerate(field_1_coeffs):
+        field_1_compare += coeff * np.power(field_1, p)
+    np.testing.assert_array_equal(eu_dataset["FIELD_1"].data, field_1_compare)
+
+    # Check that a ValueError is raised for unexpected conversion specified in
+    # conversion table "convertAs" column
+    with pytest.raises(ValueError, match=r"Unexpected conversion type: .*"):
+        eu_dataset = utils.convert_raw_to_eu(
+            dn_dataset.copy(), test_csv.absolute(), "PACKET_1", comment="#"
+        )
+
+
+@pytest.mark.parametrize(
+    "use_derived_value, expected_mode", [(True, "HVENG"), (False, 2)]
+)
+def test_packet_file_to_datasets(use_derived_value, expected_mode):
+    """
+    Test that all datatypes aren't all int64 and that we get
+    uint8/uint16 from header items as expected.
+
+    Test that we get multiple apids in the output.
+    """
+    test_file = "tests/swapi/l0_data/imap_swapi_l0_raw_20231012_v001.pkts"
+    packet_files = imap_module_directory / test_file
+    packet_definition = (
+        imap_module_directory / "swapi/packet_definitions/swapi_packet_definition.xml"
+    )
+    datasets_by_apid = utils.packet_file_to_datasets(
+        packet_files, packet_definition, use_derived_value=use_derived_value
+    )
+    # 3 apids in the test data
+    assert len(datasets_by_apid) == 3
+    data = datasets_by_apid[1188]
+    assert data["sec_hdr_flg"].dtype == np.uint8
+    assert data["pkt_apid"].dtype == np.uint16
+    np.testing.assert_array_equal(data["mode"], [expected_mode] * len(data["mode"]))
+
+
+def test_packet_file_to_datasets_flat_definition():
+    test_file = "tests/idex/imap_idex_l0_raw_20230725_v001.pkts"
+    packet_files = imap_module_directory / test_file
+    packet_definition = (
+        imap_module_directory / "idex/packet_definitions/idex_packet_definition.xml"
+    )
+    with pytest.raises(ValueError, match="Packet fields do not match"):
+        utils.packet_file_to_datasets(packet_files, packet_definition)

imap_processing/tests/ultra/test_data/l0/ultra45_raw_sc_auxdata_Ultra45_EM_SwRI_Cal_Run7_ThetaScan_20220530T225054.csv

@@ -0,0 +1,24 @@
+epoch,MET,Type,SequenceCount,SpinStartSeconds,SpinStartSubseconds,SpinDuration,SpinNumber,SpinDataTime,SpinPeriod,SpinPhase,SpinPeriodValid,SpinPhaseValid,SpinPeriodSource,CatbedHeater,Spins,IntegrationMultiplier1,IntegrationMultiplier2,HardwareMode,IMCEnb,LeftDeflection,RightDeflection
+2022-150T22:50:52.000, 391647054,AuxiliaryData,794,391647054,836,14001,26109934,391647055,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:51:06.000, 391647068,AuxiliaryData,795,391647068,837,13999,26109935,391647069,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:51:20.000, 391647082,AuxiliaryData,796,391647082,836,13999,26109936,391647083,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:51:34.000, 391647096,AuxiliaryData,797,391647096,835,14001,26109937,391647097,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:51:48.000, 391647110,AuxiliaryData,798,391647110,837,13998,26109938,391647111,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:52:02.000, 391647124,AuxiliaryData,799,391647124,835,14001,26109939,391647125,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:52:16.000, 391647138,AuxiliaryData,800,391647138,837,13998,26109940,391647139,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:52:30.000, 391647152,AuxiliaryData,801,391647152,836,13999,26109941,391647153,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:52:44.000, 391647166,AuxiliaryData,802,391647166,835,14002,26109942,391647167,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:52:58.000, 391647180,AuxiliaryData,803,391647180,837,13999,26109943,391647181,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:53:12.000, 391647194,AuxiliaryData,804,391647194,836,14000,26109944,391647195,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:53:26.000, 391647208,AuxiliaryData,805,391647208,836,13999,26109945,391647209,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:53:40.000, 391647222,AuxiliaryData,806,391647222,836,13999,26109946,391647223,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:53:54.000, 391647236,AuxiliaryData,807,391647236,835,14001,26109947,391647237,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:54:08.000, 391647250,AuxiliaryData,808,391647250,837,13998,26109948,391647251,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:54:22.000, 391647264,AuxiliaryData,809,391647264,835,14001,26109949,391647265,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:54:36.000, 391647278,AuxiliaryData,810,391647278,837,13999,26109950,391647279,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:54:50.000, 391647292,AuxiliaryData,811,391647292,836,14000,26109951,391647293,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:55:04.000, 391647306,AuxiliaryData,812,391647306,836,14000,26109952,391647307,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:55:18.000, 391647320,AuxiliaryData,813,391647320,836,13999,26109953,391647321,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:55:32.000, 391647334,AuxiliaryData,814,391647334,836,13999,26109954,391647335,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:55:46.000, 391647348,AuxiliaryData,815,391647348,835,14001,26109955,391647349,43750,766,0,1,1,0,1,1,1,0,0,0,0
+2022-150T22:56:00.000, 391647362,AuxiliaryData,816,391647362,836,14000,26109956,391647363,43750,766,0,1,1,0,1,1,1,0,0,0,0