imap-processing 0.12.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

imap_processing/tests/lo/test_lo_l1b.py

@@ -1,4 +1,5 @@
 from collections import namedtuple
+from unittest.mock import patch
 
 import numpy as np
 import pytest
@@ -8,17 +9,29 @@ from imap_processing import imap_module_directory
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.cdf.utils import load_cdf
 from imap_processing.lo.l1b.lo_l1b import (
+    calculate_tof1_for_golden_triples,
     convert_start_end_acq_times,
+    convert_tofs_to_eu,
     create_datasets,
     get_avg_spin_durations,
     get_spin_angle,
+    get_spin_start_times,
+    identify_species,
     initialize_l1b_de,
     lo_l1b,
+    set_bad_times,
+    set_coincidence_type,
+    set_each_event_epoch,
+    set_event_met,
+    set_pointing_bin,
+    set_pointing_direction,
     set_spin_bin,
+    set_spin_cycle,
 )
+from imap_processing.spice.time import met_to_ttj2000ns
 
 
-@pytest.fixture()
+@pytest.fixture
 def dependencies():
     return {
         "imap_lo_l1a_de": load_cdf(
@@ -32,16 +45,24 @@ def dependencies():
     }
 
 
-@pytest.fixture()
+@pytest.fixture
 def attr_mgr_l1b():
     attr_mgr_l1b = ImapCdfAttributes()
     attr_mgr_l1b.add_instrument_global_attrs(instrument="lo")
     attr_mgr_l1b.add_instrument_variable_attrs(instrument="lo", level="l1b")
-    attr_mgr_l1b.add_global_attribute("Data_version", "000")
     return attr_mgr_l1b
 
 
-def test_lo_l1b():
+@pytest.fixture
+def attr_mgr_l1a():
+    attr_mgr = ImapCdfAttributes()
+    attr_mgr.add_instrument_global_attrs(instrument="lo")
+    attr_mgr.add_instrument_variable_attrs(instrument="lo", level="l1a")
+    return attr_mgr
+
+
+@patch("imap_processing.lo.l1b.lo_l1b.instrument_pointing")
+def test_lo_l1b(mock_instrument_pointing):
     # Arrange
     de_file = (
         imap_module_directory / "tests/lo/test_cdfs/imap_lo_l1a_de_20241022_v002.cdf"
@@ -55,13 +76,16 @@ def test_lo_l1b():
         data[dataset.attrs["Logical_source"]] = dataset
 
     expected_logical_source = "imap_lo_l1b_de"
+    mock_instrument_pointing.return_value = np.zeros((2000, 2))
     # Act
-    output_file = lo_l1b(data, "001")
+    output_file = lo_l1b(data)
 
     # Assert
     assert expected_logical_source == output_file[0].attrs["Logical_source"]
 
 
+# @pytest.mark.external_kernel
+# @pytest.mark.use_test_metakernel("imap_ena_sim_metakernel.template")
 def test_create_datasets():
     attr_mgr = ImapCdfAttributes()
     attr_mgr.add_instrument_global_attrs(instrument="lo")
@@ -201,7 +225,6 @@ def test_get_spin_angle():
         spin_angle,
         spin_angle_expected,
         atol=1e-2,
-        err_msg=f"Spin angle: {spin_angle} vs {spin_angle_expected}",
     )
 
 
@@ -216,3 +239,345 @@ def test_spin_bin():
 
     # Assert
     np.testing.assert_array_equal(l1b_de["spin_bin"], expected_spin_bins)
+
+
+def test_spin_cycle():
+    # Arrange
+    de = xr.Dataset(
+        {
+            "de_count": ("epoch", [2, 3]),
+            "esa_step": ("direct_event", [1, 2, 3, 4, 5]),
+        },
+        coords={"epoch": [0, 1], "direct_event": [1, 2, 3, 4, 5]},
+    )
+
+    # spin_cycle = spin_start + 7 + (esa_step - 1) * 2
+    # where spin start is the spin number for the first spin
+    # in an Aggregated Science Cycle (first spin number of an epoch)
+    # and esa_step is the esa_step for a direct event
+    spin_cycle_expected = np.array([7, 9, 39, 41, 43])
+    spin_cycle_data = xr.Dataset()
+
+    # Act
+    spin_cycle_data = set_spin_cycle(de, spin_cycle_data)
+
+    # Assert
+    np.testing.assert_array_equal(spin_cycle_data["spin_cycle"], spin_cycle_expected)
+
+
+def test_get_spin_start_times():
+    # Arrange
+    l1b_de = xr.Dataset(
+        {
+            "spin_cycle": ("epoch", [0, 1, 2, 3, 4]),
+        },
+        coords={
+            "epoch": [
+                0,
+                1,
+                2,
+                3,
+                4,
+            ]
+        },
+    )
+    l1a_de = xr.Dataset(
+        {
+            "de_count": ("epoch", [2, 3]),
+            "met": ("direct_event", [0, 1, 2, 3, 4]),
+            "de_time": ("direct_event", [0000, 1000, 2000, 3000, 4000]),
+        },
+        coords={"epoch": [0, 1], "direct_event": [0, 1, 2, 3, 4]},
+    )
+    spin = xr.Dataset(
+        {
+            "start_sec_spin": (
+                ["epoch", "spin"],
+                [[20, 25, 30, 35, 40], [45, 50, 55, 60, 65]],
+            ),
+            "start_subsec_spin": (
+                ["epoch", "spin"],
+                [[2000, 3000, 4000, 5000, 6000], [1000, 1500, 2000, 3000, 4000]],
+            ),
+        }
+    )
+
+    end_acq = xr.DataArray([0, 1], dims="epoch")
+    spin_start_times_expected = np.array([20.002, 50.0015, 55.002, 60.003, 65.004])
+    spin_start_times = get_spin_start_times(l1a_de, l1b_de, spin, end_acq)
+
+    np.testing.assert_allclose(
+        spin_start_times,
+        spin_start_times_expected,
+        atol=1e-4,
+    )
+
+
+def test_set_event_met():
+    # Arrange
+    l1b_de = xr.Dataset()
+    l1a_de = xr.Dataset(
+        {
+            "de_count": ("epoch", [2, 3]),
+            "de_time": ("direct_event", [0000, 1000, 2000, 3000, 4000]),
+        },
+        coords={
+            "epoch": [0, 1],
+            "direct_event": [
+                0,
+                1,
+                2,
+                3,
+                4,
+            ],
+        },
+    )
+    avg_spin_durations = xr.DataArray([5, 10])
+    spin_start_times = xr.DataArray([10, 20, 30, 40, 50])
+    expected_event_met = np.array([10, 21.2207, 34.8828, 47.3242, 59.7656])
+
+    # Act
+    l1b_de = set_event_met(l1a_de, l1b_de, spin_start_times, avg_spin_durations)
+
+    # Assert
+    np.testing.assert_allclose(
+        l1b_de["event_met"].values,
+        expected_event_met,
+        atol=1e-4,
+    )
+
+    def test_set_each_event_epoch():
+        l1b_de = xr.Dataset(
+            {
+                "event_met": ("epoch", [10, 20, 30, 40, 50]),
+            },
+            coords={
+                "epoch": [0, 1, 2, 3, 4],
+            },
+        )
+        epoch_expected = met_to_ttj2000ns(np.array([10, 20, 30, 40, 50]))
+
+        l1b_de = set_each_event_epoch(l1b_de)
+
+        np.testing.assert_allclose(
+            l1b_de["epoch"].values,
+            epoch_expected,
+            atol=1e-4,
+        )
+
+
+def test_calculate_tof1_for_golden_triples():
+    # Arrange
+    l1a_de = xr.Dataset(
+        {
+            "coincidence_type": ("epoch", [0, 0, 0]),
+            "mode": ("epoch", [0, 0, 1]),
+            "tof0": ("epoch", [2, 4, 2]),
+            "tof1": ("epoch", [0, 0, 0]),
+            "tof2": ("epoch", [2, 6, 2]),
+            "tof3": ("epoch", [2, 8, 2]),
+            "cksm": ("epoch", [2, 12, 2]),
+        }
+    )
+
+    l1a_de_expected = xr.Dataset(
+        {
+            "coincidence_type": ("epoch", [0, 0, 0]),
+            "mode": ("epoch", [0, 0, 1]),
+            "tof0": ("epoch", [2, 4, 2]),
+            "tof1": ("epoch", [42, 36, 0]),
+            "tof2": ("epoch", [2, 6, 2]),
+            "tof3": ("epoch", [2, 8, 2]),
+            "cksm": ("epoch", [2, 12, 2]),
+        }
+    )
+
+    # Act
+    l1a_de = calculate_tof1_for_golden_triples(l1a_de)
+
+    # Assert
+    assert l1a_de_expected.equals(l1a_de)
+
+
+def test_set_coincidence_type(attr_mgr_l1a):
+    # Arrange
+    l1b_de = xr.Dataset()
+    tof_fill = attr_mgr_l1a.get_variable_attributes("tof0")["FILLVAL"]
+    ckm_fill = attr_mgr_l1a.get_variable_attributes("cksm")["FILLVAL"]
+    l1a_de = xr.Dataset(
+        {
+            "de_count": ("epoch", [3]),
+            "coincidence_type": ("direct_events", [0, 0, 4]),
+            "mode": ("direct_events", [1, 0, 1]),
+            "tof0": ("direct_events", [5, 2, 10]),
+            "tof1": ("direct_events", [10, 4, tof_fill]),
+            "tof2": ("direct_events", [15, 6, 20]),
+            "tof3": ("direct_events", [20, 8, 30]),
+            "cksm": ("direct_events", [25, ckm_fill, ckm_fill]),
+        },
+        coords={
+            "epoch": [0],
+            "direct_events": [0, 1, 2],
+        },
+    )
+
+    coincidence_type_expected = np.array(["111111", "111100", "101101"])
+
+    # Act
+    l1b_de = set_coincidence_type(l1a_de, l1b_de, attr_mgr_l1a)
+
+    # Assert
+    np.testing.assert_array_equal(
+        l1b_de["coincidence_type"].values,
+        coincidence_type_expected,
+    )
+
+
+def test_convert_tofs_to_eu(attr_mgr_l1b, attr_mgr_l1a):
+    l1b_de = xr.Dataset()
+    tof_fill_l1a = attr_mgr_l1a.get_variable_attributes("tof0")["FILLVAL"]
+    tof_fill_l1b = attr_mgr_l1b.get_variable_attributes("tof1")["FILLVAL"]
+    l1a_de = xr.Dataset(
+        {
+            "de_count": ("epoch", [2]),
+            "coincidence_type": ("direct_events", [0, 4]),
+            "mode": ("direct_events", [1, 0]),
+            "tof0": ("direct_events", [5, 2]),
+            "tof1": ("direct_events", [10, tof_fill_l1a]),
+            "tof2": ("direct_events", [15, 6]),
+            "tof3": ("direct_events", [20, 8]),
+        },
+        coords={
+            "epoch": [0],
+            "direct_events": [0, 1],
+        },
+    )
+
+    tof0_expected = np.array([1.394394, 0.889272])
+    tof1_expected = np.array([0.931059, tof_fill_l1b])
+    tof2_expected = np.array([2.870557, 1.372876])
+    tof3_expected = np.array([3.88245, 1.818162])
+
+    # Act
+    l1b_de = convert_tofs_to_eu(l1a_de, l1b_de, attr_mgr_l1a, attr_mgr_l1b)
+
+    tof_checks = [
+        ("tof0", tof0_expected),
+        ("tof1", tof1_expected),
+        ("tof2", tof2_expected),
+        ("tof3", tof3_expected),
+    ]
+    # Assert
+    for tof, expected_tof in tof_checks:
+        np.testing.assert_allclose(
+            l1b_de[tof].values,
+            expected_tof,
+            atol=1e-6,
+        )
+
+
+def test_identify_species(attr_mgr_l1b):
+    # Arrange
+    fill_val = attr_mgr_l1b.get_variable_attributes("tof2")["FILLVAL"]
+    l1b_de = xr.Dataset(
+        {
+            "tof2": ("epoch", [1, 14, 50, 80, 500, fill_val]),
+        }
+    )
+
+    expected_species = np.array(["U", "H", "U", "O", "U", "U"])
+
+    # Act
+    l1b_de = identify_species(l1b_de)
+
+    # Assert
+    np.testing.assert_array_equal(l1b_de["species"], expected_species)
+
+
+def test_set_bad_times():
+    # Arrange
+    l1b_de = xr.Dataset(
+        {},
+        coords={
+            "epoch": [0, 1, 2],
+        },
+    )
+
+    expected_bad_times = np.array([0, 0, 0])
+
+    # Act
+    l1b_de = set_bad_times(l1b_de)
+
+    # Assert
+    np.testing.assert_array_equal(l1b_de["badtimes"], expected_bad_times)
+
+
+@pytest.mark.external_kernel
+@pytest.mark.use_test_metakernel("imap_ena_sim_metakernel.template")
+def test_set_direction():
+    # Arrange
+    l1b_de = xr.Dataset(
+        {},
+        coords={
+            "epoch": [
+                7.9794907254e17,
+                # + 1 second. Should be 24deg diff from
+                # previous epoch
+                7.9794907254e17 + 1e9,
+                # + 7.5 seconds. Should be 180deg diff from
+                # first epoch
+                7.9794907254e17 + 7.5e9,
+                # + 15 seconds. Should be 360deg diff from
+                # previous epoch
+                7.9794907254e17 + 15e9,
+            ],
+        },
+    )
+    # latitudes are -90 to 90
+    expected_direction_lat = np.array([0, 0, 0, 0])
+    # longitude are -180 to 180
+    expected_direction_lon = np.array([140.5, 164.5, -39.5, 140.5])
+
+    # Act
+    l1b_de = set_pointing_direction(l1b_de)
+
+    # Assert
+    np.testing.assert_allclose(
+        l1b_de["direction_lat"].values,
+        expected_direction_lat,
+        atol=1e-1,
+    )
+    np.testing.assert_allclose(
+        l1b_de["direction_lon"].values,
+        expected_direction_lon,
+        atol=1e-1,
+    )
+
+
+def test_pointing_bins():
+    # Arrange
+    l1b_de = xr.Dataset(
+        {
+            "direction_lat": ("epoch", [0, 0, 0, 0, 0]),
+            "direction_lon": ("epoch", [-180, 91.3, 116.3, 140.5, 180]),
+        },
+        coords={
+            "epoch": [
+                7.9794907049e17,
+                7.9794907153e17,
+                7.9794907254e17,
+                7.9794907354e17,
+                7.9794907454e17,
+            ],
+        },
+    )
+
+    expected_pointing_lats = np.array([20, 20, 20, 20, 20])
+    expected_pointing_lons = np.array([0, 2712, 2962, 3205, 3600])
+
+    # Act
+    l1b_de = set_pointing_bin(l1b_de)
+
+    # Assert
+    np.testing.assert_array_equal(l1b_de["pointing_bin_lat"], expected_pointing_lats)
+    np.testing.assert_array_equal(l1b_de["pointing_bin_lon"], expected_pointing_lons)

imap_processing/tests/lo/test_lo_l1c.py

@@ -19,7 +19,7 @@ def test_lo_l1c():
 
     expected_logical_source = "imap_lo_l1c_pset"
     # Act
-    output_dataset = lo_l1c(data, "001")
+    output_dataset = lo_l1c(data)
 
     # Assert
     assert expected_logical_source == output_dataset.attrs["Logical_source"]

imap_processing/tests/lo/test_lo_science.py

@@ -17,7 +17,7 @@ from imap_processing.lo.l0.lo_science import (
 from imap_processing.utils import convert_to_binary_string, packet_file_to_datasets
 
 
-@pytest.fixture()
+@pytest.fixture
 def fake_de_dataset():
     # binary packet fields
     count = "0000000000000010"  # 2
@@ -72,7 +72,7 @@ def fake_de_dataset():
     return dataset
 
 
-@pytest.fixture()
+@pytest.fixture
 def sample_data():
     xtce_file = imap_module_directory / "lo/packet_definitions/lo_xtce.xml"
     dependency = (
@@ -86,7 +86,7 @@ def sample_data():
     return datasets_by_apid
 
 
-@pytest.fixture()
+@pytest.fixture
 def segmented_pkts_fake_data():
     dataset = xr.Dataset(
         data_vars=dict(
@@ -116,7 +116,7 @@ def segmented_pkts_fake_data():
     return dataset
 
 
-@pytest.fixture()
+@pytest.fixture
 def fake_spin_data():
     dataset = xr.Dataset(
         data_vars=dict(
@@ -148,16 +148,15 @@ def fake_spin_data():
     return dataset
 
 
-@pytest.fixture()
+@pytest.fixture
 def attr_mgr():
     attr_mgr = ImapCdfAttributes()
     attr_mgr.add_instrument_global_attrs(instrument="lo")
     attr_mgr.add_instrument_variable_attrs(instrument="lo", level="l1a")
-    attr_mgr.add_global_attribute("Data_version", "v000")
     return attr_mgr
 
 
-@pytest.fixture()
+@pytest.fixture
 def initialized_dataset(fake_de_dataset, attr_mgr):
     fake_de_dataset.attrs["bit_pos"] = 0
     de_fields = [

imap_processing/tests/lo/test_star_sensor.py

@@ -6,7 +6,7 @@ from imap_processing.ccsds.ccsds_data import CcsdsData
 from imap_processing.lo.l0.data_classes.star_sensor import StarSensor
 
 
-@pytest.fixture()
+@pytest.fixture
 def star_sensor():
     fake_data_field = namedtuple("fake_packet", ["raw_value", "derived_value"])
     star_sensor = StarSensor.__new__(StarSensor)

imap_processing/tests/mag/conftest.py

@@ -7,15 +7,17 @@ import pytest
 import xarray as xr
 
 from imap_processing.cdf.utils import load_cdf
+from imap_processing.mag.constants import VecSec
 from imap_processing.mag.l1a.mag_l1a import mag_l1a
+from imap_processing.spice.time import TTJ2000_EPOCH
 
 
-@pytest.fixture()
+@pytest.fixture
 def validation_l1a():
     current_directory = Path(__file__).parent
     test_file = current_directory / "validation" / "mag_l1_test_data.pkts"
     # Test file contains only normal packets
-    l1a = mag_l1a(test_file, "v000")
+    l1a = mag_l1a(test_file)
     return l1a
 
 
@@ -57,27 +59,74 @@ def mag_l1a_dataset_generator(length):
     return output_dataset
 
 
-@pytest.fixture()
-def mag_test_calibration_data():
+@pytest.fixture
+def mag_test_l1b_calibration_data():
     imap_dir = Path(__file__).parent
-    cal_file = imap_dir / "validation" / "imap_calibration_mag_20240229_v01.cdf"
+    cal_file = (
+        imap_dir
+        / "validation"
+        / "calibration"
+        / "imap_mag_l1b-calibration_20240229_v001.cdf"
+    )
     calibration_data = load_cdf(cal_file)
     return calibration_data
 
 
-def generate_test_epoch(end, vectors_per_second, starting_point=0, gaps=None):
-    spacing = 1 / vectors_per_second[0]
+@pytest.fixture
+def mag_test_l2_data():
+    imap_dir = Path(__file__).parent
+    cal_file = (
+        imap_dir
+        / "validation"
+        / "calibration"
+        / "imap_mag_l2-calibration-matrices_20251017_v004.cdf"
+    )
+    calibration_data = load_cdf(cal_file)
+
+    offsets_data = load_cdf(
+        imap_dir
+        / "validation"
+        / "calibration"
+        / "imap_mag_l2-offsets-norm_20251017_20251017_v001.cdf"
+    )
+
+    return calibration_data, offsets_data
+
+
+def mag_generate_l1b_from_csv(df, logical_source):
+    length = len(df.index)
+    dataset = mag_l1a_dataset_generator(length)
+
+    dataset["vectors"].data = np.array(df[["x", "y", "z", "range"]])
+    dataset["compression_flags"].data = np.array(
+        df[["compression", "compression_width"]]
+    )
+
+    epoch = [np.datetime64(t) - np.datetime64(TTJ2000_EPOCH) for t in df["t"]]
+    epoch_ns = [(e / np.timedelta64(1, "ns")).astype(np.int64) for e in epoch]
+    dataset.coords["epoch"] = xr.DataArray(epoch_ns, name="epoch", dims=["epoch"])
+
+    dataset.attrs["Logical_source"] = logical_source
+    dataset.attrs["vectors_per_second"] = f"{epoch_ns[0]}:2"
+
+    return dataset
+
+
+def generate_test_epoch(
+    end, vectors_per_second: list[VecSec], starting_point=0, gaps=None
+):
+    spacing = 1 / vectors_per_second[0].value
     output = np.array([])
     prev = starting_point
     if gaps:
         for index, gap in enumerate(gaps):
             if len(vectors_per_second) != 1:
-                spacing = 1 / vectors_per_second[index]
+                spacing = 1 / vectors_per_second[index].value
             output = np.concatenate(
                 (output, np.arange(prev, gap[0] + spacing, step=spacing) * 1e9)
             )
             prev = gap[1]
-        spacing = 1 / vectors_per_second[-1]
+        spacing = 1 / vectors_per_second[-1].value
     output = np.concatenate(
         (output, np.arange(prev, end + spacing, step=spacing) * 1e9)
     )

imap_processing/tests/mag/test_mag_decom.py

@@ -9,11 +9,12 @@ from imap_processing.mag.constants import DataMode
 from imap_processing.mag.l0.decom_mag import decom_packets, generate_dataset
 
 
-@pytest.fixture()
+@pytest.fixture
 def cdf_attrs():
     test_attrs = ImapCdfAttributes()
     test_attrs.add_instrument_global_attrs("mag")
     test_attrs.add_instrument_variable_attrs("mag", "l1a")
+    # Default v001 expected when writing to file and re-loading
     test_attrs.add_global_attribute("Data_version", "v001")
     return test_attrs
 
@@ -75,7 +76,7 @@ def test_mag_raw_xarray(cdf_attrs):
         [
             item is not None
             for key, item in norm_data.attrs.items()
-            if key != "Logical_file_id"
+            if key not in ("Logical_file_id", "Data_version")
         ]
     )
 
@@ -83,7 +84,7 @@ def test_mag_raw_xarray(cdf_attrs):
         [
             item is not None
             for key, item in burst_data.attrs.items()
-            if key != "Logical_file_id"
+            if key not in ("Logical_file_id", "Data_version")
         ]
     )
 

imap_processing/tests/mag/test_mag_l1a.py

@@ -15,7 +15,7 @@ from imap_processing.mag.l1a.mag_l1a_data import (
 from imap_processing.spice.time import met_to_ttj2000ns
 
 
-@pytest.fixture()
+@pytest.fixture
 def uncompressed_vector_bytearray():
     input_data = np.array(
         [
@@ -225,7 +225,7 @@ def uncompressed_vector_bytearray():
     return input_data
 
 
-@pytest.fixture()
+@pytest.fixture
 def expected_vectors():
     primary_expected = np.array(
         [
@@ -272,7 +272,7 @@ def expected_vectors():
     return (primary_expected, secondary_expected)
 
 
-@pytest.fixture()
+@pytest.fixture
 def raw_compressed_vectors():
     # compressed vectors, without the first starting uncompressed vector.
     # 15 primary vectors and 15 secondary vectors, corresponding to most of
@@ -883,7 +883,7 @@ def test_mag_l1a():
     current_directory = Path(__file__).parent
     test_file = current_directory / "validation" / "mag_l1_test_data.pkts"
 
-    output_data = mag_l1a(test_file, "v001")
+    output_data = mag_l1a(test_file)
 
     # Test data is one day's worth of NORM data, so it should return one raw, one MAGO
     # and one MAGI dataset
@@ -897,9 +897,6 @@ def test_mag_l1a():
     for data_type in [data.attrs["Logical_source"] for data in output_data]:
         assert data_type in expected_logical_source
 
-    for data in output_data:
-        assert data.attrs["Data_version"] == "v001"
-
     assert "vectors" in output_data[-1].variables.keys()
     assert "compression_flags" in output_data[-1].variables.keys()
 
@@ -910,6 +907,15 @@ def test_mag_l1a():
         == output_data[-1]["compression_flags"].data.shape[0]
     )
 
+    assert (
+        str(output_data[-1]["epoch"].data[0])
+        in output_data[-1].attrs["vectors_per_second"]
+    )
+    assert (
+        str(output_data[-2]["epoch"].data[0])
+        in output_data[-2].attrs["vectors_per_second"]
+    )
+
 
 def test_mag_packet_properties():
     # equal to 01001011