imap-processing 0.11.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

imap_processing/tests/mag/conftest.py

@@ -2,8 +2,11 @@
 
 from pathlib import Path
 
+import numpy as np
 import pytest
+import xarray as xr
 
+from imap_processing.cdf.utils import load_cdf
 from imap_processing.mag.l1a.mag_l1a import mag_l1a
 
 
@@ -14,3 +17,69 @@ def validation_l1a():
     # Test file contains only normal packets
     l1a = mag_l1a(test_file, "v000")
     return l1a
+
+
+def mag_l1a_dataset_generator(length):
+    epoch = xr.DataArray(np.arange(length), name="epoch", dims=["epoch"])
+    direction = xr.DataArray(np.arange(4), name="direction", dims=["direction"])
+    compression = xr.DataArray(np.arange(2), name="compression", dims=["compression"])
+
+    direction_label = xr.DataArray(
+        direction.values.astype(str),
+        name="direction_label",
+        dims=["direction_label"],
+    )
+
+    compression_label = xr.DataArray(
+        compression.values.astype(str),
+        name="compression_label",
+        dims=["compression_label"],
+    )
+
+    vectors = xr.DataArray(
+        np.zeros((length, 4)),
+        dims=["epoch", "direction"],
+        coords={"epoch": epoch, "direction": direction},
+    )
+    compression_flags = xr.DataArray(
+        np.zeros((length, 2), dtype=np.int8), dims=["epoch", "compression"]
+    )
+
+    output_dataset = xr.Dataset(
+        coords={"epoch": epoch, "direction": direction, "compression": compression},
+    )
+    output_dataset["vectors"] = vectors
+    output_dataset["compression_flags"] = compression_flags
+    output_dataset["direction_label"] = direction_label
+    output_dataset["compression_label"] = compression_label
+    output_dataset.attrs["Logical_source"] = ["imap_mag_l1a_norm-mago"]
+
+    return output_dataset
+
+
+@pytest.fixture()
+def mag_test_calibration_data():
+    imap_dir = Path(__file__).parent
+    cal_file = imap_dir / "validation" / "imap_calibration_mag_20240229_v01.cdf"
+    calibration_data = load_cdf(cal_file)
+    return calibration_data
+
+
+def generate_test_epoch(end, vectors_per_second, starting_point=0, gaps=None):
+    spacing = 1 / vectors_per_second[0]
+    output = np.array([])
+    prev = starting_point
+    if gaps:
+        for index, gap in enumerate(gaps):
+            if len(vectors_per_second) != 1:
+                spacing = 1 / vectors_per_second[index]
+            output = np.concatenate(
+                (output, np.arange(prev, gap[0] + spacing, step=spacing) * 1e9)
+            )
+            prev = gap[1]
+        spacing = 1 / vectors_per_second[-1]
+    output = np.concatenate(
+        (output, np.arange(prev, end + spacing, step=spacing) * 1e9)
+    )
+
+    return output

imap_processing/tests/mag/test_mag_decom.py

@@ -13,7 +13,7 @@ from imap_processing.mag.l0.decom_mag import decom_packets, generate_dataset
 def cdf_attrs():
     test_attrs = ImapCdfAttributes()
     test_attrs.add_instrument_global_attrs("mag")
-    test_attrs.add_instrument_variable_attrs("mag", "l1")
+    test_attrs.add_instrument_variable_attrs("mag", "l1a")
     test_attrs.add_global_attribute("Data_version", "v001")
     return test_attrs
 

imap_processing/tests/mag/test_mag_l1a.py

@@ -919,3 +919,41 @@ def test_mag_packet_properties():
     )
 
     assert packet_properties.compression_width == 18
+
+
+def test_changing_vecsec():
+    packet_one = MagL1aPacketProperties(
+        1000,
+        TimeTuple(1000, 0),
+        2,  # 2 vectors per second
+        1,  # 2 seconds of data
+        0,
+        0,
+        1,
+        0,
+    )
+    packet_two = MagL1aPacketProperties(
+        2000,
+        TimeTuple(2000, 0),
+        8,  # 8 vectors per second
+        1,  # 2 seconds of data
+        1,
+        0,
+        1,
+        0,
+    )
+
+    four_vectors = np.full((4, 5), [1, 2, 3, 4, 2])
+    sixteen_vectors = np.full((16, 5), [1, 2, 3, 4, 2])
+
+    mag_l1a = MagL1a(True, 1, 1, four_vectors, packet_one)
+    mag_l1a.append_vectors(sixteen_vectors, packet_two)
+
+    assert mag_l1a.vectors.shape[0] == 20
+    assert len(mag_l1a.packet_definitions.keys()) == 2
+    expected = [2, 8]
+    for index, value in enumerate(mag_l1a.packet_definitions.values()):
+        assert expected[index] == value.vectors_per_second
+
+    assert ":2," in mag_l1a.vectors_per_second_attribute()
+    assert ":8" in mag_l1a.vectors_per_second_attribute()

imap_processing/tests/mag/test_mag_l1b.py

@@ -4,6 +4,7 @@ import numpy as np
 import pytest
 import xarray as xr
 
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.cdf.utils import load_cdf, write_cdf
 from imap_processing.mag.l1b.mag_l1b import (
     calibrate_vector,
@@ -11,55 +12,30 @@ from imap_processing.mag.l1b.mag_l1b import (
     mag_l1b_processing,
     rescale_vector,
 )
+from imap_processing.tests.mag.conftest import (
+    mag_l1a_dataset_generator,
+)
 
 
-@pytest.fixture(scope="module")
-def mag_l1a_dataset():
-    epoch = xr.DataArray(np.arange(20), name="epoch", dims=["epoch"])
-    direction = xr.DataArray(np.arange(4), name="direction", dims=["direction"])
-    compression = xr.DataArray(np.arange(2), name="compression", dims=["compression"])
-
-    direction_label = xr.DataArray(
-        direction.values.astype(str),
-        name="direction_label",
-        dims=["direction_label"],
-    )
-
-    compression_label = xr.DataArray(
-        compression.values.astype(str),
-        name="compression_label",
-        dims=["compression_label"],
-    )
-
-    vectors = xr.DataArray(
-        np.zeros((20, 4)),
-        dims=["epoch", "direction"],
-        coords={"epoch": epoch, "direction": direction},
-    )
-    compression_flags = xr.DataArray(
-        np.zeros((20, 2), dtype=np.int8), dims=["epoch", "compression"]
-    )
-    compression_flags[1, :] = np.array([1, 18], dtype=np.int8)
-
-    vectors[0, :] = np.array([1, 1, 1, 0])
-    vectors[1, :] = np.array([7982, 48671, -68090, 0])
-
-    output_dataset = xr.Dataset(
-        coords={"epoch": epoch, "direction": direction, "compression": compression},
+def test_mag_processing(mag_test_calibration_data):
+    # All specific test values come from MAG team to accommodate various cases.
+    # Each vector is multiplied by the matrix in the calibration data for the given
+    # range to get the calibrated vector.
+    mag_l1a_dataset = mag_l1a_dataset_generator(20)
+    mag_l1a_dataset["compression_flags"].data[1, :] = np.array([1, 18], dtype=np.int8)
+
+    mag_l1a_dataset["vectors"].data[0, :] = np.array([1, 1, 1, 0])
+    mag_l1a_dataset["vectors"].data[1, :] = np.array([7982, 48671, -68090, 0])
+    mag_attributes = ImapCdfAttributes()
+    mag_attributes.add_instrument_global_attrs("mag")
+    mag_attributes.add_instrument_variable_attrs("mag", "l1b")
+    mag_l1b = mag_l1b_processing(
+        mag_l1a_dataset,
+        mag_test_calibration_data,
+        mag_attributes,
+        "imap_mag_l1b_norm-mago",
     )
-    output_dataset["vectors"] = vectors
-    output_dataset["compression_flags"] = compression_flags
-    output_dataset["direction_label"] = direction_label
-    output_dataset["compression_label"] = compression_label
-    output_dataset.attrs["Logical_source"] = ["imap_mag_l1a_norm-mago"]
 
-    return output_dataset
-
-
-def test_mag_processing(mag_l1a_dataset):
-    mag_l1a_dataset.attrs["Logical_source"] = ["imap_mag_l1a_norm-mago"]
-
-    mag_l1b = mag_l1b_processing(mag_l1a_dataset)
     np.testing.assert_allclose(
         mag_l1b["vectors"][0].values, [2.2972, 2.2415, 2.2381, 0], atol=1e-4
     )
@@ -68,13 +44,16 @@ def test_mag_processing(mag_l1a_dataset):
         [4584.1029091, 27238.73161294, -38405.22240195, 0.0],
     )
 
-    # np.testing.assert_allclose(mag_l1b["vectors"][1].values, [0, 0, 0, 0])
+    np.testing.assert_allclose(mag_l1b["vectors"][2].values, [0, 0, 0, 0])
 
     assert mag_l1b["vectors"].values.shape == mag_l1a_dataset["vectors"].values.shape
 
-    mag_l1a_dataset.attrs["Logical_source"] = ["imap_mag_l1a_norm-magi"]
-
-    mag_l1b = mag_l1b_processing(mag_l1a_dataset)
+    mag_l1b = mag_l1b_processing(
+        mag_l1a_dataset,
+        mag_test_calibration_data,
+        mag_attributes,
+        "imap_mag_l1b_norm-magi",
+    )
 
     np.testing.assert_allclose(
         mag_l1b["vectors"][0].values, [2.27538, 2.23416, 2.23682, 0], atol=1e-5
@@ -83,7 +62,9 @@ def test_mag_processing(mag_l1a_dataset):
     assert mag_l1b["vectors"].values.shape == mag_l1a_dataset["vectors"].values.shape
 
 
-def test_mag_attributes(mag_l1a_dataset):
+def test_mag_attributes():
+    mag_l1a_dataset = mag_l1a_dataset_generator(20)
+
     mag_l1a_dataset.attrs["Logical_source"] = ["imap_mag_l1a_norm-mago"]
 
     output = mag_l1b(mag_l1a_dataset, "v001")
@@ -94,8 +75,6 @@ def test_mag_attributes(mag_l1a_dataset):
     output = mag_l1b(mag_l1a_dataset, "v001")
     assert output.attrs["Logical_source"] == "imap_mag_l1b_burst-magi"
 
-    assert output.attrs["Data_level"] == "L1B"
-
 
 def test_cdf_output():
     l1a_cdf = load_cdf(
@@ -110,7 +89,9 @@ def test_cdf_output():
     assert Path.exists(output_path)
 
 
-def test_mag_compression_scale(mag_l1a_dataset):
+def test_mag_compression_scale():
+    mag_l1a_dataset = mag_l1a_dataset_generator(20)
+
     test_calibration = np.array(
         [
             [2.2972202, 0.0, 0.0],

imap_processing/tests/mag/test_mag_l1c.py

@@ -1,40 +1,271 @@
+from pathlib import Path
+
 import numpy as np
 import pytest
 import xarray as xr
+import yaml
 
-from imap_processing.mag.l1c.mag_l1c import mag_l1c
+from imap_processing.mag.l1c.interpolation_methods import InterpolationFunction
+from imap_processing.mag.l1c.mag_l1c import (
+    fill_normal_data,
+    find_all_gaps,
+    find_gaps,
+    generate_timeline,
+    interpolate_gaps,
+    mag_l1c,
+    process_mag_l1c,
+)
+from imap_processing.tests.mag.conftest import (
+    generate_test_epoch,
+    mag_l1a_dataset_generator,
+)
 
 
 @pytest.fixture(scope="module")
 def mag_l1b_dataset():
-    epoch = xr.DataArray(np.arange(20), name="epoch", dims=["epoch"])
-    direction = xr.DataArray(np.arange(4), name="direction", dims=["direction"])
-    vectors = xr.DataArray(
-        np.zeros((20, 4)),
-        dims=["epoch", "direction"],
-        coords={"epoch": epoch, "direction": direction},
-    )
+    output_dataset = mag_l1a_dataset_generator(10)
 
+    output_dataset["epoch"] = xr.DataArray(
+        np.arange(0.1, 5.1, step=0.5) * 1e9, name="epoch", dims=["epoch"]
+    )
+    vectors = np.array([[i, i, i, 2] for i in range(1, 11)])
     vectors[0, :] = np.array([1, 1, 1, 0])
+    output_dataset["vectors"].data = vectors
+
+    return output_dataset
+
+
+@pytest.fixture()
+def norm_dataset():
+    dataset = mag_l1a_dataset_generator(10)
+    epoch_vals = generate_test_epoch(6, [2, 4, 4], 0, [[2, 4], [4.25, 5.5]])
+    vectors_per_second_attr = "0:2,4000000000:4"
+    dataset.attrs["vectors_per_second"] = vectors_per_second_attr
+    dataset["epoch"] = epoch_vals
+    dataset.attrs["Logical_source"] = "imap_mag_l1b_norm-mago"
+    vectors = np.array([[i, i, i, 2] for i in range(1, 11)])
+    dataset["vectors"].data = vectors
+
+    return dataset
+
+
+@pytest.fixture()
+def burst_dataset():
+    dataset = mag_l1a_dataset_generator(17)
+    epoch_vals = generate_test_epoch(5.1, [5], 1.9)
+    dataset["epoch"] = epoch_vals
+    dataset.attrs["Logical_source"] = ["imap_mag_l1b_burst-mago"]
+    vectors = np.array([[i, i, i, 2] for i in range(1, 18)])
+    dataset["vectors"].data = vectors
+    return dataset
+
+
+def test_configuration_file():
+    with open(
+        Path(__file__).parent.parent.parent
+        / "mag"
+        / "imap_mag_sdc-configuration_v001.yaml"
+    ) as f:
+        configuration = yaml.safe_load(f)
+
+    assert configuration["L1C_interpolation_method"] in [
+        e.name for e in InterpolationFunction
+    ]
 
-    output_dataset = xr.Dataset(
-        coords={"epoch": epoch, "direction": direction},
+    # should not raise an error
+    configuration_file = InterpolationFunction[
+        configuration["L1C_interpolation_method"]
+    ]
+    configuration_file([1], [1], [1])
+
+
+def test_process_mag_l1c(norm_dataset, burst_dataset):
+    l1c = process_mag_l1c(norm_dataset, burst_dataset, InterpolationFunction.linear)
+    expected_output_timeline = (
+        np.array([0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.25, 4.75, 5.25, 5.5, 5.75, 6])
+        * 1e9
     )
-    output_dataset["vectors"] = vectors
+    assert np.array_equal(l1c[:, 0], expected_output_timeline)
+    # Every new timestamp should have data
+    assert (
+        np.count_nonzero([np.sum(l1c[i, 1:4]) for i in range(l1c.shape[0])])
+        == l1c.shape[0]
+    )
+    expected_flags = np.zeros(15)
+    # filled sections should have 1 as a flag
+    expected_flags[5:8] = 1
+    expected_flags[10:12] = 1
+    assert np.array_equal(l1c[:, 5], expected_flags)
+    assert np.array_equal(l1c[:5, 1:5], norm_dataset["vectors"].data[:5, :])
+    for i in range(5, 8):
+        e = l1c[i, 0]
+        burst_vectors = burst_dataset.sel(epoch=int(e), method="nearest")[
+            "vectors"
+        ].data
+        # We're just finding the closest burst values to the array, so they won't be
+        # identical.
+        assert np.allclose(l1c[i, 1:5], burst_vectors, rtol=0, atol=1)
 
-    return output_dataset
+    assert np.array_equal(l1c[8:10, 1:5], norm_dataset["vectors"].data[5:7, :])
+    for i in range(10, 12):
+        e = l1c[i, 0]
+        burst_vectors = burst_dataset.sel(epoch=int(e), method="nearest")[
+            "vectors"
+        ].data
+        # We're just finding the closest burst values to the array, so they won't be
+        # identical.
+        assert np.allclose(l1c[i, 1:5], burst_vectors, rtol=0, atol=1)
+
+
+def test_interpolate_gaps(norm_dataset, mag_l1b_dataset):
+    # np.array([0, 0.5, 1, 1.5, 2, 4, 4.25, 5.5, 5.75, 6]) * 1e9
+    gaps = np.array([[2, 4], [4.25, 5.5]]) * 1e9
+    generated_timeline = generate_timeline(norm_dataset["epoch"].data, gaps)
+    norm_timeline = fill_normal_data(norm_dataset, generated_timeline)
+    gaps = np.array([[2, 4]]) * 1e9
+    output = interpolate_gaps(
+        mag_l1b_dataset, gaps, norm_timeline, InterpolationFunction.linear
+    )
+    expected_output = np.array(
+        [
+            [5.8, 5.8, 5.8, 2, 1, 0, 0],
+            [6.8, 6.8, 6.8, 2, 1, 0, 0],
+            [7.8, 7.8, 7.8, 2, 1, 0, 0],
+        ]
+    )
+
+    assert np.allclose(output[5:8, 1:], expected_output)
+
+    input_norm_timeline = np.array(
+        [
+            [1.50e09, 4, 4, 4, 2, 0, 0, 0],
+            [2.00e09, 5, 5, 5, 2, 0, 0, 0],
+            [2.50e09, 0, 0, 0, 0, 1, 0, 0],
+            [3.00e09, 0, 0, 0, 0, 1, 0, 0],
+            [3.50e09, 0, 0, 0, 0, 1, 0, 0],
+            [4.00e09, 6, 6, 6, 2, 0, 0, 0],
+            [4.25e09, 7, 7, 7, 2, 0, 0, 0],
+        ]
+    )
+
+    # output - all timestamps with -1 should be filled with interpolated values.
+    expected_output = np.array(
+        [
+            [1.50e09, 4, 4, 4, 2, 0, 0, 0],
+            [2.00e09, 5, 5, 5, 2, 0, 0, 0],
+            [2.50e09, 5.8, 5.8, 5.8, 2, 1, 0, 0],
+            [3.00e09, 6.8, 6.8, 6.8, 2, 1, 0, 0],
+            [3.50e09, 7.8, 7.8, 7.8, 2, 1, 0, 0],
+            [4.00e09, 6, 6, 6, 2, 0, 0, 0],
+            [4.25e09, 7, 7, 7, 2, 0, 0, 0],
+        ]
+    )
 
+    output = interpolate_gaps(
+        mag_l1b_dataset, gaps, input_norm_timeline, InterpolationFunction.linear
+    )
+
+    assert np.allclose(output, expected_output)
+
+
+def test_mag_l1c(norm_dataset, burst_dataset):
+    l1c = mag_l1c(burst_dataset, norm_dataset, "v001")
+    assert l1c["vector_magnitude"].shape == (len(l1c["epoch"].data),)
+    assert l1c["vector_magnitude"].data[0] == np.linalg.norm(l1c["vectors"].data[0][:4])
+    assert l1c["vector_magnitude"].data[-1] == np.linalg.norm(
+        l1c["vectors"].data[-1][:4]
+    )
+
+    expected_vars = [
+        "vectors",
+        "compression_flags",
+        "vector_magnitude",
+        "generated_flag",
+    ]
+
+    for var in expected_vars:
+        assert var in l1c.data_vars
 
-def test_mag_attributes(mag_l1b_dataset):
-    # Fixture from test_mag_l1b.py, since L1A and L1B are very similar
-    mag_l1b_dataset.attrs["Logical_source"] = ["imap_mag_l1b_norm-mago"]
 
-    output = mag_l1c(mag_l1b_dataset, mag_l1b_dataset, "v001")
+def test_mag_attributes(norm_dataset, burst_dataset):
+    output = mag_l1c(norm_dataset, burst_dataset, "v001")
     assert output.attrs["Logical_source"] == "imap_mag_l1c_norm-mago"
 
-    mag_l1b_dataset.attrs["Logical_source"] = ["imap_mag_l1b_norm-magi"]
+    expected_attrs = ["missing_sequences", "interpolation_method"]
+    for attr in expected_attrs:
+        assert attr in output.attrs
+
+
+def test_find_all_gaps():
+    epoch_test = generate_test_epoch(5.5, [2, 2], 0, [[2, 5]])
+
+    vectors_per_second_attr = "0:2"
+    output = find_all_gaps(epoch_test, vectors_per_second_attr)
+    expected_gaps = np.array([[2, 5]]) * 1e9
+    assert np.array_equal(output, expected_gaps)
+
+    epoch_test = np.array([0, 0.5, 1, 1.5, 2, 4, 4.25, 4.5, 4.75, 5.5]) * 1e9
+    vectors_per_second_attr = "0:2,4000000000:4"
+    expected_gaps = np.array([[2, 4], [4.75, 5.5]]) * 1e9
+    output = find_all_gaps(epoch_test, vectors_per_second_attr)
+    assert np.array_equal(output, expected_gaps)
+
+
+def test_find_gaps():
+    # Test should be in ns
+    epoch_test = generate_test_epoch(3.5, [2], 0, [[0.5, 2], [2, 3.5]])
+    print(epoch_test)
+    gaps = find_gaps(epoch_test, 2)
+    expected_return = np.array([[0.5, 2], [2, 3.5]]) * 1e9
+
+    assert np.array_equal(gaps, expected_return)
+
+    epoch_test = generate_test_epoch(5, [2], gaps=[[0.5, 2], [3, 4]])
+    gaps = find_gaps(epoch_test, 2)
+    expected_return = np.array([[0.5, 2], [3, 4]]) * 1e9
+
+    assert np.array_equal(gaps, expected_return)
+
+    epoch_test = generate_test_epoch(3, [4], gaps=[[0.5, 1], [2, 3]])
+    gaps = find_gaps(epoch_test, 4)
+    expected_return = np.array([[0.5, 1], [2, 3]]) * 1e9
+
+    assert np.array_equal(gaps, expected_return)
+
+
+def test_generate_timeline():
+    epoch_test = generate_test_epoch(3, [4], gaps=[[0.5, 1], [2, 3]])
+
+    gaps = np.array([[0.5, 1], [2, 3]]) * 1e9
+    expected_output = np.array([0, 0.25, 0.5, 1, 1.25, 1.5, 1.75, 2, 2.5, 3]) * 1e9
+    output = generate_timeline(epoch_test, gaps)
+    assert np.array_equal(output, expected_output)
+
+    epoch_test = generate_test_epoch(5, [2], starting_point=1)
+    # Expected output from find_gaps if none are found
+    gaps = np.zeros((0, 2))
+    output = generate_timeline(epoch_test, gaps)
+    assert np.array_equal(output, epoch_test)
+
+    epoch_test = generate_test_epoch(5, [2], starting_point=1, gaps=[[3, 5]])
+    gaps = np.array([[3, 5]]) * 1e9
+
+    expected_output = np.array([1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5]) * 1e9
+    output = generate_timeline(epoch_test, gaps)
+    assert np.array_equal(output, expected_output)
+
+
+def test_fill_normal_data(mag_l1b_dataset):
+    output_timeline = np.arange(0.1, 6.1, step=0.5) * 1e9
+    output = fill_normal_data(mag_l1b_dataset, output_timeline)
 
-    output = mag_l1c(mag_l1b_dataset, mag_l1b_dataset, "v001")
-    assert output.attrs["Logical_source"] == "imap_mag_l1c_norm-magi"
+    assert output.shape == (12, 8)
+    # all vectors should be nonzero
+    assert np.count_nonzero(output[:-2, 1:4]) == 30
+    # last two vectors should be zero
+    assert np.count_nonzero(output[-2:, 1:5]) == 0
 
-    assert output.attrs["Data_level"] == "L1C"
+    # spot check
+    assert np.array_equal(output[0, 1:5], mag_l1b_dataset["vectors"].data[0, :])
+    assert np.array_equal(output[5, 1:5], mag_l1b_dataset["vectors"].data[5, :])
+    assert np.array_equal(output[9, 1:5], mag_l1b_dataset["vectors"].data[9, :])