imap-processing 0.11.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

imap_processing/tests/mag/conftest.py

@@ -2,15 +2,133 @@
 
 from pathlib import Path
 
+import numpy as np
 import pytest
+import xarray as xr
 
+from imap_processing.cdf.utils import load_cdf
+from imap_processing.mag.constants import VecSec
 from imap_processing.mag.l1a.mag_l1a import mag_l1a
+from imap_processing.spice.time import TTJ2000_EPOCH
 
 
-@pytest.fixture()
+@pytest.fixture
 def validation_l1a():
     current_directory = Path(__file__).parent
     test_file = current_directory / "validation" / "mag_l1_test_data.pkts"
     # Test file contains only normal packets
-    l1a = mag_l1a(test_file, "v000")
+    l1a = mag_l1a(test_file)
     return l1a
+
+
+def mag_l1a_dataset_generator(length):
+    epoch = xr.DataArray(np.arange(length), name="epoch", dims=["epoch"])
+    direction = xr.DataArray(np.arange(4), name="direction", dims=["direction"])
+    compression = xr.DataArray(np.arange(2), name="compression", dims=["compression"])
+
+    direction_label = xr.DataArray(
+        direction.values.astype(str),
+        name="direction_label",
+        dims=["direction_label"],
+    )
+
+    compression_label = xr.DataArray(
+        compression.values.astype(str),
+        name="compression_label",
+        dims=["compression_label"],
+    )
+
+    vectors = xr.DataArray(
+        np.zeros((length, 4)),
+        dims=["epoch", "direction"],
+        coords={"epoch": epoch, "direction": direction},
+    )
+    compression_flags = xr.DataArray(
+        np.zeros((length, 2), dtype=np.int8), dims=["epoch", "compression"]
+    )
+
+    output_dataset = xr.Dataset(
+        coords={"epoch": epoch, "direction": direction, "compression": compression},
+    )
+    output_dataset["vectors"] = vectors
+    output_dataset["compression_flags"] = compression_flags
+    output_dataset["direction_label"] = direction_label
+    output_dataset["compression_label"] = compression_label
+    output_dataset.attrs["Logical_source"] = ["imap_mag_l1a_norm-mago"]
+
+    return output_dataset
+
+
+@pytest.fixture
+def mag_test_l1b_calibration_data():
+    imap_dir = Path(__file__).parent
+    cal_file = (
+        imap_dir
+        / "validation"
+        / "calibration"
+        / "imap_mag_l1b-calibration_20240229_v001.cdf"
+    )
+    calibration_data = load_cdf(cal_file)
+    return calibration_data
+
+
+@pytest.fixture
+def mag_test_l2_data():
+    imap_dir = Path(__file__).parent
+    cal_file = (
+        imap_dir
+        / "validation"
+        / "calibration"
+        / "imap_mag_l2-calibration-matrices_20251017_v004.cdf"
+    )
+    calibration_data = load_cdf(cal_file)
+
+    offsets_data = load_cdf(
+        imap_dir
+        / "validation"
+        / "calibration"
+        / "imap_mag_l2-offsets-norm_20251017_20251017_v001.cdf"
+    )
+
+    return calibration_data, offsets_data
+
+
+def mag_generate_l1b_from_csv(df, logical_source):
+    length = len(df.index)
+    dataset = mag_l1a_dataset_generator(length)
+
+    dataset["vectors"].data = np.array(df[["x", "y", "z", "range"]])
+    dataset["compression_flags"].data = np.array(
+        df[["compression", "compression_width"]]
+    )
+
+    epoch = [np.datetime64(t) - np.datetime64(TTJ2000_EPOCH) for t in df["t"]]
+    epoch_ns = [(e / np.timedelta64(1, "ns")).astype(np.int64) for e in epoch]
+    dataset.coords["epoch"] = xr.DataArray(epoch_ns, name="epoch", dims=["epoch"])
+
+    dataset.attrs["Logical_source"] = logical_source
+    dataset.attrs["vectors_per_second"] = f"{epoch_ns[0]}:2"
+
+    return dataset
+
+
+def generate_test_epoch(
+    end, vectors_per_second: list[VecSec], starting_point=0, gaps=None
+):
+    spacing = 1 / vectors_per_second[0].value
+    output = np.array([])
+    prev = starting_point
+    if gaps:
+        for index, gap in enumerate(gaps):
+            if len(vectors_per_second) != 1:
+                spacing = 1 / vectors_per_second[index].value
+            output = np.concatenate(
+                (output, np.arange(prev, gap[0] + spacing, step=spacing) * 1e9)
+            )
+            prev = gap[1]
+        spacing = 1 / vectors_per_second[-1].value
+    output = np.concatenate(
+        (output, np.arange(prev, end + spacing, step=spacing) * 1e9)
+    )
+
+    return output

imap_processing/tests/mag/test_mag_decom.py

@@ -9,11 +9,12 @@ from imap_processing.mag.constants import DataMode
 from imap_processing.mag.l0.decom_mag import decom_packets, generate_dataset
 
 
-@pytest.fixture()
+@pytest.fixture
 def cdf_attrs():
     test_attrs = ImapCdfAttributes()
     test_attrs.add_instrument_global_attrs("mag")
-    test_attrs.add_instrument_variable_attrs("mag", "l1")
+    test_attrs.add_instrument_variable_attrs("mag", "l1a")
+    # Default v001 expected when writing to file and re-loading
     test_attrs.add_global_attribute("Data_version", "v001")
     return test_attrs
 
@@ -75,7 +76,7 @@ def test_mag_raw_xarray(cdf_attrs):
         [
             item is not None
             for key, item in norm_data.attrs.items()
-            if key != "Logical_file_id"
+            if key not in ("Logical_file_id", "Data_version")
         ]
     )
 
@@ -83,7 +84,7 @@ def test_mag_raw_xarray(cdf_attrs):
         [
             item is not None
             for key, item in burst_data.attrs.items()
-            if key != "Logical_file_id"
+            if key not in ("Logical_file_id", "Data_version")
         ]
     )
 

imap_processing/tests/mag/test_mag_l1a.py

@@ -15,7 +15,7 @@ from imap_processing.mag.l1a.mag_l1a_data import (
 from imap_processing.spice.time import met_to_ttj2000ns
 
 
-@pytest.fixture()
+@pytest.fixture
 def uncompressed_vector_bytearray():
     input_data = np.array(
         [
@@ -225,7 +225,7 @@ def uncompressed_vector_bytearray():
     return input_data
 
 
-@pytest.fixture()
+@pytest.fixture
 def expected_vectors():
     primary_expected = np.array(
         [
@@ -272,7 +272,7 @@ def expected_vectors():
     return (primary_expected, secondary_expected)
 
 
-@pytest.fixture()
+@pytest.fixture
 def raw_compressed_vectors():
     # compressed vectors, without the first starting uncompressed vector.
     # 15 primary vectors and 15 secondary vectors, corresponding to most of
@@ -883,7 +883,7 @@ def test_mag_l1a():
     current_directory = Path(__file__).parent
     test_file = current_directory / "validation" / "mag_l1_test_data.pkts"
 
-    output_data = mag_l1a(test_file, "v001")
+    output_data = mag_l1a(test_file)
 
     # Test data is one day's worth of NORM data, so it should return one raw, one MAGO
     # and one MAGI dataset
@@ -897,9 +897,6 @@ def test_mag_l1a():
     for data_type in [data.attrs["Logical_source"] for data in output_data]:
         assert data_type in expected_logical_source
 
-    for data in output_data:
-        assert data.attrs["Data_version"] == "v001"
-
     assert "vectors" in output_data[-1].variables.keys()
     assert "compression_flags" in output_data[-1].variables.keys()
 
@@ -910,6 +907,15 @@ def test_mag_l1a():
         == output_data[-1]["compression_flags"].data.shape[0]
     )
 
+    assert (
+        str(output_data[-1]["epoch"].data[0])
+        in output_data[-1].attrs["vectors_per_second"]
+    )
+    assert (
+        str(output_data[-2]["epoch"].data[0])
+        in output_data[-2].attrs["vectors_per_second"]
+    )
+
 
 def test_mag_packet_properties():
     # equal to 01001011
@@ -919,3 +925,41 @@ def test_mag_packet_properties():
     )
 
     assert packet_properties.compression_width == 18
+
+
+def test_changing_vecsec():
+    packet_one = MagL1aPacketProperties(
+        1000,
+        TimeTuple(1000, 0),
+        2,  # 2 vectors per second
+        1,  # 2 seconds of data
+        0,
+        0,
+        1,
+        0,
+    )
+    packet_two = MagL1aPacketProperties(
+        2000,
+        TimeTuple(2000, 0),
+        8,  # 8 vectors per second
+        1,  # 2 seconds of data
+        1,
+        0,
+        1,
+        0,
+    )
+
+    four_vectors = np.full((4, 5), [1, 2, 3, 4, 2])
+    sixteen_vectors = np.full((16, 5), [1, 2, 3, 4, 2])
+
+    mag_l1a = MagL1a(True, 1, 1, four_vectors, packet_one)
+    mag_l1a.append_vectors(sixteen_vectors, packet_two)
+
+    assert mag_l1a.vectors.shape[0] == 20
+    assert len(mag_l1a.packet_definitions.keys()) == 2
+    expected = [2, 8]
+    for index, value in enumerate(mag_l1a.packet_definitions.values()):
+        assert expected[index] == value.vectors_per_second
+
+    assert ":2," in mag_l1a.vectors_per_second_attribute()
+    assert ":8" in mag_l1a.vectors_per_second_attribute()

imap_processing/tests/mag/test_mag_l1b.py

@@ -4,6 +4,7 @@ import numpy as np
 import pytest
 import xarray as xr
 
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.cdf.utils import load_cdf, write_cdf
 from imap_processing.mag.l1b.mag_l1b import (
     calibrate_vector,
@@ -11,55 +12,30 @@ from imap_processing.mag.l1b.mag_l1b import (
     mag_l1b_processing,
     rescale_vector,
 )
+from imap_processing.tests.mag.conftest import (
+    mag_l1a_dataset_generator,
+)
 
 
-@pytest.fixture(scope="module")
-def mag_l1a_dataset():
-    epoch = xr.DataArray(np.arange(20), name="epoch", dims=["epoch"])
-    direction = xr.DataArray(np.arange(4), name="direction", dims=["direction"])
-    compression = xr.DataArray(np.arange(2), name="compression", dims=["compression"])
-
-    direction_label = xr.DataArray(
-        direction.values.astype(str),
-        name="direction_label",
-        dims=["direction_label"],
-    )
-
-    compression_label = xr.DataArray(
-        compression.values.astype(str),
-        name="compression_label",
-        dims=["compression_label"],
-    )
-
-    vectors = xr.DataArray(
-        np.zeros((20, 4)),
-        dims=["epoch", "direction"],
-        coords={"epoch": epoch, "direction": direction},
-    )
-    compression_flags = xr.DataArray(
-        np.zeros((20, 2), dtype=np.int8), dims=["epoch", "compression"]
-    )
-    compression_flags[1, :] = np.array([1, 18], dtype=np.int8)
-
-    vectors[0, :] = np.array([1, 1, 1, 0])
-    vectors[1, :] = np.array([7982, 48671, -68090, 0])
-
-    output_dataset = xr.Dataset(
-        coords={"epoch": epoch, "direction": direction, "compression": compression},
+def test_mag_processing(mag_test_l1b_calibration_data):
+    # All specific test values come from MAG team to accommodate various cases.
+    # Each vector is multiplied by the matrix in the calibration data for the given
+    # range to get the calibrated vector.
+    mag_l1a_dataset = mag_l1a_dataset_generator(20)
+    mag_l1a_dataset["compression_flags"].data[1, :] = np.array([1, 18], dtype=np.int8)
+
+    mag_l1a_dataset["vectors"].data[0, :] = np.array([1, 1, 1, 0])
+    mag_l1a_dataset["vectors"].data[1, :] = np.array([7982, 48671, -68090, 0])
+    mag_attributes = ImapCdfAttributes()
+    mag_attributes.add_instrument_global_attrs("mag")
+    mag_attributes.add_instrument_variable_attrs("mag", "l1b")
+    mag_l1b = mag_l1b_processing(
+        mag_l1a_dataset,
+        mag_test_l1b_calibration_data,
+        mag_attributes,
+        "imap_mag_l1b_norm-mago",
     )
-    output_dataset["vectors"] = vectors
-    output_dataset["compression_flags"] = compression_flags
-    output_dataset["direction_label"] = direction_label
-    output_dataset["compression_label"] = compression_label
-    output_dataset.attrs["Logical_source"] = ["imap_mag_l1a_norm-mago"]
 
-    return output_dataset
-
-
-def test_mag_processing(mag_l1a_dataset):
-    mag_l1a_dataset.attrs["Logical_source"] = ["imap_mag_l1a_norm-mago"]
-
-    mag_l1b = mag_l1b_processing(mag_l1a_dataset)
     np.testing.assert_allclose(
         mag_l1b["vectors"][0].values, [2.2972, 2.2415, 2.2381, 0], atol=1e-4
     )
@@ -68,13 +44,16 @@ def test_mag_processing(mag_l1a_dataset):
         [4584.1029091, 27238.73161294, -38405.22240195, 0.0],
     )
 
-    # np.testing.assert_allclose(mag_l1b["vectors"][1].values, [0, 0, 0, 0])
+    np.testing.assert_allclose(mag_l1b["vectors"][2].values, [0, 0, 0, 0])
 
     assert mag_l1b["vectors"].values.shape == mag_l1a_dataset["vectors"].values.shape
 
-    mag_l1a_dataset.attrs["Logical_source"] = ["imap_mag_l1a_norm-magi"]
-
-    mag_l1b = mag_l1b_processing(mag_l1a_dataset)
+    mag_l1b = mag_l1b_processing(
+        mag_l1a_dataset,
+        mag_test_l1b_calibration_data,
+        mag_attributes,
+        "imap_mag_l1b_norm-magi",
+    )
 
     np.testing.assert_allclose(
         mag_l1b["vectors"][0].values, [2.27538, 2.23416, 2.23682, 0], atol=1e-5
@@ -83,19 +62,19 @@ def test_mag_processing(mag_l1a_dataset):
     assert mag_l1b["vectors"].values.shape == mag_l1a_dataset["vectors"].values.shape
 
 
-def test_mag_attributes(mag_l1a_dataset):
+def test_mag_attributes():
+    mag_l1a_dataset = mag_l1a_dataset_generator(20)
+
     mag_l1a_dataset.attrs["Logical_source"] = ["imap_mag_l1a_norm-mago"]
 
-    output = mag_l1b(mag_l1a_dataset, "v001")
+    output = mag_l1b(mag_l1a_dataset)
     assert output.attrs["Logical_source"] == "imap_mag_l1b_norm-mago"
 
     mag_l1a_dataset.attrs["Logical_source"] = ["imap_mag_l1a_burst-magi"]
 
-    output = mag_l1b(mag_l1a_dataset, "v001")
+    output = mag_l1b(mag_l1a_dataset)
     assert output.attrs["Logical_source"] == "imap_mag_l1b_burst-magi"
 
-    assert output.attrs["Data_level"] == "L1B"
-
 
 def test_cdf_output():
     l1a_cdf = load_cdf(
@@ -103,14 +82,16 @@ def test_cdf_output():
         / "validation"
         / "imap_mag_l1a_norm-magi_20251017_v001.cdf"
     )
-    l1b_dataset = mag_l1b(l1a_cdf, "v001")
+    l1b_dataset = mag_l1b(l1a_cdf)
 
     output_path = write_cdf(l1b_dataset)
 
     assert Path.exists(output_path)
 
 
-def test_mag_compression_scale(mag_l1a_dataset):
+def test_mag_compression_scale():
+    mag_l1a_dataset = mag_l1a_dataset_generator(20)
+
     test_calibration = np.array(
         [
             [2.2972202, 0.0, 0.0],
@@ -129,7 +110,7 @@ def test_mag_compression_scale(mag_l1a_dataset):
     mag_l1a_dataset["compression_flags"][3, :] = np.array([1, 14], dtype=np.int8)
 
     mag_l1a_dataset.attrs["Logical_source"] = ["imap_mag_l1a_norm-mago"]
-    output = mag_l1b(mag_l1a_dataset, "v001")
+    output = mag_l1b(mag_l1a_dataset)
 
     calibrated_vectors = np.matmul(test_calibration, np.array([1, 1, 1]))
     # 16 bit width is the standard
@@ -206,9 +187,9 @@ def test_calibrate_vector():
 def test_l1a_to_l1b(validation_l1a):
     # Convert l1a input validation packet file to l1b
     with pytest.raises(ValueError, match="Raw L1A"):
-        mag_l1b(validation_l1a[0], "v000")
+        mag_l1b(validation_l1a[0])
 
-    l1b = [mag_l1b(i, "v000") for i in validation_l1a[1:]]
+    l1b = [mag_l1b(i) for i in validation_l1a[1:]]
 
     assert len(l1b) == len(validation_l1a) - 1