imap-processing 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

imap_processing/tests/hi/test_hi_l1c.py

@@ -1,56 +1,156 @@
 """Test coverage for imap_processing.hi.l1c.hi_l1c.py"""
 
+from unittest import mock
+
 import numpy as np
+import pandas as pd
 import pytest
+import xarray as xr
 
-from imap_processing.cdf.utils import write_cdf
-from imap_processing.hi.l1a.hi_l1a import hi_l1a
-from imap_processing.hi.l1b.hi_l1b import hi_l1b
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.cdf.utils import load_cdf, write_cdf
 from imap_processing.hi.l1c import hi_l1c
+from imap_processing.hi.l1c.hi_l1c import CalibrationProductConfig
 from imap_processing.hi.utils import HIAPID
 
 
-@pytest.mark.skip(
-    reason="See TODO in test comments. Need to convert this test"
-    "to use a test L1B file rather than running L1B on fake"
-    "data."
-)
-def test_generate_pset_dataset(create_de_data):
-    """Test coverage for generate_pset_dataset function"""
-    # TODO: once things are more stable, check in an L1B DE file as test data?
-    # For now, test using false de data run through l1a and l1b processing
-    bin_data_path = create_de_data(HIAPID.H45_SCI_DE.value)
-    processed_data = hi_l1a(bin_data_path, "002")
-    l1b_dataset = hi_l1b(processed_data[0], "002")
+@pytest.fixture(scope="module")
+def hi_test_cal_prod_config_path(hi_l1_test_data_path):
+    return (
+        hi_l1_test_data_path / "imap_his_pset-calibration-prod-config_20240101_v001.csv"
+    )
 
-    l1c_dataset = hi_l1c.generate_pset_dataset(l1b_dataset)
 
-    assert l1c_dataset.epoch.data[0] == l1b_dataset.epoch.data[0]
+@mock.patch("imap_processing.hi.l1c.hi_l1c.generate_pset_dataset")
+def test_hi_l1c(mock_generate_pset_dataset, hi_test_cal_prod_config_path):
+    """Test coverage for hi_l1c function"""
+    mock_generate_pset_dataset.return_value = xr.Dataset(attrs={"Data_version": None})
+    pset = hi_l1c.hi_l1c(
+        [xr.Dataset(), hi_test_cal_prod_config_path], data_version="99"
+    )
+    assert pset.attrs["Data_version"] == "99"
 
 
-def test_allocate_pset_dataset():
-    """Test coverage for allocate_pset_dataset function"""
-    n_esa_steps = 10
-    n_calibration_prods = 5
-    sensor_str = HIAPID.H90_SCI_DE.sensor
-    dataset = hi_l1c.allocate_pset_dataset(n_esa_steps, sensor_str)
+def test_hi_l1c_not_implemented():
+    """Test coverage for hi_l1c function with unrecognized dependencies"""
+    with pytest.raises(NotImplementedError):
+        hi_l1c.hi_l1c([None, None], "0")
 
-    assert dataset.epoch.size == 1
-    assert dataset.spin_angle_bin.size == 3600
-    assert dataset.esa_energy_step.size == n_esa_steps
-    assert dataset.calibration_prod.size == n_calibration_prods
-    np.testing.assert_array_equal(dataset.despun_z.data.shape, (1, 3))
-    np.testing.assert_array_equal(dataset.hae_latitude.data.shape, (1, 3600))
-    np.testing.assert_array_equal(dataset.hae_longitude.data.shape, (1, 3600))
+
+@pytest.mark.external_kernel()
+@pytest.mark.use_test_metakernel("imap_ena_sim_metakernel.template")
+def test_generate_pset_dataset(hi_l1_test_data_path, hi_test_cal_prod_config_path):
+    """Test coverage for generate_pset_dataset function"""
+    l1b_de_path = hi_l1_test_data_path / "imap_hi_l1b_45sensor-de_20250415_v999.cdf"
+    l1b_dataset = load_cdf(l1b_de_path)
+    l1c_dataset = hi_l1c.generate_pset_dataset(
+        l1b_dataset, hi_test_cal_prod_config_path
+    )
+
+    assert l1c_dataset.epoch.data[0] == np.mean(l1b_dataset.epoch.data[[0, -1]]).astype(
+        np.int64
+    )
+
+    np.testing.assert_array_equal(l1c_dataset.despun_z.data.shape, (1, 3))
+    np.testing.assert_array_equal(l1c_dataset.hae_latitude.data.shape, (1, 3600))
+    np.testing.assert_array_equal(l1c_dataset.hae_longitude.data.shape, (1, 3600))
     for var in [
         "counts",
         "exposure_times",
         "background_rates",
         "background_rates_uncertainty",
     ]:
-        np.testing.assert_array_equal(
-            dataset[var].data.shape, (1, n_esa_steps, n_calibration_prods, 3600)
-        )
-    # Verify resulting CDF is ISTP compliant by writing to disk
-    dataset.attrs["Data_version"] = 1
-    write_cdf(dataset)
+        np.testing.assert_array_equal(l1c_dataset[var].data.shape, (1, 9, 2, 3600))
+
+    # Test ISTP compliance by writing CDF
+    l1c_dataset.attrs["Data_version"] = 1
+    write_cdf(l1c_dataset)
+
+
+def test_empty_pset_dataset():
+    """Test coverage for empty_pset_dataset function"""
+    n_energy_steps = 8
+    l1b_esa_energy_steps = np.arange(n_energy_steps + 1).repeat(2)
+    n_calibration_prods = 5
+    sensor_str = HIAPID.H90_SCI_DE.sensor
+    dataset = hi_l1c.empty_pset_dataset(
+        l1b_esa_energy_steps, n_calibration_prods, sensor_str
+    )
+
+    assert dataset.epoch.size == 1
+    assert dataset.spin_angle_bin.size == 3600
+    assert dataset.esa_energy_step.size == n_energy_steps
+    np.testing.assert_array_equal(
+        dataset.esa_energy_step.data, np.arange(n_energy_steps) + 1
+    )
+    assert dataset.calibration_prod.size == n_calibration_prods
+
+    # verify that attrs defined in hi_pset_epoch have overwritten default
+    # epoch attributes
+    attr_mgr = ImapCdfAttributes()
+    attr_mgr.add_instrument_global_attrs("hi")
+    attr_mgr.add_instrument_variable_attrs(instrument="hi", level=None)
+    pset_epoch_attrs = attr_mgr.get_variable_attributes(
+        "hi_pset_epoch", check_schema=False
+    )
+    for k, v in pset_epoch_attrs.items():
+        assert k in dataset.epoch.attrs
+        assert dataset.epoch.attrs[k] == v
+
+
+@pytest.mark.parametrize("sensor_str", ["90sensor", "45sensor"])
+@mock.patch("imap_processing.spice.geometry.frame_transform")
+@mock.patch("imap_processing.hi.l1c.hi_l1c.frame_transform")
+def test_pset_geometry(mock_frame_transform, mock_geom_frame_transform, sensor_str):
+    """Test coverage for pset_geometry function"""
+    # pset_geometry uses both frame_transform and frame_transform_az_el. By mocking
+    # the frame_transform imported into hi_l1c as well as the geometry.frame_transform
+    # the underlying need for SPICE kernels is remove. Mock them both to just return
+    # the input position vectors.
+    mock_frame_transform.side_effect = lambda et, pos, from_frame, to_frame: pos
+    mock_geom_frame_transform.side_effect = lambda et, pos, from_frame, to_frame: pos
+
+    geometry_vars = hi_l1c.pset_geometry(0, sensor_str)
+
+    assert "despun_z" in geometry_vars
+    np.testing.assert_array_equal(geometry_vars["despun_z"].data, [[0, 0, 1]])
+
+    assert "hae_latitude" in geometry_vars
+    assert "hae_longitude" in geometry_vars
+    # frame_transform is mocked to return the input vectors. For Hi-90, we
+    # expect hae_latitude to be 0, and for Hi-45 we expect -45. Both sensors
+    # have an expected longitude to be 0.1 degree steps starting at 0.05
+    expected_latitude = 0 if sensor_str == "90sensor" else -45
+    np.testing.assert_array_equal(
+        geometry_vars["hae_latitude"].data, np.full((1, 3600), expected_latitude)
+    )
+    np.testing.assert_allclose(
+        geometry_vars["hae_longitude"].data,
+        np.arange(0.05, 360, 0.1, dtype=np.float32).reshape((1, 3600)),
+        atol=4e-05,
+    )
+
+
+class TestCalibrationProductConfig:
+    """
+    All test coverage for the pd.DataFrame accessor extension "cal_prod_config".
+    """
+
+    def test_wrong_columns(self):
+        """Test coverage for a dataframe with the wrong columns."""
+        required_columns = CalibrationProductConfig.required_columns
+        for exclude_column_name in required_columns:
+            include_columns = set(required_columns) - {exclude_column_name}
+            df = pd.DataFrame({col: [1, 2, 3] for col in include_columns})
+            with pytest.raises(AttributeError, match="Required column*"):
+                _ = df.cal_prod_config.number_of_products
+
+    def test_from_csv(self, hi_test_cal_prod_config_path):
+        """Test coverage for read_csv function."""
+        df = CalibrationProductConfig.from_csv(hi_test_cal_prod_config_path)
+        assert isinstance(df["coincidence_type_list"][0, 1], list)
+
+    def test_number_of_products(self, hi_test_cal_prod_config_path):
+        """Test coverage for number of products accessor."""
+        df = CalibrationProductConfig.from_csv(hi_test_cal_prod_config_path)
+        assert df.cal_prod_config.number_of_products == 2

imap_processing/tests/hi/test_l1a.py

@@ -19,8 +19,6 @@ def test_sci_de_decom(hi_l0_test_data_path):
 
     # Write to CDF
     cdf_filename = "imap_hi_l1a_90sensor-de_20241105_v001.cdf"
-    # TODO: Dropping duplicates to ignore ISTP for now. Should be fixed by #1186
-    processed_data[0] = processed_data[0].sortby("epoch").groupby("epoch").first()
     cdf_filepath = write_cdf(processed_data[0])
     assert cdf_filepath.name == cdf_filename
 

imap_processing/tests/hi/test_science_direct_event.py

@@ -37,22 +37,26 @@ def test_parse_direct_events():
 def test_create_dataset():
     """Test create_dataset"""
     # dummy data to test create_dataset
+    n_packets = 3
+    n_events = 4
     data_dict = {
-        "trigger_id": [1, 2, 3],
-        "tof_1": [512, 512, 512],
-        "tof_2": [512, 512, 512],
-        "tof_3": [512, 512, 512],
-        "de_tag": [1, 2, 3],
-        "meta_seconds": [433522962, 433522962, 433522962],
-        "meta_subseconds": [512, 512, 512],
-        "esa_step": [4, 4, 4],
-        "ccsds_met": [433522961, 433522961, 433522961],
-        "src_seq_ctr": [10, 10, 10],
-        "pkt_len": [146, 146, 146],
-        "last_spin_num": [4, 4, 4],
-        "spin_invalids": [0, 0, 0],
+        "ccsds_met": np.arange(n_packets) + 433522961,
+        "src_seq_ctr": np.arange(n_packets) + 10,
+        "pkt_len": np.full(n_packets, 146),
+        "last_spin_num": np.arange(n_packets) + 3,
+        "spin_invalids": np.zeros(n_packets),
+        "esa_step": np.full(n_packets, 4),
+        "meta_seconds": np.arange(n_packets) + 433522960,
+        "meta_subseconds": np.full(n_packets, 500),
+        "trigger_id": [1 + i % 3 for i in range(n_events)],
+        "tof_1": [450] * n_events,
+        "tof_2": [450] * n_events,
+        "tof_3": [450] * n_events,
+        "de_tag": [i for i in range(n_events)],
+        "ccsds_index": [int(i / n_events * n_packets) for i in range(n_events)],
     }
 
     # Test for good data
     dataset = create_dataset(data_dict)
-    assert dataset["epoch"].shape == (3,)
+    assert dataset["epoch"].shape == (n_packets,)
+    assert dataset["event_met"].shape == (n_events,)

imap_processing/tests/hi/test_utils.py

@@ -44,14 +44,14 @@ def test_parse_sensor_number(test_str, expected):
 
 
 @pytest.mark.parametrize(
-    "name, shape, expected_shape",
+    "name, shape, fill_value, expected_shape",
     [
-        ("despun_z", (1, 3), (1, 3)),
-        ("hae_latitude", None, (1, 360)),
-        ("counts", None, (1, 10, 5, 360)),
+        ("despun_z", (1, 3), None, (1, 3)),
+        ("hae_latitude", None, 0, (1, 360)),
+        ("counts", None, None, (1, 10, 5, 360)),
     ],
 )
-def test_full_dataarray(name, shape, expected_shape):
+def test_full_dataarray(name, shape, fill_value, expected_shape):
     """Test coverage for full_dataarray function"""
     coords = {
         "epoch": xr.DataArray(np.array([0])),
@@ -61,25 +61,28 @@ def test_full_dataarray(name, shape, expected_shape):
     }
     cdf_manager = ImapCdfAttributes()
     cdf_manager.add_instrument_variable_attrs(instrument="hi", level=None)
+    attrs = cdf_manager.get_variable_attributes(f"hi_pset_{name}")
 
     dataarray = full_dataarray(
-        name, cdf_manager.get_variable_attributes(f"hi_pset_{name}"), coords, shape
+        name, attrs, coords=coords, shape=shape, fill_value=fill_value
     )
     assert dataarray.data.shape == expected_shape
+    expected_fill_value = fill_value if fill_value is not None else attrs["FILLVAL"]
+    np.testing.assert_array_equal(dataarray.data, expected_fill_value)
 
 
 @pytest.mark.parametrize(
-    "var_names, shape, lookup_str",
+    "var_names, shape, fill_value, lookup_str",
     [
-        (["delta_t_ab", "delta_t_ac1"], 5, "hi_de_{0}"),
-        (["hae_latitude"], (3, 5), "hi_pset_{0}"),
+        (["delta_t_ab", "delta_t_ac1"], 5, None, "hi_de_{0}"),
+        (["hae_latitude"], (3, 5), 0, "hi_pset_{0}"),
     ],
 )
-def test_create_dataset_variables(var_names, shape, lookup_str):
+def test_create_dataset_variables(var_names, shape, fill_value, lookup_str):
     """Test coverage for `imap_processing.hi.utils.create_dataset_variables`"""
     var_names = ["delta_t_ab", "delta_t_ac1", "delta_t_bc1"]
     l1b_de_vars = create_dataset_variables(
-        var_names, shape, att_manager_lookup_str="hi_de_{0}"
+        var_names, shape, fill_value=fill_value, att_manager_lookup_str="hi_de_{0}"
     )
     assert len(l1b_de_vars) == len(var_names)
     attr_mgr = ImapCdfAttributes()
@@ -95,3 +98,5 @@ def test_create_dataset_variables(var_names, shape, lookup_str):
             assert data_array.size == shape
         else:
             assert data_array.shape == shape
+        expected_fill_value = fill_value if fill_value is not None else attrs["FILLVAL"]
+        np.testing.assert_array_equal(data_array, expected_fill_value)