imap-processing 0.11.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

imap_processing/tests/hi/test_hi_l1c.py

@@ -1,6 +1,8 @@
 """Test coverage for imap_processing.hi.l1c.hi_l1c.py"""
 
+from collections import namedtuple
 from unittest import mock
+from unittest.mock import MagicMock
 
 import numpy as np
 import pandas as pd
@@ -9,9 +11,10 @@ import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.cdf.utils import load_cdf, write_cdf
+from imap_processing.hi.l1a.science_direct_event import DE_CLOCK_TICK_S
 from imap_processing.hi.l1c import hi_l1c
 from imap_processing.hi.l1c.hi_l1c import CalibrationProductConfig
-from imap_processing.hi.utils import HIAPID
+from imap_processing.hi.utils import HIAPID, CoincidenceBitmap
 
 
 @pytest.fixture(scope="module")
@@ -24,46 +27,46 @@ def hi_test_cal_prod_config_path(hi_l1_test_data_path):
 @mock.patch("imap_processing.hi.l1c.hi_l1c.generate_pset_dataset")
 def test_hi_l1c(mock_generate_pset_dataset, hi_test_cal_prod_config_path):
     """Test coverage for hi_l1c function"""
-    mock_generate_pset_dataset.return_value = xr.Dataset(attrs={"Data_version": None})
-    pset = hi_l1c.hi_l1c(
-        [xr.Dataset(), hi_test_cal_prod_config_path], data_version="99"
-    )
-    assert pset.attrs["Data_version"] == "99"
+    mock_generate_pset_dataset.return_value = xr.Dataset()
+    pset = hi_l1c.hi_l1c([xr.Dataset(), hi_test_cal_prod_config_path])[0]
+    # Empty attributes, global values get added in post-processing
+    assert pset.attrs == {}
 
 
 def test_hi_l1c_not_implemented():
     """Test coverage for hi_l1c function with unrecognized dependencies"""
     with pytest.raises(NotImplementedError):
-        hi_l1c.hi_l1c([None, None], "0")
+        hi_l1c.hi_l1c([None, None])
 
 
-@pytest.mark.external_kernel()
+@pytest.mark.external_test_data
+@pytest.mark.external_kernel
 @pytest.mark.use_test_metakernel("imap_ena_sim_metakernel.template")
-def test_generate_pset_dataset(hi_l1_test_data_path, hi_test_cal_prod_config_path):
+def test_generate_pset_dataset(
+    hi_l1_test_data_path, hi_test_cal_prod_config_path, use_fake_spin_data_for_time
+):
     """Test coverage for generate_pset_dataset function"""
+    use_fake_spin_data_for_time(482372987.999)
     l1b_de_path = hi_l1_test_data_path / "imap_hi_l1b_45sensor-de_20250415_v999.cdf"
     l1b_dataset = load_cdf(l1b_de_path)
     l1c_dataset = hi_l1c.generate_pset_dataset(
         l1b_dataset, hi_test_cal_prod_config_path
     )
 
-    assert l1c_dataset.epoch.data[0] == np.mean(l1b_dataset.epoch.data[[0, -1]]).astype(
-        np.int64
-    )
+    assert l1c_dataset.epoch.data[0] == l1b_dataset.epoch.data[0].astype(np.int64)
 
     np.testing.assert_array_equal(l1c_dataset.despun_z.data.shape, (1, 3))
     np.testing.assert_array_equal(l1c_dataset.hae_latitude.data.shape, (1, 3600))
     np.testing.assert_array_equal(l1c_dataset.hae_longitude.data.shape, (1, 3600))
+    np.testing.assert_array_equal(l1c_dataset.exposure_times.data.shape, (1, 9, 3600))
     for var in [
         "counts",
-        "exposure_times",
         "background_rates",
         "background_rates_uncertainty",
     ]:
         np.testing.assert_array_equal(l1c_dataset[var].data.shape, (1, 9, 2, 3600))
 
     # Test ISTP compliance by writing CDF
-    l1c_dataset.attrs["Data_version"] = 1
     write_cdf(l1c_dataset)
 
 
@@ -74,7 +77,7 @@ def test_empty_pset_dataset():
     n_calibration_prods = 5
     sensor_str = HIAPID.H90_SCI_DE.sensor
     dataset = hi_l1c.empty_pset_dataset(
-        l1b_esa_energy_steps, n_calibration_prods, sensor_str
+        100, l1b_esa_energy_steps, n_calibration_prods, sensor_str
     )
 
     assert dataset.epoch.size == 1
@@ -131,6 +134,225 @@ def test_pset_geometry(mock_frame_transform, mock_geom_frame_transform, sensor_s
     )
 
 
+@pytest.mark.external_test_data
+def test_pset_counts(hi_l1_test_data_path, hi_test_cal_prod_config_path):
+    """Test coverage for pset_counts function."""
+    l1b_de_path = hi_l1_test_data_path / "imap_hi_l1b_45sensor-de_20250415_v999.cdf"
+    l1b_dataset = load_cdf(l1b_de_path)
+    cal_config_df = hi_l1c.CalibrationProductConfig.from_csv(
+        hi_test_cal_prod_config_path
+    )
+    empty_pset = hi_l1c.empty_pset_dataset(
+        100,
+        l1b_dataset.esa_energy_step.data,
+        cal_config_df.cal_prod_config.number_of_products,
+        HIAPID.H90_SCI_DE.sensor,
+    )
+    counts_var = hi_l1c.pset_counts(empty_pset.coords, cal_config_df, l1b_dataset)
+    assert "counts" in counts_var
+
+
+def test_get_tof_window_mask():
+    """Test coverage for get_tof_window_mask function."""
+    # Create a synthetic dataframe with required columns containing data
+    # intended to test all aspects of the function.
+    fill_vals = {
+        "tof_ab": -11,
+        "tof_ac1": -12,
+        "tof_bc1": -13,
+        "tof_c1c2": -14,
+    }
+    Row = namedtuple(
+        "Row",
+        [
+            "Index",
+            "tof_ab_low",
+            "tof_ab_high",
+            "tof_ac1_low",
+            "tof_ac1_high",
+            "tof_bc1_low",
+            "tof_bc1_high",
+            "tof_c1c2_low",
+            "tof_c1c2_high",
+        ],
+    )
+    prod_config_row = Row((1, 0), 0, 1, -1, 2, 1, 5, 4, 6)
+    synth_df = pd.DataFrame(
+        {
+            "tof_ab": np.array(
+                [0, 2, 1, 0, -1, -5, -11], dtype=np.int32
+            ),  # T, F, T, T, F, F, FILL
+            "tof_ac1": np.array(
+                [-1, 2, -2, 0, 3, 0, -12], dtype=np.int32
+            ),  # T, T, F, T, F, T, FILL
+            "tof_bc1": np.array(
+                [1, 5, 3, 0, 6, 2, -13], dtype=np.int32
+            ),  # T, T, T, F, F, T, FILL
+            "tof_c1c2": np.array(
+                [4, 6, 5, 3, 7, -9, -14], dtype=np.int32
+            ),  # T, T, T, F, F, F, FILL
+        },
+    )
+    expected_mask = np.array([True, False, False, False, False, False, True])
+    window_mask = hi_l1c.get_tof_window_mask(synth_df, prod_config_row, fill_vals)
+    np.testing.assert_array_equal(expected_mask, window_mask)
+
+
+@mock.patch("imap_processing.hi.l1c.hi_l1c.get_spin_data", return_value=None)
+@mock.patch("imap_processing.hi.l1c.hi_l1c.get_instrument_spin_phase")
+@mock.patch("imap_processing.hi.l1c.hi_l1c.get_de_clock_ticks_for_esa_step")
+@mock.patch("imap_processing.hi.l1c.hi_l1c.find_second_de_packet_data")
+def test_pset_exposure(
+    mock_find_second_de_packet_data,
+    mock_de_clock_ticks,
+    mock_spin_phase,
+    mock_spin_data,
+):
+    """Test coverage for pset_exposure function"""
+    empty_pset = hi_l1c.empty_pset_dataset(
+        100, np.arange(2) + 1, 2, HIAPID.H90_SCI_DE.sensor
+    )
+    # Set the mock of find_second_de_packet_data to return a xr.Dataset
+    # with some dummy data. ESA 1 will get binned data once, ESA 2 will get
+    # binned data twice.
+    mock_find_second_de_packet_data.return_value = xr.Dataset(
+        coords={"epoch": xr.DataArray(np.arange(3), dims=["epoch"])},
+        data_vars={
+            "ccsds_met": xr.DataArray(np.arange(3), dims=["epoch"]),
+            "esa_energy_step": xr.DataArray(np.array([1, 2, 2]), dims=["epoch"]),
+        },
+    )
+    # Set mock of get_de_clock_ticks_for_esa_step and spin phase to generate
+    # deterministic histogram values.
+    # ESA step 1 should have repeating values of 3, 1.
+    # ESA step 2 should have repeating values of 6, 2
+    mock_spin_phase.return_value = np.concat(
+        [hi_l1c.SPIN_PHASE_BIN_CENTERS, hi_l1c.SPIN_PHASE_BIN_CENTERS[::2]]
+    )
+    mock_de_clock_ticks.return_value = (
+        np.zeros(hi_l1c.N_SPIN_BINS + hi_l1c.N_SPIN_BINS // 2),
+        np.concat([np.ones(hi_l1c.N_SPIN_BINS), np.ones(hi_l1c.N_SPIN_BINS // 2) * 2]),
+    )
+
+    # The above mocks mean no data needs to be in the l1b_dataset. It
+    # only needs to provide a logical source that contains "90sensor".
+    l1b_dataset = MagicMock()
+    l1b_dataset.attrs = {"Logical_source": "90sensor"}
+
+    # All the setup is done, call the pset_exposure function
+    exposure_dict = hi_l1c.pset_exposure(empty_pset.coords, l1b_dataset)
+
+    # Based on the spin phase and clock_tick mocks, the expected clock ticks are:
+    # - Repeated values of 3, 1 for the first half of the spin bins
+    # - Repeated values of 3, 2 for the second half of the spin bins
+    expected_values = np.stack(
+        [
+            np.tile([3, 1], hi_l1c.N_SPIN_BINS // 2),
+            np.tile([6, 2], hi_l1c.N_SPIN_BINS // 2),
+        ]
+    ).astype(float)[None, :, :]
+    # Convert expected clock ticks to seconds
+    expected_values *= DE_CLOCK_TICK_S
+    np.testing.assert_allclose(
+        exposure_dict["exposure_times"].data,
+        expected_values,
+        atol=DE_CLOCK_TICK_S / 100,
+    )
+
+
+def test_find_second_de_packet_data():
+    """Test coverage for find_second_de_packet_data function"""
+    # Create a test l1b_dataset
+    # Expect to remove index 0 and 5 due to missing esa_step pair
+    # Expect to remove index 11 due to 0 being a calibration step
+    # Expect to return indices 2, 4, 7, 9, 13
+    esa_steps = np.array([1, 2, 2, 3, 3, 4, 5, 5, 6, 6, 0, 0, 7, 7])
+    l1b_dataset = xr.Dataset(
+        coords={
+            "epoch": xr.DataArray(
+                np.arange(esa_steps.size),
+                dims=["epoch"],
+            ),
+            "event_met": xr.DataArray(
+                np.arange(10),
+                dims=["event_met"],
+            ),
+        },
+        data_vars={
+            "esa_step": xr.DataArray(
+                esa_steps,
+                dims=["epoch"],
+            ),
+            "coincidence_type": xr.DataArray(
+                np.ones(10),
+                dims=["event_met"],
+            ),
+        },
+    )
+    subset = hi_l1c.find_second_de_packet_data(l1b_dataset)
+    np.testing.assert_array_equal(subset.epoch.data, np.array([2, 4, 7, 9, 13]))
+
+
+@pytest.fixture(scope="module")
+def fake_spin_df():
+    """Generate a synthetic spin dataframe"""
+    # Generate some spin periods that vary by a random fraction of a second
+    spin_period = np.full(10, 15) + np.random.randn(10) / 10
+    d = {
+        "spin_start_met": np.add.accumulate(spin_period),
+        "spin_period_sec": spin_period,
+    }
+    spin_df = pd.DataFrame.from_dict(d)
+    return spin_df
+
+
+def test_get_de_clock_ticks_for_esa_step(fake_spin_df):
+    """Test coverage for get_de_clock_ticks_for_esa_step function."""
+
+    # Test nominal cases where CCSDS met falls after 8th spin start and before
+    # the end spin in the table + 1/2 spin period
+    for _, spin_row in fake_spin_df.iloc[8:].iterrows():
+        for ccsds_met in np.linspace(
+            spin_row.spin_start_met,
+            spin_row.spin_start_met + np.floor(spin_row.spin_period_sec / 2),
+            10,
+        ):
+            clock_tick_mets, clock_tick_weights = (
+                hi_l1c.get_de_clock_ticks_for_esa_step(ccsds_met, fake_spin_df)
+            )
+            np.testing.assert_array_equal(clock_tick_mets.shape, clock_tick_mets.shape)
+            # Verify last weight entry
+            exp_final_weight = (
+                np.absolute(
+                    fake_spin_df.spin_start_met.to_numpy() - clock_tick_mets[-1]
+                ).min()
+                / DE_CLOCK_TICK_S
+            )
+            assert clock_tick_weights[-1] == exp_final_weight
+            assert np.all(clock_tick_weights[:-1] == 1)
+
+
+def test_get_de_clock_ticks_for_esa_step_exceptions(fake_spin_df):
+    """Test the exception logic in the get_de_clock_ticks_for_esa_step function."""
+    # Test the ccsds_met being > 1/2 spin period past the spin start
+    bad_ccsds_met = (
+        fake_spin_df.iloc[8].spin_start_met
+        + fake_spin_df.iloc[8].spin_period_sec / 2
+        + 0.1
+    )
+    with pytest.raises(
+        ValueError, match="The difference between ccsds_met and spin_start_met"
+    ):
+        hi_l1c.get_de_clock_ticks_for_esa_step(bad_ccsds_met, fake_spin_df)
+
+    # Test the ccsds_met being too close to the start of the spin table
+    bad_ccsds_met = fake_spin_df.iloc[7].spin_start_met
+    with pytest.raises(
+        ValueError, match="Error determining start/end time for exposure time"
+    ):
+        hi_l1c.get_de_clock_ticks_for_esa_step(bad_ccsds_met, fake_spin_df)
+
+
 class TestCalibrationProductConfig:
     """
     All test coverage for the pd.DataFrame accessor extension "cal_prod_config".
@@ -138,7 +360,7 @@ class TestCalibrationProductConfig:
 
     def test_wrong_columns(self):
         """Test coverage for a dataframe with the wrong columns."""
-        required_columns = CalibrationProductConfig.required_columns
+        required_columns = hi_l1c.CalibrationProductConfig.required_columns
         for exclude_column_name in required_columns:
             include_columns = set(required_columns) - {exclude_column_name}
             df = pd.DataFrame({col: [1, 2, 3] for col in include_columns})
@@ -147,10 +369,19 @@ class TestCalibrationProductConfig:
 
     def test_from_csv(self, hi_test_cal_prod_config_path):
         """Test coverage for read_csv function."""
+        df = hi_l1c.CalibrationProductConfig.from_csv(hi_test_cal_prod_config_path)
+        assert isinstance(df["coincidence_type_list"][0, 1], tuple)
+
+    def test_added_coincidence_type_values_column(self, hi_test_cal_prod_config_path):
         df = CalibrationProductConfig.from_csv(hi_test_cal_prod_config_path)
-        assert isinstance(df["coincidence_type_list"][0, 1], list)
+        assert "coincidence_type_values" in df.columns
+        for _, row in df.iterrows():
+            for detect_string, val in zip(
+                row["coincidence_type_list"], row["coincidence_type_values"]
+            ):
+                assert val == CoincidenceBitmap.detector_hit_str_to_int(detect_string)
 
     def test_number_of_products(self, hi_test_cal_prod_config_path):
         """Test coverage for number of products accessor."""
-        df = CalibrationProductConfig.from_csv(hi_test_cal_prod_config_path)
+        df = hi_l1c.CalibrationProductConfig.from_csv(hi_test_cal_prod_config_path)
         assert df.cal_prod_config.number_of_products == 2

imap_processing/tests/hi/test_l1a.py

@@ -1,4 +1,5 @@
 import numpy as np
+import pandas as pd
 
 from imap_processing.cdf.utils import write_cdf
 from imap_processing.hi.l1a.hi_l1a import hi_l1a
@@ -10,29 +11,56 @@ def test_sci_de_decom(hi_l0_test_data_path):
     """Test science direct event data"""
 
     bin_data_path = hi_l0_test_data_path / "H90_sci_de_20241104.bin"
-    processed_data = hi_l1a(bin_data_path, data_version="001")
+    processed_data = hi_l1a(bin_data_path)
 
     assert processed_data[0].attrs["Logical_source"] == "imap_hi_l1a_90sensor-de"
-    assert processed_data[0].attrs["Data_version"] == "001"
 
     # TODO: Verify correct unpacking of sample data. Issue: #1186
 
     # Write to CDF
-    cdf_filename = "imap_hi_l1a_90sensor-de_20241105_v001.cdf"
+    cdf_filename = "imap_hi_l1a_90sensor-de_20241105_v999.cdf"
     cdf_filepath = write_cdf(processed_data[0])
     assert cdf_filepath.name == cdf_filename
 
 
+def test_diag_fee_decom(hi_l0_test_data_path):
+    """Test diag_fee data"""
+    bin_data_path = hi_l0_test_data_path / "H45_diag_fee_20250208.bin"
+    processed_data = hi_l1a(packet_file_path=bin_data_path)
+    dataset = processed_data[0]
+    cdf_filepath = write_cdf(processed_data[0], istp=False)
+    assert cdf_filepath.name == "imap_hi_l1a_45sensor-diagfee_20250208_v999.cdf"
+
+    assert np.unique(processed_data[0]["pkt_apid"].values) == HIAPID.H45_DIAG_FEE.value
+
+    validation_df = pd.read_csv(
+        hi_l0_test_data_path / "H45_diag_fee_20250208_verify.csv"
+    )
+    val_to_test_map = {
+        "PHVERNO": "version",
+        "PHTYPE": "type",
+        "PHSHF": "sec_hdr_flg",
+        "PHAPID": "pkt_apid",
+        "PHGROUPF": "seq_flgs",
+        "PHSEQCNT": "src_seq_ctr",
+        "PHDLEN": "pkt_len",
+    }
+    for col_name, series in validation_df.items():
+        if col_name == "timestamp":
+            continue
+        ds_var_name = val_to_test_map.get(col_name, col_name.lower())
+        np.testing.assert_array_equal(series.values, dataset[ds_var_name].data)
+
+
 def test_app_nhk_decom(hi_l0_test_data_path):
     """Test housekeeping data"""
 
     # Unpack housekeeping data
     bin_data_path = hi_l0_test_data_path / "H90_NHK_20241104.bin"
-    processed_data = hi_l1a(packet_file_path=bin_data_path, data_version="001")
+    processed_data = hi_l1a(packet_file_path=bin_data_path)
 
     assert np.unique(processed_data[0]["pkt_apid"].values) == HIAPID.H90_APP_NHK.value
     assert processed_data[0].attrs["Logical_source"] == "imap_hi_l1a_90sensor-hk"
-    assert processed_data[0].attrs["Data_version"] == "001"
     # TODO: compare with validation data once we have it. Issue: #1184
 
     # Write CDF
@@ -40,13 +68,13 @@ def test_app_nhk_decom(hi_l0_test_data_path):
 
     # TODO: ask Vivek about this date mismatch between the file name
     # and the data. May get resolved when we have good sample data.
-    assert cem_raw_cdf_filepath.name == "imap_hi_l1a_90sensor-hk_20241105_v001.cdf"
+    assert cem_raw_cdf_filepath.name == "imap_hi_l1a_90sensor-hk_20241105_v999.cdf"
 
 
 def test_app_hist_decom(hi_l0_test_data_path):
     """Test histogram (SCI_CNT) data"""
     bin_data_path = hi_l0_test_data_path / "H90_sci_cnt_20241104.bin"
-    processed_data = hi_l1a(packet_file_path=bin_data_path, data_version="001")
+    processed_data = hi_l1a(packet_file_path=bin_data_path)
 
     assert processed_data[0].attrs["Logical_source"] == "imap_hi_l1a_90sensor-hist"
     # TODO: compare with validation data once we have it. Issue: #1185

imap_processing/tests/hi/test_science_direct_event.py

@@ -21,8 +21,8 @@ def test_parse_direct_events():
     # Encode the random events data into a bit-string
     bin_str = ""
     for i in range(n_events):
-        bin_str += f"{exp_dict['trigger_id'][i]:02b}"  # 2-bits for trigger_id
         bin_str += f"{exp_dict['de_tag'][i]:016b}"  # 16-bits for de_tag
+        bin_str += f"{exp_dict['trigger_id'][i]:02b}"  # 2-bits for trigger_id
         bin_str += f"{exp_dict['tof_1'][i]:010b}"  # 10-bits for tof_1
         bin_str += f"{exp_dict['tof_2'][i]:010b}"  # 10-bits for tof_2
         bin_str += f"{exp_dict['tof_3'][i]:010b}"  # 10-bits for tof_3
@@ -30,8 +30,8 @@ def test_parse_direct_events():
     bytes_obj = bytes([int(bin_str[i : i + 8], 2) for i in range(0, len(bin_str), 8)])
     # Parse the fake events and check values
     de_dict = parse_direct_events(bytes_obj)
-    for key in exp_dict.keys():
-        np.testing.assert_array_equal(de_dict[key], exp_dict[key])
+    for key, expected_val in exp_dict.items():
+        np.testing.assert_array_equal(de_dict[key], expected_val)
 
 
 def test_create_dataset():

imap_processing/tests/hi/test_utils.py

@@ -7,6 +7,7 @@ import xarray as xr
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.hi.utils import (
     HIAPID,
+    CoincidenceBitmap,
     create_dataset_variables,
     full_dataarray,
     parse_sensor_number,
@@ -74,13 +75,13 @@ def test_full_dataarray(name, shape, fill_value, expected_shape):
 @pytest.mark.parametrize(
     "var_names, shape, fill_value, lookup_str",
     [
-        (["delta_t_ab", "delta_t_ac1"], 5, None, "hi_de_{0}"),
+        (["tof_ab", "tof_ac1"], 5, None, "hi_de_{0}"),
         (["hae_latitude"], (3, 5), 0, "hi_pset_{0}"),
     ],
 )
 def test_create_dataset_variables(var_names, shape, fill_value, lookup_str):
     """Test coverage for `imap_processing.hi.utils.create_dataset_variables`"""
-    var_names = ["delta_t_ab", "delta_t_ac1", "delta_t_bc1"]
+    var_names = ["tof_ab", "tof_ac1", "tof_bc1"]
     l1b_de_vars = create_dataset_variables(
         var_names, shape, fill_value=fill_value, att_manager_lookup_str="hi_de_{0}"
     )
@@ -100,3 +101,24 @@ def test_create_dataset_variables(var_names, shape, fill_value, lookup_str):
             assert data_array.shape == shape
         expected_fill_value = fill_value if fill_value is not None else attrs["FILLVAL"]
         np.testing.assert_array_equal(data_array, expected_fill_value)
+
+
+@pytest.mark.parametrize(
+    "sensor_hit_str, expected_val",
+    [
+        ("ABC1C2", 15),
+        ("ABC1", 14),
+        ("AB", 12),
+        ("AC1C2", 11),
+        ("AC1", 10),
+        ("A", 8),
+        ("BC1C2", 7),
+        ("BC1", 6),
+        ("B", 4),
+        ("C1C2", 3),
+        ("C1", 2),
+    ],
+)
+def test_coincidence_type_string_to_int(sensor_hit_str, expected_val):
+    """Test coverage for coincidence_type_string_to_int function"""
+    assert CoincidenceBitmap.detector_hit_str_to_int(sensor_hit_str) == expected_val