imap-processing 0.11.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

imap_processing/tests/hit/helpers/l1_validation.py

@@ -48,16 +48,16 @@ RENAME_COLUMNS = {
 }
 
 MOD_VALUE_TO_SPECIES_ENERGY_MAP = {
-    0: {"species": "H", "energy_idx": 0},
-    1: {"species": "H", "energy_idx": 1},
-    2: {"species": "H", "energy_idx": 2},
-    3: {"species": "He4", "energy_idx": 0},
-    4: {"species": "He4", "energy_idx": 1},
-    5: {"species": "CNO", "energy_idx": 0},
-    6: {"species": "CNO", "energy_idx": 1},
-    7: {"species": "NeMgSi", "energy_idx": 0},
-    8: {"species": "NeMgSi", "energy_idx": 1},
-    9: {"species": "Fe", "energy_idx": 0},
+    0: {"species": "h", "energy_bin": 0},
+    1: {"species": "h", "energy_bin": 1},
+    2: {"species": "h", "energy_bin": 2},
+    3: {"species": "he4", "energy_bin": 0},
+    4: {"species": "he4", "energy_bin": 1},
+    5: {"species": "cno", "energy_bin": 0},
+    6: {"species": "cno", "energy_bin": 1},
+    7: {"species": "nemgsi", "energy_bin": 0},
+    8: {"species": "nemgsi", "energy_bin": 1},
+    9: {"species": "fe", "energy_bin": 0},
 }
 
 
@@ -124,8 +124,8 @@ def consolidate_rate_columns(
 
     The validation data has each value in a separate column. This
     function aggregates related data into arrays to match processed
-    data. Each rate column has a corresponding delta plus and delta
-    minus column for uncertainty values.
+    data. Each rate column has corresponding plus and minus columns
+    for uncertainty values.
 
     Parameters
     ----------
@@ -142,16 +142,16 @@ def consolidate_rate_columns(
     """
     for new_col, prefix in rate_columns.items():
         pattern_rates = re.compile(rf"^{prefix}\d+$")
-        pattern_delta_plus = re.compile(rf"^{prefix}\d+_DELTA_PLUS$")
-        pattern_delta_minus = re.compile(rf"^{prefix}\d+_DELTA_MINUS$")
+        pattern_uncert_plus = re.compile(rf"^{prefix}\d+_STAT_UNCERT_PLUS$")
+        pattern_uncert_minus = re.compile(rf"^{prefix}\d+_STAT_UNCERT_MINUS$")
         data[new_col] = data.filter(regex=pattern_rates.pattern).apply(
             lambda row: row.values, axis=1
         )
-        data[f"{new_col}_delta_plus"] = data.filter(
-            regex=pattern_delta_plus.pattern
+        data[f"{new_col}_stat_uncert_plus"] = data.filter(
+            regex=pattern_uncert_plus.pattern
         ).apply(lambda row: row.values, axis=1)
-        data[f"{new_col}_delta_minus"] = data.filter(
-            regex=pattern_delta_minus.pattern
+        data[f"{new_col}_stat_uncert_minus"] = data.filter(
+            regex=pattern_uncert_minus.pattern
         ).apply(lambda row: row.values, axis=1)
         if new_col == "sectorates":
             data = consolidate_sectorates(data)
@@ -159,10 +159,11 @@ def consolidate_rate_columns(
             columns=data.filter(regex=pattern_rates.pattern).columns, inplace=True
         )
         data.drop(
-            columns=data.filter(regex=pattern_delta_plus.pattern).columns, inplace=True
+            columns=data.filter(regex=pattern_uncert_plus.pattern).columns, inplace=True
         )
         data.drop(
-            columns=data.filter(regex=pattern_delta_minus.pattern).columns, inplace=True
+            columns=data.filter(regex=pattern_uncert_minus.pattern).columns,
+            inplace=True,
         )
     return data
 
@@ -170,7 +171,7 @@ def consolidate_rate_columns(
 def consolidate_sectorates(data: pd.DataFrame) -> pd.DataFrame:
     """Consolidate sector rate data into arrays.
 
-    This function distinguishes between sector rate columns with three digits
+    This function distinguishes between sectored rate columns with three digits
     and those with four digits in their names.
 
     SECTORATES_000 SECTORATES_000_0 SECTORATES_000_1 SECTORATES_000_2...SECTORATES_120_9
@@ -185,9 +186,9 @@ def consolidate_sectorates(data: pd.DataFrame) -> pd.DataFrame:
 
     Columns with four digits (e.g., SECTORATES_000_0) include the sectorate
     values with a mod 10 value appended (e.g., 0). The mod 10 value determines
-    the species and energy range the sector rates represent in the science frame.
+    the species and energy range the sectored rates represent in the science frame.
     There are 10 possible species and energy ranges, but only one has data per
-    science frame. The validation data has 10 columns per 120 sector rates,
+    science frame. The validation data has 10 columns per 120 sectored rates,
     totaling 1200 columns per science frame. Each set of 10 columns will have
     only one value, resulting in an array that looks like this:
 
@@ -207,22 +208,22 @@ def consolidate_sectorates(data: pd.DataFrame) -> pd.DataFrame:
         Validation data with sectorate columns consolidated into arrays
     """
     sectorates_three_digits = data.filter(regex=r"^SECTORATES_\d{3}$").columns
-    sectorates_delta_plus_three_digits = data.filter(
-        regex=r"^SECTORATES_\d{3}_DELTA_PLUS$"
+    sectorates_uncert_plus_three_digits = data.filter(
+        regex=r"^SECTORATES_\d{3}_STAT_UNCERT_PLUS$"
     ).columns
-    sectorates_delta_minus_three_digits = data.filter(
-        regex=r"^SECTORATES_\d{3}_DELTA_MINUS$"
+    sectorates_uncert_minus_three_digits = data.filter(
+        regex=r"^SECTORATES_\d{3}_STAT_UNCERT_MINUS$"
     ).columns
 
     data["sectorates"] = data[sectorates_three_digits].apply(
-        lambda row: row.values.reshape(8, 15), axis=1
-    )
-    data["sectorates_delta_plus"] = data[sectorates_delta_plus_three_digits].apply(
-        lambda row: row.values.reshape(8, 15), axis=1
-    )
-    data["sectorates_delta_minus"] = data[sectorates_delta_minus_three_digits].apply(
-        lambda row: row.values.reshape(8, 15), axis=1
+        lambda row: row.values.reshape(15, 8), axis=1
     )
+    data["sectorates_stat_uncert_plus"] = data[
+        sectorates_uncert_plus_three_digits
+    ].apply(lambda row: row.values.reshape(15, 8), axis=1)
+    data["sectorates_stat_uncert_minus"] = data[
+        sectorates_uncert_minus_three_digits
+    ].apply(lambda row: row.values.reshape(15, 8), axis=1)
 
     sectorates_four_digits = data.filter(regex=r"^SECTORATES_\d{3}_\d{1}$").columns
     data["sectorates_by_mod_val"] = data[sectorates_four_digits].apply(
@@ -250,15 +251,15 @@ def process_single_rates(data: pd.DataFrame) -> pd.DataFrame:
     data["sngrates"] = data.apply(
         lambda row: np.array([row["sngrates_hg"], row["sngrates_lg"]]), axis=1
     )
-    data["sngrates_delta_plus"] = data.apply(
+    data["sngrates_stat_uncert_plus"] = data.apply(
         lambda row: np.array(
-            [row["sngrates_hg_delta_plus"], row["sngrates_lg_delta_plus"]]
+            [row["sngrates_hg_stat_uncert_plus"], row["sngrates_lg_stat_uncert_plus"]]
         ),
         axis=1,
     )
-    data["sngrates_delta_minus"] = data.apply(
+    data["sngrates_stat_uncert_minus"] = data.apply(
         lambda row: np.array(
-            [row["sngrates_hg_delta_minus"], row["sngrates_lg_delta_minus"]]
+            [row["sngrates_hg_stat_uncert_minus"], row["sngrates_lg_stat_uncert_minus"]]
         ),
         axis=1,
     )
@@ -266,10 +267,10 @@ def process_single_rates(data: pd.DataFrame) -> pd.DataFrame:
         columns=[
             "sngrates_hg",
             "sngrates_lg",
-            "sngrates_hg_delta_plus",
-            "sngrates_lg_delta_plus",
-            "sngrates_hg_delta_minus",
-            "sngrates_lg_delta_minus",
+            "sngrates_hg_stat_uncert_plus",
+            "sngrates_lg_stat_uncert_plus",
+            "sngrates_hg_stat_uncert_minus",
+            "sngrates_lg_stat_uncert_minus",
         ],
         inplace=True,
     )
@@ -279,7 +280,7 @@ def process_single_rates(data: pd.DataFrame) -> pd.DataFrame:
 def add_species_energy(data: pd.DataFrame) -> pd.DataFrame:
     """Add species and energy index to the validation data.
 
-    The sector rate data is organized by species and energy index
+    The sectored rate data is organized by species and energy index
     in the processed data so this function adds this information
     to each row (i.e. science frame) in the validation data.
 
@@ -304,12 +305,12 @@ def add_species_energy(data: pd.DataFrame) -> pd.DataFrame:
         )
     )
     data["species"] = data["mod_10"].apply(
-        lambda row: MOD_VALUE_TO_SPECIES_ENERGY_MAP[row]["species"].lower()
+        lambda row: MOD_VALUE_TO_SPECIES_ENERGY_MAP[row]["species"]
         if row is not None
         else None
     )
-    data["energy_idx"] = data["mod_10"].apply(
-        lambda row: MOD_VALUE_TO_SPECIES_ENERGY_MAP[row]["energy_idx"]
+    data["energy_bin"] = data["mod_10"].apply(
+        lambda row: MOD_VALUE_TO_SPECIES_ENERGY_MAP[row]["energy_bin"]
         if row is not None
         else None
     )
@@ -336,55 +337,55 @@ def compare_data(
         if field not in [
             "sc_tick_by_frame",
             "species",
-            "energy_idx",
+            "energy_bin",
         ]:
-            assert (
-                field in actual_data.data_vars.keys()
-            ), f"Field {field} not found in actual data variables"
+            assert field in actual_data.data_vars.keys(), (
+                f"Field {field} not found in actual data variables"
+            )
         if field not in skip:
             for frame in range(expected_data.shape[0]):
                 if field == "species":
-                    # Compare sector rates data using species and energy index.
+                    # Compare sectored rates data using species and energy index.
                     # which are only present in the validation data. In the actual
-                    # data, sector rates are organized by species in 4D arrays.
-                    #    i.e. h_counts_sectored has shape
-                    #         (epoch, h_energy_index, declination, azimuth).
+                    # data, sectored rates are organized by species in 4D arrays.
+                    #    i.e. h_sectored_counts has shape
+                    #         (epoch, h_energy_index, azimuth, declination).
                     # species and energy index are used to find the correct
-                    # array of sector rate data from the actual data for comparison.
+                    # array of sectored rate data from the actual data for comparison.
                     species = expected_data[field][frame]
-                    energy_idx = expected_data["energy_idx"][frame]
-                    if "sectorates_delta_plus" in expected_data.columns:
+                    energy_bin = expected_data["energy_bin"][frame]
+                    if "sectorates_stat_uncert_plus" in expected_data.columns:
                         np.testing.assert_allclose(
-                            actual_data[f"{species}_counts_sectored_delta_plus"][frame][
-                                energy_idx
-                            ].data,
-                            expected_data["sectorates_delta_plus"][frame],
+                            actual_data[f"{species}_sectored_counts_stat_uncert_plus"][
+                                frame
+                            ][energy_bin].data,
+                            expected_data["sectorates_stat_uncert_plus"][frame],
                             rtol=1e-7,  # relative tolerance
                             atol=1e-8,  # absolute tolerance
-                            err_msg=f"Mismatch in {species}_counts_sectored_delta_"
-                            f"plus at frame {frame}, energy_idx {energy_idx}",
+                            err_msg=f"Mismatch in {species}_sectored_counts_stat_uncert"
+                            f"_plus at frame {frame}, energy_bin {energy_bin}",
                         )
-                    if "sectorates_delta_minus" in expected_data.columns:
+                    if "sectorates_stat_uncert_minus" in expected_data.columns:
                         np.testing.assert_allclose(
-                            actual_data[f"{species}_counts_sectored_delta_minus"][
+                            actual_data[f"{species}_sectored_counts_stat_uncert_minus"][
                                 frame
-                            ][energy_idx].data,
-                            expected_data["sectorates_delta_minus"][frame],
+                            ][energy_bin].data,
+                            expected_data["sectorates_stat_uncert_minus"][frame],
                             rtol=1e-7,
                             atol=1e-8,
-                            err_msg=f"Mismatch in {species}_counts_sectored_delta_"
-                            f"minus at frame {frame}, energy_idx {energy_idx}",
+                            err_msg=f"Mismatch in {species}_sectored_counts_stat_uncert"
+                            f"_minus at frame {frame}, energy_bin {energy_bin}",
                         )
                     else:
                         np.testing.assert_allclose(
-                            actual_data[f"{species}_counts_sectored"][frame][
-                                energy_idx
+                            actual_data[f"{species}_sectored_counts"][frame][
+                                energy_bin
                             ].data,
                             expected_data["sectorates"][frame],
                             rtol=1e-7,
                             atol=1e-8,
-                            err_msg=f"Mismatch in {species}_counts_sectored at"
-                            f"frame {frame}, energy_idx {energy_idx}",
+                            err_msg=f"Mismatch in {species}_sectored_counts at"
+                            f"frame {frame}, energy_bin {energy_bin}",
                         )
                 elif field == "sc_tick_by_frame":
                     # Get the sc_tick values for each frame in the actual data

imap_processing/tests/hit/test_decom_hit.py

@@ -7,6 +7,7 @@ from imap_processing import imap_module_directory
 from imap_processing.hit.hit_utils import (
     HitAPID,
 )
+from imap_processing.hit.l0.constants import AZIMUTH_ANGLES, DECLINATION_ANGLES
 from imap_processing.hit.l0.decom_hit import (
     assemble_science_frames,
     decom_hit,
@@ -20,7 +21,7 @@ from imap_processing.hit.l0.decom_hit import (
 from imap_processing.utils import packet_file_to_datasets
 
 
-@pytest.fixture()
+@pytest.fixture
 def sci_dataset():
     """Create a xarray dataset for testing from sample data."""
     packet_definition = (
@@ -122,6 +123,9 @@ def test_parse_count_rates(sci_dataset):
     if count_rate_vars in list(sci_dataset.keys()):
         assert True
 
+    assert np.allclose(sci_dataset["declination"].values, DECLINATION_ANGLES)
+    assert np.allclose(sci_dataset["azimuth"].values, AZIMUTH_ANGLES)
+
 
 def test_is_sequential():
     """Test the is_sequential function."""

imap_processing/tests/hit/test_hit_l1a.py

@@ -27,9 +27,7 @@ from imap_processing.tests.hit.helpers.l1_validation import (
 @pytest.fixture(scope="module")
 def hk_packet_filepath():
     """Set path to test data file"""
-    return (
-        imap_module_directory / "tests/hit/test_data/imap_hit_l0_raw_20100105_v001.pkts"
-    )
+    return imap_module_directory / "tests/hit/test_data/hskp_sample.ccsds"
 
 
 @pytest.fixture(scope="module")
@@ -54,7 +52,7 @@ def validation_data():
 def test_subcom_sectorates(sci_packet_filepath):
     """Test the subcom_sectorates function.
 
-    This function organizes the sector rates data
+    This function organizes the sectored rates data
     by species and adds the data as new variables
     to the dataset.
     """
@@ -64,31 +62,31 @@ def test_subcom_sectorates(sci_packet_filepath):
     sci_dataset = decom_hit(sci_dataset)
 
     # Call the function to be tested
-    subcom_sectorates(sci_dataset)
+    sci_dataset = subcom_sectorates(sci_dataset)
 
     # Number of science frames in the dataset
     frames = sci_dataset["epoch"].shape[0]
 
-    # Check if the dataset has the expected new variables
-    for species in ["h", "he4", "cno", "nemgsi", "fe"]:
-        assert f"{species}_counts_sectored" in sci_dataset
-        assert f"{species}_energy_min" in sci_dataset
-        assert f"{species}_energy_max" in sci_dataset
+    # Shape of the new data variables
+    expected_shapes = {
+        "h": (3, 15, 8),
+        "he4": (2, 15, 8),
+        "cno": (2, 15, 8),
+        "nemgsi": (2, 15, 8),
+        "fe": (1, 15, 8),
+    }
 
+    for species, shape in expected_shapes.items():
+        # Check if the dataset has the new  data variables
+        assert f"{species}_sectored_counts" in sci_dataset
+        assert f"{species}_energy_mean" in sci_dataset.coords
+        assert f"{species}_energy_delta_minus" in sci_dataset
+        assert f"{species}_energy_delta_plus" in sci_dataset
         # Check the shape of the new data variables
-        if species == "h":
-            assert sci_dataset[f"{species}_counts_sectored"].shape == (frames, 3, 8, 15)
-            assert sci_dataset[f"{species}_energy_min"].shape == (3,)
-        elif species in ("4he", "cno", "nemgsi"):
-            assert sci_dataset[f"{species}_counts_sectored"].shape == (frames, 2, 8, 15)
-            assert sci_dataset[f"{species}_energy_min"].shape == (2,)
-        elif species == "fe":
-            assert sci_dataset[f"{species}_counts_sectored"].shape == (frames, 1, 8, 15)
-            assert sci_dataset[f"{species}_energy_min"].shape == (1,)
-        assert (
-            sci_dataset[f"{species}_energy_max"].shape
-            == sci_dataset[f"{species}_energy_min"].shape
-        )
+        assert sci_dataset[f"{species}_sectored_counts"].shape == (frames, *shape)
+        assert sci_dataset[f"{species}_energy_mean"].shape == (shape[0],)
+        assert sci_dataset[f"{species}_energy_delta_minus"].shape == (shape[0],)
+        assert sci_dataset[f"{species}_energy_delta_plus"].shape == (shape[0],)
 
 
 def test_calculate_uncertainties():
@@ -119,13 +117,13 @@ def test_calculate_uncertainties():
 
     # Assertions
     np.testing.assert_array_almost_equal(
-        result["counts_delta_plus"].values, expected_delta_plus
+        result["counts_stat_uncert_plus"].values, expected_delta_plus
     )
     np.testing.assert_array_almost_equal(
-        result["counts_delta_minus"].values, expected_delta_minus
+        result["counts_stat_uncert_minus"].values, expected_delta_minus
     )
-    assert "version_delta_plus" not in result
-    assert "version_delta_minus" not in result
+    assert "version_stat_uncert_plus" not in result
+    assert "version_stat_uncert_minus" not in result
 
 
 def test_validate_l1a_housekeeping_data(hk_packet_filepath):
@@ -139,7 +137,7 @@ def test_validate_l1a_housekeeping_data(hk_packet_filepath):
     hk_packet_filepath : str
         File path to housekeeping ccsds file
     """
-    datasets = hit_l1a(hk_packet_filepath, "001")
+    datasets = hit_l1a(hk_packet_filepath)
     hk_dataset = None
     for dataset in datasets:
         if dataset.attrs["Logical_source"] == "imap_hit_l1a_hk":
@@ -151,6 +149,7 @@ def test_validate_l1a_housekeeping_data(hk_packet_filepath):
     )
     validation_data = pd.read_csv(validation_file)
     validation_data.columns = validation_data.columns.str.lower()
+    validation_data.columns = validation_data.columns.str.strip()
 
     # Get a list of leak columns in ascending order
     # (LEAK_I_00, LEAK_I_01, ..., LEAK_I_63)
@@ -166,7 +165,6 @@ def test_validate_l1a_housekeeping_data(hk_packet_filepath):
     # Define the keys that should have dropped from the housekeeping dataset
     dropped_fields = {
         "pkt_apid",
-        "sc_tick",
         "version",
         "type",
         "sec_hdr_flg",
@@ -190,7 +188,6 @@ def test_validate_l1a_housekeeping_data(hk_packet_filepath):
         "ccsds_grp_flag",
         "ccsds_seq_cnt",
         "ccsds_length",
-        "shcoarse",
     }
 
     # Check that dropped variables are not in the dataset
@@ -226,7 +223,7 @@ def test_validate_l1a_counts_data(sci_packet_filepath, validation_data):
     """
 
     # Process the sample data
-    processed_datasets = hit_l1a(sci_packet_filepath, "001")
+    processed_datasets = hit_l1a(sci_packet_filepath)
     l1a_counts_data = processed_datasets[0]
 
     # Prepare validation data for comparison with processed data
@@ -243,7 +240,7 @@ def test_validate_l1a_counts_data(sci_packet_filepath, validation_data):
         "seq_flgs",
         "src_seq_ctr",
         "pkt_len",
-        "energy_idx",
+        "energy_bin",
     ]
 
     # Compare processed data to validation data
@@ -263,7 +260,7 @@ def test_hit_l1a(hk_packet_filepath, sci_packet_filepath):
         Path to ccsds file for science data
     """
     for packet_filepath in [hk_packet_filepath, sci_packet_filepath]:
-        processed_datasets = hit_l1a(packet_filepath, "001")
+        processed_datasets = hit_l1a(packet_filepath)
         assert isinstance(processed_datasets, list)
         assert all(isinstance(ds, xr.Dataset) for ds in processed_datasets)
         if packet_filepath == hk_packet_filepath:
@@ -272,10 +269,9 @@ def test_hit_l1a(hk_packet_filepath, sci_packet_filepath):
         else:
             assert len(processed_datasets) == 2
             assert (
-                processed_datasets[0].attrs["Logical_source"]
-                == "imap_hit_l1a_count-rates"
+                processed_datasets[0].attrs["Logical_source"] == "imap_hit_l1a_counts"
             )
             assert (
                 processed_datasets[1].attrs["Logical_source"]
-                == "imap_hit_l1a_pulse-height-events"
+                == "imap_hit_l1a_direct-events"
             )