imap-processing 0.11.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

imap_processing/tests/codice/test_codice_l1a.py

@@ -1,6 +1,7 @@
 """Tests the L1a processing for decommutated CoDICE data"""
 
 import logging
+import re
 
 import numpy as np
 import pytest
@@ -9,6 +10,7 @@ import xarray as xr
 from imap_processing.cdf.utils import load_cdf, write_cdf
 from imap_processing.codice import constants
 from imap_processing.codice.codice_l1a import process_codice_l1a
+from imap_processing.tests.conftest import _download_external_data, _test_data_paths
 
 from .conftest import TEST_L0_FILE, VALIDATION_DATA
 
@@ -50,19 +52,19 @@ EXPECTED_ARRAY_SHAPES = [
     (77, 1, 128),  # lo-nsw-species
     (77, 5, 12, 128),  # lo-sw-angular
     (77, 19, 12, 128),  # lo-nsw-angular
-    (77, 1, 6, 1),  # hi-counters-aggregated
-    (77, 1, 12, 1),  # hi-counters-singles
-    (77, 15, 4, 1),  # hi-omni
+    (77,),  # hi-counters-aggregated
+    (77, 12),  # hi-counters-singles
+    (77, 15, 4),  # hi-omni
     (77, 8, 12, 12),  # hi-sectored
-    (),  # hi-priority  # TODO: Need to implement
-    (),  # lo-pha  # TODO: Need to implement
+    (77,),  # hi-priority
+    (77, 10000),  # lo-pha
     (),  # hi-pha  # TODO: Need to implement
 ]
 
 EXPECTED_NUM_VARIABLES = [
     0,  # hi-ialirt  # TODO: Need to implement
     0,  # lo-ialirt  # TODO: Need to implement
-    148,  # hskp
+    139,  # hskp
     8 + len(constants.LO_COUNTERS_AGGREGATED_VARIABLE_NAMES),  # lo-counters-aggregated
     9,  # lo-counters-singles
     13,  # lo-sw-priority
@@ -71,15 +73,37 @@ EXPECTED_NUM_VARIABLES = [
     16,  # lo-nsw-species
     12,  # lo-sw-angular
     9,  # lo-nsw-angular
-    1,  # hi-counters-aggregated
-    3,  # hi-counters-singles
+    2 + len(constants.HI_COUNTERS_AGGREGATED_VARIABLE_NAMES),  # hi-counters-aggregated
+    5,  # hi-counters-singles
     10,  # hi-omni
     6,  # hi-sectored
-    0,  # hi-priority  # TODO: Need to implement
-    0,  # lo-pha  # TODO: Need to implement
+    8,  # hi-priority
+    80,  # lo-pha
     0,  # hi-pha  # TODO: Need to implement
 ]
 
+# CoDICE-Hi products that have support variables to test
+CODICE_HI_PRODUCTS = [
+    "hi-counters-aggregated",
+    "hi-counters-singles",
+    "hi-priority",
+    "hi-sectored",
+]
+# TODO: Add hi-omni here once I sort out the array shape discrepancy with the
+#       validation data
+
+# CoDICE-Lo products that have support variables to test
+CODICE_LO_PRODUCTS = [
+    "lo-counters-aggregated",
+    "lo-counters-singles",
+    "lo-sw-priority",
+    "lo-nsw-priority",
+    "lo-sw-species",
+    "lo-nsw-species",
+    "lo-sw-angular",
+    "lo-nsw-angular",
+]
+
 
 @pytest.fixture(scope="session")
 def test_l1a_data() -> xr.Dataset:
@@ -90,8 +114,11 @@ def test_l1a_data() -> xr.Dataset:
     processed_datasets : list[xarray.Dataset]
         A list of ``xarray`` datasets containing the test data
     """
-
-    processed_datasets = process_codice_l1a(file_path=TEST_L0_FILE, data_version="001")
+    # Make sure we have the data available here. This test collection gets
+    # skipped at the module level if the mark isn't present. We can't decorate
+    # a fixture, so add the needed call directly here instead.
+    _download_external_data(_test_data_paths())
+    processed_datasets = process_codice_l1a(file_path=TEST_L0_FILE)
 
     return processed_datasets
 
@@ -113,9 +140,10 @@ def test_l1a_data_array_shape(test_l1a_data, index):
 
     # Mark currently broken/unsupported datasets as expected to fail
     # TODO: Remove these once they are supported
-    if index in [0, 1, 15, 16, 17]:
+    if index in [0, 1, 17]:
         pytest.xfail("Data product is currently unsupported")
 
+    # There are exceptions for some variables
     for variable in processed_dataset:
         # For variables with energy dimensions
         if variable in ["energy_table", "acquisition_time_per_step"]:
@@ -132,7 +160,10 @@ def test_l1a_data_array_shape(test_l1a_data, index):
             assert processed_dataset[variable].data.shape == (
                 len(processed_dataset["epoch"].data),
             )
-        # For counter variables
+        # For some direct event variables:
+        elif re.match(r"P[0-7]_(NumEvents|DataQuality)", variable):
+            assert processed_dataset[variable].data.shape == (77,)
+        # For nominal variables
         else:
             assert processed_dataset[variable].data.shape == expected_shape
 
@@ -157,7 +188,7 @@ def test_l1a_logical_sources(test_l1a_data, index):
 
     # Mark currently broken/unsupported datasets as expected to fail
     # TODO: Remove these once they are supported
-    if index in [0, 1, 15, 16, 17]:
+    if index in [0, 1, 17]:
         pytest.xfail("Data product is currently unsupported")
 
     # Write the dataset to a file to set the logical source attribute
@@ -185,7 +216,7 @@ def test_l1a_num_data_variables(test_l1a_data, index):
 
     # Mark currently broken/unsupported datasets as expected to fail
     # TODO: Remove these once they are supported
-    if index in [0, 1, 15, 16, 17]:
+    if index in [0, 1, 17]:
         pytest.xfail("Data product is currently unsupported")
 
     assert len(processed_dataset) == EXPECTED_NUM_VARIABLES[index]
@@ -211,6 +242,10 @@ def test_l1a_validate_data_arrays(test_l1a_data: xr.Dataset, index):
     # TODO: Currently only the following products can be validated, expand this
     #       to other data products as I can validate them.
     able_to_be_validated = [
+        "hi-counters-aggregated",
+        "hi-counters-singles",
+        "hi-priority",
+        "hi-sectored",
         "lo-counters-aggregated",
         "lo-counters-singles",
         "lo-sw-angular",
@@ -219,26 +254,22 @@ def test_l1a_validate_data_arrays(test_l1a_data: xr.Dataset, index):
         "lo-nsw-priority",
         "lo-sw-species",
         "lo-nsw-species",
+        "lo_pha",
     ]
+
     if descriptor in able_to_be_validated:
         counters = getattr(
-            constants, f'{descriptor.upper().replace("-","_")}_VARIABLE_NAMES'
+            constants, f"{descriptor.upper().replace('-', '_')}_VARIABLE_NAMES"
         )
         processed_dataset = test_l1a_data[index]
         validation_dataset = load_cdf(VALIDATION_DATA[index])
 
         for counter in counters:
-            # Ensure the data array shapes are equal
-            assert (
-                processed_dataset[counter].data.shape
-                == validation_dataset[counter].data.shape
+            # Ensure the data arrays are equal
+            np.testing.assert_equal(
+                processed_dataset[counter].data, validation_dataset[counter].data
             )
 
-            # TODO: Once Joey and I figure out some small discrepancies with
-            #       some data products, we should get matching data array shapes
-            #       AND values (i.e. run assert_array_equal on the arrays,
-            #       instead of just checking shape)
-
     else:
         pytest.xfail(f"Still need to implement validation for {descriptor}")
 
@@ -253,35 +284,17 @@ def test_l1a_validate_hskp_data(test_l1a_data):
     # Load the validation housekeeping data
     validation_hskp_data = load_cdf(validation_hskp_filepath)
 
-    # These variables are present in the decommed test data, but not present in
-    # the validation data
-    # TODO: Ask Joey if these can be removed from the L1a housekeeping CDFs
+    # These variables are not present in the validation dataset
     exclude_variables = [
-        "spare_1",
-        "spare_2",
-        "spare_3",
-        "spare_4",
-        "spare_5",
-        "spare_6",
-        "spare_62",
-        "spare_68",
+        "version",
+        "type",
+        "sec_hdr_flg",
+        "pkt_apid",
+        "seq_flgs",
+        "src_seq_ctr",
+        "pkt_len",
     ]
 
-    # These variables are not present in the validation data
-    # TODO: Ask joey if this is expected
-    exclude_variables.extend(
-        [
-            "version",
-            "type",
-            "sec_hdr_flg",
-            "pkt_apid",
-            "seq_flgs",
-            "src_seq_ctr",
-            "pkt_len",
-            "chksum",
-        ]
-    )
-
     for variable in hskp_data:
         if variable not in exclude_variables:
             np.testing.assert_array_equal(
@@ -289,10 +302,70 @@ def test_l1a_validate_hskp_data(test_l1a_data):
             )
 
 
+@pytest.mark.parametrize("index", range(len(DESCRIPTORS)))
+def test_l1a_validate_support_variables(test_l1a_data, index):
+    """Tests that the support variables for the generated products match the
+    validation data
+
+    Parameters
+    ----------
+    test_l1a_data : list[xarray.Dataset]
+        A list of ``xarray`` datasets containing the test data
+    index : int
+        The index of the list to test
+    """
+
+    # Hopefully I can remove this someday if Joey gives me validation data
+    # with updated naming conventions
+    variable_name_mapping = {
+        "data_quality": "DataQuality",
+        "nso_half_spin": "NSOHalfSpin",
+        "rgfo_half_spin": "RGFOHalfSpin",
+        "spin_period": "SpinPeriod",
+        "st_bias_gain_mode": "STBiasGainMode",
+        "sw_bias_gain_mode": "SWBiasGainMode",
+    }
+
+    descriptor = DESCRIPTORS[index]
+    dataset = test_l1a_data[index]
+    validation_dataset = load_cdf(VALIDATION_DATA[index])
+
+    if descriptor in CODICE_LO_PRODUCTS:
+        # Note that for the energy table and acquisition time, the validation
+        # data only carries three decimal places whereas the SDC-generated CDFs
+        # carry more significant figures
+
+        # Ensure the energy table values are (nearly) equal
+        np.testing.assert_almost_equal(
+            dataset.energy_table.data, validation_dataset.EnergyTable.data, decimal=3
+        )
+
+        # Ensure that the acquisition times are (nearly) equal
+        np.testing.assert_almost_equal(
+            dataset.acquisition_time_per_step.data,
+            validation_dataset.AcquisitionTimePerStep.data,
+            decimal=3,
+        )
+
+        # Ensure that the support variables derived from packet data are equal
+        for variable in variable_name_mapping:
+            np.testing.assert_equal(
+                dataset[variable].data,
+                validation_dataset[variable_name_mapping[variable]].data,
+            )
+
+    elif descriptor in CODICE_HI_PRODUCTS:
+        for variable in ["spin_period", "data_quality"]:
+            np.testing.assert_equal(
+                dataset[variable].data,
+                validation_dataset[variable_name_mapping[variable]].data,
+            )
+
+
 def test_l1a_multiple_packets():
     """Tests that an input L0 file containing multiple APIDs can be processed."""
 
-    processed_datasets = process_codice_l1a(file_path=TEST_L0_FILE, data_version="001")
+    processed_datasets = process_codice_l1a(file_path=TEST_L0_FILE)
 
     # TODO: Could add some more checks here?
     assert len(processed_datasets) == 18

imap_processing/tests/codice/test_codice_l1b.py

@@ -3,16 +3,16 @@
 import pytest
 import xarray as xr
 
-from imap_processing.cdf.utils import load_cdf
 from imap_processing.codice.codice_l1b import process_codice_l1b
 
 from .conftest import TEST_L1A_FILES
 
-EXPECTED_LOGICAL_SOURCE = [
+EXPECTED_LOGICAL_SOURCES = [
     "imap_codice_l1b_hskp",
     "imap_codice_l1b_hi-counters-aggregated",
     "imap_codice_l1b_hi-counters-singles",
     "imap_codice_l1b_hi-omni",
+    "imap_codice_l1b_hi-priority",
     "imap_codice_l1b_hi-sectored",
     "imap_codice_l1b_lo-counters-aggregated",
     "imap_codice_l1b_lo-counters-singles",
@@ -34,18 +34,17 @@ def test_l1b_data(request) -> xr.Dataset:
     dataset : xr.Dataset
         A ``xarray`` dataset containing the test data
     """
-    input_dataset = load_cdf(request.param)
-    dataset = process_codice_l1b(input_dataset, data_version="001")
+    dataset = process_codice_l1b(request.param)
     return dataset
 
 
 @pytest.mark.skip("Awaiting proper implementation of L1B")
 @pytest.mark.parametrize(
     "test_l1b_data, expected_logical_source",
-    list(zip(TEST_L1A_FILES, EXPECTED_LOGICAL_SOURCE)),
+    list(zip(TEST_L1A_FILES, EXPECTED_LOGICAL_SOURCES)),
     indirect=["test_l1b_data"],
 )
-def test_l1b_cdf_filenames(test_l1b_data: xr.Dataset, expected_logical_source: str):
+def test_l1b_logical_sources(test_l1b_data: xr.Dataset, expected_logical_source: str):
     """Tests that the ``process_codice_l1b`` function generates datasets
     with the expected logical source.
 
@@ -53,7 +52,7 @@ def test_l1b_cdf_filenames(test_l1b_data: xr.Dataset, expected_logical_source: s
     ----------
     test_l1b_data : xr.Dataset
         A ``xarray`` dataset containing the test data
-    expected_filename : str
+    expected_logical_source : str
         The expected CDF filename
     """
 

imap_processing/tests/codice/test_decompress.py

@@ -13,11 +13,11 @@ lzma_bytes = lzma.compress((234).to_bytes(1, byteorder="big"))
 # LZMA_EXAMPLE = "".join(format(byte, "08b") for byte in lzma_bytes)
 TEST_DATA = [
     (b"\xea", CoDICECompression.NO_COMPRESSION, [234]),
-    (b"\xea", CoDICECompression.LOSSY_A, [212992]),
-    (b"\xea", CoDICECompression.LOSSY_B, [1310720]),
+    (b"\xea", CoDICECompression.LOSSY_A, [221183]),
+    (b"\xea", CoDICECompression.LOSSY_B, [1441791]),
     (lzma_bytes, CoDICECompression.LOSSLESS, [234]),
-    (lzma_bytes, CoDICECompression.LOSSY_A_LOSSLESS, [212992]),
-    (lzma_bytes, CoDICECompression.LOSSY_B_LOSSLESS, [1310720]),
+    (lzma_bytes, CoDICECompression.LOSSY_A_LOSSLESS, [221183]),
+    (lzma_bytes, CoDICECompression.LOSSY_B_LOSSLESS, [1441791]),
 ]