imap-processing 0.6.0__py3-none-any.whl

imap_processing/tests/glows/test_glows_l1b.py

@@ -0,0 +1,348 @@
+import dataclasses
+
+import numpy as np
+import pytest
+import xarray as xr
+
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.glows.l1b.glows_l1b import glows_l1b, process_de, process_histogram
+from imap_processing.glows.l1b.glows_l1b_data import (
+    AncillaryParameters,
+    DirectEventL1B,
+    HistogramL1B,
+)
+
+
+@pytest.fixture()
+def hist_dataset():
+    variables = {
+        "flight_software_version": np.zeros((20,)),
+        "seq_count_in_pkts_file": np.zeros((20,)),
+        "last_spin_id": np.zeros((20,)),
+        "flags_set_onboard": np.zeros((20,)),
+        "is_generated_on_ground": np.zeros((20,)),
+        "number_of_spins_per_block": np.zeros((20,)),
+        "number_of_bins_per_histogram": np.zeros((20,)),
+        "number_of_events": np.zeros((20,)),
+        "filter_temperature_average": np.zeros((20,)),
+        "filter_temperature_variance": np.zeros((20,)),
+        "hv_voltage_average": np.zeros((20,)),
+        "hv_voltage_variance": np.zeros((20,)),
+        "spin_period_average": np.zeros((20,)),
+        "spin_period_variance": np.zeros((20,)),
+        "pulse_length_average": np.zeros((20,)),
+        "pulse_length_variance": np.zeros((20,)),
+        "imap_start_time": np.zeros((20,)),
+        "imap_time_offset": np.zeros((20,)),
+        "glows_start_time": np.zeros((20,)),
+        "glows_time_offset": np.zeros((20,)),
+    }
+    cdf_attrs = ImapCdfAttributes()
+    cdf_attrs.add_instrument_global_attrs("glows")
+    cdf_attrs.add_instrument_variable_attrs("glows", "l1b")
+
+    epoch = xr.DataArray(
+        np.arange(20),
+        name="epoch",
+        dims=["epoch"],
+        attrs=cdf_attrs.get_variable_attributes("epoch"),
+    )
+
+    bins = xr.DataArray(np.arange(3600), name="bins", dims=["bins"])
+
+    ds = xr.Dataset(
+        coords={"epoch": epoch},
+        attrs=cdf_attrs.get_global_attributes("imap_glows_l1b_hist"),
+    )
+
+    ds["histograms"] = xr.DataArray(
+        np.zeros((20, 3600)),
+        dims=["epoch", "bins"],
+        coords={"epoch": epoch, "bins": bins},
+    )
+
+    for var in variables:
+        ds[var] = xr.DataArray(variables[var], dims=["epoch"], coords={"epoch": epoch})
+
+    return ds
+
+
+@pytest.fixture()
+def de_dataset():
+    variables = {
+        "seq_count_in_pkts_file": np.zeros((20,)),
+        "number_of_de_packets": np.zeros((20,)),
+        "imap_sclk_last_pps": np.zeros((20,)),
+        "glows_sclk_last_pps": np.zeros((20,)),
+        "glows_ssclk_last_pps": np.zeros((20,)),
+        "imap_sclk_next_pps": np.zeros((20,)),
+        "catbed_heater_active": np.zeros((20,)),
+        "spin_period_valid": np.zeros((20,)),
+        "spin_phase_at_next_pps_valid": np.zeros((20,)),
+        "spin_period_source": np.zeros((20,)),
+        "spin_period": np.zeros((20,)),
+        "spin_phase_at_next_pps": np.zeros((20,)),
+        "number_of_completed_spins": np.zeros((20,)),
+        "filter_temperature": np.zeros((20,)),
+        "hv_voltage": np.zeros((20,)),
+        "glows_time_on_pps_valid": np.zeros((20,)),
+        "time_status_valid": np.zeros((20,)),
+        "housekeeping_valid": np.zeros((20,)),
+        "is_pps_autogenerated": np.zeros((20,)),
+        "hv_test_in_progress": np.zeros((20,)),
+        "pulse_test_in_progress": np.zeros((20,)),
+        "memory_error_detected": np.zeros((20,)),
+    }
+
+    cdf_attrs = ImapCdfAttributes()
+    cdf_attrs.add_instrument_global_attrs("glows")
+    cdf_attrs.add_instrument_variable_attrs("glows", "l1b")
+
+    epoch = xr.DataArray(
+        np.arange(20),
+        name="epoch",
+        dims=["epoch"],
+        attrs=cdf_attrs.get_variable_attributes("epoch"),
+    )
+
+    per_second = xr.DataArray(np.arange(2295), name="per_second", dims=["per_second"])
+    direct_event = xr.DataArray(
+        np.arange(4), name="direct_event", dims=["direct_event"]
+    )
+
+    de_data = np.zeros((20, 2295, 4))
+    de_data[0][0] = [1, 2_000, 3, 0]
+
+    variables["filter_temperature"][0] = 100
+
+    ds = xr.Dataset(
+        coords={"epoch": epoch, "per_second": per_second, "direct_event": direct_event},
+        attrs=cdf_attrs.get_global_attributes("imap_glows_l1b_de"),
+    )
+
+    ds["direct_events"] = xr.DataArray(
+        de_data,
+        dims=["epoch", "per_second", "direct_event"],
+        coords={"epoch": epoch, "per_second": per_second, "direct_event": direct_event},
+    )
+
+    for var in variables:
+        ds[var] = xr.DataArray(variables[var], dims=["epoch"], coords={"epoch": epoch})
+
+    return ds
+
+
+@pytest.fixture()
+def ancillary_dict():
+    dictionary = {
+        "description": "Table for conversion/decoding ancillary parameters collected "
+        "onboard by IMAP/GLOWS",
+        "version": "0.1",
+        "date_of_creation_yyyymmdd": "20230527",
+        "filter_temperature": {
+            "min": -30.0,
+            "max": 80.0,
+            "n_bits": 8,
+            "p01": 0.0,
+            "p02": 0.0,
+            "p03": 0.0,
+            "p04": 0.0,
+        },
+        "hv_voltage": {
+            "min": 0.0,
+            "max": 3500.0,
+            "n_bits": 12,
+            "p01": 0.0,
+            "p02": 0.0,
+            "p03": 0.0,
+            "p04": 0.0,
+        },
+        "spin_period": {"min": 0.0, "max": 20.9712, "n_bits": 16},
+        "spin_phase": {"min": 0.0, "max": 360.0, "n_bits": 16},
+        "pulse_length": {
+            "min": 0.0,
+            "max": 255.0,
+            "n_bits": 8,
+            "p01": 0.0,
+            "p02": 0.0,
+            "p03": 0.0,
+            "p04": 0.0,
+        },
+    }
+    return dictionary
+
+
+def test_histogram_mapping():
+    time_val = 1111111.11
+    # A = 2.318
+    # B = 69.5454
+    expected_temp = 100
+
+    test_hists = np.zeros((200, 3600))
+    # For temp
+    encoded_val = expected_temp * 2.318 + 69.5454
+
+    # For now, testing types and number of inputs
+    output = tuple(
+        dataclasses.asdict(
+            HistogramL1B(
+                test_hists,
+                "test",
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                encoded_val,
+                encoded_val,
+                encoded_val,
+                encoded_val,
+                encoded_val,
+                encoded_val,
+                encoded_val,
+                encoded_val,
+                time_val,
+                time_val,
+                time_val,
+                time_val,
+            )
+        ).values()
+    )
+
+    assert output[17] == time_val
+
+    # Correctly decoded temperature
+    assert output[9] - expected_temp < 0.1
+
+
+def test_process_histograms(hist_dataset):
+    time_val = np.single(1111111.11)
+    # A = 2.318
+    # B = 69.5454
+    expected_temp = 100
+
+    test_hists = np.zeros((200,))
+    # For temp
+    encoded_val = np.single(expected_temp * 2.318 + 69.5454)
+
+    test_l1b = HistogramL1B(
+        test_hists,
+        "test",
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        encoded_val,
+        encoded_val,
+        encoded_val,
+        encoded_val,
+        encoded_val,
+        encoded_val,
+        encoded_val,
+        encoded_val,
+        time_val,
+        time_val,
+        time_val,
+        time_val,
+    )
+
+    output = process_histogram(hist_dataset)
+    assert len(output) == len(dataclasses.asdict(test_l1b))
+
+
+def test_process_de(de_dataset, ancillary_dict):
+    output = process_de(de_dataset)
+
+    # Output has the same length as non-initvar fields in DirectEventL1B
+    assert len(output) == len(dataclasses.fields(DirectEventL1B))
+
+    # Validate timeestamp
+    assert np.isclose(output[-2][0].data[0], 1.001)
+    # Validate pulse length
+    assert output[-1][0].data[0] == 3.0
+
+    ancillary = AncillaryParameters(ancillary_dict)
+
+    expected_temp = ancillary.decode("filter_temperature", 100.0)
+
+    assert np.isclose(output[8].data[0], expected_temp)
+
+
+def test_glows_l1b(de_dataset, hist_dataset):
+    hist_output = glows_l1b(hist_dataset, "V001")
+
+    assert hist_output["histograms"].dims == ("epoch", "bins")
+    assert hist_output["histograms"].shape == (20, 3600)
+
+    # This needs to be added eventually, but is skipped for now.
+    expected_de_data = [
+        "flight_software_version",
+        "ground_software_version",
+        "pkts_file_name",
+        "seq_count_in_pkts_file",
+        "l1a_file_name",
+        "ancillary_data_files",
+    ]
+
+    # From table 11 in the algorithm document
+    expected_hist_data = [
+        # "unique_block_identifier", Left out until I determine where to put string data
+        "glows_start_time",
+        "glows_end_time_offset",
+        "imap_start_time",
+        "imap_end_time_offset",
+        "number_of_spins_per_block",
+        "number_of_bins_per_histogram",
+        "histograms",  # named histogram in alg doc
+        "number_of_events",
+        "imap_spin_angle_bin_cntr",
+        "histogram_flag_array",
+        "filter_temperature_average",
+        "filter_temperature_std_dev",
+        "hv_voltage_average",
+        "hv_voltage_std_dev",
+        "spin_period_average",
+        "spin_period_std_dev",
+        "pulse_length_average",
+        "pulse_length_std_dev",
+        "spin_period_ground_average",
+        "spin_period_ground_std_dev",
+        "position_angle_offset_average",
+        "position_angle_offset_std_dev",
+        "spin_axis_orientation_std_dev",
+        "spin_axis_orientation_average",
+        "spacecraft_location_average",
+        "spacecraft_location_std_dev",
+        "spacecraft_velocity_average",
+        "spacecraft_velocity_std_dev",
+        "flags",
+    ]
+
+    for key in expected_hist_data:
+        assert key in hist_output
+
+    de_output = glows_l1b(de_dataset, "V001")
+
+    # From table 15 in the algorithm document
+    expected_de_data = [
+        # "unique_identifier", TODO: determine the best way to add this
+        "imap_time_last_pps",
+        "imap_time_next_pps",
+        "glows_time_last_pps",
+        "number_of_completed_spins",
+        "filter_temperature",
+        "hv_voltage",
+        "spin_period",
+        "spin_phase_at_next_pps",
+        "de_flags",
+        "direct_event_glows_times",
+        "direct_event_pulse_lengths",
+    ]
+
+    for key in expected_de_data:
+        assert key in de_output

imap_processing/tests/glows/test_glows_l1b_data.py

@@ -0,0 +1,70 @@
+import numpy as np
+import pytest
+
+from imap_processing.glows.l1b.glows_l1b_data import AncillaryParameters, DirectEventL1B
+
+
+def test_glows_l1b_ancillary_file():
+    fake_good_input = {
+        "version": "0.1",
+        "filter_temperature": {
+            "min": -30.0,
+            "max": 80.0,
+            "n_bits": 8,
+            "p01": 0.0,
+            "p02": 0.0,
+            "p03": 0.0,
+            "p04": 0.0,
+        },
+        "hv_voltage": {
+            "min": 0.0,
+            "max": 3500.0,
+            "n_bits": 12,
+            "p01": 0.0,
+            "p02": 0.0,
+            "p03": 0.0,
+            "p04": 0.0,
+        },
+        "spin_period": {"min": 0.0, "max": 20.9712, "n_bits": 16},
+        "spin_phase": {"min": 0.0, "max": 360.0, "n_bits": 16},
+        "pulse_length": {
+            "min": 0.0,
+            "max": 255.0,
+            "n_bits": 8,
+            "p01": 0.0,
+            "p02": 0.0,
+            "p03": 0.0,
+            "p04": 0.0,
+        },
+    }
+
+    ancillary = AncillaryParameters(fake_good_input)
+    for key in fake_good_input:
+        assert getattr(ancillary, key) == fake_good_input[key]
+
+    fake_bad_input = {
+        "version": "0.1",
+        "filter_temperature": {
+            "min": -30.0,
+            "n_bits": 8,
+            "p01": 0.0,
+            "p02": 0.0,
+            "p03": 0.0,
+            "p04": 0.0,
+        },
+    }
+
+    with pytest.raises(KeyError):
+        ancillary = AncillaryParameters(fake_bad_input)
+
+
+def test_glows_l1b_de():
+    input_test_data = np.array([[1, 0, 3], [100, 2_000, 6]])
+    times, pulse_len = DirectEventL1B.process_direct_events(input_test_data)
+
+    expected_times = np.array([1.0, 100.001])
+
+    expected_pulse = np.array([3, 6])
+
+    assert np.allclose(times, expected_times)
+    assert np.allclose(pulse_len, expected_pulse)

imap_processing/tests/hi/conftest.py

@@ -0,0 +1,133 @@
+import random
+
+import pytest
+
+
+@pytest.fixture(scope="session")
+def hi_test_data_path(imap_tests_path):
+    return imap_tests_path / "hi" / "test_data"
+
+
+@pytest.fixture(scope="session")
+def hi_l0_test_data_path(hi_test_data_path):
+    return hi_test_data_path / "l0"
+
+
+def create_metaevent(esa_step, met_subseconds, met_seconds):
+    start_bitmask_data = 0  # META
+    return (
+        f"{start_bitmask_data:02b}{esa_step:04b}{met_subseconds:010b}{met_seconds:032b}"
+    )
+
+
+def create_directevent(tof_1, tof_2, tof_3, de_tag):
+    start_bitmask_data = random.choice([1, 2, 3])  # Detector A, B, C
+    return f"{start_bitmask_data:02b}{tof_1:010b}{tof_2:010b}{tof_3:010b}{de_tag:016b}"
+
+
+def ccsds_header_data(apid, pkt_len):
+    """Create binary data for CCSDS header with apid provided."""
+    # CCSDS primary header
+    # 3 bits - Version number
+    # 1 bit - Packet type
+    # 1 bit - Secondary header flag
+    # 16 bits - APID
+    # 2 bits - Sequence flag
+    # 14 bits - Packet sequence count
+    # 16 bits - Packet length
+    return f"{0:03b}{0:01b}{1:01b}{apid:011b}{1:02b}{0:014b}{pkt_len:016b}"
+
+
+def check_sum(bits_size):
+    """Create check test sum."""
+    return f"{0:0{bits_size}b}"
+
+
+@pytest.fixture()
+def create_de_data(tmp_path):
+    """Fixture to create fake direct event data. Note that there has been no
+    effort to make simulate this data meaning that packets are not self
+    consistent and values may not make sense.
+
+    TODO: remove this once we have good sample data.
+    TODO: remove "S311" from pyproject.toml when we remove this.
+    """
+
+    def create_data(apid):
+        """
+        The function that generates the fake direct event data and returns
+        the Path to the binary file.
+
+        Parameters
+        ----------
+        apid : int
+            CCSDS packet apid
+
+        Returns
+        -------
+        Path to the binary file with fake direct event data
+        """
+        num_packets = 4
+        packets_data = []
+        # This MET seconds evaluates to 2023-09-27T16:04:49.150
+        met_seconds = 433522962
+        esa_step = 0
+        spare = f"{0:032b}"
+        # To use a consistent random number
+        random.seed(0)
+        for index in range(num_packets):
+            current_data = []
+            if index % 2 == 0:
+                # Two packets per ESA step
+                # Every even index, event starts with metaevent
+                if esa_step == 9:
+                    esa_step = 0
+                esa_step += 1
+                met_subseconds = random.randint(0, 1023)
+                met_seconds += 3600
+                current_data.append(
+                    create_metaevent(esa_step, met_subseconds, met_seconds)
+                )
+            # Random length for direct events. It could be empty
+            num_directevents = random.randint(0, 10)
+            for _ in range(num_directevents):
+                # We don't always get triple coincidence. To make data
+                # more realistic, we will use random.choices.
+                # This line will select a random integer between 0 and 1023
+                # with a probability of 0.xx and the value 1023 with a
+                # probability of 0.xx. The `k=1` argument specifies that one
+                # item should be chosen, and `[0]` is used to extract the
+                # single item from the resulting list.
+                tof_1, tof_2, tof_3 = random.choices(
+                    [random.randint(0, 1022), 1023], weights=[0.45, 0.55], k=3
+                )
+                de_tag = random.randint(1, 65535)
+                current_data.append(create_directevent(tof_1, tof_2, tof_3, de_tag))
+
+            # create a CCSDS data using current data
+            # pkt_len = 4 byte (MET seconds) + 4 byte (spare) +
+            # 6 byte (direct event) * number of direct event + 2 byte (checksum) - 1å
+            if index % 2 == 0:
+                # Add 6 bytes for metaevent
+                pkt_len = 4 + 4 + 6 + 6 * num_directevents + 2 - 1
+            else:
+                pkt_len = 4 + 4 + 6 * num_directevents + 2 - 1
+            ccsds_data = (
+                ccsds_header_data(apid, pkt_len)
+                + f"{met_seconds:032b}"
+                + spare
+                + "".join(current_data)
+                + check_sum(bits_size=16)
+            )
+            packets_data.append(ccsds_data)
+        # Join all the events as one binary string
+        packets_data = "".join(packets_data)
+        # write data to pkts file
+        with open(tmp_path / "imap_hi_l0_sdc-test-data_20240318_v000.pkts", "wb") as f:
+            byte_data = int(packets_data, 2).to_bytes(
+                (len(packets_data) + 7) // 8, byteorder="big"
+            )
+            f.write(byte_data)
+        return tmp_path / "imap_hi_l0_sdc-test-data_20240318_v000.pkts"
+
+    return create_data