imap-processing 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

imap_processing/swe/l1b/swe_l1b_science.py

@@ -95,7 +95,7 @@ def deadtime_correction(counts: np.ndarray, acq_duration: int) -> npt.NDArray:
     counts : numpy.ndarray
         Counts data before deadtime corrections.
     acq_duration : int
-        This is ACQ_DURATION from science packet.
+        This is ACQ_DURATION from science packet. acq_duration is in microseconds.
 
     Returns
     -------
@@ -104,10 +104,10 @@ def deadtime_correction(counts: np.ndarray, acq_duration: int) -> npt.NDArray:
     """
     # deadtime is 360 ns
     deadtime = 360e-9
-    correct = 1.0 - (deadtime * counts / (acq_duration / 1000.0))
+    correct = 1.0 - (deadtime * (counts / (acq_duration * 1e-6)))
     correct = np.maximum(0.1, correct)
     corrected_count = np.divide(counts, correct)
-    return corrected_count
+    return corrected_count.astype(np.float64)
 
 
 def convert_counts_to_rate(data: np.ndarray, acq_duration: int) -> npt.NDArray:
@@ -121,17 +121,17 @@ def convert_counts_to_rate(data: np.ndarray, acq_duration: int) -> npt.NDArray:
     data : numpy.ndarray
         Counts data.
     acq_duration : int
-        Acquisition duration. acq_duration is in millieseconds.
+        Acquisition duration. acq_duration is in microseconds.
 
     Returns
     -------
     numpy.ndarray
         Count rates array in seconds.
     """
-    # convert milliseconds to seconds
-    # Todo: check with SWE team about int or float types.
-    acq_duration = int(acq_duration / 1000.0)
-    return data / acq_duration
+    # convert microseconds to seconds
+    acq_duration_sec = acq_duration * 1e-6
+    count_rate = data / acq_duration_sec
+    return count_rate.astype(np.float64)
 
 
 def calculate_calibration_factor(time: int) -> None:

imap_processing/swe/l2/swe_l2.py

@@ -1,9 +1,14 @@
-"""SWE L2 processing module."""
+"""
+SWE L2 processing module.
+
+This module contains functions to process L1B data to L2 data products.
+"""
 
 import numpy as np
 import numpy.typing as npt
 import xarray as xr
 
+from imap_processing.spice.geometry import get_spacecraft_spin_phase
 from imap_processing.swe.utils.swe_utils import read_lookup_table
 
 # TODO: add these to instrument status summary
@@ -22,6 +27,9 @@ GEOMETRIC_FACTORS = np.array(
 )
 ELECTRON_MASS = 9.10938356e-31  # kg
 
+# See doc string of calculate_phase_space_density() for more details.
+VELOCITY_CONVERSION_FACTOR = 1.237e31
+
 
 def get_particle_energy() -> npt.NDArray:
     """
@@ -44,11 +52,11 @@ def get_particle_energy() -> npt.NDArray:
     return lookup_table
 
 
-def calculate_phase_space_density(l1b_dataset: xr.Dataset) -> npt.NDArray:
+def calculate_phase_space_density(l1b_dataset: xr.Dataset) -> xr.Dataset:
     """
     Convert counts to phase space density.
 
-    Calculate phase space density, fv, in units of s^3/cm^6
+    Calculate phase space density, fv, in units of s^3/ (cm^6 * ster).
         fv = 2 * (C/tau) / (G * v^4)
         where:
             C / tau = corrected count rate. L1B science data.
@@ -56,12 +64,26 @@ def calculate_phase_space_density(l1b_dataset: xr.Dataset) -> npt.NDArray:
             v = electron speed, computed from energy, in cm/s.
                 We need to use this formula to convert energy to speed:
                     E = 0.5 * m * v^2
-                where E is electron energy, in eV
-                (result from get_particle_energy() function),
-                m is mass of electron (9.10938356e-31 kg),
-                and v is what we want to calculate. Reorganizing above
-                formula result in v = sqrt(2 * E / m). This will be used
-                to calculate electron speed.
+                    where E is electron energy, in eV
+                    (result from get_particle_energy() function),
+                    m is mass of electron (9.10938356e-31 kg),
+                    and v is what we want to calculate. Reorganizing above
+                    formula result in v = sqrt(2 * E / m). This will be used
+                    to calculate electron speed, v.
+
+                Now to convert electron speed units to cm/s:
+                    v = sqrt(2 * E / m)
+                      = sqrt(2 * E(eV) * 1.60219e-19(J/eV) / 9.10938e-31 kg)
+                        where J = kg * m^2 / s^2.
+                      = sqrt(2 * 1.60219 * 10e−19 m^2/s^2 * E(eV) / 9.10938e-31)
+                      = sqrt(2 * 1.60219 * 10e−19 * 10e4 cm^2/s^2 * E(eV) / 9.10938e-31)
+                      = sqrt(3.20438 * 10e-15 * E(eV) / 9.10938e-31) cm/s
+                      = sqrt( (3.20438 * 10e-15 / 9.10938e-31) * E(eV) ) cm/s
+        fv = 2 * (C/tau) / (G * v^4)
+           = 2 * (C/tau) / (G * (sqrt( (3.20438 * 10e-15 / 9.10938e-31) * E(eV) ))^4)
+           = 2 * (C/tau) / (G * (sqrt(3.5176e16)^4 * E(eV)^2)
+           = 2 * (C/tau) / (G * 1.237e31 * E(eV)^2)
+        Ruth Skoug also got the same result, 1.237e31.
 
     Parameters
     ----------
@@ -70,8 +92,9 @@ def calculate_phase_space_density(l1b_dataset: xr.Dataset) -> npt.NDArray:
 
     Returns
     -------
-    density : numpy.ndarray
-        Phase space density.
+    phase_space_density_dataset : xarray.Dataset
+        Phase space density. We need to call this phase space density because
+        there will be density in L3 processing.
     """
     # Get esa_table_num for each full sweep.
     esa_table_nums = l1b_dataset["esa_table_num"].values[:, 0]
@@ -87,13 +110,107 @@ def calculate_phase_space_density(l1b_dataset: xr.Dataset) -> npt.NDArray:
     )
     particle_energy_data = particle_energy_data.reshape(-1, 24, 30)
 
-    # Calculate electron speed.
-    electron_speed = np.sqrt(2 * particle_energy_data / ELECTRON_MASS)
-
-    # Calculate phase space density.
+    # Calculate phase space density using formula:
+    #   2 * (C/tau) / (G * 1.237e31 * E(eV)^2)
+    # See doc string for more details.
     density = (2 * l1b_dataset["science_data"]) / (
         GEOMETRIC_FACTORS[np.newaxis, np.newaxis, np.newaxis, :]
-        * electron_speed[:, :, :, np.newaxis] ** 4
+        * VELOCITY_CONVERSION_FACTOR
+        * particle_energy_data[:, :, :, np.newaxis] ** 2
+    )
+
+    # Return density as xr.dataset with phase space density and
+    # energy in eV value that flux calculation can use.
+    phase_space_density_dataset = xr.Dataset(
+        {
+            "phase_space_density": (["epoch", "energy", "angle", "cem"], density.data),
+            "energy_in_eV": (["epoch", "energy", "angle"], particle_energy_data),
+        },
+        coords=l1b_dataset.coords,
+    )
+
+    return phase_space_density_dataset
+
+
+def calculate_flux(l1b_dataset: xr.Dataset) -> npt.NDArray:
+    """
+    Calculate flux.
+
+    To get flux, j = 2 * m * E * f
+        where:
+        f = calculate_phase_space_density() result
+        m - mass of electron
+        E - energy in joules. E(eV) * 1.60219e-19(J/eV)
+
+    To convert flux units:
+        j = 2 * m * E * f
+          = 2 * 9.10938356e-31 kg * E(eV) * 1.60219e-19(J/eV) * (s^3/ (cm^6 * ster)
+          = 2 * 9.10938356e-31 kg * E(eV) * 1.60219e-19 (kg * m^2 / s^2) *
+            (s^3 / (cm^6 * ster)
+          = 2 * 9.10938356e-31 * E(eV) * 1.60219e-19  kg^2 * ( 10^4 cm^2 / s^2) *
+            (s^3 / (cm^6 * ster)
+          = 2 * 9.10938356e-31 * E(eV) * 1.60219e-19 * 10^4 ((kg^2 * cm^2 * s^3) /
+            (s^2 * cm^6 * ster))
+        TODO: ask Ruth Skoug what units to use for flux and work out remaining units.
+
+    Parameters
+    ----------
+    l1b_dataset : xarray.Dataset
+        The L1B dataset to process.
+
+    Returns
+    -------
+    flux : numpy.ndarray
+        Flux values.
+    """
+    phase_space_density_ds = calculate_phase_space_density(l1b_dataset)
+    # TODO: update this once Ruth sends the correct conversion factors.
+    flux = (
+        2
+        * ELECTRON_MASS
+        * phase_space_density_ds["energy_in_eV"].data[:, :, :, np.newaxis]
+        * 1.60219e-19
+        * 10e4
+        * phase_space_density_ds["phase_space_density"].data
     )
+    return flux
 
-    return density
+
+def swe_l2(l1b_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
+    """
+    Will process data to L2.
+
+    Parameters
+    ----------
+    l1b_dataset : xarray.Dataset
+        The L1B dataset to process.
+    data_version : str
+        Version of the data product being created.
+
+    Returns
+    -------
+    data : xarray.Dataset
+        Processed data to L2.
+    """
+    flux = calculate_flux(l1b_dataset)
+
+    # Calculate spin phase using SWE sci_step_acq_time_sec calculated in l1b.
+    # L1B dataset stores it by (epoch, energy, angle, cem).
+    data_acq_time = l1b_dataset["sci_step_acq_time_sec"].data.flatten()
+
+    # calculate spin phase
+    spin_phase = get_spacecraft_spin_phase(
+        query_met_times=data_acq_time,
+    ).reshape(-1, 24, 30, 7)
+    # TODO: organize flux data by energy and spin_phase.
+    # My understanding from conversation with Ruth is that this is the hardest part
+    # and last part of the L2 processing.
+
+    # TODO: Correct return value. This is just a placeholder.
+    return xr.Dataset(
+        {
+            "flux": (["epoch", "energy", "spin_phase", "cem"], flux),
+            "spin_phase": (["epoch", "energy", "spin_phase", "cem"], spin_phase),
+        },
+        coords=l1b_dataset.coords,
+    )

imap_processing/tests/ccsds/test_data/expected_output.xml

@@ -1,6 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
 <xtce:SpaceSystem xmlns:xtce="http://www.omg.org/space/xtce" name="Test Instrument">
-	<xtce:Header date="2024-07-26 00:00:00" version="v1.2" author="IMAP SDC" />
+	<xtce:Header date="2024-07-26 00:00:00" version="v1.2" author="IMAP SDC" source_file="excel_to_xtce_test_file.xlsx" />
 	<xtce:TelemetryMetaData>
 		<xtce:ParameterTypeSet>
 			<xtce:IntegerParameterType name="VERSION" signed="false">
@@ -64,6 +64,7 @@
 				<xtce:EnumerationList>
 					<xtce:Enumeration value="0" label="OFF" />
 					<xtce:Enumeration value="1" label="ON" />
+					<xtce:Enumeration value="2" label="NONE" />
 				</xtce:EnumerationList>
 			</xtce:EnumeratedParameterType>
 			<xtce:IntegerParameterType name="TEST_PACKET2.SHCOARSE" signed="false">

imap_processing/tests/ccsds/test_excel_to_xtce.py

@@ -195,10 +195,10 @@ def xtce_excel_file(tmp_path):
     }
 
     states = {
-        "packetName": ["TEST_PACKET"] * 2,
-        "mnemonic": ["VAR_STATE"] * 2,
-        "value": [0, 1],
-        "state": ["OFF", "ON"],
+        "packetName": ["TEST_PACKET"] * 3,
+        "mnemonic": ["VAR_STATE"] * 3,
+        "value": [0, 1, "0x2"],
+        "state": ["OFF", "ON", "NONE"],
     }
 
     # Write the DataFrame to an excel file

imap_processing/tests/cdf/test_imap_cdf_manager.py

@@ -1,5 +1,4 @@
 from pathlib import Path
-from unittest import mock
 
 # from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
@@ -25,15 +24,6 @@ def test_add_instrument_global_attrs():
     assert instrument2_instrument["Project"] == "STP>Solar Terrestrial Probes"
 
 
-@mock.patch(
-    "imap_processing.cdf.cdf_attribute_manager.CdfAttributeManager.load_variable_attributes"
-)
-def testing_source_dir(mock_load_variable_attributes):
-    # Create an ImapCdfAttributes object
-    imap_cdf_manager = ImapCdfAttributes(Path(__file__).parent.parent / "cdf")
-    assert str(imap_cdf_manager.source_dir) == str(Path(__file__).parent.parent / "cdf")
-
-
 def test_add_instrument_variable_attrs():
     # Create an ImapCdfAttributes object
     imap_cdf_manager = ImapCdfAttributes()

imap_processing/tests/codice/conftest.py

@@ -2,23 +2,7 @@ from imap_processing import imap_module_directory
 
 TEST_DATA_PATH = imap_module_directory / "tests" / "codice" / "data"
 
-TEST_PACKETS = [
-    TEST_DATA_PATH / "imap_codice_l0_hskp_20100101_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_hi-counters-singles_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_hi-omni_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_hi-sectored_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_hi-pha_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_lo-counters-aggregated_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_lo-counters-singles_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_lo-sw-angular_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_lo-nsw-angular_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_lo-sw-priority_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_lo-nsw-priority_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_lo-sw-species_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_lo-nsw-species_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_lo-pha_20240429_v001.pkts",
-]
+TEST_L0_FILE = TEST_DATA_PATH / "imap_codice_l0_raw_20241110_v001.pkts"
 
 TEST_L1A_FILES = [
     TEST_DATA_PATH / "imap_codice_l1a_hskp_20100101_v001.pkts",

imap_processing/tests/codice/test_codice_l0.py

@@ -37,7 +37,7 @@ def decom_test_data() -> xr.Dataset:
 
     packet_file = Path(
         f"{imap_module_directory}/tests/codice/data/"
-        f"imap_codice_l0_hskp_20100101_v001.pkts"
+        f"imap_codice_l0_raw_20241110_v001.pkts"
     )
 
     packet = codice_l0.decom_packets(packet_file)[1136]
@@ -69,6 +69,9 @@ def validation_data() -> pd.core.frame.DataFrame:
     return validation_data
 
 
+@pytest.mark.xfail(
+    reason="Need to update to validate against new validation CDFs. See issue #1154."
+)
 def test_eu_hskp_data(
     decom_test_data: xr.Dataset,
     validation_data: pd.core.frame.DataFrame,
@@ -106,6 +109,9 @@ def test_eu_hskp_data(
             assert round(eu_val, 5) == round(validation_val, 5)
 
 
+@pytest.mark.xfail(
+    reason="Need to update to validate against new validation CDFs. See issue #1154."
+)
 def test_raw_hskp_data(
     decom_test_data: xr.Dataset,
     validation_data: pd.core.frame.DataFrame,
@@ -140,5 +146,5 @@ def test_total_packets_in_data_file(decom_test_data: xr.Dataset):
         The decommutated housekeeping packet
     """
 
-    total_packets = 99
+    total_packets = 31778
     assert len(decom_test_data.epoch) == total_packets