imap-processing 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

imap_processing/swe/l1a/swe_l1a.py

@@ -42,7 +42,12 @@ def swe_l1a(packet_file: str, data_version: str) -> xr.Dataset:
         packet_file, xtce_document, use_derived_value=False
     )
 
+    if SWEAPID.SWE_SCIENCE not in datasets_by_apid:
+        logger.info("No science data found in packet file.")
+        return []
     # TODO: figure out how to handle non-science data
-    return swe_science(
-        l0_dataset=datasets_by_apid[SWEAPID.SWE_SCIENCE], data_version=data_version
-    )
+    return [
+        swe_science(
+            l0_dataset=datasets_by_apid[SWEAPID.SWE_SCIENCE], data_version=data_version
+        )
+    ]

imap_processing/swe/l1a/swe_science.py

@@ -141,45 +141,45 @@ def swe_science(l0_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
         attrs=cdf_attrs.get_variable_attributes("epoch"),
     )
 
-    spin_angle = xr.DataArray(
+    spin_sector = xr.DataArray(
         np.arange(180),
-        name="spin_angle",
-        dims=["spin_angle"],
-        attrs=cdf_attrs.get_variable_attributes("spin_angle"),
+        name="spin_sector",
+        dims=["spin_sector"],
+        attrs=cdf_attrs.get_variable_attributes("spin_sector"),
     )
 
     # NOTE: LABL_PTR_1 should be CDF_CHAR.
-    spin_angle_label = xr.DataArray(
-        spin_angle.values.astype(str),
-        name="spin_angle_label",
-        dims=["spin_angle_label"],
-        attrs=cdf_attrs.get_variable_attributes("spin_angle_label"),
+    spin_sector_label = xr.DataArray(
+        spin_sector.values.astype(str),
+        name="spin_sector_label",
+        dims=["spin_sector"],
+        attrs=cdf_attrs.get_variable_attributes("spin_sector_label"),
     )
 
-    polar_angle = xr.DataArray(
+    cem_id = xr.DataArray(
         np.arange(7),
-        name="polar_angle",
-        dims=["polar_angle"],
-        attrs=cdf_attrs.get_variable_attributes("polar_angle"),
+        name="cem_id",
+        dims=["cem_id"],
+        attrs=cdf_attrs.get_variable_attributes("cem_id"),
     )
 
     # NOTE: LABL_PTR_2 should be CDF_CHAR.
-    polar_angle_label = xr.DataArray(
-        polar_angle.values.astype(str),
-        name="polar_angle_label",
-        dims=["polar_angle_label"],
-        attrs=cdf_attrs.get_variable_attributes("polar_angle_label"),
+    cem_id_label = xr.DataArray(
+        cem_id.values.astype(str),
+        name="cem_id_label",
+        dims=["cem_id"],
+        attrs=cdf_attrs.get_variable_attributes("cem_id_label"),
     )
 
     science_xarray = xr.DataArray(
         science_array,
-        dims=["epoch", "spin_angle", "polar_angle"],
+        dims=["epoch", "spin_sector", "cem_id"],
         attrs=cdf_attrs.get_variable_attributes("science_data"),
     )
 
     raw_science_xarray = xr.DataArray(
         raw_science_array,
-        dims=["epoch", "spin_angle", "polar_angle"],
+        dims=["epoch", "spin_sector", "cem_id"],
         attrs=cdf_attrs.get_variable_attributes("raw_counts"),
     )
 
@@ -190,10 +190,10 @@ def swe_science(l0_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
     dataset = xr.Dataset(
         coords={
             "epoch": epoch_time,
-            "spin_angle": spin_angle,
-            "polar_angle": polar_angle,
-            "spin_angle_label": spin_angle_label,
-            "polar_angle_label": polar_angle_label,
+            "spin_sector": spin_sector,
+            "cem_id": cem_id,
+            "spin_sector_label": spin_sector_label,
+            "cem_id_label": cem_id_label,
         },
         attrs=l1a_global_attrs,
     )

imap_processing/swe/l1b/swe_l1b.py

@@ -46,4 +46,5 @@ def swe_l1b(l1a_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
     data = swe_l1b_science(eu_data, data_version)
     if data is None:
         logger.info("No data to write to CDF")
-    return data
+        return []
+    return [data]

imap_processing/swe/l1b/swe_l1b_science.py

@@ -8,38 +8,13 @@ import pandas as pd
 import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
-from imap_processing.swe.utils.swe_utils import read_lookup_table
+from imap_processing.swe.utils.swe_utils import (
+    ESA_VOLTAGE_ROW_INDEX_DICT,
+    read_lookup_table,
+)
 
 logger = logging.getLogger(__name__)
 
-# ESA voltage and index in the final data table
-esa_voltage_row_index_dict = {
-    0.56: 0,
-    0.78: 1,
-    1.08: 2,
-    1.51: 3,
-    2.10: 4,
-    2.92: 5,
-    4.06: 6,
-    5.64: 7,
-    7.85: 8,
-    10.92: 9,
-    15.19: 10,
-    21.13: 11,
-    29.39: 12,
-    40.88: 13,
-    56.87: 14,
-    79.10: 15,
-    110.03: 16,
-    153.05: 17,
-    212.89: 18,
-    296.14: 19,
-    411.93: 20,
-    572.99: 21,
-    797.03: 22,
-    1108.66: 23,
-}
-
 
 def get_esa_dataframe(esa_table_number: int) -> pd.DataFrame:
     """
@@ -110,7 +85,7 @@ def deadtime_correction(counts: np.ndarray, acq_duration: int) -> npt.NDArray:
     return corrected_count.astype(np.float64)
 
 
-def convert_counts_to_rate(data: np.ndarray, acq_duration: int) -> npt.NDArray:
+def convert_counts_to_rate(data: np.ndarray, acq_duration: np.ndarray) -> npt.NDArray:
     """
     Convert counts to rate using sampling time.
 
@@ -120,7 +95,7 @@ def convert_counts_to_rate(data: np.ndarray, acq_duration: int) -> npt.NDArray:
     ----------
     data : numpy.ndarray
         Counts data.
-    acq_duration : int
+    acq_duration : numpy.ndarray
         Acquisition duration. acq_duration is in microseconds.
 
     Returns
@@ -134,51 +109,107 @@ def convert_counts_to_rate(data: np.ndarray, acq_duration: int) -> npt.NDArray:
     return count_rate.astype(np.float64)
 
 
-def calculate_calibration_factor(time: int) -> None:
+def read_in_flight_cal_data() -> pd.DataFrame:
     """
-    Calculate calibration factor.
+    Read in-flight calibration data.
 
-    Steps to calculate calibration factor:
+    In-flight calibration data file will contain rows where each line
+    has 8 numbers, with the first being a time stamp in MET, and the next
+    7 being the factors for the 7 detectors.
 
-    1. Convert input time to match time format in the calibration data file.
-    2. Find the nearest in time calibration data point.
-    3. Linear interpolate between those two nearest time and get factor for input time.
+    This file will be updated weekly with new calibration data. In other
+    words, one line of data will be added each week to the existing file.
+    File will be in CSV format. Processing won't be kicked off until there
+    is in-flight calibration data that covers science data.
 
-    What this function is doing:
+    TODO: decide filename convention given this information. This function
+    is a placeholder for reading in the calibration data until we decide on
+    how to read calibration data through dependencies list.
 
-    | 1. **Reading Calibration Data**: The function first reads a file containing
-    |     calibration data for electron measurements over time. This data helps
-    |     adjust or correct the measurements based on changes in the instrument's
-    |     sensitivity.
+    Returns
+    -------
+    in_flight_cal_df : pandas.DataFrame
+        DataFrame with in-flight calibration data.
+    """
+    # TODO: Read in in-flight calibration file.
 
-    | 2. **Interpolating Calibration Factors**: Imagine you have several points on
-    |     a graph, and you want to estimate values between those points. In our case,
-    |     these points represent calibration measurements taken at different times.
-    |     The function figures out which two calibration points are closest in time
-    |     to the specific measurement time you're interested in.
+    # Define the column headers
+    columns = ["met_time", "cem1", "cem2", "cem3", "cem4", "cem5", "cem6", "cem7"]
 
-    | 3. **Calculating Factors**: Once it finds these two nearby calibration points,
-    |     the function calculates a correction factor by drawing a straight line
-    |     between them (linear interpolation). This factor helps adjust the measurement
-    |     to make it more accurate, considering how the instrument's sensitivity changed
-    |     between those two calibration points.
+    # Create an empty DataFrame with the specified columns
+    empty_df = pd.DataFrame(columns=columns)
+    return empty_df
 
-    | 4. **Returning the Correction Factor**: Finally, the function returns this
-    |     correction factor. You can then use this factor to adjust or calibrate your
-    |     measurements at the specific time you're interested in. This ensures that
-    |     your measurements are as accurate as possible, taking into account the
-    |     instrument's changing sensitivity over time.
 
-    Parameters
-    ----------
-    time : int
-        Input time.
+def calculate_calibration_factor(
+    acquisition_times: np.ndarray, cal_times: np.ndarray, cal_data: np.ndarray
+) -> npt.NDArray:
     """
-    # NOTE: waiting on fake calibration data to write this.
-    pass
+    Calculate calibration factor using linear interpolation.
+
+    Steps to calculate calibration factor:
+        1. Convert input time to match time format in the calibration data file.
+           Both times should be in S/C MET time.
+        2. Find the nearest in time calibration data point.
+        3. Linear interpolate between those two nearest time and get factor for
+           input time.
 
+    Parameters
+    ----------
+    acquisition_times : numpy.ndarray
+        Data points to interpolate. Shape is (24, 30).
+    cal_times : numpy.ndarray
+        X-coordinates data points. Calibration times. Shape is (n,).
+    cal_data : numpy.ndarray
+        Y-coordinates data points. Calibration data of corresponding cal_times.
+        Shape is (n, 7).
 
-def apply_in_flight_calibration(data: np.ndarray) -> None:
+    Returns
+    -------
+    calibration_factor : numpy.ndarray
+        Calibration factor for each CEM detector. Shape is (24, 30, 7)
+        where last 7 dimension contains calibration factor for each CEM detector.
+    """
+    # Raise error if there is no pre or post time in cal_times. SWE does not
+    # want to extrapolate calibration data.
+    if (
+        acquisition_times.min() < cal_times.min()
+        or acquisition_times.max() > cal_times.max()
+    ):
+        error_msg = (
+            f"Acquisition min/max times: {acquisition_times.min()} to "
+            f"{acquisition_times.max()}. "
+            f"Calibration min/max times: {cal_times.min()} to {cal_times.max()}. "
+            "Acquisition times should be within calibration time range."
+        )
+        raise ValueError(error_msg)
+
+    # This line of code finds the indices of acquisition_times in cal_times where
+    # acquisition_times should be inserted to maintain order. As a result, it finds
+    # its nearest pre and post time from cal_times.
+    input_time_indices = np.searchsorted(cal_times, acquisition_times)
+
+    # Assign to a variable for better readability
+    x = acquisition_times
+    xp = cal_times
+    fp = cal_data
+
+    # Given this situation which will be the case for SWE data
+    # where data will fall in between two calibration times and
+    # not be exactly equal to any calibration time,
+    #   >>> a = [1, 2, 3]
+    #   >>> np.searchsorted(a, [2.5])
+    #   array([2])
+    # we need to use (j - 1) to get pre time indices. (j-1) is
+    # pre time indices and j is post time indices.
+    j = input_time_indices
+    w = (x - xp[j - 1]) / (xp[j] - xp[j - 1])
+    return fp[j - 1] + w[..., None] * (fp[j] - fp[j - 1])
+
+
+def apply_in_flight_calibration(
+    corrected_counts: np.ndarray, acquisition_time: np.ndarray
+) -> npt.NDArray:
     """
     Apply in flight calibration to full cycle data.
 
@@ -188,12 +219,29 @@ def apply_in_flight_calibration(data: np.ndarray) -> None:
 
     Parameters
     ----------
-    data : numpy.ndarray
-        Full cycle data array.
+    corrected_counts : numpy.ndarray
+        Corrected count of full cycle data. Data shape is (24, 30, 7).
+    acquisition_time : numpy.ndarray
+        Acquisition time of full cycle data. Data shape is (24, 30).
+
+    Returns
+    -------
+    corrected_counts : numpy.ndarray
+        Corrected count of full cycle data after applying in-flight calibration.
+        Array shape is (24, 30, 7).
     """
-    # calculate calibration factor
-    # Apply to all data
-    pass
+    # Read in in-flight calibration data
+    in_flight_cal_df = read_in_flight_cal_data()
+    # calculate calibration factor.
+    # return shape of calculate_calibration_factor is (24, 30, 7) where
+    # last 7 dimension contains calibration factor for each CEM detector.
+    cal_factor = calculate_calibration_factor(
+        acquisition_time,
+        in_flight_cal_df["met_time"].values,
+        in_flight_cal_df.iloc[:, 1:].values,
+    )
+    # Apply to full cycle data
+    return corrected_counts.astype(np.float64) * cal_factor
 
 
 def populate_full_cycle_data(
@@ -231,7 +279,10 @@ def populate_full_cycle_data(
         # to use in level 2 processing to calculate
         # spin phase. This is done below by using information from
         # science packet.
-        acquisition_times = np.zeros((energy_steps, angle, cem_detectors))
+        acquisition_times = np.zeros((energy_steps, angle))
+
+        # Store acquisition duration for later calculation in this function
+        acq_duration_arr = np.zeros((energy_steps, angle))
 
         # Initialize esa_step_number and column_index.
         # esa_step_number goes from 0 to 719 range where
@@ -249,19 +300,12 @@ def populate_full_cycle_data(
             acq_duration = l1a_data["acq_duration"].data[packet_index + index]
             settle_duration = l1a_data["settle_duration"].data[packet_index + index]
             corrected_counts = deadtime_correction(decompressed_counts, acq_duration)
-            # Convert counts to rate
-            counts_rate = convert_counts_to_rate(corrected_counts, acq_duration)
 
             # Each quarter cycle data should have same acquisition start time coarse
             # and fine value. We will use that as base time to calculate each
-            # acquisition time for each count data. Acquisition time of each count
-            # data point will be calculated using this formula:
+            # acquisition time for each count data.
             #   base_quarter_cycle_acq_time = acq_start_coarse +
             #                                 acq_start_fine / 1000000
-            #   each_count_acq_time = base_quarter_cycle_acq_time +
-            #                         (step * ( acq_duration + settle_duration) / 1000 )
-            # where step goes from 0 to 179, acq_start_coarse is in seconds and
-            # acq_start_fine is in microseconds and acq_duration is in milliseconds.
             base_quarter_cycle_acq_time = (
                 l1a_data["acq_start_coarse"].data[packet_index + index]
                 + l1a_data["acq_start_fine"].data[packet_index + index] / 1000000
@@ -273,18 +317,27 @@ def populate_full_cycle_data(
                 # Get esa voltage value from esa lookup table and
                 # use that to get row index in full data array
                 esa_voltage_value = esa_lookup_table.loc[esa_step_number]["esa_v"]
-                esa_voltage_row_index = esa_voltage_row_index_dict[esa_voltage_value]
+                esa_voltage_row_index = ESA_VOLTAGE_ROW_INDEX_DICT[esa_voltage_value]
 
                 # every six steps, increment column index
                 if esa_step_number % 6 == 0:
                     column_index += 1
                 # Put counts rate in full cycle data array
-                full_cycle_data[esa_voltage_row_index][column_index] = counts_rate[step]
-                # Put acquisition time in acquisition_times array
+                full_cycle_data[esa_voltage_row_index][column_index] = corrected_counts[
+                    step
+                ]
+                # Acquisition time (in seconds) of each count data point will be
+                # using this formula:
+                #   each_count_acq_time = base_quarter_cycle_acq_time +
+                #            (step * ( acq_duration + settle_duration) / 1000000 )
+                # where step goes from 0 to 179, acq_start_coarse is in seconds and
+                # acq_start_fine is in microseconds and acq_duration is in microseconds.
                 acquisition_times[esa_voltage_row_index][column_index] = (
                     base_quarter_cycle_acq_time
-                    + (step * (acq_duration + settle_duration) / 1000)
+                    + (step * (acq_duration + settle_duration) / 1000000)
                 )
+                # Store acquisition duration for later calculation
+                acq_duration_arr[esa_voltage_row_index][column_index] = acq_duration
                 esa_step_number += 1
 
             # reset column index for next quarter cycle
@@ -295,11 +348,17 @@ def populate_full_cycle_data(
     # data. But for now, we are advice to continue with current setup and can
     # add/change it when we get real data.
 
+    # Apply calibration based on in-flight calibration.
+    calibrated_counts = apply_in_flight_calibration(full_cycle_data, acquisition_times)
+
+    # Convert counts to rate
+    counts_rate = convert_counts_to_rate(calibrated_counts, acq_duration)
+
     # Store count data and acquisition times of full cycle data in xr.Dataset
     full_cycle_ds = xr.Dataset(
         {
-            "full_cycle_data": (["energy", "angle", "cem"], full_cycle_data),
-            "sci_step_acq_time_sec": (["energy", "angle", "cem"], acquisition_times),
+            "full_cycle_data": (["esa_step", "spin_sector", "cem_id"], counts_rate),
+            "acquisition_time": (["esa_step", "spin_sector"], acquisition_times),
         }
     )
 
@@ -464,7 +523,7 @@ def swe_l1b_science(l1a_data: xr.Dataset, data_version: str) -> xr.Dataset:
 
         # save full data array to file
         full_cycle_science_data.append(full_cycle_ds["full_cycle_data"].data)
-        full_cycle_acq_times.append(full_cycle_ds["sci_step_acq_time_sec"].data)
+        full_cycle_acq_times.append(full_cycle_ds["acquisition_time"].data)
 
     # ------------------------------------------------------------------
     # Save data to dataset.
@@ -486,34 +545,34 @@ def swe_l1b_science(l1a_data: xr.Dataset, data_version: str) -> xr.Dataset:
         attrs=cdf_attrs.get_variable_attributes("epoch"),
     )
 
-    energy = xr.DataArray(
+    esa_step = xr.DataArray(
         np.arange(24),
-        name="energy",
-        dims=["energy"],
-        attrs=cdf_attrs.get_variable_attributes("energy"),
+        name="esa_step",
+        dims=["esa_step"],
+        attrs=cdf_attrs.get_variable_attributes("esa_step"),
     )
 
     # NOTE: LABL_PTR_1 should be CDF_CHAR.
-    energy_label = xr.DataArray(
-        energy.values.astype(str),
-        name="energy_label",
-        dims=["energy_label"],
-        attrs=cdf_attrs.get_variable_attributes("energy_label"),
+    esa_step_label = xr.DataArray(
+        esa_step.values.astype(str),
+        name="esa_step_label",
+        dims=["esa_step"],
+        attrs=cdf_attrs.get_variable_attributes("esa_step_label"),
     )
 
-    angle = xr.DataArray(
+    spin_sector = xr.DataArray(
         np.arange(30),
-        name="angle",
-        dims=["angle"],
-        attrs=cdf_attrs.get_variable_attributes("angle"),
+        name="spin_sector",
+        dims=["spin_sector"],
+        attrs=cdf_attrs.get_variable_attributes("spin_sector"),
     )
 
     # NOTE: LABL_PTR_2 should be CDF_CHAR.
-    angle_label = xr.DataArray(
-        angle.values.astype(str),
-        name="angle_label",
-        dims=["angle_label"],
-        attrs=cdf_attrs.get_variable_attributes("angle_label"),
+    spin_sector_label = xr.DataArray(
+        spin_sector.values.astype(str),
+        name="spin_sector_label",
+        dims=["spin_sector"],
+        attrs=cdf_attrs.get_variable_attributes("spin_sector_label"),
     )
 
     cycle = xr.DataArray(
@@ -523,19 +582,19 @@ def swe_l1b_science(l1a_data: xr.Dataset, data_version: str) -> xr.Dataset:
         attrs=cdf_attrs.get_variable_attributes("cycle"),
     )
 
-    cem = xr.DataArray(
+    cem_id = xr.DataArray(
         np.arange(7, dtype=np.float64),
-        name="cem",
-        dims=["cem"],
-        attrs=cdf_attrs.get_variable_attributes("cem"),
+        name="cem_id",
+        dims=["cem_id"],
+        attrs=cdf_attrs.get_variable_attributes("cem_id"),
     )
 
     # NOTE: LABL_PTR_3 should be CDF_CHAR.
-    cem_label = xr.DataArray(
-        cem.values.astype(str),
-        name="cem_label",
-        dims=["cem_label"],
-        attrs=cdf_attrs.get_variable_attributes("cem_label"),
+    cem_id_label = xr.DataArray(
+        cem_id.values.astype(str),
+        name="cem_id_label",
+        dims=["cem_id"],
+        attrs=cdf_attrs.get_variable_attributes("cem_id_label"),
     )
 
     # Add science data and it's associated metadata into dataset.
@@ -556,26 +615,26 @@ def swe_l1b_science(l1a_data: xr.Dataset, data_version: str) -> xr.Dataset:
     dataset = xr.Dataset(
         coords={
             "epoch": epoch_time,
-            "energy": energy,
-            "angle": angle,
-            "cem": cem,
+            "esa_step": esa_step,
+            "spin_sector": spin_sector,
+            "cem_id": cem_id,
             "cycle": cycle,
-            "energy_label": energy_label,
-            "angle_label": angle_label,
-            "cem_label": cem_label,
+            "esa_step_label": esa_step_label,
+            "spin_sector_label": spin_sector_label,
+            "cem_id_label": cem_id_label,
         },
         attrs=cdf_attrs.get_global_attributes("imap_swe_l1b_sci"),
     )
 
     dataset["science_data"] = xr.DataArray(
         full_cycle_science_data,
-        dims=["epoch", "energy", "angle", "cem"],
+        dims=["epoch", "esa_step", "spin_sector", "cem_id"],
         attrs=cdf_attrs.get_variable_attributes("science_data"),
     )
-    dataset["sci_step_acq_time_sec"] = xr.DataArray(
+    dataset["acquisition_time"] = xr.DataArray(
         full_cycle_acq_times,
-        dims=["epoch", "energy", "angle", "cem"],
-        attrs=cdf_attrs.get_variable_attributes("sci_step_acq_time_sec"),
+        dims=["epoch", "esa_step", "spin_sector"],
+        attrs=cdf_attrs.get_variable_attributes("acquisition_time"),
     )
 
     # create xarray dataset for each metadata field