imap-processing 0.12.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

imap_processing/mag/imap_mag_sdc_configuration_v001.py

@@ -0,0 +1,8 @@
+"""File describing configuration changes that are not expected to change often."""
+
+# interpolation method used in L1C processing
+L1C_INTERPOLATION_METHOD = "linear_filtered"
+
+# Setting for L2 output
+# if this setting is updated, the dependency settings for L2 will also need to change.
+ALWAYS_OUTPUT_MAGO = True

imap_processing/mag/l1a/mag_l1a.py

@@ -24,7 +24,7 @@ from imap_processing.spice.time import (
 logger = logging.getLogger(__name__)
 
 
-def mag_l1a(packet_filepath: Path, data_version: str) -> list[xr.Dataset]:
+def mag_l1a(packet_filepath: Path) -> list[xr.Dataset]:
     """
     Will process MAG L0 data into L1A CDF files at cdf_filepath.
 
@@ -32,8 +32,6 @@ def mag_l1a(packet_filepath: Path, data_version: str) -> list[xr.Dataset]:
     ----------
     packet_filepath : pathlib.Path
         Packet files for processing.
-    data_version : str
-        Data version to write to CDF files.
 
     Returns
     -------
@@ -51,8 +49,6 @@ def mag_l1a(packet_filepath: Path, data_version: str) -> list[xr.Dataset]:
     attribute_manager = ImapCdfAttributes()
     attribute_manager.add_instrument_global_attrs("mag")
     attribute_manager.add_instrument_variable_attrs("mag", "l1a")
-
-    attribute_manager.add_global_attribute("Data_version", data_version)
     attribute_manager.add_global_attribute("Input_files", str(input_files))
     attribute_manager.add_global_attribute(
         "Generation_date",
@@ -179,7 +175,7 @@ def process_packets(
         # each sensor, we can calculate how much data is in this packet and where the
         # byte boundaries are.
         primary_vectors, secondary_vectors = MagL1a.process_vector_data(
-            mag_l0.VECTORS,  # type: ignore
+            mag_l0.VECTORS,
             primary_packet_properties.total_vectors,
             secondary_packet_data.total_vectors,
             mag_l0.COMPRESSION,
@@ -297,8 +293,6 @@ def generate_dataset(
         ),
     )
 
-    # TODO: Epoch here refers to the start of the sample. Confirm that this is
-    # what mag is expecting, and if it is, CATDESC needs to be updated.
     epoch_time = xr.DataArray(
         time_data,
         name="epoch",
@@ -338,7 +332,6 @@ def generate_dataset(
         ),
     )
     global_attributes = attribute_manager.get_global_attributes(logical_file_id)
-    # TODO: this method won't work because these values are not in the schema.
     global_attributes["is_mago"] = str(bool(single_file_l1a.is_mago))
     global_attributes["is_active"] = str(bool(single_file_l1a.is_active))
     global_attributes["vectors_per_second"] = (

imap_processing/mag/l1a/mag_l1a_data.py

@@ -368,7 +368,7 @@ class MagL1a:
 
     @staticmethod
     def process_vector_data(
-        vector_data: np.ndarray,
+        vector_data: np.ndarray | bytes,
         primary_count: int,
         secondary_count: int,
         compression: int,
@@ -679,10 +679,9 @@ class MagL1a:
                             primary_boundaries[-1] - primary_boundaries[-4]
                             > MAX_COMPRESSED_VECTOR_BITS
                         )
-                        or (
-                            vector_count == 2
-                            and primary_boundaries[-1] > MAX_COMPRESSED_VECTOR_BITS
-                        )
+                    ) or (
+                        vector_count == 2
+                        and primary_boundaries[-1] > MAX_COMPRESSED_VECTOR_BITS
                     ):
                         # Since we know how long each uncompressed vector is,
                         # we can determine the end of the primary vectors.
@@ -792,7 +791,7 @@ class MagL1a:
         return primary_vectors, secondary_vectors
 
     @staticmethod
-    def _process_vector_section(  # noqa: PLR0913
+    def _process_vector_section(
         vector_bits: np.ndarray,
         split_bits: list,
         last_index: int,
@@ -998,7 +997,7 @@ class MagL1a:
         """
         if np.any(vector_data > 1):
             raise ValueError(
-                "unpack_one_vector method is expecting an array of bits as" "input."
+                "unpack_one_vector method is expecting an array of bits as input."
             )
 
         if len(vector_data) != width * AXIS_COUNT + RANGE_BIT_WIDTH * has_range:
@@ -1107,13 +1106,14 @@ class MagL1a:
         """
         output_str = ""
         last_vectors_per_second = None
-        for start_time, packet in self.packet_definitions.items():
+        for _, packet in self.packet_definitions.items():
             vecsec = packet.vectors_per_second
+            time: np.int64 = packet.start_time.to_j2000ns().astype(np.int64)
             if vecsec != last_vectors_per_second:
                 if output_str == "":
-                    output_str = f"{start_time}:{vecsec}"
+                    output_str = f"{time}:{vecsec}"
                 else:
-                    output_str += f",{start_time}:{vecsec}"
+                    output_str += f",{time}:{vecsec}"
                 last_vectors_per_second = vecsec
 
         return output_str

imap_processing/mag/l1b/mag_l1b.py

@@ -9,12 +9,13 @@ from xarray import Dataset
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.cdf.utils import load_cdf
+from imap_processing.mag.constants import vectors_per_second_from_string
 
 logger = logging.getLogger(__name__)
 
 
 def mag_l1b(
-    input_dataset: xr.Dataset, version: str, calibration_dataset: xr.Dataset = None
+    input_dataset: xr.Dataset, calibration_dataset: xr.Dataset = None
 ) -> Dataset:
     """
     Will process MAG L1B data from L1A data.
@@ -23,8 +24,6 @@ def mag_l1b(
     ----------
     input_dataset : xr.Dataset
         The input dataset to process.
-    version : str
-        The version of the output data.
     calibration_dataset : xr.Dataset
         The calibration dataset containing calibration matrices and timeshift values for
         mago and magi.
@@ -56,7 +55,6 @@ def mag_l1b(
     mag_attributes = ImapCdfAttributes()
     mag_attributes.add_instrument_global_attrs("mag")
     mag_attributes.add_instrument_variable_attrs("mag", "l1b")
-    mag_attributes.add_global_attribute("Data_version", version)
     source = source.replace("l1a", "l1b")
 
     output_dataset = mag_l1b_processing(
@@ -100,15 +98,10 @@ def mag_l1b_processing(
     output_dataset : xr.Dataset
         L1b dataset.
     """
-    dims = [["direction"], ["compression"]]
-    new_dims = [["direction"], ["compression"]]
-
     if "mago" in logical_source:
-        calibration_matrix = calibration_dataset["MFOTOURFO"]
-        time_shift = calibration_dataset["OTS"]
+        is_mago = True
     elif "magi" in logical_source:
-        calibration_matrix = calibration_dataset["MFITOURFI"]
-        time_shift = calibration_dataset["ITS"]
+        is_mago = False
     else:
         raise ValueError(
             f"Calibration matrix not found, invalid logical source "
@@ -116,6 +109,12 @@ def mag_l1b_processing(
         )
 
     # TODO: Check validity of time range for calibration
+    calibration_matrix, time_shift = retrieve_matrix_from_l1b_calibration(
+        calibration_dataset, is_mago
+    )
+
+    dims = [["direction"], ["compression"]]
+    new_dims = [["direction"], ["compression"]]
 
     l1b_fields = xr.apply_ufunc(
         update_vector,
@@ -129,6 +128,8 @@ def mag_l1b_processing(
     )
 
     epoch_time = shift_time(input_dataset["epoch"], time_shift)
+
+    # Update attributes and assemble dataset
     epoch_time.attrs = mag_attributes.get_variable_attributes("epoch")
 
     direction = xr.DataArray(
@@ -171,9 +172,9 @@ def mag_l1b_processing(
     try:
         global_attributes["is_mago"] = input_dataset.attrs["is_mago"]
         global_attributes["is_active"] = input_dataset.attrs["is_active"]
-        global_attributes["vectors_per_second"] = input_dataset.attrs[
-            "vectors_per_second"
-        ]
+        global_attributes["vectors_per_second"] = timeshift_vectors_per_second(
+            input_dataset.attrs["vectors_per_second"], time_shift
+        )
         global_attributes["missing_sequences"] = input_dataset.attrs[
             "missing_sequences"
         ]
@@ -210,6 +211,37 @@ def mag_l1b_processing(
     return output_dataset
 
 
+def retrieve_matrix_from_l1b_calibration(
+    calibration_dataset: xr.Dataset, is_mago: bool = True
+) -> tuple[xr.DataArray, xr.DataArray]:
+    """
+    Retrieve the calibration matrix and time shift from the calibration dataset.
+
+    Parameters
+    ----------
+    calibration_dataset : xarray.Dataset
+        The calibration dataset containing the calibration matrices and time shift.
+    is_mago : bool
+        Whether the calibration is for mago or magi. If True, it retrieves the mago
+        calibration matrix and time shift. If False, it retrieves the magi calibration
+        matrix and time shift.
+
+    Returns
+    -------
+    tuple[xr.DataArray, xr.DataArray]
+        The calibration matrix and time shift. These can be passed directly into
+        update_vector, calibrate_vector, and shift_time.
+    """
+    if is_mago:
+        calibration_matrix = calibration_dataset["MFOTOURFO"]
+        time_shift = calibration_dataset["OTS"]
+    else:
+        calibration_matrix = calibration_dataset["MFITOURFI"]
+        time_shift = calibration_dataset["ITS"]
+
+    return calibration_matrix, time_shift
+
+
 def update_vector(
     input_vector: np.ndarray,
     input_compression: np.ndarray,
@@ -302,13 +334,14 @@ def calibrate_vector(
     updated_vector : numpy.ndarray
         Calibrated vector.
     """
-    updated_vector: np.ndarray = input_vector.copy()
+    updated_vector: np.ndarray = input_vector.copy().astype(np.float64)
     if input_vector[3] % 1 != 0:
         raise ValueError("Range must be an integer.")
 
     range = int(input_vector[3])
-    x_y_z = input_vector[:3]
+    x_y_z = updated_vector[:3]
     updated_vector[:3] = np.dot(calibration_matrix.values[:, :, range], x_y_z)
+
     return updated_vector
 
 
@@ -336,7 +369,7 @@ def shift_time(epoch_times: xr.DataArray, time_shift: xr.DataArray) -> xr.DataAr
     Parameters
     ----------
     epoch_times : xr.DataArray
-        The input epoch times, in J2000 ns.
+        The input epoch times, in TT J2000 ns.
     time_shift : xr.DataArray
         The time shift to apply for the given sensor. This should be one value and is
         in seconds.
@@ -353,3 +386,37 @@ def shift_time(epoch_times: xr.DataArray, time_shift: xr.DataArray) -> xr.DataAr
     time_shift_ns = time_shift.data * 1e9
 
     return epoch_times + time_shift_ns
+
+
+def timeshift_vectors_per_second(
+    vectors_per_second: str, time_shift: xr.DataArray
+) -> str:
+    """
+    Shift the vectors per second attribute by the time shift value.
+
+    This ensures that the vectors per second attribute is aligned with the epoch values
+    if the time is shifted.
+
+    Parameters
+    ----------
+    vectors_per_second : str
+        The vectors per second attribute from the input dataset, in the format
+         "timestamp:rate,timestamp:rate".
+    time_shift : xr.DataArray
+        The time shift to apply for the given sensor. This should be one value and is
+        in seconds.
+
+    Returns
+    -------
+    str
+        The updated vectors per second attribute.
+    """
+    time_shift_ns = time_shift.data * 1e9
+
+    vecsec = vectors_per_second_from_string(vectors_per_second)
+    new_vecsec = ""
+    for time, rate in vecsec.items():
+        new_time = time + time_shift_ns
+        new_vecsec += f"{new_time.astype(np.int64)}:{rate},"
+
+    return new_vecsec[:-1]

imap_processing/mag/l1c/interpolation_methods.py

@@ -1,16 +1,51 @@
 # mypy: ignore-errors
 """Module containing interpolation methods for MAG L1C."""
 
+import logging
 from enum import Enum
+from typing import Optional
 
 import numpy as np
 from scipy.interpolate import make_interp_spline
+from scipy.signal import lfilter
+
+from imap_processing.mag.constants import POSSIBLE_RATES, VecSec
+
+logger = logging.getLogger(__name__)
+
+
+def remove_invalid_output_timestamps(
+    input_timestamps: np.ndarray, output_timestamps: np.ndarray
+) -> np.ndarray:
+    """
+    Remove output timestamps where we don't have input timestamps to interpolate.
+
+    I.E. We should never create science data outside of the timeline of burst data.
+
+    Parameters
+    ----------
+    input_timestamps : numpy.ndarray
+        List of input timestamps (from burst data).
+    output_timestamps : numpy.ndarray
+        List of output timestamps (from norm data) to downsample/interpolate to.
+
+    Returns
+    -------
+    numpy.ndarray
+        All valid output timestamps where there exists input data.
+    """
+    if input_timestamps[0] > output_timestamps[0]:
+        # Chop data where we don't have input timestamps to interpolate
+        output_timestamps = output_timestamps[output_timestamps >= input_timestamps[0]]
+    return output_timestamps
 
 
 def linear(
     input_vectors: np.ndarray,
     input_timestamps: np.ndarray,
     output_timestamps: np.ndarray,
+    input_rate: Optional[VecSec] = None,
+    output_rate: Optional[VecSec] = None,
 ) -> np.ndarray:
     """
     Linear interpolation of input vectors to output timestamps.
@@ -25,6 +60,10 @@ def linear(
         vectors.
     output_timestamps : numpy.ndarray
         Output timestamps of shape (m,) to generate interpolated vectors for.
+    input_rate : VecSec, optional
+        Not required for this interpolation method.
+    output_rate : VecSec, optional
+        Not required for this interpolation method.
 
     Returns
     -------
@@ -32,6 +71,7 @@ def linear(
         Interpolated vectors of shape (m, 3) where m is equal to the number of output
         timestamps. Contains x, y, z components of the vector.
     """
+    # TODO: Remove invalid timestamps using remove_invalid_output_timestamps
     spline = make_interp_spline(input_timestamps, input_vectors, k=1)
     return spline(output_timestamps)
 
@@ -40,6 +80,8 @@ def quadratic(
     input_vectors: np.ndarray,
     input_timestamps: np.ndarray,
     output_timestamps: np.ndarray,
+    input_rate: Optional[VecSec] = None,
+    output_rate: Optional[VecSec] = None,
 ) -> np.ndarray:
     """
     Quadratic interpolation of input vectors to output timestamps.
@@ -54,6 +96,10 @@ def quadratic(
         vectors.
     output_timestamps : numpy.ndarray
         Output timestamps of shape (m,) to generate interpolated vectors for.
+    input_rate : VecSec, optional
+        Not required for this interpolation method.
+    output_rate : VecSec, optional
+        Not required for this interpolation method.
 
     Returns
     -------
@@ -69,6 +115,8 @@ def cubic(
     input_vectors: np.ndarray,
     input_timestamps: np.ndarray,
     output_timestamps: np.ndarray,
+    input_rate: Optional[VecSec] = None,
+    output_rate: Optional[VecSec] = None,
 ) -> np.ndarray:
     """
     Cubic interpolation of input vectors to output timestamps.
@@ -83,6 +131,10 @@ def cubic(
         vectors.
     output_timestamps : numpy.ndarray
         Output timestamps of shape (m,) to generate interpolated vectors for.
+    input_rate : VecSec, optional
+        Not required for this interpolation method.
+    output_rate : VecSec, optional
+        Not required for this interpolation method.
 
     Returns
     -------
@@ -94,10 +146,104 @@ def cubic(
     return spline(output_timestamps)
 
 
+def estimate_rate(timestamps: np.ndarray) -> VecSec:
+    """
+    Given a set of timestamps, estimate the rate of the timestamps.
+
+    This rate will be one of the defined rates in the VecSec enum. The calculation
+    assumes there are no significant gaps in the timestamps.
+
+    Parameters
+    ----------
+    timestamps : numpy.ndarray
+        1D array of timestamps to estimate the rate of.
+
+    Returns
+    -------
+    VecSec
+        Estimated rate of the timestamps.
+    """
+    samples_per_second = timestamps.shape[0] / (timestamps[-1] - timestamps[0]) * 1e9
+    per_second = VecSec(
+        POSSIBLE_RATES[(np.abs(POSSIBLE_RATES - samples_per_second)).argmin()]
+    )
+
+    return per_second
+
+
+def cic_filter(
+    input_vectors: np.ndarray,
+    input_timestamps: np.ndarray,
+    output_timestamps: np.ndarray,
+    input_rate: Optional[VecSec],
+    output_rate: Optional[VecSec],
+):
+    """
+    Apply CIC filter to data before interpolating.
+
+    The filtering uses a Cascaded integrator-comb (CIC) filter which is used in FSW to
+    filter down the raw data to telemetered data.
+
+    This assumes that the input_vectors and input_timestamps are downsampled to
+    the output_timestamps rate. Neither input_timestamps nor output_timestamps should
+    have significant gaps.
+
+    After the CIC filter is applied, the timestamps at the beginning and end of the
+    output are invalid. Therefore, we must pass in extra values and remove them after
+    applying the filter. This needs about double the input values to work.
+
+    Parameters
+    ----------
+    input_vectors : numpy.ndarray
+        Input vectors of shape (n, 3) where n is equal to the number of input
+        timestamps. Contains x, y, z components of the vector.
+    input_timestamps : numpy.ndarray
+        Input timestamps of shape (n,) which correspond to the timestamps of the input
+        vectors.
+    output_timestamps : numpy.ndarray
+        Output timestamps of shape (m,) to generate interpolated vectors for.
+    input_rate : VecSec, optional
+        Expected rate of input timestamps.
+    output_rate : VecSec, optional
+        Expected rate of output timestamps.
+
+    Returns
+    -------
+    input_filtered, vectors_filtered : tuple[numpy.ndarray]
+        Filtered input timestamps and filtered input vectors.
+    """
+    # output rate should always be higher
+    input_rate = estimate_rate(input_timestamps) if input_rate is None else input_rate
+    output_rate = (
+        estimate_rate(output_timestamps) if output_rate is None else output_rate
+    )
+
+    if input_rate.value <= output_rate.value:
+        raise ValueError(
+            f"Burst mode input rate {input_rate} should never be less than "
+            f"the normal mode output rate {output_rate}. "
+            f"Both rates are required"
+        )
+
+    decimation_factor = int(input_rate.value / output_rate.value)
+    cic1 = np.ones(decimation_factor)
+    cic1 = cic1 / decimation_factor
+    cic2 = np.convolve(cic1, cic1)
+    delay = (len(cic2) - 1) // 2
+    input_filtered = input_timestamps
+    if delay != 0:
+        input_filtered = input_timestamps[:-delay]
+
+    vectors_filtered = lfilter(cic2, 1, input_vectors, axis=0)[delay:]
+    return input_filtered, vectors_filtered
+
+
 def linear_filtered(
     input_vectors: np.ndarray,
     input_timestamps: np.ndarray,
     output_timestamps: np.ndarray,
+    input_rate: Optional[VecSec] = None,
+    output_rate: Optional[VecSec] = None,
 ) -> np.ndarray:
     """
     Linear filtered interpolation of input vectors to output timestamps.
@@ -112,6 +258,12 @@ def linear_filtered(
         vectors.
     output_timestamps : numpy.ndarray
         Output timestamps of shape (m,) to generate interpolated vectors for.
+    input_rate : VecSec, optional
+        Expected rate of input timestamps to be passed into the CIC filter. If not
+        provided, this will be estimated.
+    output_rate : VecSec, optional
+        Expected rate of output timestamps to be passed into the CIC filter. If not
+        provided, this will be estimated.
 
     Returns
     -------
@@ -119,13 +271,18 @@ def linear_filtered(
         Interpolated vectors of shape (m, 3) where m is equal to the number of output
         timestamps. Contains x, y, z components of the vector.
     """
-    pass
+    input_filtered, vectors_filtered = cic_filter(
+        input_vectors, input_timestamps, output_timestamps, input_rate, output_rate
+    )
+    return linear(vectors_filtered, input_filtered, output_timestamps)
 
 
 def quadratic_filtered(
     input_vectors: np.ndarray,
     input_timestamps: np.ndarray,
     output_timestamps: np.ndarray,
+    input_rate: Optional[VecSec] = None,
+    output_rate: Optional[VecSec] = None,
 ) -> np.ndarray:
     """
     Quadratic filtered interpolation of input vectors to output timestamps.
@@ -140,6 +297,12 @@ def quadratic_filtered(
         vectors.
     output_timestamps : numpy.ndarray
         Output timestamps of shape (m,) to generate interpolated vectors for.
+    input_rate : VecSec, optional
+        Expected rate of input timestamps to be passed into the CIC filter. If not
+        provided, this will be estimated.
+    output_rate : VecSec, optional
+        Expected rate of output timestamps to be passed into the CIC filter. If not
+        provided, this will be estimated.
 
     Returns
     -------
@@ -147,13 +310,18 @@ def quadratic_filtered(
         Interpolated vectors of shape (m, 3) where m is equal to the number of output
         timestamps. Contains x, y, z components of the vector.
     """
-    pass
+    input_filtered, vectors_filtered = cic_filter(
+        input_vectors, input_timestamps, output_timestamps, input_rate, output_rate
+    )
+    return quadratic(vectors_filtered, input_filtered, output_timestamps)
 
 
 def cubic_filtered(
     input_vectors: np.ndarray,
     input_timestamps: np.ndarray,
     output_timestamps: np.ndarray,
+    input_rate: Optional[VecSec] = None,
+    output_rate: Optional[VecSec] = None,
 ) -> np.ndarray:
     """
     Cubic filtered interpolation of input vectors to output timestamps.
@@ -168,6 +336,12 @@ def cubic_filtered(
         vectors.
     output_timestamps : numpy.ndarray
         Output timestamps of shape (m,) to generate interpolated vectors for.
+    input_rate : VecSec, optional
+        Expected rate of input timestamps to be passed into the CIC filter. If not
+        provided, this will be estimated.
+    output_rate : VecSec, optional
+        Expected rate of output timestamps to be passed into the CIC filter. If not
+        provided, this will be estimated.
 
     Returns
     -------
@@ -175,7 +349,10 @@ def cubic_filtered(
         Interpolated vectors of shape (m, 3) where m is equal to the number of output
         timestamps. Contains x, y, z components of the vector.
     """
-    pass
+    input_filtered, vectors_filtered = cic_filter(
+        input_vectors, input_timestamps, output_timestamps, input_rate, output_rate
+    )
+    return cubic(vectors_filtered, input_filtered, output_timestamps)
 
 
 class InterpolationFunction(Enum):