imap-processing 0.17.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

imap_processing/mag/constants.py

@@ -131,6 +131,7 @@ MAX_FINE_TIME = np.iinfo(np.uint16).max  # maximum 16 bit unsigned int
 AXIS_COUNT = 3
 RANGE_BIT_WIDTH = 2
 MAX_COMPRESSED_VECTOR_BITS = 60
+FILLVAL = -1e31
 
 
 def vectors_per_second_from_string(vecsec_string: str) -> dict:

imap_processing/mag/l1a/mag_l1a.py

@@ -328,6 +328,7 @@ def generate_dataset(
     global_attributes = attribute_manager.get_global_attributes(logical_file_id)
     global_attributes["is_mago"] = str(bool(single_file_l1a.is_mago))
     global_attributes["is_active"] = str(bool(single_file_l1a.is_active))
+    global_attributes["all_vectors_primary"] = single_file_l1a.all_vectors_primary()
     global_attributes["vectors_per_second"] = (
         single_file_l1a.vectors_per_second_attribute()
     )

imap_processing/mag/l1a/mag_l1a_data.py

@@ -15,6 +15,7 @@ from imap_processing.mag.constants import (
     MAX_COMPRESSED_VECTOR_BITS,
     MAX_FINE_TIME,
     RANGE_BIT_WIDTH,
+    PrimarySensor,
 )
 from imap_processing.spice.time import met_to_ttj2000ns
 
@@ -241,6 +242,7 @@ class MagL1a:
     twos_complement()
     update_compression_array()
     vectors_per_second_attribute()
+    all_vectors_primary()
     """
 
     is_mago: bool
@@ -1117,3 +1119,27 @@ class MagL1a:
                 last_vectors_per_second = vecsec
 
         return output_str
+
+    def all_vectors_primary(self) -> bool:
+        """
+        Check if all vectors in the file are from the primary sensor.
+
+        For MAGO datasets, this checks if MAGO was consistently the primary sensor
+        across all packets. For MAGI datasets, this checks if MAGI was consistently
+        the primary sensor across all packets.
+
+        Returns
+        -------
+        bool
+            True if all vectors are from the primary sensor across all packets,
+            False otherwise.
+        """
+        expected_primary_value = (
+            PrimarySensor.MAGO.value if self.is_mago else PrimarySensor.MAGI.value
+        )
+
+        for _, packet in self.packet_definitions.items():
+            if packet.mago_is_primary != expected_primary_value:
+                return False
+
+        return True

imap_processing/mag/l1b/mag_l1b.py

@@ -74,7 +74,6 @@ def mag_l1b(
     calibration_matrix, time_shift = retrieve_matrix_from_l1b_calibration(
         calibration_dataset, day_to_process, is_mago
     )
-    print(f"Using calibration matrix: {calibration_matrix}")
 
     output_dataset = mag_l1b_processing(
         input_dataset, calibration_matrix, time_shift, mag_attributes, source
@@ -182,6 +181,9 @@ def mag_l1b_processing(
     try:
         global_attributes["is_mago"] = input_dataset.attrs["is_mago"]
         global_attributes["is_active"] = input_dataset.attrs["is_active"]
+        global_attributes["all_vectors_primary"] = input_dataset.attrs[
+            "all_vectors_primary"
+        ]
         global_attributes["vectors_per_second"] = timeshift_vectors_per_second(
             input_dataset.attrs["vectors_per_second"], time_shift
         )
@@ -245,7 +247,6 @@ def retrieve_matrix_from_l1b_calibration(
         The calibration matrix and time shift. These can be passed directly into
         update_vector, calibrate_vector, and shift_time.
     """
-    print(f"Finding data for day {day}")
     if is_mago:
         calibration_matrix = calibration_dataset.sel(epoch=day)["MFOTOURFO"]
         time_shift = calibration_dataset.sel(epoch=day)["OTS"]

imap_processing/mag/l1c/interpolation_methods.py

@@ -3,7 +3,6 @@
 
 import logging
 from enum import Enum
-from typing import Optional
 
 import numpy as np
 from scipy.interpolate import make_interp_spline
@@ -44,8 +43,8 @@ def linear(
     input_vectors: np.ndarray,
     input_timestamps: np.ndarray,
     output_timestamps: np.ndarray,
-    input_rate: Optional[VecSec] = None,
-    output_rate: Optional[VecSec] = None,
+    input_rate: VecSec | None = None,
+    output_rate: VecSec | None = None,
 ) -> np.ndarray:
     """
     Linear interpolation of input vectors to output timestamps.
@@ -80,8 +79,8 @@ def quadratic(
     input_vectors: np.ndarray,
     input_timestamps: np.ndarray,
     output_timestamps: np.ndarray,
-    input_rate: Optional[VecSec] = None,
-    output_rate: Optional[VecSec] = None,
+    input_rate: VecSec | None = None,
+    output_rate: VecSec | None = None,
 ) -> np.ndarray:
     """
     Quadratic interpolation of input vectors to output timestamps.
@@ -115,8 +114,8 @@ def cubic(
     input_vectors: np.ndarray,
     input_timestamps: np.ndarray,
     output_timestamps: np.ndarray,
-    input_rate: Optional[VecSec] = None,
-    output_rate: Optional[VecSec] = None,
+    input_rate: VecSec | None = None,
+    output_rate: VecSec | None = None,
 ) -> np.ndarray:
     """
     Cubic interpolation of input vectors to output timestamps.
@@ -175,8 +174,8 @@ def cic_filter(
     input_vectors: np.ndarray,
     input_timestamps: np.ndarray,
     output_timestamps: np.ndarray,
-    input_rate: Optional[VecSec],
-    output_rate: Optional[VecSec],
+    input_rate: VecSec | None,
+    output_rate: VecSec | None,
 ):
     """
     Apply CIC filter to data before interpolating.
@@ -242,8 +241,8 @@ def linear_filtered(
     input_vectors: np.ndarray,
     input_timestamps: np.ndarray,
     output_timestamps: np.ndarray,
-    input_rate: Optional[VecSec] = None,
-    output_rate: Optional[VecSec] = None,
+    input_rate: VecSec | None = None,
+    output_rate: VecSec | None = None,
 ) -> np.ndarray:
     """
     Linear filtered interpolation of input vectors to output timestamps.
@@ -281,8 +280,8 @@ def quadratic_filtered(
     input_vectors: np.ndarray,
     input_timestamps: np.ndarray,
     output_timestamps: np.ndarray,
-    input_rate: Optional[VecSec] = None,
-    output_rate: Optional[VecSec] = None,
+    input_rate: VecSec | None = None,
+    output_rate: VecSec | None = None,
 ) -> np.ndarray:
     """
     Quadratic filtered interpolation of input vectors to output timestamps.
@@ -320,8 +319,8 @@ def cubic_filtered(
     input_vectors: np.ndarray,
     input_timestamps: np.ndarray,
     output_timestamps: np.ndarray,
-    input_rate: Optional[VecSec] = None,
-    output_rate: Optional[VecSec] = None,
+    input_rate: VecSec | None = None,
+    output_rate: VecSec | None = None,
 ) -> np.ndarray:
     """
     Cubic filtered interpolation of input vectors to output timestamps.

imap_processing/mag/l1c/mag_l1c.py

@@ -1,7 +1,6 @@
 """MAG L1C processing module."""
 
 import logging
-from typing import Optional
 
 import numpy as np
 import xarray as xr
@@ -125,6 +124,23 @@ def mag_l1c(
     try:
         global_attributes["is_mago"] = normal_mode_dataset.attrs["is_mago"]
         global_attributes["is_active"] = normal_mode_dataset.attrs["is_active"]
+
+        # Check if all vectors are primary in both normal and burst datasets
+        is_mago = normal_mode_dataset.attrs.get("is_mago", "False") == "True"
+        normal_all_primary = normal_mode_dataset.attrs.get("all_vectors_primary", False)
+
+        # Default for missing burst dataset: 1 if MAGO (expected primary), 0 if MAGI
+        burst_all_primary = is_mago
+        if burst_mode_dataset is not None:
+            burst_all_primary = burst_mode_dataset.attrs.get(
+                "all_vectors_primary", False
+            )
+
+        # Both datasets must have all vectors primary for the combined result to be True
+        global_attributes["all_vectors_primary"] = (
+            normal_all_primary and burst_all_primary
+        )
+
         global_attributes["missing_sequences"] = normal_mode_dataset.attrs[
             "missing_sequences"
         ]
@@ -162,8 +178,9 @@ def mag_l1c(
         output_core_dims=[[]],
         vectorize=True,
     )
-    # output_dataset['vector_magnitude'].attrs =
-    # attribute_manager.get_variable_attributes("vector_magnitude_attrs")
+    output_dataset[
+        "vector_magnitude"
+    ].attrs = attribute_manager.get_variable_attributes("vector_magnitude_attrs")
 
     output_dataset["compression_flags"] = xr.DataArray(
         completed_timeline[:, 6:8],
@@ -176,14 +193,14 @@ def mag_l1c(
         completed_timeline[:, 5],
         name="generated_flag",
         dims=["epoch"],
-        # attrs=attribute_manager.get_variable_attributes("generated_flag_attrs"),
+        attrs=attribute_manager.get_variable_attributes("generated_flag_attrs"),
     )
 
     return output_dataset
 
 
 def select_datasets(
-    first_input_dataset: xr.Dataset, second_input_dataset: Optional[xr.Dataset] = None
+    first_input_dataset: xr.Dataset, second_input_dataset: xr.Dataset | None = None
 ) -> tuple[xr.Dataset, xr.Dataset]:
     """
     Given one or two datasets, assign one to norm and one to burst.
@@ -503,7 +520,7 @@ def generate_timeline(epoch_data: np.ndarray, gaps: np.ndarray) -> np.ndarray:
 
 
 def find_all_gaps(
-    epoch_data: np.ndarray, vecsec_dict: Optional[dict] = None
+    epoch_data: np.ndarray, vecsec_dict: dict | None = None
 ) -> np.ndarray:
     """
     Find all the gaps in the epoch data.

imap_processing/mag/l1d/mag_l1d.py

@@ -0,0 +1,176 @@
+"""Module for generating Level 1d magnetic field data."""
+
+import numpy as np
+import xarray as xr
+
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.mag.constants import DataMode
+from imap_processing.mag.l1d.mag_l1d_data import MagL1d, MagL1dConfiguration
+from imap_processing.mag.l2.mag_l2_data import ValidFrames
+
+
+def mag_l1d(  # noqa: PLR0912
+    science_data: list[xr.Dataset],
+    calibration_dataset: xr.Dataset,
+    day_to_process: np.datetime64,
+) -> list[xr.Dataset]:
+    """
+    Generate Level 1d magnetic field data from Level 1b/1c data.
+
+    Both norm and burst mode are calculated at the same time. Normal mode MAGO and MAGI
+    L1C data is required, burst mode MAGO and MAGI L1B data is optional.
+
+    Parameters
+    ----------
+    science_data : list[xr.Dataset]
+        The list of input datasets containing the MAG L1C and L1B data. This is required
+        to have at least one normal mode dataset for MAGo and MAGi, and optionally
+        burst mode datasets for MAGo and MAGi. There cannot be duplicates, so two
+        norm-mago files is invalid.
+    calibration_dataset : xr.Dataset
+        The calibration dataset to use for processing. Generated from multiple L1D
+        ancillary files using MagAncillaryCombiner class.
+    day_to_process : np.datetime64
+        The day to process, in np.datetime64[D] format. This is used to select the
+        correct ancillary parameters and to remove excessive data from the output.
+
+    Returns
+    -------
+    list[xr.Dataset]
+        A list containing the generated Level 1d dataset(s).
+    """
+    input_magi_norm = None
+    input_mago_norm = None
+    input_magi_burst = None
+    input_mago_burst = None
+    for dataset in science_data:
+        source = dataset.attrs.get("Logical_source", "")
+        instrument_mode = source.split("_")[-1]
+        match instrument_mode:
+            case "norm-magi":
+                input_magi_norm = dataset
+            case "norm-mago":
+                input_mago_norm = dataset
+            case "burst-magi":
+                input_magi_burst = dataset
+            case "burst-mago":
+                input_mago_burst = dataset
+            case _:
+                raise ValueError(f"Input data has invalid logical source {source}")
+
+    if input_magi_norm is None or input_mago_norm is None:
+        raise ValueError(
+            "Both MAGo and MAGi normal mode datasets are required for L1d processing."
+        )
+
+    day: np.datetime64 = day_to_process.astype("datetime64[D]")
+
+    output_datasets = []
+
+    # Read configuration out of file
+    config = MagL1dConfiguration(calibration_dataset, day)
+
+    # Only the first 3 components are used for L1d
+    mago_vectors = input_mago_norm["vectors"].data[:, :3]
+    magi_vectors = input_magi_norm["vectors"].data[:, :3]
+
+    # Verify that MAGO is primary sensor for all vectors before applying gradiometry
+    if not input_mago_norm.attrs.get("all_vectors_primary", 1):
+        config.apply_gradiometry = False
+
+    # TODO: L1D attributes
+    attributes = ImapCdfAttributes()
+    attributes.add_instrument_global_attrs("mag")
+    attributes.add_instrument_variable_attrs("mag", "l2")
+
+    l1d_norm = MagL1d(
+        vectors=mago_vectors,
+        epoch=input_mago_norm["epoch"].data,
+        range=input_mago_norm["vectors"].data[:, 3],
+        global_attributes={},
+        quality_flags=np.zeros(len(input_mago_norm["epoch"].data)),
+        quality_bitmask=np.zeros(len(input_mago_norm["epoch"].data)),
+        data_mode=DataMode.NORM,
+        magi_vectors=magi_vectors,
+        magi_range=input_magi_norm["vectors"].data[:, 3],
+        magi_epoch=input_magi_norm["epoch"].data,
+        config=config,
+        day=day,
+    )
+
+    # Nominally, this is expected to create MAGO data. However, if the configuration
+    # setting for always_output_mago is set to False, it will create MAGI data.
+
+    l1d_norm.rotate_frame(ValidFrames.SRF)
+    norm_srf_dataset = l1d_norm.generate_dataset(attributes, day_to_process)
+    l1d_norm.rotate_frame(ValidFrames.DSRF)
+    norm_dsrf_dataset = l1d_norm.generate_dataset(attributes, day_to_process)
+    l1d_norm.rotate_frame(ValidFrames.GSE)
+    norm_gse_dataset = l1d_norm.generate_dataset(attributes, day_to_process)
+    l1d_norm.rotate_frame(ValidFrames.RTN)
+    norm_rtn_dataset = l1d_norm.generate_dataset(attributes, day_to_process)
+    output_datasets.append(norm_srf_dataset)
+    output_datasets.append(norm_dsrf_dataset)
+    output_datasets.append(norm_gse_dataset)
+    output_datasets.append(norm_rtn_dataset)
+
+    if input_mago_burst is not None and input_magi_burst is not None:
+        # If burst data is provided, use it to create the burst L1d dataset
+        mago_burst_vectors = input_mago_burst["vectors"].data[:, :3]
+        magi_burst_vectors = input_magi_burst["vectors"].data[:, :3]
+
+        l1d_burst = MagL1d(
+            vectors=mago_burst_vectors,
+            epoch=input_mago_burst["epoch"].data,
+            range=input_mago_burst["vectors"].data[:, 3],
+            global_attributes={},
+            quality_flags=np.zeros(len(input_mago_burst["epoch"].data)),
+            quality_bitmask=np.zeros(len(input_mago_burst["epoch"].data)),
+            data_mode=DataMode.BURST,
+            magi_vectors=magi_burst_vectors,
+            magi_range=input_magi_burst["vectors"].data[:, 3],
+            magi_epoch=input_magi_burst["epoch"].data,
+            config=config,
+            spin_offsets=l1d_norm.spin_offsets,
+            day=day,
+        )
+
+        # TODO: frame specific attributes may be required
+        l1d_burst.rotate_frame(ValidFrames.SRF)
+        burst_srf_dataset = l1d_burst.generate_dataset(attributes, day_to_process)
+        l1d_burst.rotate_frame(ValidFrames.DSRF)
+        burst_dsrf_dataset = l1d_burst.generate_dataset(attributes, day_to_process)
+        l1d_burst.rotate_frame(ValidFrames.GSE)
+        burst_gse_dataset = l1d_burst.generate_dataset(attributes, day_to_process)
+        l1d_burst.rotate_frame(ValidFrames.RTN)
+        burst_rtn_dataset = l1d_burst.generate_dataset(attributes, day_to_process)
+        output_datasets.append(burst_srf_dataset)
+        output_datasets.append(burst_dsrf_dataset)
+        output_datasets.append(burst_gse_dataset)
+        output_datasets.append(burst_rtn_dataset)
+
+    # Output ancillary files
+    # Add spin offsets dataset from normal mode processing
+    if l1d_norm.spin_offsets is not None:
+        spin_offset_dataset = l1d_norm.generate_spin_offset_dataset()
+        spin_offset_dataset.attrs["Logical_source"] = "imap_mag_l1d-spin-offsets"
+        output_datasets.append(spin_offset_dataset)
+
+    # Add gradiometry offsets dataset if gradiometry was applied
+    if l1d_norm.config.apply_gradiometry and hasattr(l1d_norm, "gradiometry_offsets"):
+        gradiometry_dataset = l1d_norm.gradiometry_offsets.copy()
+        gradiometry_dataset.attrs["Logical_source"] = (
+            "imap_mag_l1d-gradiometry-offsets-norm"
+        )
+        output_datasets.append(gradiometry_dataset)
+
+        # Also add burst gradiometry offsets if burst data was processed
+        if input_mago_burst is not None and input_magi_burst is not None:
+            if hasattr(l1d_burst, "gradiometry_offsets"):
+                burst_gradiometry_dataset = l1d_burst.gradiometry_offsets.copy()
+                burst_gradiometry_dataset.attrs["Logical_source"] = (
+                    "imap_mag_l1d-gradiometry-offsets-burst"
+                )
+                output_datasets.append(burst_gradiometry_dataset)
+
+    return output_datasets