imap-processing 0.18.0__py3-none-any.whl → 0.19.2__py3-none-any.whl

imap_processing/lo/lo_ancillary.py

@@ -0,0 +1,55 @@
+"""Ancillary file reading for IMAP-Lo processing."""
+
+from pathlib import Path
+
+import pandas as pd
+
+# convert the YYYYDDD datetime format directly upon reading
+_CONVERTERS = {
+    "YYYYDDD": lambda x: pd.to_datetime(str(x), format="%Y%j"),
+    "#YYYYDDD": lambda x: pd.to_datetime(str(x), format="%Y%j"),
+    "YYYYDDD_strt": lambda x: pd.to_datetime(str(x), format="%Y%j"),
+    "YYYYDDD_end": lambda x: pd.to_datetime(str(x), format="%Y%j"),
+}
+
+# Columns in the csv files to rename for consistency
+_RENAME_COLUMNS = {
+    "YYYYDDD": "Date",
+    "#YYYYDDD": "Date",
+    "#Comments": "Comments",
+    "YYYYDDD_strt": "StartDate",
+    "YYYYDDD_end": "EndDate",
+}
+
+
+def read_ancillary_file(ancillary_file: str | Path) -> pd.DataFrame:
+    """
+    Read a generic ancillary CSV file into a pandas DataFrame.
+
+    Parameters
+    ----------
+    ancillary_file : str or Path
+        Path to the ancillary CSV file.
+
+    Returns
+    -------
+    pd.DataFrame
+        DataFrame containing the ancillary data.
+    """
+    skiprows = None
+    if "esa-mode-lut" in str(ancillary_file):
+        # skip the first row which is a comment
+        skiprows = [0]
+    elif "geometric-factor" in str(ancillary_file):
+        # skip the rows with comment headers indicating Hi_Res and Hi_Thr
+        skiprows = [1, 38]
+    df = pd.read_csv(ancillary_file, converters=_CONVERTERS, skiprows=skiprows)
+    df = df.rename(columns=_RENAME_COLUMNS)
+
+    if "geometric-factor" in str(ancillary_file):
+        # Add an ESA mode column based on the known structure of the file.
+        # The first 36 rows are ESA mode 0 (HiRes), the second 36 are ESA mode 1 (HiThr)
+        df["esa_mode"] = 0
+        df.loc[36:, "esa_mode"] = 1
+
+    return df

imap_processing/mag/l1a/mag_l1a.py

@@ -328,6 +328,7 @@ def generate_dataset(
     global_attributes = attribute_manager.get_global_attributes(logical_file_id)
     global_attributes["is_mago"] = str(bool(single_file_l1a.is_mago))
     global_attributes["is_active"] = str(bool(single_file_l1a.is_active))
+    global_attributes["all_vectors_primary"] = single_file_l1a.all_vectors_primary()
     global_attributes["vectors_per_second"] = (
         single_file_l1a.vectors_per_second_attribute()
     )

imap_processing/mag/l1a/mag_l1a_data.py

@@ -15,6 +15,7 @@ from imap_processing.mag.constants import (
     MAX_COMPRESSED_VECTOR_BITS,
     MAX_FINE_TIME,
     RANGE_BIT_WIDTH,
+    PrimarySensor,
 )
 from imap_processing.spice.time import met_to_ttj2000ns
 
@@ -241,6 +242,7 @@ class MagL1a:
     twos_complement()
     update_compression_array()
     vectors_per_second_attribute()
+    all_vectors_primary()
     """
 
     is_mago: bool
@@ -1117,3 +1119,27 @@ class MagL1a:
                 last_vectors_per_second = vecsec
 
         return output_str
+
+    def all_vectors_primary(self) -> bool:
+        """
+        Check if all vectors in the file are from the primary sensor.
+
+        For MAGO datasets, this checks if MAGO was consistently the primary sensor
+        across all packets. For MAGI datasets, this checks if MAGI was consistently
+        the primary sensor across all packets.
+
+        Returns
+        -------
+        bool
+            True if all vectors are from the primary sensor across all packets,
+            False otherwise.
+        """
+        expected_primary_value = (
+            PrimarySensor.MAGO.value if self.is_mago else PrimarySensor.MAGI.value
+        )
+
+        for _, packet in self.packet_definitions.items():
+            if packet.mago_is_primary != expected_primary_value:
+                return False
+
+        return True

imap_processing/mag/l1b/mag_l1b.py

@@ -74,7 +74,6 @@ def mag_l1b(
     calibration_matrix, time_shift = retrieve_matrix_from_l1b_calibration(
         calibration_dataset, day_to_process, is_mago
     )
-    print(f"Using calibration matrix: {calibration_matrix}")
 
     output_dataset = mag_l1b_processing(
         input_dataset, calibration_matrix, time_shift, mag_attributes, source
@@ -182,6 +181,9 @@ def mag_l1b_processing(
     try:
         global_attributes["is_mago"] = input_dataset.attrs["is_mago"]
         global_attributes["is_active"] = input_dataset.attrs["is_active"]
+        global_attributes["all_vectors_primary"] = input_dataset.attrs[
+            "all_vectors_primary"
+        ]
         global_attributes["vectors_per_second"] = timeshift_vectors_per_second(
             input_dataset.attrs["vectors_per_second"], time_shift
         )
@@ -245,7 +247,6 @@ def retrieve_matrix_from_l1b_calibration(
         The calibration matrix and time shift. These can be passed directly into
         update_vector, calibrate_vector, and shift_time.
     """
-    print(f"Finding data for day {day}")
     if is_mago:
         calibration_matrix = calibration_dataset.sel(epoch=day)["MFOTOURFO"]
         time_shift = calibration_dataset.sel(epoch=day)["OTS"]

imap_processing/mag/l1c/interpolation_methods.py

@@ -3,7 +3,6 @@
 
 import logging
 from enum import Enum
-from typing import Optional
 
 import numpy as np
 from scipy.interpolate import make_interp_spline
@@ -44,8 +43,8 @@ def linear(
     input_vectors: np.ndarray,
     input_timestamps: np.ndarray,
     output_timestamps: np.ndarray,
-    input_rate: Optional[VecSec] = None,
-    output_rate: Optional[VecSec] = None,
+    input_rate: VecSec | None = None,
+    output_rate: VecSec | None = None,
 ) -> np.ndarray:
     """
     Linear interpolation of input vectors to output timestamps.
@@ -80,8 +79,8 @@ def quadratic(
     input_vectors: np.ndarray,
     input_timestamps: np.ndarray,
     output_timestamps: np.ndarray,
-    input_rate: Optional[VecSec] = None,
-    output_rate: Optional[VecSec] = None,
+    input_rate: VecSec | None = None,
+    output_rate: VecSec | None = None,
 ) -> np.ndarray:
     """
     Quadratic interpolation of input vectors to output timestamps.
@@ -115,8 +114,8 @@ def cubic(
     input_vectors: np.ndarray,
     input_timestamps: np.ndarray,
     output_timestamps: np.ndarray,
-    input_rate: Optional[VecSec] = None,
-    output_rate: Optional[VecSec] = None,
+    input_rate: VecSec | None = None,
+    output_rate: VecSec | None = None,
 ) -> np.ndarray:
     """
     Cubic interpolation of input vectors to output timestamps.
@@ -175,8 +174,8 @@ def cic_filter(
     input_vectors: np.ndarray,
     input_timestamps: np.ndarray,
     output_timestamps: np.ndarray,
-    input_rate: Optional[VecSec],
-    output_rate: Optional[VecSec],
+    input_rate: VecSec | None,
+    output_rate: VecSec | None,
 ):
     """
     Apply CIC filter to data before interpolating.
@@ -242,8 +241,8 @@ def linear_filtered(
     input_vectors: np.ndarray,
     input_timestamps: np.ndarray,
     output_timestamps: np.ndarray,
-    input_rate: Optional[VecSec] = None,
-    output_rate: Optional[VecSec] = None,
+    input_rate: VecSec | None = None,
+    output_rate: VecSec | None = None,
 ) -> np.ndarray:
     """
     Linear filtered interpolation of input vectors to output timestamps.
@@ -281,8 +280,8 @@ def quadratic_filtered(
     input_vectors: np.ndarray,
     input_timestamps: np.ndarray,
     output_timestamps: np.ndarray,
-    input_rate: Optional[VecSec] = None,
-    output_rate: Optional[VecSec] = None,
+    input_rate: VecSec | None = None,
+    output_rate: VecSec | None = None,
 ) -> np.ndarray:
     """
     Quadratic filtered interpolation of input vectors to output timestamps.
@@ -320,8 +319,8 @@ def cubic_filtered(
     input_vectors: np.ndarray,
     input_timestamps: np.ndarray,
     output_timestamps: np.ndarray,
-    input_rate: Optional[VecSec] = None,
-    output_rate: Optional[VecSec] = None,
+    input_rate: VecSec | None = None,
+    output_rate: VecSec | None = None,
 ) -> np.ndarray:
     """
     Cubic filtered interpolation of input vectors to output timestamps.

imap_processing/mag/l1c/mag_l1c.py

@@ -1,7 +1,6 @@
 """MAG L1C processing module."""
 
 import logging
-from typing import Optional
 
 import numpy as np
 import xarray as xr
@@ -125,6 +124,23 @@ def mag_l1c(
     try:
         global_attributes["is_mago"] = normal_mode_dataset.attrs["is_mago"]
         global_attributes["is_active"] = normal_mode_dataset.attrs["is_active"]
+
+        # Check if all vectors are primary in both normal and burst datasets
+        is_mago = normal_mode_dataset.attrs.get("is_mago", "False") == "True"
+        normal_all_primary = normal_mode_dataset.attrs.get("all_vectors_primary", False)
+
+        # Default for missing burst dataset: 1 if MAGO (expected primary), 0 if MAGI
+        burst_all_primary = is_mago
+        if burst_mode_dataset is not None:
+            burst_all_primary = burst_mode_dataset.attrs.get(
+                "all_vectors_primary", False
+            )
+
+        # Both datasets must have all vectors primary for the combined result to be True
+        global_attributes["all_vectors_primary"] = (
+            normal_all_primary and burst_all_primary
+        )
+
         global_attributes["missing_sequences"] = normal_mode_dataset.attrs[
             "missing_sequences"
         ]
@@ -162,8 +178,9 @@ def mag_l1c(
         output_core_dims=[[]],
         vectorize=True,
     )
-    # output_dataset['vector_magnitude'].attrs =
-    # attribute_manager.get_variable_attributes("vector_magnitude_attrs")
+    output_dataset[
+        "vector_magnitude"
+    ].attrs = attribute_manager.get_variable_attributes("vector_magnitude_attrs")
 
     output_dataset["compression_flags"] = xr.DataArray(
         completed_timeline[:, 6:8],
@@ -176,14 +193,14 @@ def mag_l1c(
         completed_timeline[:, 5],
         name="generated_flag",
         dims=["epoch"],
-        # attrs=attribute_manager.get_variable_attributes("generated_flag_attrs"),
+        attrs=attribute_manager.get_variable_attributes("generated_flag_attrs"),
     )
 
     return output_dataset
 
 
 def select_datasets(
-    first_input_dataset: xr.Dataset, second_input_dataset: Optional[xr.Dataset] = None
+    first_input_dataset: xr.Dataset, second_input_dataset: xr.Dataset | None = None
 ) -> tuple[xr.Dataset, xr.Dataset]:
     """
     Given one or two datasets, assign one to norm and one to burst.
@@ -503,7 +520,7 @@ def generate_timeline(epoch_data: np.ndarray, gaps: np.ndarray) -> np.ndarray:
 
 
 def find_all_gaps(
-    epoch_data: np.ndarray, vecsec_dict: Optional[dict] = None
+    epoch_data: np.ndarray, vecsec_dict: dict | None = None
 ) -> np.ndarray:
     """
     Find all the gaps in the epoch data.

imap_processing/mag/l1d/mag_l1d.py

@@ -9,7 +9,7 @@ from imap_processing.mag.l1d.mag_l1d_data import MagL1d, MagL1dConfiguration
 from imap_processing.mag.l2.mag_l2_data import ValidFrames
 
 
-def mag_l1d(
+def mag_l1d(  # noqa: PLR0912
     science_data: list[xr.Dataset],
     calibration_dataset: xr.Dataset,
     day_to_process: np.datetime64,
@@ -45,16 +45,18 @@ def mag_l1d(
     input_mago_burst = None
     for dataset in science_data:
         source = dataset.attrs.get("Logical_source", "")
-        if "norm-magi" in source:
-            input_magi_norm = dataset
-        elif "norm-mago" in source:
-            input_mago_norm = dataset
-        elif "burst-magi" in source:
-            input_magi_burst = dataset
-        elif "burst-mago" in source:
-            input_mago_burst = dataset
-        else:
-            raise ValueError(f"Input data has invalid logical source {source}")
+        instrument_mode = source.split("_")[-1]
+        match instrument_mode:
+            case "norm-magi":
+                input_magi_norm = dataset
+            case "norm-mago":
+                input_mago_norm = dataset
+            case "burst-magi":
+                input_magi_burst = dataset
+            case "burst-mago":
+                input_mago_burst = dataset
+            case _:
+                raise ValueError(f"Input data has invalid logical source {source}")
 
     if input_magi_norm is None or input_mago_norm is None:
         raise ValueError(
@@ -72,8 +74,9 @@ def mag_l1d(
     mago_vectors = input_mago_norm["vectors"].data[:, :3]
     magi_vectors = input_magi_norm["vectors"].data[:, :3]
 
-    # TODO: verify that MAGO is primary sensor for all vectors before applying
-    #  gradiometry
+    # Verify that MAGO is primary sensor for all vectors before applying gradiometry
+    if not input_mago_norm.attrs.get("all_vectors_primary", 1):
+        config.apply_gradiometry = False
 
     # TODO: L1D attributes
     attributes = ImapCdfAttributes()
@@ -95,12 +98,21 @@ def mag_l1d(
         day=day,
     )
 
+    # Nominally, this is expected to create MAGO data. However, if the configuration
+    # setting for always_output_mago is set to False, it will create MAGI data.
+
     l1d_norm.rotate_frame(ValidFrames.SRF)
     norm_srf_dataset = l1d_norm.generate_dataset(attributes, day_to_process)
     l1d_norm.rotate_frame(ValidFrames.DSRF)
     norm_dsrf_dataset = l1d_norm.generate_dataset(attributes, day_to_process)
+    l1d_norm.rotate_frame(ValidFrames.GSE)
+    norm_gse_dataset = l1d_norm.generate_dataset(attributes, day_to_process)
+    l1d_norm.rotate_frame(ValidFrames.RTN)
+    norm_rtn_dataset = l1d_norm.generate_dataset(attributes, day_to_process)
     output_datasets.append(norm_srf_dataset)
     output_datasets.append(norm_dsrf_dataset)
+    output_datasets.append(norm_gse_dataset)
+    output_datasets.append(norm_rtn_dataset)
 
     if input_mago_burst is not None and input_magi_burst is not None:
         # If burst data is provided, use it to create the burst L1d dataset
@@ -122,12 +134,43 @@ def mag_l1d(
             spin_offsets=l1d_norm.spin_offsets,
             day=day,
         )
+
+        # TODO: frame specific attributes may be required
         l1d_burst.rotate_frame(ValidFrames.SRF)
         burst_srf_dataset = l1d_burst.generate_dataset(attributes, day_to_process)
         l1d_burst.rotate_frame(ValidFrames.DSRF)
         burst_dsrf_dataset = l1d_burst.generate_dataset(attributes, day_to_process)
+        l1d_burst.rotate_frame(ValidFrames.GSE)
+        burst_gse_dataset = l1d_burst.generate_dataset(attributes, day_to_process)
+        l1d_burst.rotate_frame(ValidFrames.RTN)
+        burst_rtn_dataset = l1d_burst.generate_dataset(attributes, day_to_process)
         output_datasets.append(burst_srf_dataset)
         output_datasets.append(burst_dsrf_dataset)
+        output_datasets.append(burst_gse_dataset)
+        output_datasets.append(burst_rtn_dataset)
+
+    # Output ancillary files
+    # Add spin offsets dataset from normal mode processing
+    if l1d_norm.spin_offsets is not None:
+        spin_offset_dataset = l1d_norm.generate_spin_offset_dataset()
+        spin_offset_dataset.attrs["Logical_source"] = "imap_mag_l1d-spin-offsets"
+        output_datasets.append(spin_offset_dataset)
+
+    # Add gradiometry offsets dataset if gradiometry was applied
+    if l1d_norm.config.apply_gradiometry and hasattr(l1d_norm, "gradiometry_offsets"):
+        gradiometry_dataset = l1d_norm.gradiometry_offsets.copy()
+        gradiometry_dataset.attrs["Logical_source"] = (
+            "imap_mag_l1d-gradiometry-offsets-norm"
+        )
+        output_datasets.append(gradiometry_dataset)
+
+        # Also add burst gradiometry offsets if burst data was processed
+        if input_mago_burst is not None and input_magi_burst is not None:
+            if hasattr(l1d_burst, "gradiometry_offsets"):
+                burst_gradiometry_dataset = l1d_burst.gradiometry_offsets.copy()
+                burst_gradiometry_dataset.attrs["Logical_source"] = (
+                    "imap_mag_l1d-gradiometry-offsets-burst"
+                )
+                output_datasets.append(burst_gradiometry_dataset)
 
-    # TODO: Output ancillary files
     return output_datasets