imap-processing 0.14.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

imap_processing/ialirt/utils/constants.py

@@ -0,0 +1,62 @@
+"""Keys for I-ALiRT data products."""
+
+IALIRT_KEYS = [
+    # H intensities in 15 energy ranges and binned into 4 azimuths and 4 spin angle bins
+    "codicehi_h",
+    # C/O abundance ratio
+    "codicelo_c_over_o_abundance",
+    # Mg/O abundance ratio
+    "codicelo_mg_over_o_abundance",
+    # Fe/O abundance ratio
+    "codicelo_fe_over_o_abundance",
+    # C+6/C+5 charge state ratio
+    "codicelo_c_plus_6_over_c_plus_5_ratio",
+    # O+7/O+6 charge state ratio
+    "codicelo_o_plus_7_over_o_plus_6_ratio",
+    # Fe low/Fe high charge state ratio
+    "codicelo_fe_low_over_fe_high_ratio",
+    # Low energy (~300 keV) electrons (A-side)
+    "hit_e_a_side_low_en",
+    # Medium energy (~3 MeV) electrons (A-side)
+    "hit_e_a_side_med_en",
+    # High energy (>3 MeV) electrons (A-side)
+    "hit_e_a_side_high_en",
+    # Low energy (~300 keV) electrons (B-side)
+    "hit_e_b_side_low_en",
+    # Medium energy (~3 MeV) electrons (B-side)
+    "hit_e_b_side_med_en",
+    # High energy (>3 MeV) electrons (B-side)
+    "hit_e_b_side_high_en",
+    # Medium energy (12 to 70 MeV) protons (Omnidirectional)
+    "hit_h_omni_med_en",
+    # High energy (>70 MeV) protons (A-side)
+    "hit_h_a_side_high_en",
+    # High energy (>70 MeV) protons (B-side)
+    "hit_h_b_side_high_en",
+    # Low energy (6 to 8 MeV/nuc) He (Omnidirectional)
+    "hit_he_omni_low_en",
+    # High energy (15 to 70 MeV/nuc) He (Omnidirectional)
+    "hit_he_omni_high_en",
+    # Magnetic field vector in GSE coordinates
+    "mag_4s_b_gse",
+    # Magnetic field vector in GSM coordinates
+    "mag_4s_b_gsm",
+    # Magnetic field vector in RTN coordinates
+    "mag_4s_b_rtn",
+    # Azimuth angle (φ) of the magnetic field in GSM coordinates
+    "mag_phi_4s_b_gsm",
+    # Elevation angle (θ) of the magnetic field in GSM coordinates
+    "mag_theta_4s_b_gsm",
+    # Pseudo density of solar wind protons
+    "swapi_pseudo_proton_density",
+    # Pseudo speed of solar wind protons in solar inertial frame
+    "swapi_pseudo_proton_speed",
+    # Pseudo temperature of solar wind protons in plasma frame
+    "swapi_pseudo_proton_temperature",
+    # SWE Normalized Counts - Half Cycle 1
+    "swe_normalized_counts_half_1",
+    # SWE Normalized Counts - Half Cycle 2
+    "swe_normalized_counts_half_2",
+    # SWE Counterstreaming flag
+    "swe_counterstreaming_electrons",
+]

imap_processing/ialirt/utils/create_xarray.py

@@ -0,0 +1,135 @@
+"""Creates xarray based on structure of queried DynamoDB."""
+
+import numpy as np
+import xarray as xr
+
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.ialirt.utils.constants import IALIRT_KEYS
+
+
+def create_xarray_from_records(records: list[dict]) -> xr.Dataset:  # noqa: PLR0912
+    """
+    Create dataset from a list of records.
+
+    Parameters
+    ----------
+    records : list of dict
+       Output of querying DynamoDB.
+
+    Returns
+    -------
+    dataset : xarray.Dataset
+        Dataset in standard format.
+    """
+    cdf_manager = ImapCdfAttributes()
+    cdf_manager.add_instrument_global_attrs("ialirt")
+    cdf_manager.add_instrument_variable_attrs("ialirt", "l1")
+
+    instrument_keys: set[str] = set(IALIRT_KEYS)
+    n = len(records)
+    attrs = cdf_manager.get_variable_attributes("default_int64_attrs")
+    fillval = attrs.get("FILLVAL")
+    ttj2000ns_values = np.full(n, fillval, dtype=np.int64)
+
+    # Collect all keys that start with the instrument prefixes.
+    for i, record in enumerate(records):
+        ttj2000ns_values[i] = record["ttj2000ns"]
+
+    epoch = xr.DataArray(
+        data=ttj2000ns_values,
+        name="epoch",
+        dims=["epoch"],
+        attrs=cdf_manager.get_variable_attributes("epoch", check_schema=False),
+    )
+    component = xr.DataArray(
+        ["x", "y", "z"],
+        name="component",
+        dims=["component"],
+        attrs=cdf_manager.get_variable_attributes("component", check_schema=False),
+    )
+
+    esa_step = xr.DataArray(
+        data=np.arange(8, dtype=np.uint8),
+        name="esa_step",
+        dims=["esa_step"],
+        attrs=cdf_manager.get_variable_attributes("esa_step", check_schema=False),
+    )
+
+    energy_ranges = xr.DataArray(
+        data=np.arange(15, dtype=np.uint8),
+        name="energy_ranges",
+        dims=["energy_ranges"],
+        attrs=cdf_manager.get_variable_attributes("energy_ranges", check_schema=False),
+    )
+
+    azimuth = xr.DataArray(
+        data=np.arange(4, dtype=np.uint8),
+        name="azimuth",
+        dims=["azimuth"],
+        attrs=cdf_manager.get_variable_attributes("azimuth", check_schema=False),
+    )
+
+    spin_angle_bin = xr.DataArray(
+        data=np.arange(4, dtype=np.uint8),
+        name="spin_angle_bin",
+        dims=["spin_angle_bin"],
+        attrs=cdf_manager.get_variable_attributes("spin_angle_bin", check_schema=False),
+    )
+
+    coords = {
+        "epoch": epoch,
+        "component": component,
+        "esa_step": esa_step,
+        "energy_ranges": energy_ranges,
+        "azimuth": azimuth,
+        "spin_angle_bin": spin_angle_bin,
+    }
+    dataset = xr.Dataset(
+        coords=coords,
+        attrs=cdf_manager.get_global_attributes("imap_ialirt_l1_realtime"),
+    )
+
+    # Create empty dataset for each key.
+    for key in instrument_keys:
+        attrs = cdf_manager.get_variable_attributes(key, check_schema=False)
+        fillval = attrs.get("FILLVAL")
+        if key.startswith("mag"):
+            data = np.full((n, 3), fillval, dtype=np.float32)
+            dims = ["epoch", "component"]
+            dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs)
+        elif key.startswith("codicehi"):
+            data = np.full((n, 15, 4, 4), fillval, dtype=np.float32)
+            dims = ["epoch", "energy", "azimuth", "spin_angle_bin"]
+            dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs)
+        elif key == "swe_counterstreaming_electrons":
+            data = np.full(n, fillval, dtype=np.uint8)
+            dims = ["epoch"]
+            dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs)
+        elif key.startswith("swe"):
+            data = np.full((n, 8), fillval, dtype=np.uint32)
+            dims = ["epoch", "esa_step"]
+            dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs)
+        elif key.startswith("hit"):
+            data = np.full(n, fillval, dtype=np.uint32)
+            dims = ["epoch"]
+            dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs)
+        else:
+            data = np.full(n, fillval, dtype=np.float32)
+            dims = ["epoch"]
+            dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs)
+
+    # Populate the dataset variables
+    for i, record in enumerate(records):
+        for key, val in record.items():
+            if key in ["apid", "met", "met_in_utc", "ttj2000ns"]:
+                continue
+            elif key.startswith("mag"):
+                dataset[key].data[i, :] = val
+            elif key.startswith("swe_normalized_counts"):
+                dataset[key].data[i, :] = val
+            elif key.startswith("codicehi"):
+                dataset[key].data[i, :, :, :] = val
+            else:
+                dataset[key].data[i] = val
+
+    return dataset

imap_processing/idex/idex_l2c.py

@@ -125,7 +125,7 @@ def idex_healpix_map(
     counts = np.histogram(hpix_idx, bins=n_pix, range=(0, n_pix))[0]
     # Add epoch dimension
     counts_da = xr.DataArray(
-        counts[np.newaxis, :].astype(np.uint16),
+        counts[np.newaxis, :].astype(np.int64),
         name="counts",
         dims=("epoch", CoordNames.HEALPIX_INDEX.value),
         attrs=idex_attrs.get_variable_attributes("healpix_counts"),
@@ -147,10 +147,10 @@ def idex_healpix_map(
     )
     map_attrs = {
         "Sky_tiling_type": SkyTilingType.HEALPIX.value,
-        "HEALPix_nside": nside,
-        "HEALPix_nest": nested,
-        "Spice_reference_frame": IDEX_EVENT_REFERENCE_FRAME,
-        "num_points": n_pix,
+        "HEALPix_nside": str(nside),
+        "HEALPix_nest": str(nested),
+        "Spice_reference_frame": IDEX_EVENT_REFERENCE_FRAME.name,
+        "num_points": str(n_pix),
     } | idex_attrs.get_global_attributes("imap_idex_l2c_sci-healpix")
     l2c_dataset.attrs.update(map_attrs)
 
@@ -193,7 +193,7 @@ def idex_rectangular_map(
         longitude_wrapped, latitude, bins=[grid.az_bin_edges, grid.el_bin_edges]
     )
     counts_da = xr.DataArray(
-        counts[np.newaxis, :, :].astype(np.uint16),
+        counts[np.newaxis, :, :].astype(np.int64),
         name="counts",
         dims=("epoch", "rectangular_lon_pixel", "rectangular_lat_pixel"),
         attrs=idex_attrs.get_variable_attributes("rectangular_counts"),
@@ -241,9 +241,9 @@ def idex_rectangular_map(
     )
     map_attrs = {
         "sky_tiling_type": SkyTilingType.RECTANGULAR.value,
-        "Spacing_degrees": spacing_deg,
-        "Spice_reference_frame": IDEX_EVENT_REFERENCE_FRAME,
-        "num_points": counts.size,
+        "Spacing_degrees": str(spacing_deg),
+        "Spice_reference_frame": IDEX_EVENT_REFERENCE_FRAME.name,
+        "num_points": str(counts.size),
     } | idex_attrs.get_global_attributes("imap_idex_l2c_sci-rectangular")
 
     l2c_dataset.attrs.update(map_attrs)

imap_processing/lo/l1b/lo_l1b.py

@@ -1,5 +1,6 @@
 """IMAP-Lo L1B Data Processing."""
 
+import logging
 from dataclasses import Field
 from pathlib import Path
 from typing import Any, Union
@@ -17,6 +18,9 @@ from imap_processing.lo.l1b.tof_conversions import (
 from imap_processing.spice.geometry import SpiceFrame, instrument_pointing
 from imap_processing.spice.time import met_to_ttj2000ns, ttj2000ns_to_et
 
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
 
 def lo_l1b(dependencies: dict) -> list[Path]:
     """
@@ -39,9 +43,10 @@ def lo_l1b(dependencies: dict) -> list[Path]:
     # create the attribute manager to access L1A fillval attributes
     attr_mgr_l1a = ImapCdfAttributes()
     attr_mgr_l1a.add_instrument_variable_attrs(instrument="lo", level="l1a")
-
+    logger.info(f"\n Dependencies: {list(dependencies.keys())}\n")
     # if the dependencies are used to create Annotated Direct Events
     if "imap_lo_l1a_de" in dependencies and "imap_lo_l1a_spin" in dependencies:
+        logger.info("\nProcessing IMAP-Lo L1B Direct Events...")
         logical_source = "imap_lo_l1b_de"
         # get the dependency dataset for l1b direct events
         l1a_de = dependencies["imap_lo_l1a_de"]

imap_processing/lo/l1c/lo_l1c.py

@@ -67,6 +67,19 @@ def lo_l1c(sci_dependencies: dict, anc_dependencies: list) -> list[xr.Dataset]:
         pset["exposure_time"] = calculate_exposure_times(
             full_counts, l1b_goodtimes_only
         )
+    pset.attrs = attr_mgr.get_global_attributes(logical_source)
+    # TODO: Temp fix before adding attribute variables.
+    #  CDF won't open if DEPEND_0 is not deleted currently.
+    del pset["epoch"].attrs["DEPEND_0"]
+
+    pset = pset.assign_coords(
+        {
+            "energy": np.arange(1, 8),
+            "longitude": np.arange(3600),
+            "latitude": np.arange(40),
+        }
+    )
+
     return [pset]
 
 
@@ -127,12 +140,8 @@ def filter_goodtimes(l1b_de: xr.Dataset, anc_dependencies: list) -> xr.Dataset:
     l1b_de : xarray.Dataset
         Filtered L1B Direct Event dataset.
     """
-    # Get the sweep table from the ancillary dependencies
-    goodtimes_table = next(
-        (item for item in anc_dependencies if "goodtimes" in item), None
-    )
-    # sweep table is a dependency so this should always be in the list
-    goodtimes_table_df = pd.read_csv(goodtimes_table)
+    # the goodtimes are currently the only ancillary file needed for L1C processing
+    goodtimes_table_df = pd.read_csv(anc_dependencies[0])
 
     # convert goodtimes from MET to TTJ2000
     goodtimes_start = met_to_ttj2000ns(goodtimes_table_df["GoodTime_strt"])
@@ -220,9 +229,9 @@ def create_pset_counts(
     # stack the filtered data into the 3D array
     data = np.column_stack(
         (
+            de_filtered["esa_step"],
             de_filtered["pointing_bin_lon"],
             de_filtered["pointing_bin_lat"],
-            de_filtered["esa_step"],
         )
     )
     # Create the histogram with 3600 longitude bins, 40 latitude bins, and 7 energy bins
@@ -232,7 +241,7 @@ def create_pset_counts(
 
     hist, edges = np.histogramdd(
         data,
-        bins=[lon_edges, lat_edges, energy_edges],
+        bins=[energy_edges, lon_edges, lat_edges],
     )
 
     # add a new axis of size 1 for the epoch
@@ -240,7 +249,7 @@ def create_pset_counts(
 
     counts = xr.DataArray(
         data=hist.astype(np.int16),
-        dims=["epoch", "lon_bins", "lat_bins", "energy_bins"],
+        dims=["epoch", "energy", "longitude", "latitude"],
     )
 
     return counts
@@ -272,7 +281,7 @@ def calculate_exposure_times(counts: xr.DataArray, l1b_de: xr.Dataset) -> xr.Dat
     energy_edges = np.arange(8)
 
     data = np.column_stack(
-        (l1b_de["pointing_bin_lon"], l1b_de["pointing_bin_lat"], l1b_de["esa_step"])
+        (l1b_de["esa_step"], l1b_de["pointing_bin_lon"], l1b_de["pointing_bin_lat"])
     )
 
     result = binned_statistic_dd(
@@ -280,14 +289,14 @@ def calculate_exposure_times(counts: xr.DataArray, l1b_de: xr.Dataset) -> xr.Dat
         # exposure time equation from Lo Alg Document 10.1.1.4
         4 * l1b_de["avg_spin_durations"].to_numpy() / 3600,
         statistic="mean",
-        bins=[lon_edges, lat_edges, energy_edges],
+        bins=[energy_edges, lon_edges, lat_edges],
     )
 
     stat = result.statistic[np.newaxis, :, :, :]
 
     exposure_time = xr.DataArray(
         data=stat.astype(np.float16),
-        dims=["epoch", "lon_bins", "lat_bins", "energy_bins"],
+        dims=["epoch", "energy", "longitude", "latitude"],
     )
 
     return exposure_time
@@ -317,7 +326,7 @@ def create_datasets(
     #  and relative L1A DE time to calculate the absolute DE time,
     #  this epoch conversion will go away and the time in the DE dataclass
     #  can be used direction
-    epoch_converted_time = [met_to_ttj2000ns(1)]
+    epoch_converted_time = [1]
 
     # Create a data array for the epoch time
     # TODO: might need to update the attrs to use new YAML file

imap_processing/lo/l2/lo_l2.py

@@ -0,0 +1,213 @@
+"""IMAP-Lo L2 data processing."""
+
+import numpy as np
+import xarray as xr
+
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.ena_maps import ena_maps
+from imap_processing.ena_maps.ena_maps import RectangularSkyMap
+from imap_processing.spice import geometry
+from imap_processing.spice.geometry import SpiceFrame
+
+
+def lo_l2(sci_dependencies: dict, anc_dependencies: list) -> list[xr.Dataset]:
+    """
+    Will process IMAP-Lo L1C data into Le CDF data products.
+
+    Parameters
+    ----------
+    sci_dependencies : dict
+        Dictionary of datasets needed for L2 data product creation in xarray Datasets.
+    anc_dependencies : list
+        Ancillary files needed for L2 data product creation.
+
+    Returns
+    -------
+    created_file_paths : list[Path]
+        Location of created CDF files.
+    """
+    # create the attribute manager for this data level
+    attr_mgr = ImapCdfAttributes()
+    attr_mgr.add_instrument_global_attrs(instrument="lo")
+    attr_mgr.add_instrument_variable_attrs(instrument="enamaps", level="l2-common")
+    attr_mgr.add_instrument_variable_attrs(instrument="enamaps", level="l2-rectangular")
+
+    # if the dependencies are used to create Annotated Direct Events
+    if "imap_lo_l1c_pset" in sci_dependencies:
+        logical_source = "imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-3mo"
+        psets = sci_dependencies["imap_lo_l1c_pset"]
+
+        # Create the rectangular sky map from the pointing set.
+        lo_rect_map = project_pset_to_rect_map(
+            psets, spacing_deg=6, spice_frame=geometry.SpiceFrame.ECLIPJ2000
+        )
+        # Add the hydrogen rates to the rectangular map dataset.
+        lo_rect_map.data_1d["h_rate"] = calculate_rates(
+            lo_rect_map.data_1d["h_counts"], lo_rect_map.data_1d["exposure_time"]
+        )
+        # Add the hydrogen flux to the rectangular map dataset.
+        lo_rect_map.data_1d["h_flux"] = calculate_fluxes(lo_rect_map.data_1d["h_rate"])
+        # Create the dataset from the rectangular map.
+        lo_rect_map_ds = lo_rect_map.to_dataset()
+        # Add the attributes to the dataset.
+        lo_rect_map_ds = add_attributes(
+            lo_rect_map_ds, attr_mgr, logical_source=logical_source
+        )
+
+    return [lo_rect_map_ds]
+
+
+def project_pset_to_rect_map(
+    psets: list[xr.Dataset], spacing_deg: int, spice_frame: SpiceFrame
+) -> RectangularSkyMap:
+    """
+    Project the pointing set to a rectangular sky map.
+
+    This function is used to create a rectangular sky map from the pointing set
+    data in the L1C dataset.
+
+    Parameters
+    ----------
+    psets : list[xr.Dataset]
+        List of pointing sets in xarray Dataset format.
+    spacing_deg : int
+        The spacing in degrees for the rectangular sky map.
+    spice_frame : SpiceFrame
+        The SPICE frame to use for the rectangular sky map projection.
+
+    Returns
+    -------
+    RectangularSkyMap
+        The rectangular sky map created from the pointing set data.
+    """
+    lo_rect_map = ena_maps.RectangularSkyMap(
+        spacing_deg=spacing_deg,
+        spice_frame=spice_frame,
+    )
+    for pset in psets:
+        lo_pset = ena_maps.LoPointingSet(pset)
+        lo_rect_map.project_pset_values_to_map(
+            pointing_set=lo_pset,
+            value_keys=["h_counts", "exposure_time"],
+            index_match_method=ena_maps.IndexMatchMethod.PUSH,
+        )
+    return lo_rect_map
+
+
+def calculate_rates(counts: xr.DataArray, exposure_time: xr.DataArray) -> xr.DataArray:
+    """
+    Calculate the hydrogen rates from the counts and exposure time.
+
+    Parameters
+    ----------
+    counts : xr.DataArray
+        The counts of hydrogen or oxygen ENAs.
+    exposure_time : xr.DataArray
+        The exposure time for the counts.
+
+    Returns
+    -------
+    xr.DataArray
+        The calculated hydrogen rates.
+    """
+    # Calculate the rates based on the h_counts and exposure_time
+    rate = counts / exposure_time
+    return rate
+
+
+def calculate_fluxes(rates: xr.DataArray) -> xr.DataArray:
+    """
+    Calculate the flux from the hydrogen rate.
+
+    Parameters
+    ----------
+    rates : xr.Dataset
+        The hydrogen or oxygen rates.
+
+    Returns
+    -------
+    xr.DataArray
+        The calculated flux.
+    """
+    # Temporary values. These will all come from ancillary data when
+    # the data is available and integrated.
+    geometric_factor = 1.0
+    efficiency_factor = 1.0
+    energy_dict = {1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6, 7: 7}
+    energies = np.array([energy_dict[i] for i in range(1, 8)])
+    energies = energies.reshape(1, 7, 1)
+
+    flux = rates / (geometric_factor * energies * efficiency_factor)
+    return flux
+
+
+def add_attributes(
+    lo_map: xr.Dataset, attr_mgr: ImapCdfAttributes, logical_source: str
+) -> xr.Dataset:
+    """
+    Add attributes to the map dataset.
+
+    Parameters
+    ----------
+    lo_map : xr.Dataset
+        The dataset to add attributes to.
+    attr_mgr : ImapCdfAttributes
+        The attribute manager to use for adding attributes.
+    logical_source : str
+        The logical source for the dataset.
+
+    Returns
+    -------
+    xr.Dataset
+        The dataset with added attributes.
+    """
+    # Add the global attributes to the dataset.
+    lo_map.attrs.update(attr_mgr.get_global_attributes(logical_source))
+
+    # TODO: Lo is using different field names than what's in the attributes.
+    #  check if the Lo should use exposure factor instead of exposure time.
+    #  check if hydrogen and oxygen specific ena intensities should be added
+    #  to the attributes or if general ena intensities can be used or updated
+    #  in the code. This dictionary is temporary solution for SIT-4
+    map_fields = {
+        "epoch": "epoch",
+        "h_flux": "ena_intensity",
+        "h_rate": "ena_rate",
+        "h_counts": "ena_count",
+        "exposure_time": "exposure_factor",
+        "energy": "energy",
+        "solid_angle": "solid_angle",
+        "longitude": "longitude",
+        "latitude": "latitude",
+    }
+
+    # TODO: The mapping utility is supposed to handle at least some of these
+    #  attributes but is not working. Need to investigate this after SIT-4
+    # Add the attributes to the dataset variables.
+    for field, attr_name in map_fields.items():
+        if field in lo_map.data_vars or field in lo_map.coords:
+            lo_map[field].attrs.update(
+                attr_mgr.get_variable_attributes(attr_name, check_schema=False)
+            )
+
+    labels = {
+        "energy": np.arange(1, 8).astype(str),
+        "longitude": lo_map["longitude"].values.astype(str),
+        "latitude": lo_map["latitude"].values.astype(str),
+    }
+    # add the coordinate labels to the dataset
+    for dim, values in labels.items():
+        lo_map = lo_map.assign_coords(
+            {
+                f"{dim}_label": xr.DataArray(
+                    values,
+                    name=f"{dim}_label",
+                    dims=[dim],
+                    attrs=attr_mgr.get_variable_attributes(
+                        f"{dim}_label", check_schema=False
+                    ),
+                )
+            }
+        )
+
+    return lo_map

imap_processing/mag/l1c/mag_l1c.py

@@ -415,6 +415,13 @@ def interpolate_gaps(
             f"difference between gap start and burst start: "
             f"{gap_timeline[0] - burst_epochs[burst_start]}"
         )
+
+        short = (gap_timeline >= burst_epochs[burst_start]) & (
+            gap_timeline <= burst_epochs[burst_gap_end]
+        )
+        if len(gap_timeline) != (short).sum():
+            print(f"Chopping timeline from {len(gap_timeline)} to {short.sum()}")
+
         # Limit timestamps to only include the areas with burst data
         gap_timeline = gap_timeline[
             (
@@ -573,7 +580,7 @@ def find_gaps(timeline_data: np.ndarray, vectors_per_second: int) -> np.ndarray:
     diffs = abs(np.diff(timeline_data))
     # 3.5e7 == 7.5% of 0.5s in nanoseconds, a common gap. In the future, this number
     # will be calculated from the expected gap.
-    gap_index = np.asarray(diffs - expected_gap > 3.5e7).nonzero()[0]
+    gap_index = np.asarray(diffs - expected_gap > expected_gap * 0.075).nonzero()[0]
     output: np.ndarray = np.zeros((len(gap_index), 3))
 
     for index, gap in enumerate(gap_index):

imap_processing/mag/l2/mag_l2.py

@@ -15,6 +15,7 @@ def mag_l2(
     offsets_dataset: xr.Dataset,
     input_data: xr.Dataset,
     day_to_process: np.datetime64,
+    mode: DataMode = DataMode.NORM,
 ) -> list[xr.Dataset]:
     """
     Complete MAG L2 processing.
@@ -63,6 +64,9 @@ def mag_l2(
     day_to_process : numpy.datetime64['D']
         The 24 hour day to process. This should match the day of the input data and
         the offset file.
+    mode : DataMode
+        The data mode to process. Default is DataMode.NORM (normal mode).
+        Can also be DataMode.BURST for burst mode processing.
 
     Returns
     -------
@@ -96,14 +100,14 @@ def mag_l2(
         {},
         np.zeros(len(input_data["epoch"].data)),
         np.zeros(len(input_data["epoch"].data)),
-        DataMode.NORM,
+        mode,
         offsets=offsets_dataset["offsets"].data,
         timedelta=offsets_dataset["timedeltas"].data,
     )
     attributes = ImapCdfAttributes()
     attributes.add_instrument_global_attrs("mag")
     # temporarily point to l1c
-    attributes.add_instrument_variable_attrs("mag", "l1c")
+    attributes.add_instrument_variable_attrs("mag", "l2")
     return [input_data.generate_dataset(attributes, day)]
 
 

imap_processing/mag/l2/mag_l2_data.py

@@ -222,7 +222,9 @@ class MagL2:
             self.epoch,
             name="epoch",
             dims=["epoch"],
-            attrs=attribute_manager.get_variable_attributes("epoch"),
+            attrs=attribute_manager.get_variable_attributes(
+                "epoch", check_schema=False
+            ),
         )
 
         vectors = xr.DataArray(
@@ -236,14 +238,14 @@ class MagL2:
             self.quality_flags,
             name="quality_flags",
             dims=["epoch"],
-            attrs=attribute_manager.get_variable_attributes("compression"),
+            attrs=attribute_manager.get_variable_attributes("qf_bitmask"),
         )
 
         quality_bitmask = xr.DataArray(
             self.quality_flags,
             name="quality_flags",
             dims=["epoch"],
-            attrs=attribute_manager.get_variable_attributes("compression"),
+            attrs=attribute_manager.get_variable_attributes("qf"),
         )
 
         rng = xr.DataArray(
@@ -251,14 +253,14 @@ class MagL2:
             name="range",
             dims=["epoch"],
             # TODO temp attrs
-            attrs=attribute_manager.get_variable_attributes("compression_width"),
+            attrs=attribute_manager.get_variable_attributes("fill"),
         )
 
         magnitude = xr.DataArray(
             self.magnitude,
             name="magnitude",
             dims=["epoch"],
-            attrs=attribute_manager.get_variable_attributes("compression_width"),
+            attrs=attribute_manager.get_variable_attributes("fill"),
         )
 
         global_attributes = (