imap-processing 0.6.0__py3-none-any.whl

imap_processing/mag/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "01"
+cdf_format_version = "v0.1"

imap_processing/mag/constants.py

@@ -0,0 +1,108 @@
+"""Collection of constant types or values for MAG."""
+
+from enum import Enum
+
+import numpy as np
+
+
+class DataMode(Enum):
+    """
+    Enum for MAG data modes: burst and normal (BURST + NORM).
+
+    Attributes
+    ----------
+    BURST: str
+        Burst data mode - higher frequency data
+    NORM: str
+        Normal data mode - lower frequency data (downsampled from burst)
+    """
+
+    BURST = "BURST"
+    NORM = "NORM"
+
+
+class Sensor(Enum):
+    """
+    Enum for MAG sensors: raw, MAGo, and MAGi (RAW, MAGO, MAGI).
+
+    Attributes
+    ----------
+    MAGO : str
+        MAGo sensor - for the outboard sensor. This is nominally expected to be the
+        primary sensor.
+    MAGI : str
+        MAGi sensor - for the inboard sensor.
+    RAW : str
+        RAW data - contains both sensors. Here, the vectors are unprocessed.
+    """
+
+    MAGO = "MAGO"
+    MAGI = "MAGI"
+    RAW = "RAW"
+
+
+class PrimarySensor(Enum):
+    """
+    Enum for primary sensor: MAGo and MAGi (MAGO, MAGI).
+
+    This corresponds to the PRI_SENS field in the MAG Level 0 data.
+
+    Attributes
+    ----------
+    MAGO : int
+        Primary sensor is MAGo.
+    MAGI : int
+        Primary sensor is MAGi.
+    """
+
+    MAGO = 0
+    MAGI = 1
+
+
+FIBONACCI_SEQUENCE = [
+    1,
+    2,
+    3,
+    5,
+    8,
+    13,
+    21,
+    34,
+    55,
+    89,
+    144,
+    233,
+    377,
+    610,
+    987,
+    1597,
+    2584,
+    4181,
+    6765,
+    10946,
+    17711,
+    28657,
+    46368,
+    75025,
+    121393,
+    196418,
+    317811,
+    514229,
+    832040,
+    1346269,
+    2178309,
+    3524578,
+    5702887,
+    9227465,
+    14930352,
+    24157817,
+    39088169,
+    63245986,
+    102334155,
+    165580141,
+]
+
+MAX_FINE_TIME = np.iinfo(np.uint16).max  # maximum 16 bit unsigned int
+AXIS_COUNT = 3
+RANGE_BIT_WIDTH = 2
+MAX_COMPRESSED_VECTOR_BITS = 60

imap_processing/mag/l0/decom_mag.py

@@ -0,0 +1,170 @@
+"""Methods for processing raw MAG packets into CDF files for level 0 and level 1a."""
+
+from __future__ import annotations
+
+import dataclasses
+import logging
+from collections import defaultdict
+from pathlib import Path
+
+import numpy as np
+import xarray as xr
+from space_packet_parser import parser, xtcedef
+
+from imap_processing import imap_module_directory
+from imap_processing.ccsds.ccsds_data import CcsdsData
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.mag.constants import DataMode
+from imap_processing.mag.l0.mag_l0_data import MagL0, Mode
+from imap_processing.spice.time import met_to_j2000ns
+
+logger = logging.getLogger(__name__)
+
+
+def decom_packets(packet_file_path: str | Path) -> dict[str, list[MagL0]]:
+    """
+    Decom MAG data packets using MAG packet definition.
+
+    Parameters
+    ----------
+    packet_file_path : str
+        Path to data packet path with filename.
+
+    Returns
+    -------
+    data_dict : dict[str, list[MagL0]]
+        A dict with 2 keys pointing to lists of MAG L0 data classes. "norm" corresponds
+        to  normal mode packets, "burst" corresponds to burst mode packets.
+    """
+    # Define paths
+    xtce_document = Path(
+        f"{imap_module_directory}/mag/packet_definitions/MAG_SCI_COMBINED.xml"
+    )
+
+    packet_definition = xtcedef.XtcePacketDefinition(xtce_document)
+    mag_parser = parser.PacketParser(packet_definition)
+
+    norm_data = []
+    burst_data = []
+
+    with open(packet_file_path, "rb") as binary_data:
+        mag_packets = mag_parser.generator(binary_data)
+
+        for packet in mag_packets:
+            apid = packet.header["PKT_APID"].derived_value
+            if apid in (Mode.BURST, Mode.NORMAL):
+                values = [
+                    item.derived_value
+                    if item.derived_value is not None
+                    else item.raw_value
+                    for item in packet.data.values()
+                ]
+                if apid == Mode.NORMAL:
+                    norm_data.append(MagL0(CcsdsData(packet.header), *values))
+                else:
+                    burst_data.append(MagL0(CcsdsData(packet.header), *values))
+
+    return {"norm": norm_data, "burst": burst_data}
+
+
+def generate_dataset(
+    l0_data: list[MagL0], mode: DataMode, attribute_manager: ImapCdfAttributes
+) -> xr.Dataset:
+    """
+    Generate a CDF dataset from the sorted raw L0 MAG data.
+
+    Parameters
+    ----------
+    l0_data : list[MagL0]
+        List of sorted L0 MAG data.
+
+    mode : DataMode
+        The mode of the CDF file - burst or norm.
+
+    attribute_manager : ImapCdfAttributes
+        Attribute manager for the dataset, including all MAG L1A attributes.
+
+    Returns
+    -------
+    dataset : xarray.Dataset
+        The xarray dataset with proper CDF attributes and shape.
+    """
+    # TODO: Correct CDF attributes from email
+
+    vector_data = np.zeros((len(l0_data), len(l0_data[0].VECTORS)))
+    shcoarse_data = np.zeros(len(l0_data), dtype="datetime64[ns]")
+
+    support_data = defaultdict(list)
+
+    for index, datapoint in enumerate(l0_data):
+        vector_len = len(datapoint.VECTORS)
+        if vector_len > vector_data.shape[1]:
+            # If the new vector is longer than the existing shape, first reshape
+            # vector_data and pad the existing vectors with zeros.
+            vector_data = np.pad(
+                vector_data,
+                (
+                    (
+                        0,
+                        0,
+                    ),
+                    (0, vector_len - vector_data.shape[1]),
+                ),
+                "constant",
+                constant_values=(0,),
+            )
+        vector_data[index, :vector_len] = datapoint.VECTORS
+
+        shcoarse_data[index] = met_to_j2000ns(datapoint.SHCOARSE)
+        # Add remaining pieces to arrays
+        for key, value in dataclasses.asdict(datapoint).items():
+            if key not in ("ccsds_header", "VECTORS", "SHCOARSE"):
+                support_data[key].append(value)
+            if key == "ccsds_header":
+                for ccsds_key, ccsds_value in value.items():
+                    support_data[ccsds_key].append(ccsds_value)
+
+    # Used in L1A vectors
+    direction = xr.DataArray(
+        np.arange(vector_data.shape[1]),
+        name="direction",
+        dims=["direction"],
+        attrs=attribute_manager.get_variable_attributes("raw_direction_attrs"),
+    )
+
+    # TODO: Epoch here refers to the start of the sample. Confirm that this is
+    # what mag is expecting, and if it is, CATDESC needs to be updated.
+    epoch_time = xr.DataArray(
+        shcoarse_data,
+        name="epoch",
+        dims=["epoch"],
+        attrs=attribute_manager.get_variable_attributes("epoch"),
+    )
+    # TODO: raw vectors units
+    raw_vectors = xr.DataArray(
+        vector_data,
+        name="raw_vectors",
+        dims=["epoch", "direction"],
+        attrs=attribute_manager.get_variable_attributes("raw_vector_attrs"),
+    )
+
+    logical_id = f"imap_mag_l1a_{mode.value.lower()}-raw"
+
+    output = xr.Dataset(
+        coords={"epoch": epoch_time, "direction": direction},
+        attrs=attribute_manager.get_global_attributes(logical_id),
+    )
+
+    output["raw_vectors"] = raw_vectors
+
+    for key, value in support_data.items():
+        # Time varying values
+        if key not in ["SHCOARSE", "VECTORS"]:
+            output[key] = xr.DataArray(
+                value,
+                name=key.lower(),
+                dims=["epoch"],
+                attrs=attribute_manager.get_variable_attributes(key),
+            )
+
+    return output

imap_processing/mag/l0/mag_l0_data.py

@@ -0,0 +1,118 @@
+"""Dataclasses for Level 0 MAG data."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import IntEnum
+
+import numpy as np
+
+from imap_processing.ccsds.ccsds_data import CcsdsData
+
+
+class Mode(IntEnum):
+    """
+    Enum class for MAG mode.
+
+    Attributes
+    ----------
+    BURST : int
+        APID for Burst mode data.
+    NORMAL : int
+        ApID for Normal mode data.
+    """
+
+    BURST = 1068
+    NORMAL = 1052
+
+
+@dataclass
+class MagL0:
+    """
+    Data class for MAG Level 0 data.
+
+    Attributes
+    ----------
+    ccsds_header: CcsdsData
+        CCSDS Header data.
+    SHCOARSE: int
+        Mission elapsed time.
+    PUS_SPARE1: int
+        ESA standard headers, PUS Spare 1.
+    PUS_VERSION: int
+        PUS Version Number.
+    PUS_SPARE2: int
+        PUS Spare 2.
+    PUS_STYPE: int
+        PUS Service Type.
+    PUS_SSUBTYPE: int
+        PUS Service Subtype - tells number of seconds of data.
+    COMPRESSION: int
+        Science Data Compression Flag - indicates if the data compressed - throw error
+        if 1.
+    MAGO_ACT: int
+        MAGO Active Status - if MAGo is active. May also be referred to as "FOB".
+    MAGI_ACT: int
+        MAGI Active Status - if MAGi is active.  May also be referred to as "FIB".
+    PRI_SENS: int
+        Primary Sensor - 0 is MAGo, 1 is MAGi.
+    SPARE1: int
+        Spare.
+    PRI_VECSEC: int
+        Primary Vectors per Second - lookup for L1b.
+    SEC_VECSEC: int
+        Secondary Vectors per second.
+    SPARE2: int
+        Spare.
+    PRI_COARSETM: int
+        Primary Coarse Time for first vector, seconds.
+    PRI_FNTM: int
+        Primary Fine Time for first vector, subseconds.
+    SEC_COARSETM: int
+        Secondary Coarse Time for first vector, seconds.
+    SEC_FNTM: int
+        Secondary Fine Time for first vector, subseconds.
+    VECTORS: numpy.ndarray or str
+        MAG Science Vectors - divide based on PRI_VECSEC and PUS_SSUBTYPE for vector
+        counts. There is a post init call to convert a string into a numpy array -
+        the only place it is a string is in the class initialization.
+    """
+
+    ccsds_header: CcsdsData
+    SHCOARSE: int
+    PUS_SPARE1: int
+    PUS_VERSION: int
+    PUS_SPARE2: int
+    PUS_STYPE: int
+    PUS_SSUBTYPE: int
+    COMPRESSION: int
+    MAGO_ACT: int
+    MAGI_ACT: int
+    PRI_SENS: int
+    SPARE1: int
+    PRI_VECSEC: int
+    SEC_VECSEC: int
+    SPARE2: int
+    PRI_COARSETM: int
+    PRI_FNTM: int
+    SEC_COARSETM: int
+    SEC_FNTM: int
+    VECTORS: np.ndarray | str
+
+    def __post_init__(self) -> None:
+        """
+        Convert Vectors attribute from string to bytearray if needed.
+
+        Also convert encoded "VECSEC" (vectors per second) into proper vectors per
+        second values.
+        """
+        if isinstance(self.VECTORS, str):
+            # Convert string output from space_packet_parser to numpy array of
+            # big-endian bytes
+            self.VECTORS = np.frombuffer(
+                int(self.VECTORS, 2).to_bytes(len(self.VECTORS) // 8, "big"),
+                dtype=np.dtype(">B"),
+            )
+
+        self.PRI_VECSEC = 2**self.PRI_VECSEC
+        self.SEC_VECSEC = 2**self.SEC_VECSEC

imap_processing/mag/l1a/mag_l1a.py

@@ -0,0 +1,317 @@
+"""Methods for decomming packets, processing to level 1A, and writing CDFs for MAG."""
+
+import dataclasses
+import logging
+from pathlib import Path
+
+import numpy as np
+import xarray as xr
+
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.cdf.utils import J2000_EPOCH
+from imap_processing.mag.constants import DataMode, PrimarySensor
+from imap_processing.mag.l0 import decom_mag
+from imap_processing.mag.l0.mag_l0_data import MagL0
+from imap_processing.mag.l1a.mag_l1a_data import (
+    MagL1a,
+    MagL1aPacketProperties,
+    TimeTuple,
+)
+from imap_processing.spice.time import met_to_j2000ns
+
+logger = logging.getLogger(__name__)
+
+
+def mag_l1a(packet_filepath: Path, data_version: str) -> list[xr.Dataset]:
+    """
+    Will process MAG L0 data into L1A CDF files at cdf_filepath.
+
+    Parameters
+    ----------
+    packet_filepath : pathlib.Path
+        Packet files for processing.
+    data_version : str
+        Data version to write to CDF files.
+
+    Returns
+    -------
+    generated_files : list[xarray.Dataset]
+        A list of generated filenames.
+    """
+    packets = decom_mag.decom_packets(packet_filepath)
+
+    norm_data = packets["norm"]
+    burst_data = packets["burst"]
+
+    input_files = [packet_filepath.name]
+
+    # Create attribute manager and add MAG L1A attributes and global variables
+    attribute_manager = ImapCdfAttributes()
+    attribute_manager.add_instrument_global_attrs("mag")
+    attribute_manager.add_instrument_variable_attrs("mag", "l1")
+
+    attribute_manager.add_global_attribute("Data_version", data_version)
+    attribute_manager.add_global_attribute("Input_files", str(input_files))
+    attribute_manager.add_global_attribute(
+        "Generation_date",
+        np.datetime64(
+            "now",
+        ).astype(str),
+    )
+
+    generated_datasets = create_l1a(norm_data, DataMode.NORM, attribute_manager)
+    generated_datasets += create_l1a(burst_data, DataMode.BURST, attribute_manager)
+
+    return generated_datasets
+
+
+def create_l1a(
+    packet_data: list[MagL0], data_mode: DataMode, attribute_manager: ImapCdfAttributes
+) -> list[xr.Dataset]:
+    """
+    Will process MAG L0 data into L1A, then create and write out CDF files.
+
+    Norm and burst mode descriptors are distinguished with the passed in attrs.
+
+    Parameters
+    ----------
+    packet_data : list[MagL0]
+        List of MagL0 packets to process, containing primary and secondary sensor data.
+
+    data_mode : DataMode
+        Enum for distinguishing between norm and burst mode data.
+
+    attribute_manager : ImapCdfAttributes
+        Attribute manager for CDF files for MAG L1A.
+
+    Returns
+    -------
+    generated_files : list[xarray.Dataset]
+        A list of generated filenames.
+    """
+    if not packet_data:
+        return []
+
+    mag_raw = decom_mag.generate_dataset(packet_data, data_mode, attribute_manager)
+
+    generated_datasets = [mag_raw]
+
+    l1a = process_packets(packet_data)
+
+    # TODO: Rearrange generate_dataset to combine these two for loops
+    # Split into MAGo and MAGi
+    for _, mago in l1a["mago"].items():
+        logical_file_id = f"imap_mag_l1a_{data_mode.value.lower()}-mago"
+        norm_mago_output = generate_dataset(mago, logical_file_id, attribute_manager)
+        generated_datasets.append(norm_mago_output)
+
+    for _, magi in l1a["magi"].items():
+        logical_file_id = f"imap_mag_l1a_{data_mode.value.lower()}-magi"
+        norm_magi_output = generate_dataset(
+            magi,
+            logical_file_id,
+            attribute_manager,
+        )
+        generated_datasets.append(norm_magi_output)
+
+    return generated_datasets
+
+
+def process_packets(
+    mag_l0_list: list[MagL0],
+) -> dict[str, dict[np.datetime64, MagL1a]]:
+    """
+    Given a list of MagL0 packets, process them into MagO and MagI L1A data classes.
+
+    This splits the MagL0 packets into MagO and MagI data, returning a dictionary with
+    keys "mago" and "magi."
+
+    Parameters
+    ----------
+    mag_l0_list : list[MagL0]
+        List of Mag L0 packets to process.
+
+    Returns
+    -------
+    packet_dict : dict[str, dict[numpy.datetime64, MagL1a]]
+        Dictionary containing two keys: "mago" which points to a dictionary of mago
+         MagL1A objects, and "magi" which points to a dictionary of magi MagL1A objects.
+         Each dictionary has keys of days and values of MagL1A objects, so each day
+         corresponds to one MagL1A object.
+    """
+    magi = {}
+    mago = {}
+
+    for mag_l0 in mag_l0_list:
+        primary_start_time = TimeTuple(mag_l0.PRI_COARSETM, mag_l0.PRI_FNTM)
+        secondary_start_time = TimeTuple(mag_l0.SEC_COARSETM, mag_l0.SEC_FNTM)
+
+        mago_is_primary = mag_l0.PRI_SENS == PrimarySensor.MAGO.value
+
+        primary_day = (
+            J2000_EPOCH
+            + met_to_j2000ns(primary_start_time.to_seconds()).astype("timedelta64[ns]")
+        ).astype("datetime64[D]")
+        secondary_day = (
+            J2000_EPOCH
+            + met_to_j2000ns(secondary_start_time.to_seconds()).astype(
+                "timedelta64[ns]"
+            )
+        ).astype("datetime64[D]")
+
+        primary_packet_data = MagL1aPacketProperties(
+            mag_l0.SHCOARSE,
+            primary_start_time,
+            mag_l0.PRI_VECSEC,
+            mag_l0.PUS_SSUBTYPE,
+            mag_l0.ccsds_header.SRC_SEQ_CTR,
+            mag_l0.COMPRESSION,
+            mago_is_primary,
+        )
+
+        secondary_packet_data = dataclasses.replace(
+            primary_packet_data,
+            start_time=secondary_start_time,
+            vectors_per_second=mag_l0.SEC_VECSEC,
+            pus_ssubtype=mag_l0.PUS_SSUBTYPE,
+        )
+        # now we know the number of secs of data in the packet, and the data rates of
+        # each sensor, we can calculate how much data is in this packet and where the
+        # byte boundaries are.
+        primary_vectors, secondary_vectors = MagL1a.process_vector_data(
+            mag_l0.VECTORS,  # type: ignore
+            primary_packet_data.total_vectors,
+            secondary_packet_data.total_vectors,
+            mag_l0.COMPRESSION,
+        )
+
+        primary_timestamped_vectors = MagL1a.calculate_vector_time(
+            primary_vectors,
+            primary_packet_data.vectors_per_second,
+            primary_packet_data.start_time,
+        )
+        secondary_timestamped_vectors = MagL1a.calculate_vector_time(
+            secondary_vectors,
+            secondary_packet_data.vectors_per_second,
+            secondary_packet_data.start_time,
+        )
+
+        # Sort primary and secondary into MAGo and MAGi by 24 hour chunks
+        mago_day = primary_day if mago_is_primary else secondary_day
+        magi_day = primary_day if not mago_is_primary else secondary_day
+
+        if mago_day not in mago:
+            mago[mago_day] = MagL1a(
+                True,
+                mag_l0.MAGO_ACT,
+                mag_l0.SHCOARSE,
+                primary_timestamped_vectors
+                if mago_is_primary
+                else secondary_timestamped_vectors,
+                primary_packet_data if mago_is_primary else secondary_packet_data,
+            )
+        else:
+            mago[mago_day].append_vectors(
+                (
+                    primary_timestamped_vectors
+                    if mago_is_primary
+                    else secondary_timestamped_vectors
+                ),
+                primary_packet_data if mago_is_primary else secondary_packet_data,
+            )
+
+        if magi_day not in magi:
+            magi[magi_day] = MagL1a(
+                False,
+                mag_l0.MAGI_ACT,
+                mag_l0.SHCOARSE,
+                primary_timestamped_vectors
+                if not mago_is_primary
+                else secondary_timestamped_vectors,
+                primary_packet_data if not mago_is_primary else secondary_packet_data,
+            )
+        else:
+            magi[magi_day].append_vectors(
+                (
+                    primary_timestamped_vectors
+                    if not mago_is_primary
+                    else secondary_timestamped_vectors
+                ),
+                primary_packet_data if not mago_is_primary else secondary_packet_data,
+            )
+
+    return {"mago": mago, "magi": magi}
+
+
+def generate_dataset(
+    single_file_l1a: MagL1a,
+    logical_file_id: str,
+    attribute_manager: ImapCdfAttributes,
+) -> xr.Dataset:
+    """
+    Generate a Xarray dataset for L1A data to output to CDF files.
+
+    Global_attrs should contain all info about mago/magi and burst/norm distinction, as
+     well as any general info in the global attributes.
+
+     Assumes each MagL1a object is a single day of data, so one MagL1a object has one
+     CDF file output.
+
+    Parameters
+    ----------
+    single_file_l1a : MagL1a
+        L1A data covering one day to process into a xarray dataset.
+    logical_file_id : str
+        Indicates which sensor (MagO or MAGi) and mode (burst or norm) the data is from.
+        This is used to retrieve the global attributes from attribute_manager.
+    attribute_manager : ImapCdfAttributes
+        Attributes for the dataset, as created by ImapCdfAttributes.
+
+    Returns
+    -------
+    dataset : xarray.Dataset
+        One xarray dataset with proper CDF attributes and shape containing MAG L1A data.
+    """
+    # TODO: add:
+    # gaps_in_data global attr
+    # magl1avectordefinition data
+
+    # TODO: Just leave time in datetime64 type with vector as dtype object to avoid this
+    # Get the timestamp from the end of the vector
+    time_data = single_file_l1a.vectors[:, 4].astype(
+        np.dtype("datetime64[ns]"), copy=False
+    )
+
+    direction = xr.DataArray(
+        np.arange(4),
+        name="direction",
+        dims=["direction"],
+        attrs=attribute_manager.get_variable_attributes("direction_attrs"),
+    )
+
+    # TODO: Epoch here refers to the start of the sample. Confirm that this is
+    # what mag is expecting, and if it is, CATDESC needs to be updated.
+    epoch_time = xr.DataArray(
+        time_data,
+        name="epoch",
+        dims=["epoch"],
+        attrs=attribute_manager.get_variable_attributes("epoch"),
+    )
+
+    vectors = xr.DataArray(
+        single_file_l1a.vectors[:, :4],
+        name="vectors",
+        dims=["epoch", "direction"],
+        attrs=attribute_manager.get_variable_attributes("vector_attrs"),
+    )
+
+    output = xr.Dataset(
+        coords={"epoch": epoch_time, "direction": direction},
+        attrs=attribute_manager.get_global_attributes(logical_file_id),
+    )
+
+    output["vectors"] = vectors
+
+    # TODO: Put is_mago and active in the header
+
+    return output