imap-processing 0.15.0__py3-none-any.whl → 0.16.1__py3-none-any.whl

imap_processing/hit/l0/constants.py

@@ -100,7 +100,9 @@ COUNTS_DATA_STRUCTURE = {
     "penfgrates": HITPacking(16, 528, (33,)),  # range 4 foreground rates
     "penbgrates": HITPacking(16, 240, (15,)),  # range 4 background rates
     "ialirtrates": HITPacking(16, 320, (20,)),  # ialirt rates
-    "sectorates": HITPacking(16, 1920, (15, 8)),  # sectored rates
+    "sectorates": HITPacking(
+        16, 1920, (8, 15)
+    ),  # sectored rates (8 zenith angles, 15 azimuth angles)
     "l4fgrates": HITPacking(16, 768, (48,)),  # all range foreground rates
     "l4bgrates": HITPacking(16, 384, (24,)),  # all range foreground rates
 }

imap_processing/hit/l0/decom_hit.py

@@ -89,11 +89,14 @@ def parse_count_rates(sci_dataset: xr.Dataset) -> None:
         if all(x not in field for x in ["hdr", "spare", "pha"]):
             parsed_data = np.vectorize(decompress_rates_16_to_32)(parsed_data)
 
-        # Get dims for data variables (yaml file not created yet)
+        # Get dims for data variables
         if len(field_meta.shape) > 1:
             if "sectorates" in field:
-                # Reshape data to 15x8 for azimuth and zenith look directions
+                # Reshape data into (num_frames, 8, 15) for zenith and azimuth
+                # look directions
                 parsed_data = np.array(parsed_data).reshape((-1, *field_meta.shape))
+                # Transpose data to (num_frames, 15, 8) for flipped look directions
+                parsed_data = np.transpose(parsed_data, axes=(0, 2, 1))
                 dims = ["epoch", "azimuth", "zenith"]
                 # Add angle values to coordinates
                 sci_dataset.coords["zenith"] = xr.DataArray(
@@ -202,9 +205,12 @@ def update_ccsds_header_dims(sci_dataset: xr.Dataset) -> xr.Dataset:
     it will be updated later in the process to represent
     time per science frame, so another time dimension is
     needed for the ccsds header fields.This function
-    updates the dimension for these fields to use sc_tick
-    instead of epoch. sc_tick is the time the packet was
-    created.
+    updates the dimension for all data vars to use sc_tick
+    instead of epoch. It also temporarily sets sc_tick as the
+    dimension for the epoch coordinate (to be updated later
+    in the assemble_science_frames function).
+
+    Note: sc_tick is the time the packet was created.
 
     Parameters
     ----------
@@ -294,7 +300,7 @@ def assemble_science_frames(sci_dataset: xr.Dataset) -> xr.Dataset:
     # Extract data per science frame and organize by L1A data products
     count_rates = []
     pha = []
-    epoch_per_science_frame = np.array([])
+    epoch_per_science_frame = []
     for idx in starting_indices:
         # Data from 20 packets in a science frame
         science_data_frame = science_data[idx : idx + FRAME_SIZE]
@@ -302,12 +308,15 @@ def assemble_science_frames(sci_dataset: xr.Dataset) -> xr.Dataset:
         count_rates.append("".join(science_data_frame[:6]))
         # Last 14 packets contain pulse height event data in binary
         pha.append("".join(science_data_frame[6:]))
-        # Get first packet's epoch for the science frame
-        epoch_per_science_frame = np.append(epoch_per_science_frame, epoch_data[idx])
+        # Get the mean epoch in the frame to use as the data collection time
+        epoch_per_science_frame.append(
+            calculate_epoch_mean(epoch_data, idx, FRAME_SIZE)
+        )
 
-    # Add new data variables to the dataset
-    sci_dataset = sci_dataset.drop_vars("epoch")
-    sci_dataset.coords["epoch"] = epoch_per_science_frame
+    # Add new data variables to the dataset and update epoch coordinate
+    sci_dataset.coords["epoch"] = xr.DataArray(
+        np.array(epoch_per_science_frame, dtype=np.int64), dims=["epoch"]
+    )
     sci_dataset["count_rates_raw"] = xr.DataArray(
         count_rates, dims=["epoch"], name="count_rates_raw"
     )
@@ -373,6 +382,31 @@ def decompress_rates_16_to_32(packed: int) -> int:
     return decompressed_int
 
 
+def calculate_epoch_mean(
+    epoch_data: np.ndarray, idx: int, frame_size: int
+) -> np.floating:
+    """
+    Calculate the mean epoch for a science frame.
+
+    This function is used to get the center collection time for science data.
+
+    Parameters
+    ----------
+    epoch_data : np.ndarray
+        Array of epoch values for every science packet.
+    idx : int
+        Starting index of the science frame.
+    frame_size : int
+        Number of packets in the science frame.
+
+    Returns
+    -------
+    float
+        Mean epoch value for the science frame.
+    """
+    return np.mean([epoch_data[idx], epoch_data[idx + frame_size - 1]])
+
+
 def decom_hit(sci_dataset: xr.Dataset) -> xr.Dataset:
     """
     Group and decode HIT science data packets.

imap_processing/hit/l1a/hit_l1a.py

@@ -13,7 +13,11 @@ from imap_processing.hit.hit_utils import (
     get_datasets_by_apid,
     process_housekeeping_data,
 )
-from imap_processing.hit.l0.constants import MOD_10_MAPPING
+from imap_processing.hit.l0.constants import (
+    AZIMUTH_ANGLES,
+    MOD_10_MAPPING,
+    ZENITH_ANGLES,
+)
 from imap_processing.hit.l0.decom_hit import decom_hit
 
 logger = logging.getLogger(__name__)
@@ -104,12 +108,16 @@ def subcom_sectorates(sci_dataset: xr.Dataset) -> xr.Dataset:
     hdr_min_count_mod_10 = updated_dataset.hdr_minute_cnt.values % 10
 
     # Reference mod 10 mapping to initialize data structure for species and
-    # energy ranges and add 15x8 arrays with fill values for each science frame.
+    # energy ranges and add arrays with fill values for each science frame.
     num_frames = len(hdr_min_count_mod_10)
     data_by_species_and_energy_range = {
         key: {
             **value,
-            "counts": np.full((num_frames, 15, 8), fill_value=fillval, dtype=np.int64),
+            "counts": np.full(
+                (num_frames, len(AZIMUTH_ANGLES), len(ZENITH_ANGLES)),
+                fill_value=fillval,
+                dtype=np.int64,
+            ),
         }
         for key, value in MOD_10_MAPPING.items()
     }
@@ -279,39 +287,38 @@ def process_science(
     # Calculate uncertainties for count rates
     count_rates_dataset = calculate_uncertainties(count_rates_dataset)
 
-    # Logical sources for the two products.
-    logical_sources = ["imap_hit_l1a_counts", "imap_hit_l1a_direct-events"]
+    l1a_datasets: dict = {
+        "imap_hit_l1a_counts": count_rates_dataset,
+        "imap_hit_l1a_direct-events": pha_raw_dataset,
+    }
 
-    datasets = []
     # Update attributes and dimensions
-    for ds, logical_source in zip(
-        [count_rates_dataset, pha_raw_dataset], logical_sources
-    ):
+    for logical_source, ds in l1a_datasets.items():
         ds.attrs = attr_mgr.get_global_attributes(logical_source)
 
-        # TODO: Add CDF attributes to yaml once they're defined for L1A science data
         # Assign attributes and dimensions to each data array in the Dataset
         for field in ds.data_vars.keys():
             try:
-                # Create a dict of dimensions using the DEPEND_I keys in the
-                # attributes
-                dims = {
-                    key: value
-                    for key, value in attr_mgr.get_variable_attributes(field).items()
-                    if "DEPEND" in key
-                }
                 ds[field].attrs = attr_mgr.get_variable_attributes(field)
-                ds[field].assign_coords(dims)
             except KeyError:
                 print(f"Field {field} not found in attribute manager.")
                 logger.warning(f"Field {field} not found in attribute manager.")
 
-        # Skip schema check for epoch to prevent attr_mgr from adding the
-        # DEPEND_0 attribute which isn't required for epoch
-        ds.epoch.attrs = attr_mgr.get_variable_attributes("epoch", check_schema=False)
-
-        datasets.append(ds)
+        # check_schema=False to avoid attr_mgr adding stuff dimensions don't need
+        for dim in ds.dims:
+            ds[dim].attrs = attr_mgr.get_variable_attributes(dim, check_schema=False)
+            # TODO: should labels be added as coordinates? Check with SPDF
+            if dim != "epoch":
+                label_array = xr.DataArray(
+                    ds[dim].values.astype(str),
+                    name=f"{dim}_label",
+                    dims=[dim],
+                    attrs=attr_mgr.get_variable_attributes(
+                        f"{dim}_label", check_schema=False
+                    ),
+                )
+                ds.coords[f"{dim}_label"] = label_array
 
         logger.info(f"HIT L1A dataset created for {logical_source}")
 
-    return datasets
+    return list(l1a_datasets.values())

imap_processing/hit/l1b/hit_l1b.py

@@ -366,7 +366,8 @@ def subset_data_for_sectored_counts(
     A set of sectored data starts with hydrogen and ends with iron and correspond to
     the mod 10 values 0-9. The livetime values from the previous 10 minutes are used
     to calculate the rates for each set since those counts are transmitted 10 minutes
-    after they were collected.
+    after they were collected. Therefore, only complete sets of sectored counts where
+    livetime from the previous 10 minutes are available are included in the output.
 
     Parameters
     ----------
@@ -378,7 +379,7 @@ def subset_data_for_sectored_counts(
     Returns
     -------
     tuple[xr.Dataset, xr.DataArray]
-        Subsetted L1A counts dataset and corresponding livetime values.
+        Dataset of complete sectored counts and corresponding livetime values.
     """
     # Identify 10-minute intervals of complete sectored counts.
     bin_size = 10
@@ -392,16 +393,34 @@ def subset_data_for_sectored_counts(
     start_indices = np.where(matches)[0]
 
     # Filter out start indices that are less than or equal to the bin size
-    # since the previous 10 minutes are needed
-    start_indices = start_indices[start_indices > bin_size]
-    data_slice = slice(start_indices[0], start_indices[-1] + bin_size)
-
-    # Subset data to include only complete sets of sectored counts
-    l1b_sectored_rates_dataset = l1a_counts_dataset.isel(epoch=data_slice)
+    # since the previous 10 minutes are needed for calculating rates
+    if start_indices.size == 0:
+        logger.error(
+            "No data to process - valid start indices not found for "
+            "complete sectored counts."
+        )
+        raise ValueError("No valid start indices found for complete sectored counts.")
+    else:
+        start_indices = start_indices[start_indices >= bin_size]
+
+    # Subset data for complete sets of sectored counts.
+    # Each set of sectored counts is 10 minutes long, so we take the indices
+    # starting from the start indices and extend to the bin size of 10.
+    # This creates a 1D array of indices that correspond to the complete
+    # sets of sectored counts which is used to filter the L1A dataset and
+    # create the L1B sectored rates dataset.
+    data_indices = np.concatenate(
+        [np.arange(idx, idx + bin_size) for idx in start_indices]
+    )
+    l1b_sectored_rates_dataset = l1a_counts_dataset.isel(epoch=data_indices)
 
-    # Subset livetime staggered from sectored counts by 10 minutes
-    livetime_slice = slice(start_indices[0] - bin_size, start_indices[-1])
-    livetime = livetime[livetime_slice]
+    # Subset livetime values corresponding to the previous 10 minutes
+    # for each start index. This ensures the livetime data aligns correctly
+    # with the sectored counts for rate calculations.
+    livetime_indices = np.concatenate(
+        [np.arange(idx - bin_size, idx) for idx in start_indices]
+    )
+    livetime = livetime.isel(epoch=livetime_indices)
 
     return l1b_sectored_rates_dataset, livetime
 

imap_processing/hit/l2/hit_l2.py

@@ -134,18 +134,15 @@ def add_cdf_attributes(
         dataset[dim].attrs = attr_mgr.get_variable_attributes(dim, check_schema=False)
         # TODO: should labels be added as coordinates? Check with SPDF
         if dim != "epoch":
-            dataset = dataset.assign_coords(
-                {
-                    f"{dim}_label": xr.DataArray(
-                        dataset[dim].values.astype(str),
-                        name=f"{dim}_label",
-                        dims=[dim],
-                        attrs=attr_mgr.get_variable_attributes(
-                            f"{dim}_label", check_schema=False
-                        ),
-                    )
-                }
+            label_array = xr.DataArray(
+                dataset[dim].values.astype(str),
+                name=f"{dim}_label",
+                dims=[dim],
+                attrs=attr_mgr.get_variable_attributes(
+                    f"{dim}_label", check_schema=False
+                ),
             )
+            dataset.coords[f"{dim}_label"] = label_array
 
     return dataset
 

imap_processing/ialirt/constants.py

@@ -0,0 +1,38 @@
+"""Module for constants and useful shared classes used in I-ALiRT processing."""
+
+from dataclasses import dataclass
+
+import numpy as np
+
+
+@dataclass(frozen=True)
+class IalirtSwapiConstants:
+    """
+    Constants for I-ALiRT SWAPI which can be used across different levels or classes.
+
+    Attributes
+    ----------
+    BOLTZ: float
+        Boltzmann constant [J/K]
+    AT_MASS: float
+        Atomic mass [kg]
+    PROT_MASS: float
+        Mass of proton [kg]
+    EFF_AREA: float
+        Instrument effective area [m^2]
+    AZ_FOV: float
+        Azimuthal width of the field of view for solar wind [radians]
+    FWHM_WIDTH: float
+        Full Width at Half Maximum of energy width [unitless]
+    SPEED_EW: float
+        Speed width of energy passband [unitless]
+    """
+
+    # Scientific constants used in optimization model
+    boltz = 1.380649e-23  # Boltzmann constant, J/K
+    at_mass = 1.6605390666e-27  # atomic mass, kg
+    prot_mass = 1.007276466621 * at_mass  # mass of proton, kg
+    eff_area = 3.3e-5 * 1e-4  # effective area, meters squared
+    az_fov = np.deg2rad(30)  # azimuthal width of the field of view, radians
+    fwhm_width = 0.085  # FWHM of energy width
+    speed_ew = 0.5 * fwhm_width  # speed width of energy passband

imap_processing/ialirt/l0/parse_mag.py

@@ -390,7 +390,7 @@ def process_packet(
             {
                 "apid": 478,
                 "met": int(met.values.min()),
-                "utc": met_to_utc(met.values.min()).split(".")[0],
+                "met_in_utc": met_to_utc(met.values.min()).split(".")[0],
                 "ttj2000ns": int(met_to_ttj2000ns(met.values.min())),
                 "mag_4s_b_gse": [Decimal("0.0") for _ in range(3)],
                 "mag_4s_b_gsm": [Decimal("0.0") for _ in range(3)],

imap_processing/ialirt/l0/process_codice.py

@@ -0,0 +1,91 @@
+"""Functions to support I-ALiRT CoDICE processing."""
+
+import logging
+from decimal import Decimal
+from typing import Any
+
+import xarray as xr
+
+from imap_processing.codice import constants
+from imap_processing.ialirt.utils.time import calculate_time
+from imap_processing.spice.time import met_to_ttj2000ns, met_to_utc
+
+logger = logging.getLogger(__name__)
+
+FILLVAL_FLOAT32 = Decimal(str(-1.0e31))
+
+
+def process_codice(
+    dataset: xr.Dataset,
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    """
+    Create final data products.
+
+    Parameters
+    ----------
+    dataset : xr.Dataset
+        Decommed L0 data.
+
+    Returns
+    -------
+    codice_data : list[dict]
+        Dictionary of final data product.
+
+    Notes
+    -----
+    This function is incomplete and will need to be updated to include the
+    necessary calculations and data products.
+    - Calculate rates (assume 4 minutes per group)
+    - Calculate L2 CoDICE pseudodensities (pg 37 of Algorithm Document)
+    - Calculate the public data products
+    """
+    # For I-ALiRT SIT, the test data being used has all zeros and thus no
+    # groups can be found, thus there is no data to process
+    # TODO: Once I-ALiRT test data is acquired that actually has data in it,
+    #       this can be turned back on
+    # codicelo_data = create_ialirt_dataset(CODICEAPID.COD_LO_IAL, dataset)
+    # codicehi_data = create_ialirt_dataset(CODICEAPID.COD_HI_IAL, dataset)
+
+    # TODO: calculate rates
+    #       This will be done in codice.codice_l1b
+
+    # TODO: calculate L2 CoDICE pseudodensities
+    #       This will be done in codice.codice_l2
+
+    # TODO: calculate the public data products
+    #       This will be done in this module
+
+    # Create mock dataset for I-ALiRT SIT
+    # TODO: Once I-ALiRT test data is acquired that actually has data in it,
+    #       we should be able to properly populate the I-ALiRT data, but for
+    #       now, just create lists of dicts with FILLVALs
+    cod_lo_data = []
+    cod_hi_data = []
+
+    for epoch in range(len(dataset.epoch.data)):
+        sc_sclk_sec = dataset.sc_sclk_sec.data[epoch]
+        sc_sclk_sub_sec = dataset.sc_sclk_sub_sec.data[epoch]
+        met = calculate_time(sc_sclk_sec, sc_sclk_sub_sec, 256)
+        utc = met_to_utc(met).split(".")[0]
+        ttj2000ns = int(met_to_ttj2000ns(met))
+
+        epoch_data = {
+            "apid": int(dataset.pkt_apid[epoch].data),
+            "met": int(met),
+            "met_to_utc": utc,
+            "ttj2000ns": ttj2000ns,
+        }
+
+        # Add in CoDICE-Lo specific data
+        cod_lo_epoch_data = epoch_data.copy()
+        for field in constants.CODICE_LO_IAL_DATA_FIELDS:
+            cod_lo_epoch_data[f"codicelo_{field}"] = []
+        cod_lo_data.append(cod_lo_epoch_data)
+
+        # Add in CoDICE-Hi specific data
+        cod_hi_epoch_data = epoch_data.copy()
+        for field in constants.CODICE_HI_IAL_DATA_FIELDS:
+            cod_hi_epoch_data[f"codicehi_{field}"] = []
+        cod_hi_data.append(cod_hi_epoch_data)
+
+    return cod_lo_data, cod_hi_data

imap_processing/ialirt/l0/process_hit.py

@@ -1,7 +1,6 @@
 """Functions to support HIT processing."""
 
 import logging
-from decimal import Decimal
 
 import numpy as np
 import xarray as xr
@@ -170,27 +169,19 @@ def process_hit(xarray_data: xr.Dataset) -> list[dict]:
             {
                 "apid": 478,
                 "met": int(met),
-                "utc": met_to_utc(met).split(".")[0],
+                "met_in_utc": met_to_utc(met).split(".")[0],
                 "ttj2000ns": int(met_to_ttj2000ns(met)),
-                "hit_e_a_side_low_en": Decimal(
-                    str(l1["IALRT_RATE_1"] + l1["IALRT_RATE_2"])
-                ),
-                "hit_e_a_side_med_en": Decimal(
-                    str(l1["IALRT_RATE_5"] + l1["IALRT_RATE_6"])
-                ),
-                "hit_e_a_side_high_en": Decimal(str(l1["IALRT_RATE_7"])),
-                "hit_e_b_side_low_en": Decimal(
-                    str(l1["IALRT_RATE_11"] + l1["IALRT_RATE_12"])
-                ),
-                "hit_e_b_side_med_en": Decimal(
-                    str(l1["IALRT_RATE_15"] + l1["IALRT_RATE_16"])
-                ),
-                "hit_e_b_side_high_en": Decimal(str(l1["IALRT_RATE_17"])),
-                "hit_h_omni_med_en": Decimal(str(l1["H_12_15"] + l1["H_15_70"])),
-                "hit_h_a_side_high_en": Decimal(str(l1["IALRT_RATE_8"])),
-                "hit_h_b_side_high_en": Decimal(str(l1["IALRT_RATE_18"])),
-                "hit_he_omni_low_en": Decimal(str(l1["HE4_06_08"])),
-                "hit_he_omni_high_en": Decimal(str(l1["HE4_15_70"])),
+                "hit_e_a_side_low_en": int(l1["IALRT_RATE_1"] + l1["IALRT_RATE_2"]),
+                "hit_e_a_side_med_en": int(l1["IALRT_RATE_5"] + l1["IALRT_RATE_6"]),
+                "hit_e_a_side_high_en": int(l1["IALRT_RATE_7"]),
+                "hit_e_b_side_low_en": int(l1["IALRT_RATE_11"] + l1["IALRT_RATE_12"]),
+                "hit_e_b_side_med_en": int(l1["IALRT_RATE_15"] + l1["IALRT_RATE_16"]),
+                "hit_e_b_side_high_en": int(l1["IALRT_RATE_17"]),
+                "hit_h_omni_med_en": int(l1["H_12_15"] + l1["H_15_70"]),
+                "hit_h_a_side_high_en": int(l1["IALRT_RATE_8"]),
+                "hit_h_b_side_high_en": int(l1["IALRT_RATE_18"]),
+                "hit_he_omni_low_en": int(l1["HE4_06_08"]),
+                "hit_he_omni_high_en": int(l1["HE4_15_70"]),
             }
         )