imap-processing 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

imap_processing/hi/utils.py

@@ -4,6 +4,7 @@ import re
 from collections.abc import Sequence
 from dataclasses import dataclass
 from enum import IntEnum
+from numbers import Number
 from typing import Optional, Union
 
 import numpy as np
@@ -99,6 +100,7 @@ def full_dataarray(
     attrs: dict,
     coords: Optional[dict[str, xr.DataArray]] = None,
     shape: Optional[Union[int, Sequence[int]]] = None,
+    fill_value: Optional[Number] = None,
 ) -> xr.DataArray:
     """
     Generate an empty xarray.DataArray with appropriate attributes.
@@ -117,6 +119,9 @@ def full_dataarray(
         Coordinate variables for the Dataset.
     shape : int or tuple
         Shape of ndarray data array to instantiate in the xarray.DataArray.
+    fill_value : optional, float
+        Override the fill value that the DataArray will be filled with. If not
+        supplied, the "FILLVAL" value from `attrs` will be used.
 
     Returns
     -------
@@ -133,9 +138,11 @@ def full_dataarray(
         shape = [coords[k].data.size for k in dims]  # type: ignore
     if hasattr(shape, "__len__") and len(shape) > len(dims):
         dims.append("")
+    if fill_value is None:
+        fill_value = _attrs["FILLVAL"]
 
     data_array = xr.DataArray(
-        np.full(shape, _attrs["FILLVAL"], dtype=dtype),
+        np.full(shape, fill_value, dtype=dtype),
         name=name,
         dims=dims,
         attrs=_attrs,
@@ -146,6 +153,8 @@ def full_dataarray(
 def create_dataset_variables(
     variable_names: list[str],
     variable_shape: Union[int, Sequence[int]],
+    coords: Optional[dict[str, xr.DataArray]] = None,
+    fill_value: Optional[Number] = None,
     att_manager_lookup_str: str = "{0}",
 ) -> dict[str, xr.DataArray]:
     """
@@ -157,8 +166,14 @@ def create_dataset_variables(
     ----------
     variable_names : list[str]
         List of variable names to create.
-    variable_shape : tuple[int]
+    variable_shape : int or sequence of int
         Shape of the new variables data ndarray.
+    coords : dict
+        Coordinate variables for the Dataset.
+    fill_value : optional, number
+        Value to fill the new variables data arrays with. If not supplied,
+        the fill value is pulled from the CDF variable attributes "FILLVAL"
+        attribute.
     att_manager_lookup_str : str
         String defining how to build the string passed to the
         CdfAttributeManager in order to retrieve the CdfAttributes for each
@@ -181,5 +196,7 @@ def create_dataset_variables(
         attrs = attr_mgr.get_variable_attributes(
             att_manager_lookup_str.format(var), check_schema=False
         )
-        new_variables[var] = full_dataarray(var, attrs, shape=variable_shape)
+        new_variables[var] = full_dataarray(
+            var, attrs, shape=variable_shape, coords=coords, fill_value=fill_value
+        )
     return new_variables

imap_processing/hit/l0/constants.py

@@ -47,7 +47,7 @@ COUNTS_DATA_STRUCTURE = {
     "spare": HITPacking(24, 24, (1,)),
     # ------------------------------------------
     # erates - contains livetime counters
-    "livetime": HITPacking(16, 16, (1,)),  # livetime counter
+    "livetime_counter": HITPacking(16, 16, (1,)),  # livetime counter
     "num_trig": HITPacking(16, 16, (1,)),  # number of triggers
     "num_reject": HITPacking(16, 16, (1,)),  # number of rejected events
     "num_acc_w_pha": HITPacking(
@@ -92,7 +92,7 @@ COUNTS_DATA_STRUCTURE = {
     # -------------------------------------------
     # other count rates
     "coinrates": HITPacking(16, 416, (26,)),  # coincidence rates
-    "bufrates": HITPacking(16, 512, (32,)),  # priority buffer rates
+    "pbufrates": HITPacking(16, 512, (32,)),  # priority buffer rates
     "l2fgrates": HITPacking(16, 2112, (132,)),  # range 2 foreground rates
     "l2bgrates": HITPacking(16, 192, (12,)),  # range 2 background rates
     "l3fgrates": HITPacking(16, 2672, (167,)),  # range 3 foreground rates

imap_processing/hit/l0/decom_hit.py

@@ -129,7 +129,7 @@ def is_sequential(counters: np.ndarray) -> np.bool_:
     bool
         True if the sequence counters are sequential, False otherwise.
     """
-    return np.all(np.diff(counters) == 1)
+    return np.all((np.diff(counters) % 16384) == 1)
 
 
 def get_valid_starting_indices(flags: np.ndarray, counters: np.ndarray) -> np.ndarray:

imap_processing/hit/l1a/hit_l1a.py

@@ -19,6 +19,9 @@ logger = logging.getLogger(__name__)
 
 # TODO review logging levels to use (debug vs. info)
 
+# Fill value for missing data
+fillval = -9223372036854775808
+
 
 def hit_l1a(packet_file: str, data_version: str) -> list[xr.Dataset]:
     """
@@ -91,14 +94,6 @@ def subcom_sectorates(sci_dataset: xr.Dataset) -> None:
     sci_dataset : xarray.Dataset
         Xarray dataset containing parsed HIT science data.
     """
-    # TODO:
-    #  - Update to use fill values defined in attribute manager which
-    #    isn't defined for L1A science data yet
-    #  - fix issues with fe_counts_sectored. The array has shape
-    #      (epoch: 28, fe_energy_index: 1, declination: 8, azimuth: 15),
-    #      but cdflib drops second dimension of size 1 and recognizes
-    #      only 3 total dimensions. Are dimensions of 1 ignored?
-
     # Calculate mod 10 values
     hdr_min_count_mod_10 = sci_dataset.hdr_minute_cnt.values % 10
 
@@ -107,7 +102,10 @@ def subcom_sectorates(sci_dataset: xr.Dataset) -> None:
     num_frames = len(hdr_min_count_mod_10)
     # TODO: add more specific dtype for rates (ex. int16) once this is defined by HIT
     data_by_species_and_energy_range = {
-        key: {**value, "rates": np.full((num_frames, 8, 15), fill_value=-1, dtype=int)}
+        key: {
+            **value,
+            "rates": np.full((num_frames, 8, 15), fill_value=fillval, dtype=int),
+        }
         for key, value in MOD_10_MAPPING.items()
     }
 
@@ -162,6 +160,85 @@ def subcom_sectorates(sci_dataset: xr.Dataset) -> None:
         )
 
 
+def calculate_uncertainties(dataset: xr.Dataset) -> xr.Dataset:
+    """
+    Calculate uncertainties for each counts data variable in the dataset.
+
+    Calculate the upper and lower uncertainties. The uncertainty for
+    the raw Lev1A HIT data will be calculated as asymmetric Poisson
+    uncertainty as prescribed in Gehrels 1986 (DOI: 10.1086/164079).
+    See section 5.5 in the algorithm document for details.
+
+    The upper uncertainty will be calculated as
+        DELTA_PLUS = sqrt(counts + 1) + 1
+
+    The lower uncertainty will be calculated as
+        DELTA_MINUS = sqrt(counts)
+
+    Parameters
+    ----------
+    dataset : xarray.Dataset
+        The dataset containing counts data.
+
+    Returns
+    -------
+    dataset : xarray.Dataset
+        The dataset with added uncertainties for each counts data variable.
+    """
+    # Variables that aren't counts data and should be skipped in the calculation
+    ignore_vars = [
+        "version",
+        "type",
+        "sec_hdr_flg",
+        "pkt_apid",
+        "seq_flgs",
+        "src_seq_ctr",
+        "pkt_len",
+        "hdr_unit_num",
+        "hdr_frame_version",
+        "hdr_dynamic_threshold_state",
+        "hdr_leak_conv",
+        "hdr_heater_duty_cycle",
+        "hdr_code_ok",
+        "hdr_minute_cnt",
+        "livetime_counter",
+        "h_energy_min",
+        "h_energy_max",
+        "he4_energy_min",
+        "he4_energy_max",
+        "cno_energy_min",
+        "cno_energy_max",
+        "nemgsi_energy_min",
+        "nemgsi_energy_max",
+        "fe_energy_min",
+        "fe_energy_max",
+    ]
+
+    # Counts data that need uncertainties calculated
+    count_vars = set(dataset.data_vars) - set(ignore_vars)
+
+    # Calculate uncertainties for counts data variables.
+    # Arrays with fill values (i.e. missing data) are skipped in this calculation
+    # but are kept in the new data arrays to retain shape and dimensions.
+    for var in count_vars:
+        mask = dataset[var] != fillval  # Mask for valid values
+        # Ensure that the values are positive before taking the square root
+        safe_values_plus = np.maximum(dataset[var] + 1, 0).astype(np.float32)
+        safe_values_minus = np.maximum(dataset[var], 0).astype(np.float32)
+
+        dataset[f"{var}_delta_plus"] = xr.DataArray(
+            np.where(
+                mask, np.sqrt(safe_values_plus) + 1, dataset[var].astype(np.float32)
+            ),
+            dims=dataset[var].dims,
+        )
+        dataset[f"{var}_delta_minus"] = xr.DataArray(
+            np.where(mask, np.sqrt(safe_values_minus), dataset[var].astype(np.float32)),
+            dims=dataset[var].dims,
+        )
+    return dataset
+
+
 def process_science(
     dataset: xr.Dataset, attr_mgr: ImapCdfAttributes
 ) -> list[xr.Dataset]:
@@ -201,6 +278,9 @@ def process_science(
     )
     count_rates_dataset = sci_dataset.drop_vars("pha_raw")
 
+    # Calculate uncertainties for count rates
+    count_rates_dataset = calculate_uncertainties(count_rates_dataset)
+
     # Logical sources for the two products.
     logical_sources = ["imap_hit_l1a_count-rates", "imap_hit_l1a_pulse-height-events"]
 
@@ -228,10 +308,11 @@ def process_science(
                 print(f"Field {field} not found in attribute manager.")
                 logger.warning(f"Field {field} not found in attribute manager.")
 
-        dataset.epoch.attrs = attr_mgr.get_variable_attributes("epoch")
-        # Remove DEPEND_0 attribute from epoch variable added by attr_mgr.
-        # Not required for epoch
-        del dataset["epoch"].attrs["DEPEND_0"]
+        # Skip schema check for epoch to prevent attr_mgr from adding the
+        # DEPEND_0 attribute which isn't required for epoch
+        dataset.epoch.attrs = attr_mgr.get_variable_attributes(
+            "epoch", check_schema=False
+        )
 
         datasets.append(dataset)
 

imap_processing/hit/l1b/hit_l1b.py

@@ -4,6 +4,7 @@ import logging
 
 import xarray as xr
 
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.hit.hit_utils import (
     HitAPID,
     get_attribute_manager,
@@ -26,7 +27,7 @@ def hit_l1b(dependencies: dict, data_version: str) -> list[xr.Dataset]:
     ----------
     dependencies : dict
         Dictionary of dependencies that are L1A xarray datasets
-        for science data and a file path string to a CCSDS file
+        for science data and a file path string to an L0 file
         for housekeeping data.
     data_version : str
         Version of the data product being created.
@@ -34,26 +35,174 @@ def hit_l1b(dependencies: dict, data_version: str) -> list[xr.Dataset]:
     Returns
     -------
     processed_data : list[xarray.Dataset]
-        List of L1B datasets.
+        List of four L1B datasets.
     """
     # Create the attribute manager for this data level
     attr_mgr = get_attribute_manager(data_version, "l1b")
 
     # Create L1B datasets
-    datasets: list = []
+    l1b_datasets: list = []
     if "imap_hit_l0_raw" in dependencies:
         # Unpack ccsds file to xarray datasets
         packet_file = dependencies["imap_hit_l0_raw"]
         datasets_by_apid = get_datasets_by_apid(packet_file, derived=True)
-        # Process housekeeping to l1b.
-        datasets.append(
+        # Process housekeeping to L1B.
+        l1b_datasets.append(
             process_housekeeping_data(
                 datasets_by_apid[HitAPID.HIT_HSKP], attr_mgr, "imap_hit_l1b_hk"
             )
         )
         logger.info("HIT L1B housekeeping dataset created")
-    if "imap_hit_l1a_countrates" in dependencies:
-        # TODO: process science data. placeholder for future code
-        pass
+    if "imap_hit_l1a_count-rates" in dependencies:
+        # Process science data to L1B datasets
+        l1a_counts_dataset = dependencies["imap_hit_l1a_count-rates"]
+        l1b_datasets.extend(process_science_data(l1a_counts_dataset, attr_mgr))
+        logger.info("HIT L1B science datasets created")
 
-    return datasets
+    return l1b_datasets
+
+
+def process_science_data(
+    raw_counts_dataset: xr.Dataset, attr_mgr: ImapCdfAttributes
+) -> list[xr.Dataset]:
+    """
+    Will create L1B science datasets for CDF products.
+
+    Process L1A raw counts data to create L1B science data for
+    CDF creation. This function will create three L1B science
+    datasets: standard rates, summed rates, and sectored rates.
+    It will also update dataset attributes, coordinates and
+    data variable dimensions according to specifications in
+    a CDF yaml file.
+
+    Parameters
+    ----------
+    raw_counts_dataset : xr.Dataset
+        The L1A counts dataset.
+    attr_mgr : AttributeManager
+        The attribute manager for the L1B data level.
+
+    Returns
+    -------
+    dataset : list
+        The processed L1B science datasets as xarray datasets.
+    """
+    logger.info("Creating HIT L1B science datasets")
+
+    # Logical sources for the three L1B science products.
+    # TODO: add logical sources for other l1b products once processing functions
+    #  are written. "imap_hit_l1b_summed-rates", "imap_hit_l1b_sectored-rates"
+    logical_sources = ["imap_hit_l1b_standard-rates"]
+
+    # TODO: Write functions to create the following datasets
+    #  Process summed rates dataset
+    #  Process sectored rates dataset
+
+    # Create a standard rates dataset
+    standard_rates_dataset = process_standard_rates_data(raw_counts_dataset)
+
+    l1b_science_datasets = []
+    # Update attributes and dimensions
+    for dataset, logical_source in zip([standard_rates_dataset], logical_sources):
+        dataset.attrs = attr_mgr.get_global_attributes(logical_source)
+
+        # TODO: Add CDF attributes to yaml once they're defined for L1B science data
+        # Assign attributes and dimensions to each data array in the Dataset
+        for field in dataset.data_vars.keys():
+            try:
+                # Create a dict of dimensions using the DEPEND_I keys in the
+                # attributes
+                dims = {
+                    key: value
+                    for key, value in attr_mgr.get_variable_attributes(field).items()
+                    if "DEPEND" in key
+                }
+                dataset[field].attrs = attr_mgr.get_variable_attributes(field)
+                dataset[field].assign_coords(dims)
+            except KeyError:
+                print(f"Field {field} not found in attribute manager.")
+                logger.warning(f"Field {field} not found in attribute manager.")
+
+        # Skip schema check for epoch to prevent attr_mgr from adding the
+        # DEPEND_0 attribute which isn't required for epoch
+        dataset.epoch.attrs = attr_mgr.get_variable_attributes(
+            "epoch", check_schema=False
+        )
+
+        l1b_science_datasets.append(dataset)
+
+        logger.info(f"HIT L1B dataset created for {logical_source}")
+
+    return l1b_science_datasets
+
+
+def process_standard_rates_data(raw_counts_dataset: xr.Dataset) -> xr.Dataset:
+    """
+    Will process L1B standard rates data from raw L1A counts data.
+
+    Parameters
+    ----------
+    raw_counts_dataset : xr.Dataset
+        The L1A counts dataset.
+
+    Returns
+    -------
+    xr.Dataset
+        The processed L1B standard rates dataset.
+    """
+    # Create a new dataset to store the L1B standard rates
+    l1b_standard_rates_dataset = xr.Dataset()
+
+    # Add required coordinates from the raw_counts_dataset
+    coords = [
+        "epoch",
+        "gain",
+        "sngrates_index",
+        "coinrates_index",
+        "pbufrates_index",
+        "l2fgrates_index",
+        "l2bgrates_index",
+        "l3fgrates_index",
+        "l3bgrates_index",
+        "penfgrates_index",
+        "penbgrates_index",
+        "ialirtrates_index",
+    ]
+    l1b_standard_rates_dataset = l1b_standard_rates_dataset.assign_coords(
+        {coord: raw_counts_dataset.coords[coord] for coord in coords}
+    )
+
+    # Add dynamic threshold field
+    l1b_standard_rates_dataset["dynamic_threshold_state"] = raw_counts_dataset[
+        "hdr_dynamic_threshold_state"
+    ]
+
+    # Define fields from the raw_counts_dataset to calculate standard rates from
+    standard_rate_fields = [
+        "sngrates",
+        "coinrates",
+        "pbufrates",
+        "l2fgrates",
+        "l2bgrates",
+        "l3fgrates",
+        "l3bgrates",
+        "penfgrates",
+        "penbgrates",
+        "ialirtrates",
+        "l4fgrates",
+        "l4bgrates",
+    ]
+
+    # Calculate livetime from the livetime counter
+    livetime = raw_counts_dataset["livetime_counter"] / 270
+
+    # Calculate standard rates by dividing the raw counts by livetime for
+    # data variables with names that contain a substring from a defined
+    # list of field names.
+    for var in raw_counts_dataset.data_vars:
+        if var != "livetime_counter" and any(
+            base_var in var for base_var in standard_rate_fields
+        ):
+            l1b_standard_rates_dataset[var] = raw_counts_dataset[var] / livetime
+
+    return l1b_standard_rates_dataset

imap_processing/ialirt/l0/process_codicehi.py

@@ -0,0 +1,156 @@
+"""Functions to support I-ALiRT CoDICE Hi processing."""
+
+import logging
+from typing import Any
+
+import numpy as np
+import xarray as xr
+
+logger = logging.getLogger(__name__)
+
+
+def find_groups(data: xr.Dataset) -> xr.Dataset:
+    """
+    Find all occurrences of the sequential set of 234 values 0-233.
+
+    If a value is missing, or we are starting/ending
+    in the middle of a sequence we do not count that as a valid group.
+
+    Parameters
+    ----------
+    data : xr.Dataset
+        CoDICE Hi Dataset.
+
+    Returns
+    -------
+    grouped_data : xr.Dataset
+        Grouped data.
+    """
+    subcom_range = (0, 233)
+
+    data = data.sortby("cod_hi_acq", ascending=True)
+
+    # Use cod_hi_counter == 0 to define the beginning of the group.
+    # Find cod_hi_acq at this index and use it as the beginning time for the group.
+    start_sc_ticks = data["cod_hi_acq"][(data["cod_hi_counter"] == subcom_range[0])]
+    start_sc_tick = start_sc_ticks.min()
+    # Use cod_hi_counter == 233 to define the end of the group.
+    last_sc_ticks = data["cod_hi_acq"][
+        ([data["cod_hi_counter"] == subcom_range[-1]][-1])
+    ]
+    last_sc_tick = last_sc_ticks.max()
+
+    # Filter out data before the first cod_hi_counter=0 and
+    # after the last cod_hi_counter=233 and cod_hi_counter values != 0-233.
+    grouped_data = data.where(
+        (data["cod_hi_acq"] >= start_sc_tick)
+        & (data["cod_hi_acq"] <= last_sc_tick)
+        & (data["cod_hi_counter"] >= subcom_range[0])
+        & (data["cod_hi_counter"] <= subcom_range[-1]),
+        drop=True,
+    )
+
+    # Assign labels based on the cod_hi_acq times.
+    group_labels = np.searchsorted(
+        start_sc_ticks, grouped_data["cod_hi_acq"], side="right"
+    )
+    # Example:
+    # grouped_data.coords
+    # Coordinates:
+    #   * epoch    (epoch) int64 7kB 315922822184000000 ... 315923721184000000
+    #   * group    (group) int64 7kB 1 1 1 1 1 1 1 1 1 ... 15 15 15 15 15 15 15 15 15
+    grouped_data["group"] = ("group", group_labels)
+
+    return grouped_data
+
+
+def append_cod_hi_data(dataset: xr.Dataset) -> xr.Dataset:
+    """
+    Append the cod_hi_## data values and create an xarray.
+
+    Parameters
+    ----------
+    dataset : xr.Dataset
+        Original dataset of group.
+
+    Returns
+    -------
+    appended_dataset : xr.Dataset
+        Dataset with cod_hi_## stacked.
+    """
+    # Number of codice hi data rows
+    num_cod_hi_rows = 5
+    cod_hi_data = np.stack(
+        [dataset[f"cod_hi_data_{i:02}"].values for i in range(num_cod_hi_rows)], axis=1
+    )
+
+    repeated_data = {
+        var: np.repeat(dataset[var].values, num_cod_hi_rows)
+        for var in dataset.data_vars
+        if not var.startswith("cod_hi_data_")
+    }
+
+    repeated_data["cod_hi_appended"] = cod_hi_data.flatten()
+    repeated_epoch = np.repeat(dataset["epoch"].values, num_cod_hi_rows)
+
+    appended_dataset = xr.Dataset(
+        data_vars={name: ("epoch", values) for name, values in repeated_data.items()},
+        coords={"epoch": repeated_epoch},
+    )
+
+    return appended_dataset
+
+
+def process_codicehi(xarray_data: xr.Dataset) -> list[dict]:
+    """
+    Create final data products.
+
+    Parameters
+    ----------
+    xarray_data : xr.Dataset
+        Parsed data.
+
+    Returns
+    -------
+    codicehi_data : list[dict]
+        Dictionary of final data product.
+
+    Notes
+    -----
+    This function is incomplete and will need to be updated to include the
+    necessary calculations and data products.
+    - Calculate species counts (pg 27 of Algorithm Document)
+    - Calculate rates (assume 4 minutes per group)
+    - Calculate L2 CoDICE pseudodensities (pg 37 of Algorithm Document)
+    - Calculate the public data products
+    """
+    grouped_data = find_groups(xarray_data)
+    unique_groups = np.unique(grouped_data["group"])
+    codicehi_data: list[dict[str, Any]] = [{}]
+
+    for group in unique_groups:
+        # cod_hi_counter values for the group should be 0-233 with no duplicates.
+        subcom_values = grouped_data["cod_hi_counter"][
+            (grouped_data["group"] == group).values
+        ]
+
+        # Ensure no duplicates and all values from 0 to 233 are present
+        if not np.array_equal(subcom_values, np.arange(234)):
+            logger.warning(
+                f"Group {group} does not contain all values from 0 to "
+                f"233 without duplicates."
+            )
+            continue
+
+        mask = grouped_data["group"] == group
+        filtered_indices = np.where(mask)[0]
+        group_data = grouped_data.isel(epoch=filtered_indices)
+
+        append_cod_hi_data(group_data)
+
+        # TODO: calculate species counts
+        # TODO: calculate rates
+        # TODO: calculate L2 CoDICE pseudodensities
+        # TODO: calculate the public data products
+
+    return codicehi_data

imap_processing/ialirt/l0/process_codicelo.py

@@ -41,9 +41,12 @@ def find_groups(data: xr.Dataset) -> xr.Dataset:
     last_sc_tick = last_sc_ticks.max()
 
     # Filter out data before the first cod_lo_counter=0 and
-    # after the last cod_lo_counter=232.
+    # after the last cod_lo_counter=232 and cod_lo_counter values != 0-232.
     grouped_data = data.where(
-        (data["cod_lo_acq"] >= start_sc_tick) & (data["cod_lo_acq"] <= last_sc_tick),
+        (data["cod_lo_acq"] >= start_sc_tick)
+        & (data["cod_lo_acq"] <= last_sc_tick)
+        & (data["cod_lo_counter"] >= subcom_range[0])
+        & (data["cod_lo_counter"] <= subcom_range[-1]),
         drop=True,
     )