imap-processing 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

imap_processing/hit/l1a/hit_l1a.py

@@ -1,39 +1,25 @@
 """Decommutate HIT CCSDS data and create L1a data products."""
 
 import logging
-from enum import IntEnum
 
 import numpy as np
 import xarray as xr
 
-from imap_processing import imap_module_directory
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
-from imap_processing.utils import packet_file_to_datasets
+from imap_processing.hit.hit_utils import (
+    HitAPID,
+    get_attribute_manager,
+    get_datasets_by_apid,
+    process_housekeeping_data,
+)
+from imap_processing.hit.l0.constants import MOD_10_MAPPING
+from imap_processing.hit.l0.decom_hit import decom_hit
 
 logger = logging.getLogger(__name__)
 
 # TODO review logging levels to use (debug vs. info)
 
 
-class HitAPID(IntEnum):
-    """
-    HIT APID Mappings.
-
-    Attributes
-    ----------
-    HIT_HSKP: int
-        Housekeeping
-    HIT_SCIENCE : int
-        Science
-    HIT_IALRT : int
-        I-ALiRT
-    """
-
-    HIT_HSKP = 1251
-    HIT_SCIENCE = 1252
-    HIT_IALRT = 1253
-
-
 def hit_l1a(packet_file: str, data_version: str) -> list[xr.Dataset]:
     """
     Will process HIT L0 data into L1A data products.
@@ -50,80 +36,137 @@ def hit_l1a(packet_file: str, data_version: str) -> list[xr.Dataset]:
     processed_data : list[xarray.Dataset]
         List of Datasets of L1A processed data.
     """
-    # TODO add logging
-
-    # Unpack ccsds file
-    packet_definition = (
-        imap_module_directory / "hit/packet_definitions/hit_packet_definitions.xml"
-    )
-    datasets_by_apid = packet_file_to_datasets(
-        packet_file=packet_file,
-        xtce_packet_definition=packet_definition,
-        use_derived_value=False,
-    )
+    # Unpack ccsds file to xarray datasets
+    datasets_by_apid = get_datasets_by_apid(packet_file)
 
     # Create the attribute manager for this data level
-    attr_mgr = ImapCdfAttributes()
-    attr_mgr.add_instrument_global_attrs(instrument="hit")
-    attr_mgr.add_instrument_variable_attrs(instrument="hit", level="l1a")
-    attr_mgr.add_global_attribute("Data_version", data_version)
+    attr_mgr = get_attribute_manager(data_version, "l1a")
 
-    # Process science to l1a.
+    l1a_datasets = []
+
+    # Process l1a data products
     if HitAPID.HIT_HSKP in datasets_by_apid:
-        datasets_by_apid[HitAPID.HIT_HSKP] = process_housekeeping(
-            datasets_by_apid[HitAPID.HIT_HSKP], attr_mgr
+        logger.info("Creating HIT L1A housekeeping dataset")
+        l1a_datasets.append(
+            process_housekeeping_data(
+                datasets_by_apid[HitAPID.HIT_HSKP], attr_mgr, "imap_hit_l1a_hk"
+            )
         )
     if HitAPID.HIT_SCIENCE in datasets_by_apid:
-        # TODO complete science data processing
-        print("Skipping science data for now")
-        datasets_by_apid[HitAPID.HIT_SCIENCE] = process_science(
-            datasets_by_apid[HitAPID.HIT_SCIENCE], attr_mgr
+        l1a_datasets.extend(
+            process_science(datasets_by_apid[HitAPID.HIT_SCIENCE], attr_mgr)
         )
-
-    return list(datasets_by_apid.values())
+    return l1a_datasets
 
 
-def concatenate_leak_variables(
-    dataset: xr.Dataset, adc_channels: xr.DataArray
-) -> xr.Dataset:
+def subcom_sectorates(sci_dataset: xr.Dataset) -> None:
     """
-    Concatenate leak variables in the dataset.
-
-    Updates the housekeeping dataset to replace the individual
-    leak_i_00, leak_i_01, ..., leak_i_63 variables with a single
-    leak_i variable as a 2D array. "i" here represents current
-    in the leakage current [Voltage] data.
+    Subcommutate sectorates data.
+
+    Sector rates data contains rates for 5 species and 10
+    energy ranges. This function subcommutates the sector
+    rates data by organizing the rates by species. Which
+    species and energy range the data belongs to is determined
+    by taking the mod 10 value of the corresponding header
+    minute count value in the dataset. A mapping of mod 10
+    values to species and energy ranges is provided in constants.py.
+
+    MOD_10_MAPPING = {
+        0: {"species": "H", "energy_min": 1.8, "energy_max": 3.6},
+        1: {"species": "H", "energy_min": 4, "energy_max": 6},
+        2: {"species": "H", "energy_min": 6, "energy_max": 10},
+        3: {"species": "4He", "energy_min": 4, "energy_max": 6},
+        ...
+        9: {"species": "Fe", "energy_min": 4, "energy_max": 12}}
+
+    The data is added to the dataset as new data fields named
+    according to their species. They have 4 dimensions: epoch
+    energy index, declination, and azimuth. The energy index
+    dimension is used to distinguish between the different energy
+    ranges the data belongs to. The energy min and max values for
+    each species are also added to the dataset as new data fields.
 
     Parameters
     ----------
-    dataset : xarray.Dataset
-        Dataset containing 64 leak variables.
-    adc_channels : xarray.DataArray
-        DataArray to be used as a dimension for the concatenated leak variables.
-
-    Returns
-    -------
-    dataset : xarray.Dataset
-        Updated dataset with concatenated leak variables.
+    sci_dataset : xarray.Dataset
+        Xarray dataset containing parsed HIT science data.
     """
-    # Stack 64 leak variables (leak_00, leak_01, ..., leak_63)
-    leak_vars = [dataset[f"leak_i_{i:02d}"] for i in range(64)]
-
-    # Concatenate along 'adc_channels' and reorder dimensions
-    stacked_leaks = xr.concat(leak_vars, dim=adc_channels).transpose(
-        "epoch", "adc_channels"
-    )
-    dataset["leak_i"] = stacked_leaks
-
-    # Drop the individual leak variables
-    updated_dataset = dataset.drop_vars([f"leak_i_{i:02d}" for i in range(64)])
-
-    return updated_dataset
+    # TODO:
+    #  - Update to use fill values defined in attribute manager which
+    #    isn't defined for L1A science data yet
+    #  - fix issues with fe_counts_sectored. The array has shape
+    #      (epoch: 28, fe_energy_index: 1, declination: 8, azimuth: 15),
+    #      but cdflib drops second dimension of size 1 and recognizes
+    #      only 3 total dimensions. Are dimensions of 1 ignored?
+
+    # Calculate mod 10 values
+    hdr_min_count_mod_10 = sci_dataset.hdr_minute_cnt.values % 10
+
+    # Reference mod 10 mapping to initialize data structure for species and
+    # energy ranges and add 8x15 arrays with fill values for each science frame.
+    num_frames = len(hdr_min_count_mod_10)
+    # TODO: add more specific dtype for rates (ex. int16) once this is defined by HIT
+    data_by_species_and_energy_range = {
+        key: {**value, "rates": np.full((num_frames, 8, 15), fill_value=-1, dtype=int)}
+        for key, value in MOD_10_MAPPING.items()
+    }
+
+    # Update rates for science frames where data is available
+    for i, mod_10 in enumerate(hdr_min_count_mod_10):
+        data_by_species_and_energy_range[mod_10]["rates"][i] = sci_dataset[
+            "sectorates"
+        ].values[i]
+
+    # H has 3 energy ranges, 4He, CNO, NeMgSi have 2, and Fe has 1.
+    # Aggregate sector rates and energy min/max values for each species.
+    # First, initialize dictionaries to store rates and min/max energy values by species
+    data_by_species: dict = {
+        value["species"]: {"rates": [], "energy_min": [], "energy_max": []}
+        for value in data_by_species_and_energy_range.values()
+    }
+
+    for value in data_by_species_and_energy_range.values():
+        species = value["species"]
+        data_by_species[species]["rates"].append(value["rates"])
+        data_by_species[species]["energy_min"].append(value["energy_min"])
+        data_by_species[species]["energy_max"].append(value["energy_max"])
+
+    # Add sector rates by species to the dataset
+    for species_type, data in data_by_species.items():
+        # Rates data has shape: energy_index, epoch, declination, azimuth
+        # Convert rates to numpy array and transpose axes to get
+        # shape: epoch, energy_index, declination, azimuth
+        rates_data = np.transpose(np.array(data["rates"]), axes=(1, 0, 2, 3))
+
+        species = species_type.lower()
+        sci_dataset[f"{species}_counts_sectored"] = xr.DataArray(
+            data=rates_data,
+            dims=["epoch", f"{species}_energy_index", "declination", "azimuth"],
+            name=f"{species}_counts_sectored",
+        )
+        sci_dataset[f"{species}_energy_min"] = xr.DataArray(
+            data=np.array(data["energy_min"], dtype=np.int8),
+            dims=[f"{species}_energy_index"],
+            name=f"{species}_energy_min",
+        )
+        sci_dataset[f"{species}_energy_max"] = xr.DataArray(
+            data=np.array(data["energy_max"], dtype=np.int8),
+            dims=[f"{species}_energy_index"],
+            name=f"{species}_energy_max",
+        )
+        # add energy index coordinate to the dataset
+        sci_dataset.coords[f"{species}_energy_index"] = xr.DataArray(
+            np.arange(sci_dataset.sizes[f"{species}_energy_index"], dtype=np.int8),
+            dims=[f"{species}_energy_index"],
+            name=f"{species}_energy_index",
+        )
 
 
-def process_science(dataset: xr.Dataset, attr_mgr: ImapCdfAttributes) -> xr.Dataset:
+def process_science(
+    dataset: xr.Dataset, attr_mgr: ImapCdfAttributes
+) -> list[xr.Dataset]:
     """
-    Will process science dataset for CDF product.
+    Will process science datasets for CDF products.
 
     Process binary science data for CDF creation. The data is
     grouped into science frames, decommutated and decompressed,
@@ -134,121 +177,64 @@ def process_science(dataset: xr.Dataset, attr_mgr: ImapCdfAttributes) -> xr.Data
     Parameters
     ----------
     dataset : xarray.Dataset
-        Dataset containing HIT science data.
+        A dataset containing HIT science data.
 
     attr_mgr : ImapCdfAttributes
         Attribute manager used to get the data product field's attributes.
 
     Returns
     -------
-    dataset : xarray.Dataset
-        An updated dataset ready for CDF conversion.
+    dataset : list
+        A list of science datasets ready for CDF conversion.
     """
     logger.info("Creating HIT L1A science datasets")
 
-    # Logical sources for the two products.
-    # logical_sources = ["imap_hit_l1a_sci-counts", "imap_hit_l1a_pulse-height-event"]
-
-    # TODO: Complete this function
-    #  - call decom_hit.py to decommutate the science data
-    #  - split the science data into count rates and event datasets
-    #  - update dimensions and add attributes to the dataset and data arrays
-    #  - return list of two datasets (count rates and events)?
+    # Decommutate and decompress the science data
+    sci_dataset = decom_hit(dataset)
 
-    # logger.info("HIT L1A event dataset created")
-    # logger.info("HIT L1A count rates dataset created")
+    # Organize sector rates by species type
+    subcom_sectorates(sci_dataset)
 
-    return dataset
-
-
-def process_housekeeping(
-    dataset: xr.Dataset, attr_mgr: ImapCdfAttributes
-) -> xr.Dataset:
-    """
-    Will process housekeeping dataset for CDF product.
-
-    Updates the housekeeping dataset to replace with a single
-    leak_i variable as a 2D array. Also updates the dataset
-    attributes and coordinates and data variable dimensions
-    according to specifications in a cdf yaml file.
-
-    Parameters
-    ----------
-    dataset : xarray.Dataset
-        Dataset containing HIT housekeeping data.
-
-    attr_mgr : ImapCdfAttributes
-        Attribute manager used to get the data product field's attributes.
-
-    Returns
-    -------
-    dataset : xarray.Dataset
-        An updated dataset ready for CDF conversion.
-    """
-    logger.info("Creating HIT L1A housekeeping dataset")
-
-    logical_source = "imap_hit_l1a_hk"
-
-    # Drop keys that are not CDF data variables
-    drop_keys = [
-        "pkt_apid",
-        "sc_tick",
-        "version",
-        "type",
-        "sec_hdr_flg",
-        "seq_flgs",
-        "src_seq_ctr",
-        "pkt_len",
-        "hskp_spare1",
-        "hskp_spare2",
-        "hskp_spare3",
-        "hskp_spare4",
-        "hskp_spare5",
-    ]
-
-    # Drop variables not needed for CDF
-    dataset = dataset.drop_vars(drop_keys)
-
-    # Create data arrays for dependencies
-    adc_channels = xr.DataArray(
-        np.arange(64, dtype=np.uint8),
-        name="adc_channels",
-        dims=["adc_channels"],
-        attrs=attr_mgr.get_variable_attributes("adc_channels"),
-    )
-
-    # NOTE: LABL_PTR_1 should be CDF_CHAR.
-    adc_channels_label = xr.DataArray(
-        adc_channels.values.astype(str),
-        name="adc_channels_label",
-        dims=["adc_channels_label"],
-        attrs=attr_mgr.get_variable_attributes("adc_channels_label"),
+    # Split the science data into count rates and event datasets
+    pha_raw_dataset = xr.Dataset(
+        {"pha_raw": sci_dataset["pha_raw"]}, coords={"epoch": sci_dataset["epoch"]}
     )
+    count_rates_dataset = sci_dataset.drop_vars("pha_raw")
 
-    # Update dataset coordinates and attributes
-    dataset = dataset.assign_coords(
-        {
-            "adc_channels": adc_channels,
-            "adc_channels_label": adc_channels_label,
-        }
-    )
-    dataset.attrs = attr_mgr.get_global_attributes(logical_source)
-
-    # Stack 64 leak variables (leak_00, leak_01, ..., leak_63)
-    dataset = concatenate_leak_variables(dataset, adc_channels)
-
-    # Assign attributes and dimensions to each data array in the Dataset
-    for field in dataset.data_vars.keys():
-        # Create a dict of dimensions using the DEPEND_I keys in the
-        # attributes
-        dims = {
-            key: value
-            for key, value in attr_mgr.get_variable_attributes(field).items()
-            if "DEPEND" in key
-        }
-        dataset[field].attrs = attr_mgr.get_variable_attributes(field)
-        dataset[field].assign_coords(dims)
-
-    dataset.epoch.attrs = attr_mgr.get_variable_attributes("epoch")
-
-    return dataset
+    # Logical sources for the two products.
+    logical_sources = ["imap_hit_l1a_count-rates", "imap_hit_l1a_pulse-height-events"]
+
+    datasets = []
+    # Update attributes and dimensions
+    for dataset, logical_source in zip(
+        [count_rates_dataset, pha_raw_dataset], logical_sources
+    ):
+        dataset.attrs = attr_mgr.get_global_attributes(logical_source)
+
+        # TODO: Add CDF attributes to yaml once they're defined for L1A science data
+        # Assign attributes and dimensions to each data array in the Dataset
+        for field in dataset.data_vars.keys():
+            try:
+                # Create a dict of dimensions using the DEPEND_I keys in the
+                # attributes
+                dims = {
+                    key: value
+                    for key, value in attr_mgr.get_variable_attributes(field).items()
+                    if "DEPEND" in key
+                }
+                dataset[field].attrs = attr_mgr.get_variable_attributes(field)
+                dataset[field].assign_coords(dims)
+            except KeyError:
+                print(f"Field {field} not found in attribute manager.")
+                logger.warning(f"Field {field} not found in attribute manager.")
+
+        dataset.epoch.attrs = attr_mgr.get_variable_attributes("epoch")
+        # Remove DEPEND_0 attribute from epoch variable added by attr_mgr.
+        # Not required for epoch
+        del dataset["epoch"].attrs["DEPEND_0"]
+
+        datasets.append(dataset)
+
+        logger.info(f"HIT L1A dataset created for {logical_source}")
+
+    return datasets

imap_processing/hit/l1b/hit_l1b.py

@@ -1,179 +1,59 @@
 """IMAP-HIT L1B data processing."""
 
 import logging
-from dataclasses import fields
 
-import numpy as np
 import xarray as xr
 
-from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
-from imap_processing.hit.l0.data_classes.housekeeping import Housekeeping
-from imap_processing.spice.time import met_to_j2000ns
+from imap_processing.hit.hit_utils import (
+    HitAPID,
+    get_attribute_manager,
+    get_datasets_by_apid,
+    process_housekeeping_data,
+)
 
 logger = logging.getLogger(__name__)
 
 # TODO review logging levels to use (debug vs. info)
 
 
-def hit_l1b(l1a_dataset: xr.Dataset, data_version: str) -> list[xr.Dataset]:
+def hit_l1b(dependencies: dict, data_version: str) -> list[xr.Dataset]:
     """
     Will process HIT data to L1B.
 
+    Processes dependencies needed to create L1B data products.
+
     Parameters
     ----------
-    l1a_dataset : xarray.Dataset
-        L1A data.
+    dependencies : dict
+        Dictionary of dependencies that are L1A xarray datasets
+        for science data and a file path string to a CCSDS file
+        for housekeeping data.
     data_version : str
         Version of the data product being created.
 
     Returns
     -------
-    cdf_filepaths : xarray.Dataset
-        L1B processed data.
+    processed_data : list[xarray.Dataset]
+        List of L1B datasets.
     """
-    # create the attribute manager for this data level
-    attr_mgr = ImapCdfAttributes()
-    attr_mgr.add_instrument_global_attrs(instrument="hit")
-    attr_mgr.add_instrument_variable_attrs(instrument="hit", level="l1b")
-    attr_mgr.add_global_attribute("Data_version", data_version)
-
-    # TODO: Check for type of L1A dataset and determine what L1B products to make
-    #   Need more info from instrument teams. Work with housekeeping data for now
-    logical_source = "imap_hit_l1b_hk"
-
-    # Create datasets
-    datasets = []
-    if "_hk" in logical_source:
-        dataset = create_hk_dataset(attr_mgr)
-        datasets.append(dataset)
-    elif "_sci" in logical_source:
-        # process science data. placeholder for future code
+    # Create the attribute manager for this data level
+    attr_mgr = get_attribute_manager(data_version, "l1b")
+
+    # Create L1B datasets
+    datasets: list = []
+    if "imap_hit_l0_raw" in dependencies:
+        # Unpack ccsds file to xarray datasets
+        packet_file = dependencies["imap_hit_l0_raw"]
+        datasets_by_apid = get_datasets_by_apid(packet_file, derived=True)
+        # Process housekeeping to l1b.
+        datasets.append(
+            process_housekeeping_data(
+                datasets_by_apid[HitAPID.HIT_HSKP], attr_mgr, "imap_hit_l1b_hk"
+            )
+        )
+        logger.info("HIT L1B housekeeping dataset created")
+    if "imap_hit_l1a_countrates" in dependencies:
+        # TODO: process science data. placeholder for future code
         pass
 
     return datasets
-
-
-# TODO: This is going to work differently when we have sample data
-def create_hk_dataset(attr_mgr: ImapCdfAttributes) -> xr.Dataset:
-    """
-    Create a housekeeping dataset.
-
-    Parameters
-    ----------
-    attr_mgr : ImapCdfAttributes
-        Attribute manager used to get the data product field's attributes.
-
-    Returns
-    -------
-    hk_dataset : xarray.dataset
-        Dataset with all data product fields in xarray.DataArray.
-    """
-    logger.info("Creating datasets for HIT L1B data")
-
-    # TODO: TEMPORARY. Need to update to use the L1B data class once that exists.
-    #  Using l1a housekeeping data class for now since l1b housekeeping has the
-    #  same data fields
-    data_fields = fields(Housekeeping)
-
-    # TODO define keys to skip. This will change later.
-    skip_keys = [
-        "shcoarse",
-        "ground_sw_version",
-        "packet_file_name",
-        "ccsds_header",
-        "leak_i_raw",
-    ]
-
-    logical_source = "imap_hit_l1b_hk"
-
-    # Create fake data for now
-
-    # Convert integers into datetime64[s]
-    epoch_converted_time = met_to_j2000ns([0, 1, 2])
-
-    # Shape for dims
-    n_epoch = 3
-    n_channels = 64
-
-    # Create xarray data arrays for dependencies
-    epoch_time = xr.DataArray(
-        data=epoch_converted_time,
-        name="epoch",
-        dims=["epoch"],
-        attrs=attr_mgr.get_variable_attributes("epoch"),
-    )
-
-    adc_channels = xr.DataArray(
-        np.arange(n_channels, dtype=np.uint16),
-        name="adc_channels",
-        dims=["adc_channels"],
-        attrs=attr_mgr.get_variable_attributes("adc_channels"),
-    )
-
-    # Create xarray dataset
-    hk_dataset = xr.Dataset(
-        coords={"epoch": epoch_time, "adc_channels": adc_channels},
-        attrs=attr_mgr.get_global_attributes(logical_source),
-    )
-
-    # Create xarray data array for each data field
-    for data_field in data_fields:
-        field = data_field.name.lower()
-        if field not in skip_keys:
-            # Create a list of all the dimensions using the DEPEND_I keys in the
-            # attributes
-            dims = [
-                value
-                for key, value in attr_mgr.get_variable_attributes(field).items()
-                if "DEPEND" in key
-            ]
-
-            # TODO: This is temporary.
-            #  The data will be set in the data class when that's created
-            if field == "leak_i":
-                # 2D array - needs two dims
-                hk_dataset[field] = xr.DataArray(
-                    np.ones((n_epoch, n_channels), dtype=np.uint16),
-                    dims=dims,
-                    attrs=attr_mgr.get_variable_attributes(field),
-                )
-            elif field in [
-                "preamp_l234a",
-                "preamp_l1a",
-                "preamp_l1b",
-                "preamp_l234b",
-                "temp0",
-                "temp1",
-                "temp2",
-                "temp3",
-                "analog_temp",
-                "hvps_temp",
-                "idpu_temp",
-                "lvps_temp",
-                "ebox_3d4vd",
-                "ebox_5d1vd",
-                "ebox_p12va",
-                "ebox_m12va",
-                "ebox_p5d7va",
-                "ebox_m5d7va",
-                "ref_p5v",
-                "l1ab_bias",
-                "l2ab_bias",
-                "l34a_bias",
-                "l34b_bias",
-                "ebox_p2d0vd",
-            ]:
-                hk_dataset[field] = xr.DataArray(
-                    np.ones(3, dtype=np.float16),
-                    dims=dims,
-                    attrs=attr_mgr.get_variable_attributes(field),
-                )
-            else:
-                hk_dataset[field] = xr.DataArray(
-                    [1, 1, 1],
-                    dims=dims,
-                    attrs=attr_mgr.get_variable_attributes(field),
-                )
-
-    logger.info("HIT L1B datasets created")
-    return hk_dataset