imap-processing 0.16.2__py3-none-any.whl → 0.18.0__py3-none-any.whl

imap_processing/idex/idex_l2c.py

@@ -20,231 +20,65 @@ Examples
 
 import logging
 
-import astropy_healpix.healpy as hp
-import numpy as np
 import xarray as xr
 
-from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.ena_maps.ena_maps import SkyTilingType
-from imap_processing.ena_maps.utils.coordinates import CoordNames
-from imap_processing.ena_maps.utils.spatial_utils import AzElSkyGrid
 from imap_processing.idex.idex_constants import (
     IDEX_EVENT_REFERENCE_FRAME,
-    IDEX_HEALPIX_NESTED,
-    IDEX_HEALPIX_NSIDE,
     IDEX_SPACING_DEG,
 )
-from imap_processing.idex.idex_utils import get_idex_attrs
+from imap_processing.idex.idex_utils import get_idex_attrs, setup_dataset
 
 logger = logging.getLogger(__name__)
 
 
-def idex_l2c(l2b_dataset: xr.Dataset) -> list[xr.Dataset]:
+def idex_l2c(l2b_datasets: list[xr.Dataset]) -> xr.Dataset:
     """
     Will process IDEX l2b data to create l2c data products.
 
     Parameters
     ----------
-    l2b_dataset : xarray.Dataset
-        IDEX L2b dataset.
+    l2b_datasets : list[xarray.Dataset]
+        IDEX L2b datasets.
 
     Returns
     -------
-    l2b_dataset : list[xarray.Dataset]
+    l2b_dataset : xarray.Dataset
         The``xarray`` dataset containing the science data and supporting metadata.
     """
-    logger.info(
-        f"Running IDEX L2C processing on datasets: "
-        f"{l2b_dataset.attrs['Logical_source']}"
-    )
+    logger.info("Running IDEX L2C processing")
     # create the attribute manager for this data level
     idex_attrs = get_idex_attrs("l2c")
-    # Epoch should be the start of the collection period.
-    # TODO should epoch be start of sci acquisition?
-    epoch = xr.DataArray(
-        l2b_dataset["epoch"].data[0:1].astype(np.int64),
-        name="epoch",
-        dims=["epoch"],
-        attrs=idex_attrs.get_variable_attributes(
-            "epoch_collection_set", check_schema=False
-        ),
-    )
-    l2c_healpix_dataset = idex_healpix_map(l2b_dataset, epoch, idex_attrs)
-    l2c_rectangular_dataset = idex_rectangular_map(l2b_dataset, epoch, idex_attrs)
-
-    # TODO exposure time
-    logger.info("IDEX L2C science data processing completed.")
-    return [l2c_healpix_dataset, l2c_rectangular_dataset]
-
-
-def idex_healpix_map(
-    l1b_dataset: xr.Dataset,
-    epoch_da: xr.DataArray,
-    idex_attrs: ImapCdfAttributes,
-    nside: int = IDEX_HEALPIX_NSIDE,
-    nested: bool = IDEX_HEALPIX_NESTED,
-) -> xr.Dataset:
-    """
-    Create a healpix map out of a l1b dataset.
-
-    Parameters
-    ----------
-    l1b_dataset : xarray.Dataset
-        IDEX L2b dataset.
-    epoch_da : xarray.DataArray
-        Epoch data array of the collection. Size: (1,).
-    idex_attrs : ImapCdfAttributes
-        The attribute manager for this data level.
-    nside : int
-        Healpix nside parameter.
-    nested : bool
-        Healpix nested parameter.
-
-    Returns
-    -------
-    map : xarray.Dataset
-        Spatially binned dust counts in a healpix map format.
-    """
-    longitude = l1b_dataset["longitude"]
-    latitude = l1b_dataset["latitude"]
-
-    # Get the healpix indices
-    hpix_idx = hp.ang2pix(
-        nside, nest=nested, lonlat=True, theta=longitude, phi=latitude
-    )
-
-    n_pix = hp.nside2npix(nside)
-    healpix = xr.DataArray(
-        np.arange(n_pix),
-        name=CoordNames.HEALPIX_INDEX.value,
-        dims=CoordNames.HEALPIX_INDEX.value,
-        attrs=idex_attrs.get_variable_attributes("pixel_index", check_schema=False),
-    )
-
-    # Create a histogram of the raw dust event counts for each pixel
-    counts = np.histogram(hpix_idx, bins=n_pix, range=(0, n_pix))[0]
-    # Add epoch dimension
-    counts_da = xr.DataArray(
-        counts[np.newaxis, :].astype(np.int64),
-        name="counts",
-        dims=("epoch", CoordNames.HEALPIX_INDEX.value),
-        attrs=idex_attrs.get_variable_attributes("healpix_counts"),
-    )
-    pixel_label = xr.DataArray(
-        healpix.astype(str),
-        name="pixel_label",
-        dims="pixel_index",
-        attrs=idex_attrs.get_variable_attributes("pixel_label", check_schema=False),
-    )
-    l2c_dataset = xr.Dataset(
-        coords={CoordNames.HEALPIX_INDEX.value: healpix, "epoch": epoch_da},
-        data_vars={
-            "counts": counts_da,
-            "longitude": longitude,
-            "latitude": latitude,
-            "pixel_label": pixel_label,
-        },
+    # Concat the list of l2b datasets into a single dataset
+    # Only concat the variables that have "epoch" as a dimension
+    l2b_dataset = xr.concat(
+        l2b_datasets, "epoch", data_vars="minimal", coords="minimal"
     )
-    map_attrs = {
-        "Sky_tiling_type": SkyTilingType.HEALPIX.value,
-        "HEALPix_nside": str(nside),
-        "HEALPix_nest": str(nested),
-        "Spice_reference_frame": IDEX_EVENT_REFERENCE_FRAME.name,
-        "num_points": str(n_pix),
-    } | idex_attrs.get_global_attributes("imap_idex_l2c_sci-healpix")
-    l2c_dataset.attrs.update(map_attrs)
-
-    return l2c_dataset
-
 
-def idex_rectangular_map(
-    l1b_dataset: xr.Dataset,
-    epoch_da: xr.DataArray,
-    idex_attrs: ImapCdfAttributes,
-    spacing_deg: int = IDEX_SPACING_DEG,
-) -> xr.Dataset:
-    """
-    Create a rectangular map out of a l1b dataset.
-
-    Parameters
-    ----------
-    l1b_dataset : xarray.Dataset
-        IDEX L2b dataset.
-    epoch_da : xarray.DataArray
-        Epoch data array of the collection. Size: (1,).
-    idex_attrs : ImapCdfAttributes
-        The attribute manager for this data level.
-    spacing_deg : int
-        The spacing in degrees for the rectangular grid.
-
-    Returns
-    -------
-    map : xarray.Dataset
-        Spatially binned dust counts in a rectangular map format.
-    """
-    # Get the rectangular grid with the specified spacing
-    grid = AzElSkyGrid(spacing_deg)
-    # Make sure longitude values are in the range [0, 360)
-    longitude_wrapped = np.mod(l1b_dataset["longitude"], 360)
-    latitude = l1b_dataset["latitude"]
-    # Create a 2d histogram of the raw dust event counts for each pixel using the grid
-    # bin edges
-    counts, _, _ = np.histogram2d(
-        longitude_wrapped, latitude, bins=[grid.az_bin_edges, grid.el_bin_edges]
-    )
-    counts_da = xr.DataArray(
-        counts[np.newaxis, :, :].astype(np.int64),
-        name="counts",
-        dims=("epoch", "rectangular_lon_pixel", "rectangular_lat_pixel"),
-        attrs=idex_attrs.get_variable_attributes("rectangular_counts"),
-    )
-    rec_lon_pixels = xr.DataArray(
-        name="rectangular_lon_pixel",
-        data=grid.az_bin_midpoints,
-        dims="rectangular_lon_pixel",
-        attrs=idex_attrs.get_variable_attributes(
-            "rectangular_lon_pixel", check_schema=False
-        ),
-    )
-    rec_lat_pixels = xr.DataArray(
-        name="rectangular_lat_pixel",
-        data=grid.el_bin_midpoints,
-        dims="rectangular_lat_pixel",
-        attrs=idex_attrs.get_variable_attributes(
-            "rectangular_lat_pixel", check_schema=False
-        ),
-    )
-
-    l2c_dataset = xr.Dataset(
-        coords={
-            "epoch": epoch_da,
-            "rectangular_lon_pixel": rec_lon_pixels,
-            "rectangular_lat_pixel": rec_lat_pixels,
-        },
-        data_vars={
-            "counts": counts_da,
-            "longitude": longitude_wrapped,
-            "latitude": latitude,
-            "rectangular_lon_pixel_label": rec_lon_pixels.astype(str),
-            "rectangular_lat_pixel_label": rec_lat_pixels.astype(str),
-        },
-    )
-    l2c_dataset[
-        "rectangular_lon_pixel_label"
-    ].attrs = idex_attrs.get_variable_attributes(
-        "rectangular_lon_pixel_label", check_schema=False
-    )
-    l2c_dataset[
-        "rectangular_lat_pixel_label"
-    ].attrs = idex_attrs.get_variable_attributes(
-        "rectangular_lat_pixel_label", check_schema=False
-    )
+    arrays_to_copy = [
+        "counts_by_charge_map",
+        "counts_by_mass_map",
+        "rate_by_charge_map",
+        "rate_by_mass_map",
+        "epoch",
+        "impact_day_of_year",
+        "impact_charge_bins",
+        "mass_bins",
+        "charge_labels",
+        "mass_labels",
+        "rectangular_lon_pixel_label",
+        "rectangular_lat_pixel_label",
+    ]
+
+    l2c_dataset = setup_dataset(l2b_dataset, arrays_to_copy, idex_attrs)
+
+    # Add map attributes
     map_attrs = {
         "sky_tiling_type": SkyTilingType.RECTANGULAR.value,
-        "Spacing_degrees": str(spacing_deg),
+        "Spacing_degrees": str(IDEX_SPACING_DEG),
         "Spice_reference_frame": IDEX_EVENT_REFERENCE_FRAME.name,
-        "num_points": str(counts.size),
     } | idex_attrs.get_global_attributes("imap_idex_l2c_sci-rectangular")
 
     l2c_dataset.attrs.update(map_attrs)
+    logger.info("IDEX L2C science data processing completed.")
     return l2c_dataset

imap_processing/lo/l0/lo_apid.py

@@ -13,3 +13,4 @@ class LoAPID(IntEnum):
     ILO_SCI_DE = 706  # Science direct event data
     ILO_STAR = 707  # Science star sensor data, every spin
     ILO_SPIN = 708  # Spin information for each science cycle (28 spins)
+    ILO_DIAG_PCC = 725  # Diagnostic pivot platform information

imap_processing/lo/l0/lo_star_sensor.py

@@ -0,0 +1,48 @@
+"""Processing function for Lo star sensor data."""
+
+import logging
+
+import numpy as np
+import xarray as xr
+
+from imap_processing.lo.l0.utils.bit_decompression import (
+    DECOMPRESSION_TABLES,
+    Decompress,
+)
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+def process_star_sensor(ds: xr.Dataset) -> xr.Dataset:
+    """
+    Process Lo star sensor data.
+
+    Parameters
+    ----------
+    ds : xr.Dataset
+        The packet dataset containing Lo star sensor data.
+
+    Returns
+    -------
+    xr.Dataset
+        Processed dataset with a decompressed data field.
+    """
+    # Make one long flat buffer
+    # This assumes that all data_compressed entries are of the same length
+    # but allows for only one frombuffer call
+    buffer = b"".join(ds["data_compressed"].values)
+    data = np.frombuffer(buffer, dtype=np.uint8).reshape(-1, 720)
+
+    # Decompress from 8 -> 12 bits using the decompression tables
+    decompression = DECOMPRESSION_TABLES[Decompress.DECOMPRESS8TO12].astype(np.uint16)
+    # Use the mean value column (2)
+    data = decompression[data, 2]
+
+    # There is already a variable called "count" in the dataset that
+    # came with the packet
+    ds["data_index"] = xr.DataArray(np.arange(720), dims="data_index")
+    ds["data"] = xr.DataArray(data, dims=("epoch", "data_index"))
+    # Remove the original compressed data field
+    ds = ds.drop_vars("data_compressed")
+    return ds

imap_processing/lo/l1a/lo_l1a.py

@@ -15,6 +15,7 @@ from imap_processing.lo.l0.lo_science import (
     parse_events,
     parse_histogram,
 )
+from imap_processing.lo.l0.lo_star_sensor import process_star_sensor
 from imap_processing.utils import convert_to_binary_string, packet_file_to_datasets
 
 logger = logging.getLogger(__name__)
@@ -44,66 +45,109 @@ def lo_l1a(dependency: Path) -> list[xr.Dataset]:
         xtce_packet_definition=xtce_file.resolve(),
         use_derived_value=False,
     )
+    datasets_by_apid_derived = packet_file_to_datasets(
+        packet_file=dependency.resolve(),
+        xtce_packet_definition=xtce_file.resolve(),
+        use_derived_value=True,
+    )
 
     # create the attribute manager for this data level
     attr_mgr = ImapCdfAttributes()
     attr_mgr.add_instrument_global_attrs(instrument="lo")
     attr_mgr.add_instrument_variable_attrs(instrument="lo", level="l1a")
 
+    datasets_to_return = []
+
     if LoAPID.ILO_SPIN in datasets_by_apid:
         logger.info(
             f"\nProcessing {LoAPID(LoAPID.ILO_SPIN).name} "
             f"packet (APID: {LoAPID.ILO_SPIN.value})"
         )
         logical_source = "imap_lo_l1a_spin"
-        datasets_by_apid[LoAPID.ILO_SPIN] = organize_spin_data(
-            datasets_by_apid[LoAPID.ILO_SPIN], attr_mgr
-        )
-
-        datasets_by_apid[LoAPID.ILO_SPIN] = add_dataset_attrs(
-            datasets_by_apid[LoAPID.ILO_SPIN], attr_mgr, logical_source
-        )
+        ds = datasets_by_apid[LoAPID.ILO_SPIN]
+        ds = organize_spin_data(ds, attr_mgr)
+        ds = add_dataset_attrs(ds, attr_mgr, logical_source)
+        datasets_to_return.append(ds)
     if LoAPID.ILO_SCI_CNT in datasets_by_apid:
         logger.info(
             f"\nProcessing {LoAPID(LoAPID.ILO_SCI_CNT).name} "
             f"packet (APID: {LoAPID.ILO_SCI_CNT.value})"
         )
         logical_source = "imap_lo_l1a_histogram"
-        datasets_by_apid[LoAPID.ILO_SCI_CNT] = parse_histogram(
-            datasets_by_apid[LoAPID.ILO_SCI_CNT], attr_mgr
-        )
-        datasets_by_apid[LoAPID.ILO_SCI_CNT] = add_dataset_attrs(
-            datasets_by_apid[LoAPID.ILO_SCI_CNT], attr_mgr, logical_source
-        )
+        ds = datasets_by_apid[LoAPID.ILO_SCI_CNT]
+        ds = parse_histogram(ds, attr_mgr)
+        ds = add_dataset_attrs(ds, attr_mgr, logical_source)
+        datasets_to_return.append(ds)
     if LoAPID.ILO_SCI_DE in datasets_by_apid:
         logger.info(
             f"\nProcessing {LoAPID(LoAPID.ILO_SCI_DE).name} "
             f"packet (APID: {LoAPID.ILO_SCI_DE.value})"
         )
         logical_source = "imap_lo_l1a_de"
-        datasets_by_apid[LoAPID.ILO_SCI_DE]["data"] = xr.DataArray(
-            [
-                convert_to_binary_string(data)
-                for data in datasets_by_apid[LoAPID.ILO_SCI_DE]["data"].values
-            ],
-            dims=datasets_by_apid[LoAPID.ILO_SCI_DE]["data"].dims,
-            attrs=datasets_by_apid[LoAPID.ILO_SCI_DE]["data"].attrs,
+        ds = datasets_by_apid[LoAPID.ILO_SCI_DE]
+        # Process the "data" array into a string
+        ds["data"] = xr.DataArray(
+            [convert_to_binary_string(data) for data in ds["data"].values],
+            dims=ds["data"].dims,
+            attrs=ds["data"].attrs,
         )
 
-        datasets_by_apid[LoAPID.ILO_SCI_DE] = combine_segmented_packets(
-            datasets_by_apid[LoAPID.ILO_SCI_DE]
+        ds = combine_segmented_packets(ds)
+        ds = parse_events(ds, attr_mgr)
+        ds = add_dataset_attrs(ds, attr_mgr, logical_source)
+        datasets_to_return.append(ds)
+    if LoAPID.ILO_STAR in datasets_by_apid:
+        logger.info(
+            f"\nProcessing {LoAPID(LoAPID.ILO_STAR).name} "
+            f"packet (APID: {LoAPID.ILO_STAR.value})"
         )
-
-        datasets_by_apid[LoAPID.ILO_SCI_DE] = parse_events(
-            datasets_by_apid[LoAPID.ILO_SCI_DE], attr_mgr
+        logical_source = "imap_lo_l1a_star"
+        ds = datasets_by_apid[LoAPID.ILO_STAR]
+        ds = process_star_sensor(ds)
+        ds = add_dataset_attrs(ds, attr_mgr, logical_source)
+        datasets_to_return.append(ds)
+    if LoAPID.ILO_DIAG_PCC in datasets_by_apid:
+        logger.info(
+            f"\nProcessing {LoAPID(LoAPID.ILO_DIAG_PCC).name} "
+            f"packet (APID: {LoAPID.ILO_DIAG_PCC.value})"
         )
-        datasets_by_apid[LoAPID.ILO_SCI_DE] = add_dataset_attrs(
-            datasets_by_apid[LoAPID.ILO_SCI_DE], attr_mgr, logical_source
+        logical_source = "imap_lo_l1a_pcc"
+        ds = datasets_by_apid[LoAPID.ILO_DIAG_PCC]
+        ds = add_dataset_attrs(ds, attr_mgr, logical_source)
+        datasets_to_return.append(ds)
+    if LoAPID.ILO_APP_NHK in datasets_by_apid:
+        logger.info(
+            f"\nProcessing {LoAPID(LoAPID.ILO_APP_NHK).name} "
+            f"packet (APID: {LoAPID.ILO_APP_NHK.value})"
         )
+        logical_source = "imap_lo_l1a_nhk"
+        ds = datasets_by_apid[LoAPID.ILO_APP_NHK]
+        ds = add_dataset_attrs(ds, attr_mgr, logical_source)
+        datasets_to_return.append(ds)
+
+        # Engineering units conversion
+        logical_source = "imap_lo_l1b_nhk"
+        ds = datasets_by_apid_derived[LoAPID.ILO_APP_NHK]
+        ds = add_dataset_attrs(ds, attr_mgr, logical_source)
+        datasets_to_return.append(ds)
+    if LoAPID.ILO_APP_SHK in datasets_by_apid:
+        logger.info(
+            f"\nProcessing {LoAPID(LoAPID.ILO_APP_SHK).name} "
+            f"packet (APID: {LoAPID.ILO_APP_SHK.value})"
+        )
+        logical_source = "imap_lo_l1a_shk"
+        ds = datasets_by_apid[LoAPID.ILO_APP_SHK]
+        ds = add_dataset_attrs(ds, attr_mgr, logical_source)
+        datasets_to_return.append(ds)
+
+        # Engineering units conversion
+        logical_source = "imap_lo_l1b_shk"
+        ds = datasets_by_apid_derived[LoAPID.ILO_APP_SHK]
+        ds = add_dataset_attrs(ds, attr_mgr, logical_source)
+        datasets_to_return.append(ds)
 
-    good_apids = [LoAPID.ILO_SPIN, LoAPID.ILO_SCI_CNT, LoAPID.ILO_SCI_DE]
-    logger.info(f"\nReturning datasets: {[LoAPID(apid) for apid in good_apids]}")
-    return [datasets_by_apid[good_apid] for good_apid in good_apids]
+    logger.info(f"Returning [{len(datasets_to_return)}] datasets")
+    return datasets_to_return
 
 
 def add_dataset_attrs(