imap-processing 0.16.2__py3-none-any.whl → 0.18.0__py3-none-any.whl

imap_processing/ultra/l1c/helio_pset.py

@@ -1,10 +1,17 @@
 """Calculate Pointing Set Grids."""
 
-import astropy_healpix.healpy as hp
 import numpy as np
+import pandas as pd
 import xarray as xr
 
-from imap_processing.ultra.l1c.ultra_l1c_pset_bins import build_energy_bins
+from imap_processing.spice.time import sct_to_et
+from imap_processing.ultra.l1c.ultra_l1c_pset_bins import (
+    build_energy_bins,
+    get_helio_background_rates,
+    get_helio_exposure_times,
+    get_helio_sensitivity,
+    get_spacecraft_histogram,
+)
 from imap_processing.ultra.utils.ultra_l1_utils import create_dataset
 
 
@@ -36,17 +43,57 @@ def calculate_helio_pset(
     dataset : xarray.Dataset
         Dataset containing the data.
     """
-    # TODO: Fill in the rest of this later.
     pset_dict: dict[str, np.ndarray] = {}
-    healpix = np.arange(hp.nside2npix(128))
-    _, _, energy_bin_geometric_means = build_energy_bins()
 
+    v_mag_helio_spacecraft = np.linalg.norm(
+        de_dataset["velocity_dps_helio"].values, axis=1
+    )
+    vhat_dps_helio = (
+        de_dataset["velocity_dps_helio"].values / v_mag_helio_spacecraft[:, np.newaxis]
+    )
+    intervals, _, energy_bin_geometric_means = build_energy_bins()
+    counts, latitude, longitude, n_pix = get_spacecraft_histogram(
+        vhat_dps_helio,
+        de_dataset["energy_heliosphere"].values,
+        intervals,
+        nside=128,
+    )
+
+    healpix = np.arange(n_pix)
+
+    # calculate background rates
+    background_rates = get_helio_background_rates()
+
+    efficiencies = ancillary_files["l1c-90sensor-efficiencies"]
+    geometric_function = ancillary_files["l1c-90sensor-gf"]
+
+    df_efficiencies = pd.read_csv(efficiencies)
+    df_geometric_function = pd.read_csv(geometric_function)
+    mid_time = sct_to_et(np.median(de_dataset["event_times"].data))
+    sensitivity = get_helio_sensitivity(
+        mid_time,
+        df_efficiencies,
+        df_geometric_function,
+    )
+
+    # Calculate exposure
+    constant_exposure = ancillary_files["l1c-90sensor-dps-exposure"]
+    df_exposure = pd.read_csv(constant_exposure)
+    exposure_pointing = get_helio_exposure_times(mid_time, df_exposure)
+
+    # For ISTP, epoch should be the center of the time bin.
     pset_dict["epoch"] = de_dataset.epoch.data[:1].astype(np.int64)
-    pset_dict["pixel_index"] = healpix
+    pset_dict["counts"] = counts[np.newaxis, ...]
+    pset_dict["latitude"] = latitude[np.newaxis, ...]
+    pset_dict["longitude"] = longitude[np.newaxis, ...]
     pset_dict["energy_bin_geometric_mean"] = energy_bin_geometric_means
-    pset_dict["exposure_factor"] = np.zeros(len(healpix), dtype=np.uint8)[
+    pset_dict["background_rates"] = background_rates[np.newaxis, ...]
+    pset_dict["helio_exposure_factor"] = exposure_pointing[np.newaxis, ...]
+    pset_dict["pixel_index"] = healpix
+    pset_dict["energy_bin_delta"] = np.diff(intervals, axis=1).squeeze()[
         np.newaxis, ...
     ]
+    pset_dict["sensitivity"] = sensitivity[np.newaxis, ...]
 
     dataset = create_dataset(pset_dict, name, "l1c")
 

imap_processing/ultra/l1c/spacecraft_pset.py

@@ -18,6 +18,8 @@ def calculate_spacecraft_pset(
     de_dataset: xr.Dataset,
     extendedspin_dataset: xr.Dataset,
     cullingmask_dataset: xr.Dataset,
+    rates_dataset: xr.Dataset,
+    params_dataset: xr.Dataset,
     name: str,
     ancillary_files: dict,
 ) -> xr.Dataset:
@@ -32,6 +34,10 @@ def calculate_spacecraft_pset(
         Dataset containing extendedspin data.
     cullingmask_dataset : xarray.Dataset
         Dataset containing cullingmask data.
+    rates_dataset : xarray.Dataset
+        Dataset containing image rates data.
+    params_dataset : xarray.Dataset
+        Dataset containing image parameters data.
     name : str
         Name of the dataset.
     ancillary_files : dict
@@ -71,7 +77,9 @@ def calculate_spacecraft_pset(
     # Calculate exposure
     constant_exposure = ancillary_files["l1c-90sensor-dps-exposure"]
     df_exposure = pd.read_csv(constant_exposure)
-    exposure_pointing = get_spacecraft_exposure_times(df_exposure)
+    exposure_pointing = get_spacecraft_exposure_times(
+        df_exposure, rates_dataset, params_dataset
+    )
 
     # For ISTP, epoch should be the center of the time bin.
     pset_dict["epoch"] = de_dataset.epoch.data[:1].astype(np.int64)

imap_processing/ultra/l1c/ultra_l1c.py

@@ -63,6 +63,8 @@ def ultra_l1c(
                 data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"],
                 data_dict[f"imap_ultra_l1b_{instrument_id}sensor-extendedspin"],
                 data_dict[f"imap_ultra_l1b_{instrument_id}sensor-cullingmask"],
+                data_dict[f"imap_ultra_{instrument_id}sensor-rates"],
+                data_dict[f"imap_ultra_{instrument_id}sensor-params"],
                 f"imap_ultra_l1c_{instrument_id}sensor-spacecraftpset",
                 ancillary_files,
             )

imap_processing/ultra/l1c/ultra_l1c_pset_bins.py

@@ -4,6 +4,7 @@ import astropy_healpix.healpy as hp
 import numpy as np
 import pandas
 import pandas as pd
+import xarray as xr
 from numpy.typing import NDArray
 from scipy.interpolate import interp1d
 
@@ -151,110 +152,6 @@ def get_spacecraft_histogram(
     return hist, latitude, longitude, n_pix
 
 
-def get_helio_histogram(
-    time: NDArray,
-    vhat: NDArray,
-    energy: NDArray,
-    energy_bin_edges: list[tuple[float, float]],
-    nside: int = 128,
-    nested: bool = False,
-) -> tuple[NDArray, NDArray, NDArray, NDArray]:
-    """
-    Compute a 3D histogram of the particle data using HEALPix binning.
-
-    Parameters
-    ----------
-    time : np.ndarray
-        Median time of pointing in et.
-    vhat : tuple[np.ndarray, np.ndarray, np.ndarray]
-        The x,y,z-components of the unit velocity vector.
-    energy : np.ndarray
-        The particle energy.
-    energy_bin_edges : list[tuple[float, float]]
-        Array of energy bin edges.
-    nside : int, optional
-        The nside parameter of the Healpix tessellation.
-        Default is 128.
-    nested : bool, optional
-        Whether the Healpix tessellation is nested. Default is False.
-
-    Returns
-    -------
-    hist : np.ndarray
-        A 3D histogram array with shape (n_pix, n_energy_bins).
-    latitude : np.ndarray
-        Array of latitude values.
-    longitude : np.ndarray
-        Array of longitude values.
-    n_pix : int
-        Number of healpix pixels.
-
-    Notes
-    -----
-    The histogram will work properly for overlapping energy bins, i.e.
-    the same energy value can fall into multiple bins if the intervals overlap.
-
-    azimuthal angle [0, 360], elevation angle [-90, 90]
-    """
-    # Compute number of HEALPix pixels that cover the sphere
-    n_pix = hp.nside2npix(nside)
-
-    # Calculate the corresponding longitude (az) latitude (el)
-    # center coordinates
-    longitude, latitude = hp.pix2ang(nside, np.arange(n_pix), lonlat=True)
-
-    # The Cartesian state vector representing the position and velocity of the
-    # IMAP spacecraft.
-    state = imap_state(time, ref_frame=SpiceFrame.IMAP_DPS)
-
-    # Extract the velocity part of the state vector
-    spacecraft_velocity = state[3:6]
-
-    # Initialize histogram: (n_energy_bins, n_HEALPix pixels)
-    hist = np.zeros((len(energy_bin_edges), n_pix))
-
-    # Bin data in energy & HEALPix space
-    for i, (e_min, e_max) in enumerate(energy_bin_edges):
-        # Convert the midpoint energy to a velocity (km/s).
-        # Based on kinetic energy equation: E = 1/2 * m * v^2.
-        energy_midpoint = (e_min + e_max) / 2
-        energy_velocity = (
-            np.sqrt(2 * energy_midpoint * UltraConstants.KEV_J / UltraConstants.MASS_H)
-            / 1e3
-        )
-
-        # Use Galilean Transform to transform the velocity wrt spacecraft
-        # to the velocity wrt heliosphere.
-        # energy_velocity * cartesian -> apply the magnitude of the velocity
-        # to every position on the grid in the despun grid.
-        mask = (energy >= e_min) & (energy < e_max)
-        vx, vy, vz = vhat.T
-
-        # Select only the particles that fall within the energy bin.
-        vx_bin, vy_bin, vz_bin = vx[mask], vy[mask], vz[mask]
-        vhat_bin = np.stack((vx_bin, vy_bin, vz_bin), axis=1)
-        helio_velocity = spacecraft_velocity.reshape(1, 3) + energy_velocity * vhat_bin
-
-        # Normalized vectors representing the direction of the heliocentric velocity.
-        helio_normalized = -helio_velocity / np.linalg.norm(
-            helio_velocity, axis=1, keepdims=True
-        )
-
-        # Convert Cartesian heliocentric vectors into spherical coordinates.
-        # Result: azimuth (longitude) and elevation (latitude) in degrees.
-        helio_spherical = cartesian_to_spherical(np.squeeze(helio_normalized))
-        helio_spherical = np.atleast_2d(helio_spherical)
-        az, el = helio_spherical[:, 1], helio_spherical[:, 2]
-
-        # Convert azimuth/elevation directions to HEALPix pixel indices.
-        hpix_idx = hp.ang2pix(nside, az, el, nest=nested, lonlat=True)
-
-        # Only count the events that fall within the energy bin
-        hist[i, :] += np.bincount(hpix_idx, minlength=n_pix).astype(np.float64)
-
-    return hist, latitude, longitude, n_pix
-
-
 def get_spacecraft_background_rates(
     nside: int = 128,
 ) -> NDArray:
@@ -309,7 +206,98 @@ def get_helio_background_rates(
     return background
 
 
-def get_spacecraft_exposure_times(constant_exposure: pandas.DataFrame) -> NDArray:
+def get_deadtime_correction_factors(sectored_rates_ds: xr.Dataset) -> xr.DataArray:
+    """
+    Compute the dead time correction factor at each sector.
+
+    Further description is available in section 3.4.3 of the IMAP-Ultra Algorithm
+    Document.
+
+    Parameters
+    ----------
+    sectored_rates_ds : xarray.Dataset
+        Dataset containing sector mode image rates data.
+
+    Returns
+    -------
+    dead_time_ratio : xarray.DataArray
+        Dead time correction factor for each sector.
+    """
+    # Compute the correction factor at each sector
+    a = sectored_rates_ds.fifo_valid_events / (
+        1
+        - (sectored_rates_ds.event_active_time + 2 * sectored_rates_ds.start_pos) * 1e-7
+    )
+
+    start_full = sectored_rates_ds.start_rf + sectored_rates_ds.start_lf
+    b = a * np.exp(start_full * 1e-7 * 5)
+
+    coin_stop_nd = (
+        sectored_rates_ds.coin_tn
+        + sectored_rates_ds.coin_bn
+        - sectored_rates_ds.stop_tn
+        - sectored_rates_ds.stop_bn
+    )
+
+    corrected_valid_events = b * np.exp(1e-7 * 8 * coin_stop_nd)
+
+    # Compute dead time ratio
+    dead_time_ratios = sectored_rates_ds.fifo_valid_events / corrected_valid_events
+
+    return dead_time_ratios
+
+
+def get_sectored_rates(rates_ds: xr.Dataset, params_ds: xr.Dataset) -> xr.Dataset:
+    """
+    Filter rates dataset to only include sector mode data.
+
+    Parameters
+    ----------
+    rates_ds : xarray.Dataset
+        Dataset containing image rates data.
+    params_ds : xarray.Dataset
+        Dataset containing image parameters data.
+
+    Returns
+    -------
+    rates : xarray.Dataset
+        Rates dataset with only the sector mode data.
+    """
+    # Find indices in which the parameters dataset, indicates that ULTRA was in
+    # sector mode. At the normal 15-second spin period, each 24° sector takes ~1 second.
+
+    # This means that data was collected as a function of spin allowing for fine grained
+    # rate analysis.
+    sector_mode_start_inds = np.where(params_ds["imageratescadence"] == 3)[0]
+    # get the sector mode start and stop indices
+    sector_mode_stop_inds = sector_mode_start_inds + 1
+    # get the sector mode start and stop times
+    mode_3_start = params_ds["epoch"].values[sector_mode_start_inds]
+
+    # if the last mode is a sector mode, we can assume that the sector data goes through
+    # the end of the dataset, so we append np.inf to the end of the last time range.
+    if sector_mode_stop_inds[-1] == len(params_ds["epoch"]):
+        mode_3_end = np.append(
+            params_ds["epoch"].values[sector_mode_stop_inds[:-1]], np.inf
+        )
+    else:
+        mode_3_end = params_ds["epoch"].values[sector_mode_stop_inds]
+
+    # Build a list of conditions for each sector mode time range
+    conditions = [
+        (rates_ds["epoch"] >= start) & (rates_ds["epoch"] < end)
+        for start, end in zip(mode_3_start, mode_3_end)
+    ]
+
+    sector_mode_mask = np.logical_or.reduce(conditions)
+    return rates_ds.isel(epoch=sector_mode_mask)
+
+
+def get_spacecraft_exposure_times(
+    constant_exposure: pandas.DataFrame,
+    rates_dataset: xr.Dataset,
+    params_dataset: xr.Dataset,
+) -> NDArray:
     """
     Compute exposure times for HEALPix pixels.
 
@@ -317,6 +305,10 @@ def get_spacecraft_exposure_times(constant_exposure: pandas.DataFrame) -> NDArra
     ----------
     constant_exposure : pandas.DataFrame
         Exposure data.
+    rates_dataset : xarray.Dataset
+        Dataset containing image rates data.
+    params_dataset : xarray.Dataset
+        Dataset containing image parameters data.
 
     Returns
     -------
@@ -325,6 +317,11 @@ def get_spacecraft_exposure_times(constant_exposure: pandas.DataFrame) -> NDArra
         Healpix tessellation of the sky
         in the pointing (dps) frame.
     """
+    # TODO: uncomment these lines when the deadtime correction is implemented
+    # sectored_rates = get_sectored_rates(rates_dataset, params_dataset)
+    # get_deadtime_correction_factors(sectored_rates)
+    # TODO: calculate the deadtime correction function
+    # TODO: Apply the deadtime correction to the exposure times
     # TODO: use the universal spin table and
     #  universal pointing table here to determine actual number of spins
     exposure_pointing = (
@@ -384,7 +381,7 @@ def get_helio_exposure_times(
     # Initialize output array.
     # Each row corresponds to a HEALPix pixel, and each column to an energy bin.
     npix = hp.nside2npix(nside)
-    helio_exposure = np.zeros((npix, len(energy_midpoints)))
+    helio_exposure = np.zeros((len(energy_midpoints), npix))
 
     # Loop through energy bins and compute transformed exposure.
     for i, energy_midpoint in enumerate(energy_midpoints):
@@ -415,7 +412,7 @@ def get_helio_exposure_times(
         hpix_idx = hp.ang2pix(nside, az, el, nest=nested, lonlat=True)
 
         # Accumulate exposure values into HEALPix pixels for this energy bin.
-        helio_exposure[:, i] = np.bincount(
+        helio_exposure[i, :] = np.bincount(
             hpix_idx, weights=exposure_flat, minlength=npix
         )
 
@@ -597,7 +594,7 @@ def get_helio_sensitivity(
     # Initialize output array.
     # Each row corresponds to a HEALPix pixel, and each column to an energy bin.
     npix = hp.nside2npix(nside)
-    helio_sensitivity = np.zeros((npix, len(energy_midpoints)))
+    helio_sensitivity = np.zeros((len(energy_midpoints), npix))
 
     # Loop through energy bins and compute transformed sensitivity.
     for i, energy in enumerate(energy_midpoints):
@@ -628,7 +625,7 @@ def get_helio_sensitivity(
         gridded_sensitivity = grid_sensitivity(efficiencies, geometric_function, energy)
 
         # Accumulate sensitivity values into HEALPix pixels for this energy bin.
-        helio_sensitivity[:, i] = np.bincount(
+        helio_sensitivity[i, :] = np.bincount(
             hpix_idx, weights=gridded_sensitivity, minlength=npix
         )
 

imap_processing/ultra/packet_definitions/ULTRA_SCI_COMBINED.xml

@@ -4726,15 +4726,15 @@
 			<xtce:IntegerParameterType name="U45_IMG_ENA_PHXTOF_HI_TIME.SPIN" signed="false">
 				<xtce:IntegerDataEncoding sizeInBits="8" encoding="unsigned" />
 			</xtce:IntegerParameterType>
+			<xtce:IntegerParameterType name="U45_IMG_ENA_PHXTOF_HI_TIME.ABORTFLAG" signed="false">
+				<xtce:IntegerDataEncoding sizeInBits="1" encoding="unsigned" />
+			</xtce:IntegerParameterType>
 			<xtce:IntegerParameterType name="U45_IMG_ENA_PHXTOF_HI_TIME.STARTDELAY" signed="false">
 				<xtce:IntegerDataEncoding sizeInBits="15" encoding="unsigned" />
 			</xtce:IntegerParameterType>
 			<xtce:IntegerParameterType name="U45_IMG_ENA_PHXTOF_HI_TIME.P00" signed="false">
 				<xtce:IntegerDataEncoding sizeInBits="8" encoding="unsigned" />
 			</xtce:IntegerParameterType>
-			<xtce:IntegerParameterType name="U45_IMG_ENA_PHXTOF_HI_TIME.ABORTFLAG" signed="false">
-				<xtce:IntegerDataEncoding sizeInBits="8" encoding="unsigned" />
-			</xtce:IntegerParameterType>
 			<xtce:BinaryParameterType name="U45_IMG_ENA_PHXTOF_HI_TIME.PACKETDATA">
 				<xtce:BinaryDataEncoding bitOrder="mostSignificantBitFirst">
 					<xtce:SizeInBits>

imap_processing/ultra/utils/ultra_l1_utils.py

@@ -91,6 +91,7 @@ def create_dataset(  # noqa: PLR0912
         "ena_rates_threshold",
         "quality_ena_rates",
     }
+    rates_pulse_keys = {"start_per_spin", "stop_per_spin", "coin_per_spin"}
 
     for key, data in data_dict.items():
         # Skip keys that are coordinates.
@@ -114,6 +115,12 @@ def create_dataset(  # noqa: PLR0912
                 dims=["epoch", "energy_bin_geometric_mean"],
                 attrs=cdf_manager.get_variable_attributes(key, check_schema=False),
             )
+        elif key in rates_pulse_keys:
+            dataset[key] = xr.DataArray(
+                data,
+                dims=["spin_number"],
+                attrs=cdf_manager.get_variable_attributes(key, check_schema=False),
+            )
         elif key in rates_keys:
             dataset[key] = xr.DataArray(
                 data,
@@ -126,7 +133,12 @@ def create_dataset(  # noqa: PLR0912
                 dims=["epoch", "pixel_index"],
                 attrs=cdf_manager.get_variable_attributes(key, check_schema=False),
             )
-        elif key in {"counts", "background_rates", "sensitivity"}:
+        elif key in {
+            "counts",
+            "background_rates",
+            "helio_exposure_factor",
+            "sensitivity",
+        }:
             dataset[key] = xr.DataArray(
                 data,
                 dims=["epoch", "energy_bin_geometric_mean", "pixel_index"],

imap_processing/utils.py

@@ -15,48 +15,6 @@ from imap_processing.spice.time import met_to_ttj2000ns
 logger = logging.getLogger(__name__)
 
 
-def sort_by_time(packets: list, time_key: str) -> list:
-    """
-    Sort packets by specified key.
-
-    Parameters
-    ----------
-    packets : list
-        Decom data packets.
-    time_key : str
-        Key to sort by. Must be a key in the packets data dictionary.
-        e.g. "SHCOARSE" or "MET_TIME" or "ACQ_START_COARSE".
-
-    Returns
-    -------
-    sorted_packets : list
-        Sorted packets.
-    """
-    sorted_packets = sorted(packets, key=lambda x: x[time_key])
-    return sorted_packets
-
-
-def group_by_apid(packets: list) -> dict:
-    """
-    Group data by apid.
-
-    Parameters
-    ----------
-    packets : list
-        Packet list.
-
-    Returns
-    -------
-    grouped_packets : dict
-        Grouped data by apid.
-    """
-    grouped_packets: dict[list] = collections.defaultdict(list)
-    for packet in packets:
-        apid = packet["PKT_APID"]
-        grouped_packets.setdefault(apid, []).append(packet)
-    return grouped_packets
-
-
 def convert_raw_to_eu(
     dataset: xr.Dataset,
     conversion_table_path: str,
@@ -347,6 +305,26 @@ def packet_file_to_datasets(
             coords={"epoch": time_data},
         )
         ds = ds.sortby("epoch")
+        # We may get duplicate packets within the packet file if packets were
+        # ingested multiple times by the POC. We want to drop packets where
+        # apid, epoch, and src_seq_ctr are the same.
+
+        # xarray only supports dropping duplicates by index, so we instead go
+        # to pandas multi-index dataframe to identify the unique positions
+        unique_indices = (
+            ds[["src_seq_ctr"]]
+            .to_dataframe()
+            .reset_index()
+            .drop_duplicates()
+            .index.values
+        )
+        nduplicates = len(ds["epoch"]) - len(unique_indices)
+        if nduplicates != 0:
+            logger.warning(
+                f"Found [{nduplicates}] duplicate packets for APID {apid}. "
+                "Dropping duplicate packets and continuing processing."
+            )
+            ds = ds.isel(epoch=unique_indices)
 
         # Strip any leading characters before "." from the field names which was due
         # to the packet_name being a part of the variable name in the XTCE definition

{imap_processing-0.16.2.dist-info → imap_processing-0.18.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: imap-processing
-Version: 0.16.2
+Version: 0.18.0
 Summary: IMAP Science Operations Center Processing
 License: MIT
 Keywords: IMAP,SDC,SOC,Science Operations
@@ -30,7 +30,7 @@ Provides-Extra: tools
 Requires-Dist: astropy-healpix (>=1.0)
 Requires-Dist: cdflib (>=1.3.1,<2.0.0)
 Requires-Dist: healpy (>=1.18.0,<2.0.0) ; extra == "map-visualization"
-Requires-Dist: imap-data-access (>=0.31.0)
+Requires-Dist: imap-data-access (>=0.32.0)
 Requires-Dist: mypy (==1.10.1) ; extra == "dev"
 Requires-Dist: netcdf4 (>=1.7.2,<2.0.0) ; extra == "test"
 Requires-Dist: numpy (<=3)