imap-processing 0.11.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

imap_processing/ultra/l1c/ultra_l1c_pset_bins.py

@@ -1,9 +1,8 @@
 """Module to create pointing sets."""
 
-from pathlib import Path
-
-import cdflib
+import astropy_healpix.healpy as hp
 import numpy as np
+import pandas
 from numpy.typing import NDArray
 
 from imap_processing.ena_maps.utils.spatial_utils import build_spatial_bins
@@ -18,7 +17,7 @@ from imap_processing.ultra.constants import UltraConstants
 # TODO: add species binning.
 
 
-def build_energy_bins() -> tuple[list[tuple[float, float]], np.ndarray]:
+def build_energy_bins() -> tuple[list[tuple[float, float]], np.ndarray, np.ndarray]:
     """
     Build energy bin boundaries.
 
@@ -28,6 +27,8 @@ def build_energy_bins() -> tuple[list[tuple[float, float]], np.ndarray]:
         Energy bins.
     energy_midpoints : np.ndarray
         Array of energy bin midpoints.
+    energy_bin_geometric_means : np.ndarray
+        Array of geometric means of energy bins.
     """
     # Calculate energy step
     energy_step = (1 + UltraConstants.ALPHA / 2) / (1 - UltraConstants.ALPHA / 2)
@@ -44,19 +45,20 @@ def build_energy_bins() -> tuple[list[tuple[float, float]], np.ndarray]:
         (float(energy_bin_edges[i]), float(energy_bin_edges[i + 1]))
         for i in range(len(energy_bin_edges) - 1)
     ]
+    energy_bin_geometric_means = np.sqrt(energy_bin_edges[:-1] * energy_bin_edges[1:])
 
-    return intervals, energy_midpoints
+    return intervals, energy_midpoints, energy_bin_geometric_means
 
 
-def get_histogram(
+def get_spacecraft_histogram(
     vhat: tuple[np.ndarray, np.ndarray, np.ndarray],
     energy: np.ndarray,
-    az_bin_edges: np.ndarray,
-    el_bin_edges: np.ndarray,
     energy_bin_edges: list[tuple[float, float]],
-) -> NDArray:
+    nside: int = 128,
+    nested: bool = False,
+) -> tuple[NDArray, NDArray, NDArray, NDArray]:
     """
-    Compute a 3D histogram of the particle data.
+    Compute a 3D histogram of the particle data using HEALPix binning.
 
     Parameters
     ----------
@@ -64,71 +66,110 @@ def get_histogram(
         The x,y,z-components of the unit velocity vector.
     energy : np.ndarray
         The particle energy.
-    az_bin_edges : np.ndarray
-        Array of azimuth bin boundary values.
-    el_bin_edges : np.ndarray
-        Array of elevation bin boundary values.
     energy_bin_edges : list[tuple[float, float]]
         Array of energy bin edges.
+    nside : int, optional
+        The nside parameter of the Healpix tessellation.
+        Default is 32.
+    nested : bool, optional
+        Whether the Healpix tessellation is nested. Default is False.
 
     Returns
     -------
     hist : np.ndarray
-        A 3D histogram array.
+        A 3D histogram array with shape (n_pix, n_energy_bins).
+    latitude : np.ndarray
+        Array of latitude values.
+    longitude : np.ndarray
+        Array of longitude values.
+    n_pix : int
+        Number of healpix pixels.
 
     Notes
     -----
-    The histogram will now work properly for overlapping energy bins, i.e.
+    The histogram will work properly for overlapping energy bins, i.e.
     the same energy value can fall into multiple bins if the intervals overlap.
+
+    azimuthal angle [0, 360], elevation angle [-90, 90]
     """
-    spherical_coords = cartesian_to_spherical(vhat)
+    # vhat = direction in which particle is traveling
+    # Make negative to see where it came from
+    spherical_coords = cartesian_to_spherical(-np.array(vhat), degrees=True)
     az, el = (
         spherical_coords[..., 1],
         spherical_coords[..., 2],
     )
 
-    # Initialize histogram
-    hist_total = np.zeros(
-        (len(az_bin_edges) - 1, len(el_bin_edges) - 1, len(energy_bin_edges))
-    )
+    # Compute number of HEALPix pixels that cover the sphere
+    n_pix = hp.nside2npix(nside)
+
+    # Calculate the corresponding longitude (az) latitude (el)
+    # center coordinates
+    longitude, latitude = hp.pix2ang(nside, np.arange(n_pix), lonlat=True)
+
+    # Get HEALPix pixel indices for each event
+    # HEALPix expects latitude in [-90, 90] so we don't need to change elevation
+    hpix_idx = hp.ang2pix(nside, az, el, nest=nested, lonlat=True)
+
+    # Initialize histogram: (n_energy_bins, n_HEALPix pixels)
+    hist = np.zeros((len(energy_bin_edges), n_pix))
 
+    # Bin data in energy & HEALPix space
     for i, (e_min, e_max) in enumerate(energy_bin_edges):
-        # Filter data for current energy bin.
         mask = (energy >= e_min) & (energy < e_max)
-        hist, _ = np.histogramdd(
-            sample=(az[mask], el[mask], energy[mask]),
-            bins=[az_bin_edges, el_bin_edges, [e_min, e_max]],
-        )
-        # Assign 2D histogram to current energy bin.
-        hist_total[:, :, i] = hist[:, :, 0]
+        # Only count the events that fall within the energy bin
+        hist[i, :] += np.bincount(hpix_idx[mask], minlength=n_pix).astype(np.float64)
 
-    return hist_total
+    return hist, latitude, longitude, n_pix
 
 
-def get_pointing_frame_exposure_times(
-    constant_exposure: Path, n_spins: int, sensor: str
+def get_background_rates(
+    nside: int = 128,
 ) -> NDArray:
     """
-    Compute a 2D array of the exposure.
+    Calculate background rates.
 
     Parameters
     ----------
-    constant_exposure : Path
-        Path to file containing constant exposure data.
-    n_spins : int
-        Number of spins per pointing.
-    sensor : str
-        Sensor (45 or 90).
+    nside : int, optional
+        The nside parameter of the Healpix tessellation (default is 128).
 
     Returns
     -------
-    exposure : np.ndarray
-        A 2D array with dimensions (az, el).
+    background_rates : np.ndarray
+        Array of background rates.
+
+    Notes
+    -----
+    This is a placeholder.
     """
-    with cdflib.CDF(constant_exposure) as cdf_file:
-        exposure = cdf_file.varget(f"dps_grid{sensor}") * n_spins
+    n_pix = hp.nside2npix(nside)
+    return np.zeros(n_pix)
 
-    return exposure
+
+def get_spacecraft_exposure_times(constant_exposure: pandas.DataFrame) -> NDArray:
+    """
+    Compute exposure times for HEALPix pixels.
+
+    Parameters
+    ----------
+    constant_exposure : pandas.DataFrame
+        Exposure data.
+
+    Returns
+    -------
+    exposure_pointing : np.ndarray
+        Total exposure times of pixels in a
+        Healpix tessellation of the sky
+        in the pointing (dps) frame.
+    """
+    # TODO: use the universal spin table and
+    #  universal pointing table here to determine actual number of spins
+    exposure_pointing = (
+        constant_exposure["Exposure Time"] * 5760
+    )  # 5760 spins per pointing (for now)
+
+    return exposure_pointing
 
 
 def get_helio_exposure_times(
@@ -154,10 +195,10 @@ def get_helio_exposure_times(
     -----
     These calculations are performed once per pointing.
     """
-    # Get bins and midpoints, and convert from radians to degrees.
-    _, energy_midpoints = build_energy_bins()
+    # Get bins and midpoints, with angles in degrees.
+    _, energy_midpoints, _ = build_energy_bins()
     az_bin_edges, el_bin_edges, az_bin_midpoints, el_bin_midpoints = (
-        np.rad2deg(angle_radians) for angle_radians in (build_spatial_bins())
+        build_spatial_bins()
     )
 
     # Initialize the exposure grid.
@@ -171,7 +212,7 @@ def get_helio_exposure_times(
 
     # Radial distance.
     r = np.ones(el_grid.shape)
-    spherical_coords = np.stack((r, np.radians(az_grid), np.radians(el_grid)), axis=-1)
+    spherical_coords = np.stack((r, az_grid, el_grid), axis=-1)
     cartesian_coords = spherical_to_cartesian(spherical_coords)
     cartesian = cartesian_coords.reshape(-1, 3, order="F").T
 
@@ -222,27 +263,37 @@ def get_helio_exposure_times(
     return exposure_3d
 
 
-def get_pointing_frame_sensitivity(
-    constant_sensitivity: Path, n_spins: int, sensor: str
-) -> NDArray:
+def get_spacecraft_sensitivity(
+    efficiencies: pandas.DataFrame,
+    geometric_function: pandas.DataFrame,
+) -> pandas.DataFrame:
     """
-    Compute a 3D array of the sensitivity.
+    Compute sensitivity.
 
     Parameters
     ----------
-    constant_sensitivity : Path
-        Path to file containing constant sensitivity data.
-    n_spins : int
-        Number of spins per pointing.
-    sensor : str
-        Sensor (45 or 90).
+    efficiencies : pandas.DataFrame
+        Efficiencies at different energy levels.
+    geometric_function : pandas.DataFrame
+        Geometric function.
 
     Returns
     -------
-    sensitivity : np.ndarray
-        A 3D array with dimensions (az, el, energy).
+    pointing_sensitivity : pandas.DataFrame
+        Sensitivity with dimensions (HEALPIX pixel_number, energy).
     """
-    with cdflib.CDF(constant_sensitivity) as cdf_file:
-        sensitivity = cdf_file.varget(f"dps_sensitivity{sensor}") * n_spins
+    # Exclude "Right Ascension (deg)" and "Declination (deg)" from the multiplication
+    energy_columns = efficiencies.columns.difference(
+        ["Right Ascension (deg)", "Declination (deg)"]
+    )
+    sensitivity = efficiencies[energy_columns].mul(
+        geometric_function["Response (cm2-sr)"].values, axis=0
+    )
+
+    # Add "Right Ascension (deg)" and "Declination (deg)" to the result
+    sensitivity.insert(
+        0, "Right Ascension (deg)", efficiencies["Right Ascension (deg)"]
+    )
+    sensitivity.insert(1, "Declination (deg)", efficiencies["Declination (deg)"])
 
     return sensitivity

imap_processing/ultra/utils/ultra_l1_utils.py

@@ -32,13 +32,24 @@ def create_dataset(
     cdf_manager.add_instrument_variable_attrs("ultra", level)
     cdf_manager.add_global_attribute("Data_version", data_version)
 
+    # L1b extended spin, badtimes, and cullingmask data products
     if "spin_number" in data_dict.keys():
         coords = {
             "spin_number": data_dict["spin_number"],
             "energy_bin_geometric_mean": data_dict["energy_bin_geometric_mean"],
+            # Start time aligns with the universal spin table
+            "epoch": data_dict["spin_start_time"],
         }
         default_dimension = "spin_number"
-
+    # L1c pset data products
+    elif "healpix" in data_dict:
+        coords = {
+            "healpix": data_dict["healpix"],
+            "energy_bin_geometric_mean": data_dict["energy_bin_geometric_mean"],
+            "epoch": data_dict["epoch"],
+        }
+        default_dimension = "healpix"
+    # L1b de data product
     else:
         epoch_time = xr.DataArray(
             data_dict["epoch"],
@@ -76,7 +87,8 @@ def create_dataset(
     }
 
     for key in data_dict.keys():
-        if key in ["epoch", "spin_number", "energy_bin_geometric_mean"]:
+        # Skip keys that are coordinates.
+        if key in ["epoch", "spin_number", "energy_bin_geometric_mean", "healpix"]:
             continue
         elif key in velocity_keys:
             dataset[key] = xr.DataArray(
@@ -96,6 +108,12 @@ def create_dataset(
                 dims=["energy_bin_geometric_mean", "spin_number"],
                 attrs=cdf_manager.get_variable_attributes(key),
             )
+        elif key == "counts":
+            dataset[key] = xr.DataArray(
+                data_dict[key],
+                dims=["energy_bin_geometric_mean", "healpix"],
+                attrs=cdf_manager.get_variable_attributes(key),
+            )
         else:
             dataset[key] = xr.DataArray(
                 data_dict[key],

imap_processing/utils.py

@@ -79,6 +79,19 @@ def convert_raw_to_eu(
         informational columns: ('packetName', 'mnemonic', 'convertAs') and
         conversion columns named 'c0', 'c1', 'c2', etc. Conversion columns
         specify the array of polynomial coefficients used for the conversion.
+        If the column 'convertAs' is 'SEGMENTED_POLY' then there must be columns
+        'dn_range_start' and 'dn_range_stop' that specifies the raw DN range and the
+        coefficients that should be used for the conversion.
+
+        E.g.:
+
+        mnemonic       convertAs …       dn_range_start   dn_range_stop  c0    c1…
+        -------------------------------------------------------------------------
+        temperature  |  SEGMENTED_POLY | 0              | 2063         | 0.1  | 0.2
+        temperature  |  SEGMENTED_POLY | 2064           | 3853         | 0    | 0.1
+        temperature  |  SEGMENTED_POLY | 3854           | 4094         | 0.6  | 0.3
+        sensor_v     |  UNSEGMENTED_POLY |              |              | 0.04 | .110
+
         Comment lines are allowed in the csv file specified by starting with
         the '#' character.
     packet_name : str
@@ -86,7 +99,8 @@ def convert_raw_to_eu(
     **read_csv_kwargs : dict
         In order to allow for some flexibility in the format of the csv
         conversion table, any additional keywords passed to this function are
-        passed in the call to `pandas.read_csv()`. See pandas documentation
+        passed in the call to `pandas.read_csv()`.
+        See pandas documentation
         for a list of keywords and their functionality:
         https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html.
 
@@ -95,47 +109,73 @@ def convert_raw_to_eu(
     dataset : xr.Dataset
         Raw data converted to engineering unit as needed.
     """
-    # Make sure there is column called "index" with unique
-    # value such as 0, 1, 2, 3, ...
     eu_conversion_df = pd.read_csv(
         conversion_table_path,
         **read_csv_kwargs,
     )
 
-    # Look up all metadata fields for the packet name
-    packet_df = eu_conversion_df.loc[eu_conversion_df["packetName"] == packet_name]
-
-    # for each metadata field, convert raw value to engineering unit
-    for _, row in packet_df.iterrows():
-        if row["convertAs"] == "UNSEGMENTED_POLY":
+    # Iterate through every variable in the dataset and check if there is an entry for
+    # That variable in the conversion table.
+    for var in dataset.variables:
+        packet_df = eu_conversion_df.loc[
+            (eu_conversion_df["packetName"] == packet_name)
+            &
+            # Filter for mnemonic case-insensitive
+            (eu_conversion_df["mnemonic"].str.lower() == var.lower())
+        ].reset_index(drop=True)
+
+        if packet_df.empty:
+            continue
+
+        if np.all(packet_df["convertAs"] == "UNSEGMENTED_POLY"):
+            if len(packet_df.index) > 1:
+                raise ValueError(
+                    "For unsegmented polynomial conversions, there should "
+                    "only be one row per mnemonic and packet name."
+                )
+            row = packet_df.iloc[0]
             # On this line, we are getting the coefficients from the
             # table and then reverse them because the np.polyval is
             # expecting coefficient in descending order
             # coeff columns must have names 'c0', 'c1', 'c2', ...
             coeff_values = row.filter(regex=r"c\d").values[::-1]
-            row_key = row["mnemonic"]
-            # TODO: remove this check once everyone has lowercase
-            # all of CDF data variable names to match SPDF requirement.
-            # Right now, check if dataset mnemonics is lowercase of row mnemonics.
-            # If so, make them match
-            mnemonics = row_key.lower() if row_key.lower() in dataset else row_key
-            try:
-                # Convert the raw value to engineering unit
-                dataset[mnemonics].data = np.polyval(
-                    coeff_values, dataset[mnemonics].data
+            # Convert the raw value to engineering unit
+            dataset[var].data = np.polyval(coeff_values, dataset[var].data)
+
+        elif np.all(packet_df["convertAs"] == "SEGMENTED_POLY"):
+            data = dataset[var].data
+            # Check if any of the raw DN values fall outside the ranges
+            bad_mask = np.logical_or(
+                data < packet_df["dn_range_start"].min(),
+                data > packet_df["dn_range_stop"].max(),
+            )
+            if np.any(data[bad_mask]):
+                raise ValueError(
+                    "Raw DN values found outside of the expected range"
+                    f"for mnemonic: {var}"
                 )
-                # Modify units attribute
-                if "unit" in row:
-                    dataset[mnemonics].attrs.update({"units": row["unit"]})
-            except KeyError:
-                # TODO: Don't catch this error once packet definitions stabilize
-                logger.warning(f"Input dataset does not contain key: {row_key}")
+            # Create conditions and corresponding functions for np.piecewise
+            conditions = [
+                (data >= row["dn_range_start"]) & (data <= row["dn_range_stop"])
+                for _, row in packet_df.iterrows()
+            ]
+            functions = [
+                lambda x, r=row: np.polyval(r.filter(regex=r"c\d").values[::-1], x)
+                for _, row in packet_df.iterrows()
+            ]
+            # Convert the raw value to engineering unit
+            dataset[var].data = np.piecewise(data, conditions, functions)
+
         else:
             raise ValueError(
-                f"Unexpected conversion type: {row['convertAs']} encountered in"
-                f" engineering unit conversion table: {conversion_table_path}"
+                "Column 'convertAs' must all be UNSEGMENTED_POLY or "
+                "SEGMENTED_POLY for a packet name and mnemonic"
             )
 
+        # Modify units attribute
+        if "unit" in packet_df:
+            dataset[var].attrs.update({"UNITS": packet_df.iloc[0]["unit"]})
+
     return dataset
 
 
@@ -339,5 +379,5 @@ def convert_to_binary_string(data: bytes) -> str:
     binary_data : str
         The binary data as a string.
     """
-    binary_str_data = f"{int.from_bytes(data, byteorder='big'):0{len(data)*8}b}"
+    binary_str_data = f"{int.from_bytes(data, byteorder='big'):0{len(data) * 8}b}"
     return binary_str_data

{imap_processing-0.11.0.dist-info → imap_processing-0.12.0.dist-info}/METADATA

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: imap-processing
-Version: 0.11.0
+Version: 0.12.0
 Summary: IMAP Science Operations Center Processing
 License: MIT
 Keywords: IMAP,SDC,SOC,Science Operations
 Author: IMAP SDC Developers
 Author-email: imap-sdc@lists.lasp.colorado.edu
-Requires-Python: >=3.9,<4
+Requires-Python: >=3.10,<4
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -16,7 +16,6 @@ Classifier: Operating System :: Microsoft :: Windows
 Classifier: Operating System :: POSIX
 Classifier: Operating System :: Unix
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
@@ -25,11 +24,15 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Provides-Extra: dev
 Provides-Extra: doc
+Provides-Extra: map-visualization
 Provides-Extra: test
 Provides-Extra: tools
+Requires-Dist: astropy-healpix (>=1.0)
 Requires-Dist: cdflib (>=1.3.1,<2.0.0)
+Requires-Dist: healpy (>=1.18.0,<2.0.0) ; extra == "map-visualization"
 Requires-Dist: imap-data-access (>=0.10.1)
 Requires-Dist: mypy (==1.10.1) ; extra == "dev"
+Requires-Dist: netcdf4 (>=1.7.2,<2.0.0) ; extra == "test"
 Requires-Dist: numpy (<=3)
 Requires-Dist: numpydoc (>=1.5.0,<2.0.0) ; extra == "doc"
 Requires-Dist: openpyxl (>=3.0.7) ; extra == "test" or extra == "tools"
@@ -38,15 +41,15 @@ Requires-Dist: pre-commit (>=3.3.3,<4.0.0) ; extra == "dev"
 Requires-Dist: pydata-sphinx-theme ; extra == "doc"
 Requires-Dist: pytest (>=6.2.5) ; extra == "test"
 Requires-Dist: pytest-cov (>=4.0.0,<5.0.0) ; extra == "test"
-Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0) ; extra == "test"
 Requires-Dist: ruff (==0.2.1) ; extra == "dev"
 Requires-Dist: sammi-cdf (>=1.0,<2.0)
+Requires-Dist: scipy (>=1.13,<2.0)
 Requires-Dist: space_packet_parser (>=5.0.1,<6.0.0)
 Requires-Dist: sphinx ; extra == "doc"
 Requires-Dist: sphinxcontrib-openapi (>=0.8.3,<0.9.0) ; extra == "doc"
 Requires-Dist: spiceypy (>=6.0.0)
-Requires-Dist: xarray (>=2023.0.0)
+Requires-Dist: xarray (>=2024.10.0)
 Description-Content-Type: text/markdown
 
 # IMAP (Interstellar Mapping and Acceleration Probe)