imap-processing 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

imap_processing/ena_maps/utils/spatial_utils.py

@@ -0,0 +1,226 @@
+"""IMAP utils for spatial binning and az/el grid creation."""
+
+from __future__ import annotations
+
+import typing
+
+import numpy as np
+from numpy.typing import NDArray
+
+
+def build_spatial_bins(
+    az_spacing_deg: float = 0.5,
+    el_spacing_deg: float = 0.5,
+) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
+    """
+    Build spatial bin boundaries for azimuth and elevation.
+
+    Input angles in degrees for consistency with map inputs,
+    output angles in radians for internal use.
+
+    Parameters
+    ----------
+    az_spacing_deg : float, optional
+        The azimuth bin spacing in degrees (default is 0.5 degrees).
+    el_spacing_deg : float, optional
+        The elevation bin spacing in degrees (default is 0.5 degrees).
+
+    Returns
+    -------
+    az_bin_edges : np.ndarray
+        Array of azimuth bin boundary values in radians.
+    el_bin_edges : np.ndarray
+        Array of elevation bin boundary values in radians.
+    az_bin_midpoints : np.ndarray
+        Array of azimuth bin midpoint values in radians.
+    el_bin_midpoints : np.ndarray
+        Array of elevation bin midpoint values in radians.
+    """
+    # Azimuth bins from 0 to 360 degrees.
+    az_bin_edges = np.arange(0, 360 + az_spacing_deg, az_spacing_deg)
+    az_bin_midpoints = az_bin_edges[:-1] + az_spacing_deg / 2  # Midpoints between edges
+
+    # Elevation bins from -90 to 90 degrees.
+    el_bin_edges = np.arange(-90, 90 + el_spacing_deg, el_spacing_deg)
+    el_bin_midpoints = el_bin_edges[:-1] + el_spacing_deg / 2  # Midpoints between edges
+
+    # Convert all angles to radians and return them
+    return (
+        np.deg2rad(az_bin_edges),
+        np.deg2rad(el_bin_edges),
+        np.deg2rad(az_bin_midpoints),
+        np.deg2rad(el_bin_midpoints),
+    )
+
+
+def build_solid_angle_map(
+    spacing_deg: float,
+) -> NDArray:
+    """
+    Build a solid angle map in steradians for a given spacing in degrees.
+
+    Parameters
+    ----------
+    spacing_deg : float
+        The bin spacing in degrees.
+
+    Returns
+    -------
+    solid_angle_grid : np.ndarray
+        The solid angle map grid in steradians.
+        First index is latitude/el, second index is longitude/az.
+    """
+    # Degrees are the preferred input units of angle, given map definitions,
+    # but we'll convert to radians for internal calculations and output steradians.
+    spacing = np.deg2rad(spacing_deg)
+
+    if spacing <= 0:
+        raise ValueError("Spacing must be positive valued, non-zero.")
+
+    if not np.isclose((np.pi / spacing) % 1, 0):
+        raise ValueError("Spacing must divide evenly into pi radians.")
+
+    latitudes = np.arange(-np.pi / 2, np.pi / 2 + spacing, step=spacing)
+    sine_latitudes = np.sin(latitudes)
+    delta_sine_latitudes = np.diff(sine_latitudes)
+    solid_angle_by_latitude = np.abs(spacing * delta_sine_latitudes)
+
+    # Order ensures agreement with build_az_el_grid's order of tiling az/el grid.
+    solid_angle_grid = np.repeat(
+        solid_angle_by_latitude[np.newaxis, :], (2 * np.pi) / spacing, axis=0
+    )
+
+    return solid_angle_grid
+
+
+@typing.no_type_check
+def rewrap_even_spaced_az_el_grid(
+    raveled_values: NDArray,
+    shape: tuple[int] | None = None,
+    order: typing.Literal["C"] | typing.Literal["F"] = "C",
+) -> NDArray:
+    """
+    Take an unwrapped (raveled) 1D array and reshapes it into a 2D az/el grid.
+
+    Assumes the following must be true of the original grid:
+    1. Grid was evenly spaced in angular space,
+    2. Grid had the same spacing in both azimuth and elevation.
+    3. Azimuth is axis 0 (and extends a total of 360 degrees).
+    4. Elevation is axis 1 (and extends a total of 180 degrees),
+
+    Parameters
+    ----------
+    raveled_values : NDArray
+        1D array of values to be reshaped into a 2D grid.
+    shape : tuple[int], optional
+        The shape of the original grid, if known, by default None.
+        If None, the shape will be inferred from the size of the input array.
+    order : {'C', 'F'}, optional
+        The order in which to rewrap the values, by default 'C'.
+
+    Returns
+    -------
+    NDArray
+        The reshaped 2D grid of values.
+
+    Raises
+    ------
+    ValueError
+        If the input is not a 1D array or 2D array with an 'extra' non-spatial axis.
+    """
+    if raveled_values.ndim > 2:
+        raise ValueError(
+            "Input must be a 1D array or 2D array with only one spatial axis as axis 0."
+        )
+
+    # We can infer the shape if its evenly spaced and 2D
+    if not shape:
+        spacing_deg = 1 / np.sqrt(raveled_values.shape[0] / (360 * 180))
+        shape = (int(360 // spacing_deg), int(180 // spacing_deg))
+
+    if raveled_values.ndim == 2:
+        shape = (shape[0], shape[1], raveled_values.shape[1])
+    return raveled_values.reshape(shape, order=order)
+
+
+class AzElSkyGrid:
+    """
+    Representation of a 2D grid of azimuth and elevation angles covering the sky.
+
+    All angles are stored internally in radians.
+    Azimuth is within the range [0, 2*pi) radians,
+    elevation is within the range [-pi/2, pi/2) radians.
+
+    Parameters
+    ----------
+    spacing_deg : float, optional
+        Spacing of the grid in degrees, by default 0.5.
+    reversed_elevation : bool, optional
+        Whether the elevation grid should be reversed, by default False.
+        If False, the elevation grid will be from -pi/2 to pi/2 radians (-90 to 90 deg).
+        If True, the elevation grid will be from pi/2 to -pi/2 radians (90 to -90 deg).
+
+    Raises
+    ------
+    ValueError
+        If the spacing is not positive or does not divide evenly into pi radians.
+    """
+
+    def __init__(
+        self,
+        spacing_deg: float = 0.5,
+        reversed_elevation: bool = False,
+    ) -> None:
+        # Store grid properties
+        self.reversed_elevation = reversed_elevation
+
+        # Internally, work in radians, regardless of desired output units
+        self.spacing = np.deg2rad(spacing_deg)
+
+        # Ensure valid grid spacing (positive, divides evenly into pi radians)
+        if self.spacing <= 0:
+            raise ValueError("Spacing must be positive valued, non-zero.")
+
+        if not np.isclose((np.pi / self.spacing) % 1, 0):
+            raise ValueError("Spacing must divide evenly into pi radians.")
+
+        # build_spacial_bins creates the bin edges and centers for azimuth and elevation
+        # E.g. for spacing=1, az_bin_edges = [0, 1, 2, ..., 359, 360] deg.
+        # However returned values are in radians.
+        (
+            self.az_bin_edges,
+            self.el_bin_edges,
+            self.az_bin_midpoints,
+            self.el_bin_midpoints,
+        ) = build_spatial_bins(az_spacing_deg=spacing_deg, el_spacing_deg=spacing_deg)
+
+        # If desired, reverse the elevation range so that the grid is in the order
+        # defined by the Ultra prototype code (`build_dps_grid.m`).
+        if self.reversed_elevation:
+            self.el_bin_midpoints = self.el_bin_midpoints[::-1]
+            self.el_bin_edges = self.el_bin_edges[::-1]
+
+        # Deriving our az/el grids with indexing "ij" allows for ravel_multi_index
+        # to work correctly with 1D digitized indices in each az and el,
+        # using the same ravel order ('C' or 'F') as the grid points were unwrapped.
+        self.az_grid, self.el_grid = np.meshgrid(
+            self.az_bin_midpoints, self.el_bin_midpoints, indexing="ij"
+        )
+
+        # Keep track of number of points on the grid
+        self.grid_shape = self.az_grid.shape
+        self.grid_size = self.az_grid.size
+
+    def __repr__(self) -> str:
+        """
+        Return a string representation of the AzElSkyGrid.
+
+        Returns
+        -------
+        str
+            A string representation of the AzElSkyGrid.
+        """
+        return (
+            f"AzElSkyGrid with a spacing of {self.spacing:.4e} radians. "
+            f"{self.grid_shape} Grid."
+        )

imap_processing/glows/__init__.py

@@ -1 +1,4 @@
 __version__ = "v001"
+
+# Quality flag list length. Used in L1B and L2.
+FLAG_LENGTH = 17

imap_processing/glows/ancillary/imap_glows_pipeline_settings_v001.json

@@ -0,0 +1,52 @@
+{
+    "description": "Settings for ground-processing pipeline for IMAP/GLOWS data",
+    "version": "0.1",
+    "date_of_creation_yyyymmdd": "20230527",
+    "filter_based_on_daily_statistical_error": {
+        "n_sigma_threshold_lower": 3.0,
+        "n_sigma_threshold_upper": 3.0
+    },
+    "filter_based_on_comparison_of_spin_periods": {
+        "relative_difference_threshold": 7.0e-5
+    },
+    "filter_based_on_temperature_std_dev": {
+        "std_dev_threshold__celsius_deg": 2.03
+    },
+    "filter_based_on_hv_voltage_std_dev": {
+        "std_dev_threshold__volt": 50.0
+    },
+    "filter_based_on_spin_period_std_dev": {
+        "std_dev_threshold__sec": 0.033333
+    },
+    "filter_based_on_pulse_length_std_dev": {
+        "std_dev_threshold__usec": 1.0
+    },
+    "filter_based_on_maps": {
+        "angular_distance_for_flaging__deg": 2.5
+    },
+    "active_bad_time_flags": {
+        "is_pps_missing": true,
+        "is_time_status_missing": true,
+        "is_phase_missing": true,
+        "is_spin_period_missing": true,
+        "is_overexposed": true,
+        "is_direct_event_non_monotonic": true,
+        "is_night": true,
+        "is_hv_test_in_progress": true,
+        "is_test_pulse_in_progress": true,
+        "is_memory_error_detected": true,
+        "is_generated_on_ground": true,
+        "is_beyond_daily_statistical_error": true,
+        "is_temperature_std_dev_beyond_threshold": true,
+        "is_hv_voltage_std_dev_beyond_threshold": true,
+        "is_spin_period_std_dev_beyond_threshold": true,
+        "is_pulse_length_std_dev_beyond_threshold": true,
+        "is_spin_period_difference_beyond_threshold": false
+    },
+    "active_bad_angle_flags": {
+        "is_close_to_uv_source": true,
+        "is_inside_excluded_region": true,
+        "is_excluded_by_instr_team": true,
+        "is_suspected_transient": true
+    }
+}

imap_processing/glows/l1a/glows_l1a.py

@@ -2,15 +2,20 @@
 
 from collections import defaultdict
 from pathlib import Path
+from typing import Optional
 
 import numpy as np
 import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.glows.l0.decom_glows import decom_packets
-from imap_processing.glows.l0.glows_l0_data import DirectEventL0
+from imap_processing.glows.l0.glows_l0_data import DirectEventL0, HistogramL0
 from imap_processing.glows.l1a.glows_l1a_data import DirectEventL1A, HistogramL1A
-from imap_processing.spice.time import J2000_EPOCH, met_to_j2000ns
+from imap_processing.glows.l1b.glows_l1b_data import HistogramL1B
+from imap_processing.spice.time import (
+    met_to_datetime64,
+    met_to_ttj2000ns,
+)
 
 
 def create_glows_attr_obj(data_version: str) -> ImapCdfAttributes:
@@ -66,19 +71,25 @@ def glows_l1a(packet_filepath: Path, data_version: str) -> list[xr.Dataset]:
     # Create dictionaries to group data by day
     de_by_day = process_de_l0(de_l0)
     hists_by_day = defaultdict(list)
+    # Assume the observational day starts with the first packet, then find any new
+    # observation days.
+    # TODO: replace determine_observational_day with spin table API
+    obs_days = [hist_l0[0].SEC]
+    obs_days += determine_observational_day(hist_l0)
 
-    # TODO: Make this its own function?
     for hist in hist_l0:
         hist_l1a = HistogramL1A(hist)
-        # Split by IMAP start time
-        # TODO: Should this be MET?
-        hist_day = (J2000_EPOCH + met_to_j2000ns(hist.SEC)).astype("datetime64[D]")
+        # Determine the day the histogram belongs to. This finds the observation
+        # day in obs_day that is nearest the histogram timestamp without going over.
+        hist_day = next(
+            (day for day in reversed(obs_days) if day <= hist.SEC), obs_days[-1]
+        )
         hists_by_day[hist_day].append(hist_l1a)
 
     # Generate CDF files for each day
     output_datasets = []
-    for hist_l1a_list in hists_by_day.values():
-        dataset = generate_histogram_dataset(hist_l1a_list, glows_attrs)
+    for obs_day, hist_l1a_list in hists_by_day.items():
+        dataset = generate_histogram_dataset(hist_l1a_list, glows_attrs, obs_day)
         output_datasets.append(dataset)
 
     for de_l1a_list in de_by_day.values():
@@ -88,6 +99,36 @@ def glows_l1a(packet_filepath: Path, data_version: str) -> list[xr.Dataset]:
     return output_datasets
 
 
+def determine_observational_day(hist_l0: list[HistogramL0]) -> list:
+    """
+    Find the timestamps for each observational day.
+
+    This function temporarily uses the is_night flag to determine the start of a new
+    observational day, but should eventually use the spin table APIs.
+
+    Parameters
+    ----------
+    hist_l0 : list[HistogramL0]
+        List of HistogramL0 objects.
+
+    Returns
+    -------
+    list
+        List of start times for each observational day.
+    """
+    prev_is_night = -1
+    obs_day_change = []
+    for hist in hist_l0:
+        flags = HistogramL1B.deserialize_flags(hist.FLAGS)
+        is_night: int = int(flags[6])
+        if prev_is_night and not is_night:
+            obs_day_change.append(hist.SEC)
+
+        prev_is_night = is_night
+
+    return obs_day_change
+
+
 def process_de_l0(
     de_l0: list[DirectEventL0],
 ) -> dict[np.datetime64, list[DirectEventL1A]]:
@@ -111,7 +152,7 @@ def process_de_l0(
     de_by_day = dict()
 
     for de in de_l0:
-        de_day = (J2000_EPOCH + met_to_j2000ns(de.MET)).astype("datetime64[D]")
+        de_day = (met_to_datetime64(de.MET)).astype("datetime64[D]")
         if de_day not in de_by_day:
             de_by_day[de_day] = [DirectEventL1A(de)]
         # Putting not first data int o last direct event list.
@@ -186,7 +227,7 @@ def generate_de_dataset(
 
     for index, de in enumerate(de_l1a_list):
         # Set the timestamp to the first timestamp of the direct event list
-        epoch_time = met_to_j2000ns(de.l0.MET).astype("datetime64[ns]")
+        epoch_time = met_to_ttj2000ns(de.l0.MET).astype("datetime64[ns]")
 
         # determine if the length of the direct_events numpy array is long enough,
         # and extend the direct_events length dimension if necessary.
@@ -297,7 +338,9 @@ def generate_de_dataset(
 
 
 def generate_histogram_dataset(
-    hist_l1a_list: list[HistogramL1A], glows_cdf_attributes: ImapCdfAttributes
+    hist_l1a_list: list[HistogramL1A],
+    glows_cdf_attributes: ImapCdfAttributes,
+    obs_day: Optional[int] = None,
 ) -> xr.Dataset:
     """
     Generate a dataset for GLOWS L1A histogram data CDF files.
@@ -308,6 +351,9 @@ def generate_histogram_dataset(
         List of HistogramL1A objects for a given day.
     glows_cdf_attributes : ImapCdfAttributes
         Object containing l1a CDF attributes for instrument glows.
+    obs_day : int, optional
+        Observational day counter. If supplied, it will be included in the
+        output file name.
 
     Returns
     -------
@@ -315,7 +361,7 @@ def generate_histogram_dataset(
         Dataset containing the GLOWS L1A histogram CDF output.
     """
     # Store timestamps for each HistogramL1A object.
-    time_data = np.zeros(len(hist_l1a_list), dtype="datetime64[ns]")
+    time_data = np.zeros(len(hist_l1a_list), dtype="int64")
     # TODO Add daily average of histogram counts
     # TODO compute average temperature etc
     # Data in lists, for each of the 25 time varying datapoints in HistogramL1A
@@ -352,7 +398,7 @@ def generate_histogram_dataset(
 
     for index, hist in enumerate(hist_l1a_list):
         # TODO: Should this be MET?
-        epoch_time = met_to_j2000ns(hist.imap_start_time.to_seconds())
+        epoch_time = met_to_ttj2000ns(hist.imap_start_time.to_seconds())
         hist_data[index] = hist.histogram
 
         support_data["flags_set_onboard"].append(hist.flags["flags_set_onboard"])
@@ -384,6 +430,15 @@ def generate_histogram_dataset(
         attrs=glows_cdf_attributes.get_variable_attributes("bins_attrs"),
     )
 
+    bin_label = xr.DataArray(
+        bins.data.astype(str),
+        name="bins_label",
+        dims=["bins_label"],
+        attrs=glows_cdf_attributes.get_variable_attributes(
+            "bins_label", check_schema=False
+        ),
+    )
+
     hist = xr.DataArray(
         hist_data,
         name="histogram",
@@ -395,9 +450,12 @@ def generate_histogram_dataset(
     )
 
     attrs = glows_cdf_attributes.get_global_attributes("imap_glows_l1a_hist")
+    if obs_day:
+        # this needs to be 5 digits, so truncate it from the temporary obs day
+        attrs["Repointing"] = int(str(obs_day)[-5:])
 
     output = xr.Dataset(
-        coords={"epoch": epoch_time, "bins": bins},
+        coords={"epoch": epoch_time, "bins": bins, "bins_label": bin_label},
         attrs=attrs,
     )
 

imap_processing/glows/l1b/glows_l1b.py

@@ -6,6 +6,7 @@ import numpy as np
 import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.glows import FLAG_LENGTH
 from imap_processing.glows.l1b.glows_l1b_data import DirectEventL1B, HistogramL1B
 
 
@@ -45,7 +46,7 @@ def glows_l1b(input_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
 
     if "hist" in logical_source:
         flag_data = xr.DataArray(
-            np.arange(17),
+            np.arange(FLAG_LENGTH),
             name="flag_dim",
             dims=["flag_dim"],
             attrs=cdf_attrs.get_variable_attributes("flag_dim"),

imap_processing/glows/l1b/glows_l1b_data.py

@@ -9,6 +9,7 @@ from typing import Optional
 
 import numpy as np
 
+from imap_processing.glows import FLAG_LENGTH
 from imap_processing.glows.utils.constants import TimeTuple
 
 
@@ -595,7 +596,7 @@ class HistogramL1B:
         # self.unique_block_identifier = np.datetime_as_string(
         #     np.datetime64(int(self.imap_start_time), "ns"), "s"
         # )
-        self.flags = np.ones((17,), dtype=np.uint8)
+        self.flags = np.ones((FLAG_LENGTH,), dtype=np.uint8)
 
     def output_data(self) -> tuple:
         """
@@ -610,3 +611,26 @@ class HistogramL1B:
             A tuple containing each attribute value in the class.
         """
         return tuple(getattr(self, out.name) for out in dataclasses.fields(self))
+
+    @staticmethod
+    def deserialize_flags(raw: int) -> np.ndarray[int]:
+        """
+        Deserialize the flags into a list.
+
+        Parameters
+        ----------
+        raw : int
+            16 bit integer containing the on-board flags to deserialize.
+
+        Returns
+        -------
+        flags : np.ndarray
+            Array of flags as a boolean.
+        """
+        # there are only 10 flags in the on-board flag array, additional flags are added
+        # later.
+        flags: np.ndarray[bool] = np.array(
+            [bool((raw >> i) & 1) for i in range(10)], dtype=bool
+        )
+
+        return flags