imap-processing 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

imap_processing/glows/l2/glows_l2.py

@@ -0,0 +1,324 @@
+"""Module for GLOWS Level 2 processing."""
+
+import dataclasses
+
+import numpy as np
+import xarray as xr
+from numpy.typing import NDArray
+
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.glows import FLAG_LENGTH
+from imap_processing.glows.l1b.glows_l1b_data import HistogramL1B
+from imap_processing.glows.l2.glows_l2_data import DailyLightcurve, HistogramL2
+
+
+def glows_l2(input_dataset: xr.Dataset, data_version: str) -> list[xr.Dataset]:
+    """
+    Will process GLoWS L2 data from L1 data.
+
+    Parameters
+    ----------
+    input_dataset : xarray.Dataset
+        Input L1B dataset.
+    data_version : str
+        Version for output.
+
+    Returns
+    -------
+    xarray.Dataset
+     Glows L2 Dataset.
+    """
+    cdf_attrs = ImapCdfAttributes()
+    cdf_attrs.add_instrument_global_attrs("glows")
+    cdf_attrs.add_instrument_variable_attrs("glows", "l2")
+    cdf_attrs.add_global_attribute("Data_version", data_version)
+
+    split_data = split_data_by_observational_day(input_dataset)
+    l2_output = []
+    for data in split_data:
+        l2 = generate_l2(data)
+        l2_output.append(create_l2_dataset(l2, cdf_attrs))
+
+    return l2_output
+
+
+# TODO: filter good times out
+def generate_l2(l1b_dataset: xr.Dataset) -> HistogramL2:
+    """
+    Generate L2 data from L1B data.
+
+    Returns L2 data in the form of a HistogramL2 dataclass.
+
+    Parameters
+    ----------
+    l1b_dataset : xarray.Dataset
+        Input L1B dataset.
+
+    Returns
+    -------
+    HistogramL2
+        L2 data in the form of a HistogramL2 dataclass.
+    """
+    # most of the values from L1B are averaged over a day
+    good_data = l1b_dataset.isel(
+        epoch=return_good_times(l1b_dataset["flags"], np.ones((FLAG_LENGTH,)))
+    )
+    # todo: bad angle filter
+    # TODO filter bad bins out. Needs to happen here while everything is still
+    # per-timestamp.
+
+    # one dataset collects multiple epoch values which need to be averaged down into
+    # one value.
+    all_variables = dataclasses.fields(HistogramL1B)
+
+    daily_lightcurve = DailyLightcurve(good_data)
+
+    var_outputs = {
+        "total_l1b_inputs": len(good_data["epoch"]),
+        "number_of_good_l1b_inputs": len(good_data["epoch"]),
+        # TODO replace post-filter
+        "identifier": 100,  # TODO: retrieve from spin table
+        # TODO fill this in
+        "bad_time_flag_occurrences": np.zeros((1, FLAG_LENGTH)),
+        # Accumulate all the histograms from good times from the day into one
+        "daily_lightcurve": daily_lightcurve,
+    }
+
+    if len(good_data["epoch"]) != 0:
+        # Generate outputs that are passed in directly from L1B
+        var_outputs["start_time"] = good_data["epoch"].data[0]
+        var_outputs["end_time"] = good_data["epoch"].data[-1]
+
+    else:
+        # No good times in the file
+        var_outputs["start_time"] = l1b_dataset["imap_start_time"].data[0]
+        var_outputs["end_time"] = (
+            l1b_dataset["imap_start_time"].data[0]
+            + l1b_dataset["imap_time_offset"].data[0]
+        )
+
+    for field in all_variables:
+        var_name = field.name
+        if "average" in var_name:
+            # This results in a scalar value, so `keepdims=True` ensures we keep the
+            # epoch dimension.
+            var_outputs[var_name] = (
+                l1b_dataset[var_name].mean(dim="epoch", keepdims=True).data
+            )
+
+            var_outputs[var_name.replace("average", "std_dev")] = (
+                l1b_dataset[var_name].std(dim="epoch", keepdims=True).data
+            )
+
+    # l1b stuff is done
+    output = HistogramL2(**var_outputs)
+
+    return output
+
+
+def filter_bad_bins(histograms: NDArray, bin_exclusions: NDArray) -> NDArray:
+    """
+    Filter out bad bins from the histogram.
+
+    Parameters
+    ----------
+    histograms : numpy.ndarray
+        Histogram data, with shape (n_timestamps, n_bins).
+    bin_exclusions : numpy.ndarray
+        Array of bin exclusions. This 2d array has a timestamp and bin filter array
+        pair. The bin filter array indicates "1" if a bin is to be excluded.
+
+    Returns
+    -------
+    numpy.ndarray
+        Histogram data with bad bins marked with -1.
+    """
+    # TODO: will need ancillary file imap_glows_exclusions_by_instr_team
+    # TODO: complete once unique_block_identifier is implemented
+    # file contains timestamp & bin filter array pairs. For the timestamp, the
+    # filter should be applied such that 1 excludes the bin.
+
+    # excluded bins can be marked with -1
+    return histograms
+
+
+def split_data_by_observational_day(input_dataset: xr.Dataset) -> list[xr.Dataset]:
+    """
+    Return L1B data array for an observational day, given start and stop times.
+
+    Parameters
+    ----------
+    input_dataset : xarray.Dataset
+        Input L1B dataset.
+
+    Returns
+    -------
+    list : xarray.Dataset
+        List of L1B datasets, each representing a day of data.
+    """
+    # TODO: replace this with a query to the spin table to get the observational days
+
+    # Find the range of epoch values within the observational day.
+    # This should be replaced with a query to the spin table to get the observational
+    # days within the time range of the file and when those observational days
+    # start and stop
+
+    # Note: slice is inclusive on both sides.
+    # data_by_day = [
+    #     input_dataset.sel(epoch=slice(day_ends[i], day_ends[i + 1]))
+    #     for i in range(len(day_ends) - 1)
+    # ]
+    return [input_dataset]
+
+
+def create_l2_dataset(
+    histogram_l2: HistogramL2, attrs: ImapCdfAttributes
+) -> xr.Dataset:
+    """
+    Create a xarray dataset from a HistogramL2 dataclass.
+
+    This dataset should include all the CDF attributes.
+
+    Parameters
+    ----------
+    histogram_l2 : HistogramL2
+        L2 data.
+    attrs : ImapCdfAttributes
+        CDF attributes for GLOWS L2.
+
+    Returns
+    -------
+    xarray.Dataset
+        L2 dataset for output to CDF file.
+    """
+    # Each L2 file only has one timestamp.
+    # TODO: If we want this to point to the start time, we need to set the attribute
+    #  variable BIN_LOCATION to 0. Otherwise, we need this to be halfway between start
+    #  time and end time.
+    time_data = np.array([histogram_l2.start_time], dtype=np.float64)
+    # TODO: Create CDF attributes
+    epoch_time = xr.DataArray(
+        time_data,
+        name="epoch",
+        dims=["epoch"],
+        attrs=attrs.get_variable_attributes("epoch", check_schema=False),
+    )
+
+    bins = xr.DataArray(
+        np.arange(histogram_l2.daily_lightcurve.number_of_bins),
+        name="bins",
+        dims=["bins"],
+        attrs=attrs.get_variable_attributes("bins_dim", check_schema=False),
+    )
+
+    bins_label = xr.DataArray(
+        -1,
+        name="bins_label",
+        attrs=attrs.get_variable_attributes("bins_label", check_schema=False),
+    )
+
+    flags = xr.DataArray(
+        np.ones(FLAG_LENGTH),
+        dims=["flags"],
+        attrs=attrs.get_variable_attributes("flags_dim", check_schema=False),
+    )
+
+    flags_label = xr.DataArray(
+        -1,
+        name="flags_label",
+        attrs=attrs.get_variable_attributes("flags_label", check_schema=False),
+    )
+
+    eclipic_data = xr.DataArray(
+        np.arange(3),
+        name="ecliptic",
+        dims=["ecliptic"],
+        attrs=attrs.get_variable_attributes("ecliptic_dim", check_schema=False),
+    )
+
+    output = xr.Dataset(
+        data_vars={"bins_label": bins_label, "flags_label": flags_label},
+        coords={
+            "epoch": epoch_time,
+            "bins": bins,
+            "flags": flags,
+            "ecliptic": eclipic_data,
+        },
+        attrs=attrs.get_global_attributes("imap_glows_l2_hist"),
+    )
+
+    ecliptic_variables = [
+        "spacecraft_location_average",
+        "spacecraft_location_std_dev",
+        "spacecraft_velocity_average",
+        "spacecraft_velocity_std_dev",
+    ]
+
+    for key, value in dataclasses.asdict(histogram_l2).items():
+        if key in ecliptic_variables:
+            output[key] = xr.DataArray(
+                value,
+                dims=["epoch", "ecliptic"],
+                attrs=attrs.get_variable_attributes(key),
+            )
+        elif key == "bad_time_flag_occurrences":
+            output[key] = xr.DataArray(
+                value,
+                dims=["epoch", "flags"],
+                attrs=attrs.get_variable_attributes(key),
+            )
+
+        elif key != "daily_lightcurve":
+            val = value
+            if type(value) != np.ndarray:
+                val = np.array([value])
+            output[key] = xr.DataArray(
+                val,
+                dims=["epoch"],
+                attrs=attrs.get_variable_attributes(key),
+            )
+
+    for key, value in dataclasses.asdict(histogram_l2.daily_lightcurve).items():
+        if key == "number_of_bins":
+            # number_of_bins does not have n_bins dimensions.
+            output[key] = xr.DataArray(
+                np.array([value]),
+                dims=["epoch"],
+                attrs=attrs.get_variable_attributes(key),
+            )
+        else:
+            output[key] = xr.DataArray(
+                np.array([value]),
+                dims=["epoch", "bins"],
+                attrs=attrs.get_variable_attributes(key),
+            )
+
+    return output
+
+
+def return_good_times(flags: xr.DataArray, active_flags: NDArray) -> NDArray:
+    """
+    Return the good times based on the input flags.
+
+    Parameters
+    ----------
+    flags : xarray.DataArray
+        Flags dataset with shape (n_timestamps, n_flags). If a flag is active and set
+        to 1, the timestamp is considered good.
+
+    active_flags : numpy.ndarray
+        Array of active flags. If the flag is set to 1, it is considered active.
+
+    Returns
+    -------
+    numpy.ndarray
+        An array of indices for good times.
+    """
+    if len(active_flags) != flags.shape[1]:
+        print("Active flags don't matched expected length")
+
+    # A good time is where all the active flags are equal to one.
+    # Here, we mask the active indices using active_flags, and then return the times
+    # where all the active indices == 1.
+    good_times = np.where(np.all(flags[:, active_flags == 1] == 1, axis=1))[0]
+    return good_times

imap_processing/glows/l2/glows_l2_data.py

@@ -3,6 +3,139 @@
 from dataclasses import InitVar, dataclass, field
 
 import numpy as np
+import xarray as xr
+from numpy.typing import NDArray
+
+
+@dataclass
+class DailyLightcurve:
+    """
+    Dataclass describing the daily lightcurve.
+
+    Used inside the HistogramL2 class.
+
+    Attributes
+    ----------
+    spin_angle : numpy.ndarray
+        values of spin angle [deg] for bin centers (measured from the north)
+    photon_flux : numpy.ndarray
+        observational-day-averaged photon flux [Rayleigh]
+    raw_histograms : numpy.ndarray
+        sum of histograms across all timestamps
+    exposure_times : numpy.ndarray
+        exposure times for bins [s]
+    flux_uncertainties : numpy.ndarray
+        statistical uncertainties for photon flux [Rayleigh]
+    histogram_flag_array : numpy.ndarray
+        flags for bins
+    ecliptic_lon : numpy.ndarray
+        ecliptic longitude of bin centers [deg]
+    ecliptic_lat : numpy.ndarray
+        ecliptic latitude of bin centers [deg]
+    number_of_bins : int
+        number of bins in lightcurve
+    raw_uncertainties : numpy.ndarray
+        statistical uncertainties for raw histograms (sqrt of self.raw_histograms)
+    l1b_data : xarray.Dataset
+        L1B data filtered by good times, good angles, and good bins.
+    """
+
+    # All variables should have n_bin elements
+    spin_angle: np.ndarray = field(init=False)
+    photon_flux: np.ndarray = field(init=False)
+    raw_histograms: np.ndarray = field(init=False)
+    exposure_times: np.ndarray = field(init=False)
+    flux_uncertainties: np.ndarray = field(init=False)
+    # TODO: flag array
+    histogram_flag_array: np.ndarray = field(init=False)
+    # TODO: ecliptic coordinates
+    ecliptic_lon: np.ndarray = field(init=False)
+    ecliptic_lat: np.ndarray = field(init=False)
+    number_of_bins: int = field(init=False)
+    raw_uncertainties: np.ndarray = field(init=False)
+    l1b_data: InitVar[xr.Dataset]
+
+    def __post_init__(self, l1b_data: xr.Dataset) -> None:
+        """
+        Compute all the daily lightcurve variables from L1B data.
+
+        Parameters
+        ----------
+        l1b_data : xarray.Dataset
+            L1B data filtered by good times, good angles, and good bins for one
+            observation day.
+        """
+        self.raw_histograms = self.calculate_histogram_sums(l1b_data["histogram"].data)
+
+        exposure_times_per_timestamp = (
+            l1b_data["spin_period_average"]
+            * l1b_data["number_of_spins_per_block"]
+            / 3600
+        )
+
+        self.exposure_times = self.calculate_exposure_times(
+            l1b_data, exposure_times_per_timestamp
+        )
+        self.raw_uncertainties = np.sqrt(self.raw_histograms)
+        self.photon_flux = np.zeros(len(self.raw_histograms))
+        self.flux_uncertainties = np.zeros(len(self.raw_histograms))
+
+        # TODO: Only where exposure counts != 0
+        if len(self.exposure_times) != 0:
+            self.photon_flux = self.raw_histograms / self.exposure_times
+            self.flux_uncertainties = self.raw_uncertainties / self.exposure_times
+
+        # TODO: Average this, or should they all be the same?
+        self.spin_angle = np.average(l1b_data["imap_spin_angle_bin_cntr"].data, axis=0)
+
+        # TODO: is the first number here ok? Would it change mid-obs day?
+        self.number_of_bins = len(self.spin_angle)
+
+        self.histogram_flag_array = np.zeros(self.number_of_bins)
+        self.ecliptic_lon = np.zeros(self.number_of_bins)
+        self.ecliptic_lat = np.zeros(self.number_of_bins)
+
+    @staticmethod
+    def calculate_exposure_times(
+        good_times: xr.Dataset, exposure_count: xr.DataArray
+    ) -> NDArray:
+        """
+        Calculate exposure times for each bin across all the timestamps.
+
+        Parameters
+        ----------
+        good_times : xarray.Dataset
+            Dataset with only good times.
+        exposure_count : float
+            Exposure count for each valid timestamp and bin.
+
+        Returns
+        -------
+        numpy.ndarray
+            Array of summed exposure times for each bin.
+        """
+        weighted_sum = (good_times["histogram"].data != -1) * exposure_count.data[
+            :, np.newaxis
+        ]
+        return np.sum(weighted_sum, axis=0)
+
+    @staticmethod
+    def calculate_histogram_sums(histograms: NDArray) -> NDArray:
+        """
+        Calculate the sum of histograms across all timestamps.
+
+        Parameters
+        ----------
+        histograms : numpy.ndarray
+            Array of histograms across all timestamps.
+
+        Returns
+        -------
+        numpy.ndarray
+            Sum of valid histograms across all timestamps.
+        """
+        histograms[histograms == -1] = 0
+        return np.sum(histograms, axis=0)
 
 
 @dataclass
@@ -12,68 +145,68 @@ class HistogramL2:
 
     This class collects multiple HistogramL1B classes into one L2 per observational day.
 
-    flight_software_version : str
+    Attributes
+    ----------
     number_of_good_l1b_inputs : int
-        number of good-time Level-1B times used for generation of Level-2 data
+        number of good-time Level-1B times used for generation of Level-2 data.
     total_l1b_inputs : int
-      number of all Level-1B times for observational day
+      number of all Level-1B times for observational day.
     identifier : int
         unique Level-2 histogram identifier
     start_time : numpy.double
-        UTC start time of a given observational day
+        J2000 start time of a given observational day
     end_time : numpy.double
-        UTC end time of a given observational day
+        J2000 end time of a given observational day
     daily_lightcurve : numpy.ndarray
         arrays for observational-day-accumulated lightcurve
     filter_temperature_average : numpy.ndarray
         observational-day-averaged filter temperature [Celsius deg]
-    filter_temperature_variance : numpy.ndarray
+    filter_temperature_std_dev : numpy.ndarray
         standard deviation for filter temperature [Celsius deg]
     hv_voltage_average : numpy.ndarray
         observational-day-averaged channeltron voltage [volt]
-    hv_voltage_variance : numpy.ndarray
+    hv_voltage_std_dev : numpy.ndarray
         standard deviation for channeltron voltage [volt]
     spin_period_average : numpy.ndarray
         observational-day-averaged spin period [s] (onboard value)
-    spin_period_variance : numpy.ndarray
+    spin_period_std_dev : numpy.ndarray
         a standard deviation for spin period [s]
     pulse_length_average : numpy.ndarray
         observational-day-averaged pulse length [μs]
-    pulse_length_variance : numpy.ndarray
+    pulse_length_std_dev : numpy.ndarray
         standard deviation for pulse length [μs]
     spin_period_ground_average : numpy.ndarray
         observational-day-averaged spin period [s] (ground value)
-    spin_period_ground_variance : numpy.ndarray
+    spin_period_ground_std_dev : numpy.ndarray
         a standard deviation for spin period [s]
     position_angle_offset_average : numpy.ndarray
         observational-day-averaged GLOWS angular offset [deg]
-    position_angle_offset_variance : numpy.ndarray
+    position_angle_offset_std_dev : numpy.ndarray
         standard deviation for GLOWS angular offset [seg]
-    spin_axis_orientation_variance : numpy.ndarray
+    spin_axis_orientation_std_dev : numpy.ndarray
         standard deviation for spin-axis longitude and latitude [deg]
-    spacecraft_location_variance : numpy.ndarray
-        standard deviation for ecliptic coordinates [km] of IMAP
-    spacecraft_velocity_variance : numpy.ndarray
-        standard deviation for IMAP velocity components [km/s]
-    spin_axis_orientation_average : numpy.ndarray
-        observational-day-averaged spin-axis ecliptic longitude and latitude [deg]
     spacecraft_location_average : numpy.ndarray
         observational-day-averaged Cartesian ecliptic coordinates ⟨X⟩, ⟨Y ⟩, ⟨Z⟩ [km]
         of IMAP
+    spacecraft_location_std_dev : numpy.ndarray
+        standard deviation for ecliptic coordinates [km] of IMAP
     spacecraft_velocity_average : numpy.ndarray
         observational-day-averaged values ⟨VX ⟩, ⟨VY ⟩, ⟨VZ ⟩ of IMAP velocity
         components [km/s] (Cartesian ecliptic frame)
+    spacecraft_velocity_std_dev : numpy.ndarray
+        standard deviation for IMAP velocity components [km/s]
+    spin_axis_orientation_average : numpy.ndarray
+        observational-day-averaged spin-axis ecliptic longitude and latitude [deg]
     bad_time_flag_occurrences : numpy.ndarray
         numbers of occurrences of blocks for each bad-time flag during observational day
     """
 
-    flight_software_version: str
     number_of_good_l1b_inputs: int
     total_l1b_inputs: int
-    # identifier: int  # comes from unique_block_identifier
+    identifier: int  # TODO: Should be the official pointing number
     start_time: np.double
     end_time: np.double
-    daily_lightcurve: np.ndarray = field(init=False)
+    daily_lightcurve: DailyLightcurve
     filter_temperature_average: np.ndarray[np.double]
     filter_temperature_std_dev: np.ndarray[np.double]
     hv_voltage_average: np.ndarray[np.double]
@@ -87,37 +220,9 @@ class HistogramL2:
     position_angle_offset_average: np.ndarray[np.double]
     position_angle_offset_std_dev: np.ndarray[np.double]
     spin_axis_orientation_std_dev: np.ndarray[np.double]
+    spacecraft_location_average: np.ndarray[np.double]
     spacecraft_location_std_dev: np.ndarray[np.double]
+    spacecraft_velocity_average: np.ndarray[np.double]
     spacecraft_velocity_std_dev: np.ndarray[np.double]
     spin_axis_orientation_average: np.ndarray[np.double]
-    spacecraft_location_average: np.ndarray[np.double]
-    spacecraft_velocity_average: np.ndarray[np.double]
     bad_time_flag_occurrences: np.ndarray
-    histogram: InitVar[np.ndarray]
-
-    def __post_init__(self, histogram: np.ndarray) -> None:
-        """
-        Post-initialization method to generate the daily light curve from one histogram.
-
-        Parameters
-        ----------
-        histogram : numpy.ndarray
-            Histogram data from L1B, of shape (bins,) where bins is nominally 3600.
-        """
-        self.daily_lightcurve = self.generate_lightcurve(histogram)
-
-    def generate_lightcurve(self, histogram: np.ndarray) -> np.ndarray:
-        """
-        Given an array of (n, bins) histograms, generate one lightcurve of size (bins).
-
-        Parameters
-        ----------
-        histogram : numpy.ndarray
-            Histogram data from L1B, of shape (bins,) where bins is nominally 3600.
-
-        Returns
-        -------
-        numpy.ndarray
-            Lightcurve of size (bins).
-        """
-        return np.zeros(3600)  # type: ignore[no-any-return]