imap-processing 1.0.27__tar.gz → 1.0.29__tar.gz

{imap_processing-1.0.27 → imap_processing-1.0.29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: imap-processing
-Version: 1.0.27
+Version: 1.0.29
 Summary: IMAP Science Operations Center Processing
 License: MIT
 Keywords: IMAP,SDC,SOC,Science Operations

{imap_processing-1.0.27 → imap_processing-1.0.29}/imap_processing/_version.py

@@ -1,3 +1,3 @@
 # These version placeholders will be replaced later during substitution.
-__version__ = "1.0.27"
-__version_tuple__ = (1, 0, 27)
+__version__ = "1.0.29"
+__version_tuple__ = (1, 0, 29)

{imap_processing-1.0.27 → imap_processing-1.0.29}/imap_processing/cdf/config/imap_ultra_global_cdf_attrs.yaml

@@ -212,6 +212,54 @@ imap_ultra_l1b_90sensor-de:
     Logical_source: imap_ultra_l1b_90sensor-de
     Logical_source_description: IMAP-Ultra Instrument Level-1B Direct Event Data.
 
+imap_ultra_l1b_45sensor-priority-1-de:
+    <<: *instrument_base
+    Data_type: L1B_PRIORITY_1_DE>Level-1B Priority 1 Direct Event
+    Logical_source: imap_ultra_l1b_45sensor-priority-1-de
+    Logical_source_description: IMAP-Ultra Instrument Level-1B Priority 1 Direct Event Data.
+
+imap_ultra_l1b_90sensor-priority-1-de:
+    <<: *instrument_base
+    Data_type: L1B_PRIORITY_1_DE>Level-1B Priority 1 Direct Event
+    Logical_source: imap_ultra_l1b_90sensor-priority-1-de
+    Logical_source_description: IMAP-Ultra Instrument Level-1B Priority 1 Direct Event Data.
+
+imap_ultra_l1b_45sensor-priority-2-de:
+    <<: *instrument_base
+    Data_type: L1B_PRIORITY_2_DE>Level-1B Priority 2 Direct Event
+    Logical_source: imap_ultra_l1b_45sensor-priority-2-de
+    Logical_source_description: IMAP-Ultra Instrument Level-1B Priority 2 Direct Event Data.
+
+imap_ultra_l1b_90sensor-priority-2-de:
+    <<: *instrument_base
+    Data_type: L1B_PRIORITY_2_DE>Level-1B Priority 2 Direct Event
+    Logical_source: imap_ultra_l1b_90sensor-priority-2-de
+    Logical_source_description: IMAP-Ultra Instrument Level-1B Priority 2 Direct Event Data.
+
+imap_ultra_l1b_45sensor-priority-3-de:
+    <<: *instrument_base
+    Data_type: L1B_PRIORITY_3_DE>Level-1B Priority 3 Direct Event
+    Logical_source: imap_ultra_l1b_45sensor-priority-3-de
+    Logical_source_description: IMAP-Ultra Instrument Level-1B Priority 3 Direct Event Data.
+
+imap_ultra_l1b_90sensor-priority-3-de:
+    <<: *instrument_base
+    Data_type: L1B_PRIORITY_3_DE>Level-1B Priority 3 Direct Event
+    Logical_source: imap_ultra_l1b_90sensor-priority-3-de
+    Logical_source_description: IMAP-Ultra Instrument Level-1B Priority 3 Direct Event Data.
+
+imap_ultra_l1b_45sensor-priority-4-de:
+    <<: *instrument_base
+    Data_type: L1B_PRIORITY_4_DE>Level-1B Priority 4 Direct Event
+    Logical_source: imap_ultra_l1b_45sensor-priority-4-de
+    Logical_source_description: IMAP-Ultra Instrument Level-1B Priority 4 Direct Event Data.
+
+imap_ultra_l1b_90sensor-priority-4-de:
+    <<: *instrument_base
+    Data_type: L1B_PRIORITY_4_DE>Level-1B Priority 4 Direct Event
+    Logical_source: imap_ultra_l1b_90sensor-priority-4-de
+    Logical_source_description: IMAP-Ultra Instrument Level-1B Priority 4 Direct Event Data.
+
 imap_ultra_l1b_45sensor-extendedspin:
     <<: *instrument_base
     Data_type: L1B_45Sensor-ExtendedSpin>Level-1B Extended Spin for Ultra45

{imap_processing-1.0.27 → imap_processing-1.0.29}/imap_processing/cli.py

@@ -879,11 +879,36 @@ class Hi(ProcessInstrument):
                         f"Expected exactly one DE science dependency. "
                         f"Got {l1b_de_paths}"
                     )
-                anc_paths = dependencies.get_file_paths(data_type="ancillary")
-                if len(anc_paths) != 1:
+
+                # Get ancillary dependencies
+                anc_dependencies = dependencies.get_processing_inputs(
+                    data_type="ancillary"
+                )
+                if len(anc_dependencies) != 2:
+                    raise ValueError(
+                        f"Expected two ancillary dependencies (cal-prod and "
+                        f"backgrounds). Got "
+                        f"{[anc_dep.descriptor for anc_dep in anc_dependencies]}"
+                    )
+
+                # Create mapping from descriptor to path
+                anc_path_dict = {
+                    dep.descriptor.split("-", 1)[1]: dep.imap_file_paths[
+                        0
+                    ].construct_path()
+                    for dep in anc_dependencies
+                }
+
+                # Verify we have both required ancillary files
+                if (
+                    "cal-prod" not in anc_path_dict
+                    or "backgrounds" not in anc_path_dict
+                ):
                     raise ValueError(
-                        f"Expected exactly one ancillary dependency. Got {anc_paths}"
+                        f"Missing required ancillary files. Expected 'cal-prod' and "
+                        f"'backgrounds', got {list(anc_path_dict.keys())}"
                     )
+
                 # Load goodtimes dependency
                 goodtimes_paths = dependencies.get_file_paths(
                     source="hi", data_type="l1b", descriptor="goodtimes"
@@ -893,10 +918,12 @@ class Hi(ProcessInstrument):
                         f"Expected exactly one goodtimes dependency. "
                         f"Got {goodtimes_paths}"
                     )
+
                 datasets = hi_l1c.hi_l1c(
                     load_cdf(l1b_de_paths[0]),
-                    anc_paths[0],
+                    anc_path_dict["cal-prod"],
                     load_cdf(goodtimes_paths[0]),
+                    anc_path_dict["backgrounds"],
                 )
         elif self.data_level == "l2":
             science_paths = dependencies.get_file_paths(source="hi", data_type="l1c")

{imap_processing-1.0.27 → imap_processing-1.0.29}/imap_processing/codice/codice_l1a_hi_sectored.py

@@ -234,7 +234,13 @@ def l1a_hi_sectored(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:
 
         # Extract species data from decompressed data:
         # - (num_packets, energy_bins, spin_sector, inst_az)
+        # Unpacked data is in this order:
+        #   (epoch, number_species, energy_{species}, inst_az, spin_sector)
         species_data = decompressed_data[:, species_index, :, :, :]
+        # Now transpose to put data in the correct order for writing to CDF:
+        #   (epoch, energy_{species}, spin_sector, inst_az)
+        species_data = np.transpose(species_data, [0, 1, 3, 2])
+
         species_attrs = cdf_attrs.get_variable_attributes("hi-species-attrs")
         species_attrs = apply_replacements_to_attrs(
             species_attrs, {"species": species_name}

{imap_processing-1.0.27 → imap_processing-1.0.29}/imap_processing/codice/codice_l2.py

@@ -1166,6 +1166,16 @@ def process_hi_sectored(dependencies: ProcessingInputCollection) -> xr.Dataset:
     base_angles = np.asarray(L2_HI_SECTORED_ANGLE, dtype=float).reshape(n_spin, 1)
 
     spin_angle = (base_angles + elevation_offsets) % 360.0
+    # We need to transpose spin_angle to put spin_angle data into correct
+    # dimensions. Eg.
+    #   spin_angle[0,0] - 285
+    #   spin_angle[1,0] - 315
+    #   ....
+    # This is expected behavior per CoDICE because the spin angle should increments
+    # at 30 degree spin angle per elevation angle. Due to that, in the example, the
+    # column remained same but spin angle incremented by 30 degrees for each
+    # elevation angle.
+    spin_angle = spin_angle.T
     # Add spin angle variable using the new elevation_angle dimension
     l2_dataset["spin_angle"] = (("spin_sector", "elevation_angle"), spin_angle)
     l2_dataset["spin_angle"].attrs = cdf_attrs.get_variable_attributes(

{imap_processing-1.0.27 → imap_processing-1.0.29}/imap_processing/ena_maps/ena_maps.py

@@ -627,6 +627,14 @@ class UltraPointingSet(HealpixPointingSet):
         nside of the input pset counts variable (e.g. 128) to the nside of the pset.
         """
         pset_data = self.data
+        # TODO remove this check once we reprocess all psets
+        #  going forward, all psets should have counts_pixel_index as a coordinate.
+        if "counts_pixel_index" not in pset_data.dims:
+            logger.info(
+                "No counts_pixel_index found in the dataset. Skipping counts "
+                "downsampling."
+            )
+            return
         counts_n_pix = pset_data.sizes["counts_pixel_index"]
         pset_n_pix = hp.nside2npix(self.nside)
         if counts_n_pix != pset_n_pix:
@@ -647,9 +655,11 @@ class UltraPointingSet(HealpixPointingSet):
             # Get counts in nested ordering. In nested ordering, the
             # pixels that need to be binned together to go from the counts nside to
             # the pset nside are contiguous in the array.
+            # Use nest2ring to get the indices to convert from ring to nest ordering if
+            # necessary.
             if not self.nested:
                 counts_n = counts[
-                    :, hp.ring2nest(counts_nside, np.arange(counts_n_pix))
+                    :, hp.nest2ring(counts_nside, np.arange(counts_n_pix))
                 ]
             else:
                 counts_n = counts
@@ -667,7 +677,7 @@ class UltraPointingSet(HealpixPointingSet):
                 # convert back to ring ordering if necessary and store in the
                 # downsampled counts array
                 binned_counts_n = binned_counts_n[
-                    :, hp.nest2ring(self.nside, np.arange(pset_n_pix))
+                    :, hp.ring2nest(self.nside, np.arange(pset_n_pix))
                 ]
 
             self.data["counts"] = xr.DataArray(

{imap_processing-1.0.27 → imap_processing-1.0.29}/imap_processing/glows/l2/glows_l2.py

@@ -59,7 +59,7 @@ def glows_l2(
         pipeline_settings_dataset.sel(epoch=day, method="nearest")
     )
 
-    l2 = HistogramL2(input_dataset, pipeline_settings)
+    l2 = HistogramL2(input_dataset, pipeline_settings, calibration_dataset)
     if l2.number_of_good_l1b_inputs == 0:
         logger.warning("No good data found in L1B dataset. Returning empty list.")
         return []

{imap_processing-1.0.27 → imap_processing-1.0.29}/imap_processing/glows/l2/glows_l2_data.py

@@ -14,7 +14,12 @@ from imap_processing.spice.geometry import (
     frame_transform_az_el,
     get_instrument_mounting_az_el,
 )
-from imap_processing.spice.time import met_to_sclkticks, sct_to_et
+from imap_processing.spice.time import (
+    et_to_datetime64,
+    met_to_sclkticks,
+    sct_to_et,
+    ttj2000ns_to_et,
+)
 
 
 @dataclass
@@ -46,6 +51,8 @@ class DailyLightcurve:
         number of bins in lightcurve
     l1b_data : xarray.Dataset
         L1B data filtered by good times, good angles, and good bins.
+    calibration_factor : float | None
+        Rayleigh calibration factor used for flux calculations.
     """
 
     # All variables should have n_bin elements
@@ -55,14 +62,19 @@ class DailyLightcurve:
     exposure_times: np.ndarray = field(init=False)
     flux_uncertainties: np.ndarray = field(init=False)
     histogram_flag_array: np.ndarray = field(init=False)
-    # TODO: ecliptic coordinates
     ecliptic_lon: np.ndarray = field(init=False)
     ecliptic_lat: np.ndarray = field(init=False)
     number_of_bins: int = field(init=False)
     l1b_data: InitVar[xr.Dataset]
     position_angle: InitVar[float]
-
-    def __post_init__(self, l1b_data: xr.Dataset, position_angle: float) -> None:
+    calibration_factor: InitVar[float | None]
+
+    def __post_init__(
+        self,
+        l1b_data: xr.Dataset,
+        position_angle: float,
+        calibration_factor: float | None,
+    ) -> None:
         """
         Compute all the daily lightcurve variables from L1B data.
 
@@ -74,6 +86,10 @@ class DailyLightcurve:
         position_angle : float
             The offset angle of the GLOWS instrument from the north spin point - this
             is used in spin angle calculations.
+        calibration_factor : float
+            Calibration factor used for flux calculations, in units of counts per second
+            per Rayleigh. This is used to convert from raw histograms and exposure times
+            to physical photon flux units.
         """
         # number_of_bins_per_histogram is the count of valid (non-FILLVAL) bins.
         # Histogram arrays from L1B are always GlowsConstants.STANDARD_BIN_COUNT
@@ -107,12 +123,16 @@ class DailyLightcurve:
         self.flux_uncertainties = np.zeros(self.number_of_bins)
 
         if (
-            len(self.exposure_times) != 0
+            self.number_of_bins > 0
             and self.exposure_times[0] > 0
-            and len(np.unique(self.exposure_times)) == 1
+            and calibration_factor
         ):
-            self.photon_flux = self.raw_histograms / self.exposure_times
-            self.flux_uncertainties = raw_uncertainties / self.exposure_times
+            self.photon_flux = (
+                self.raw_histograms / self.exposure_times
+            ) / calibration_factor
+            self.flux_uncertainties = (
+                raw_uncertainties / self.exposure_times
+            ) / calibration_factor
 
         self.spin_angle = np.zeros(0)
 
@@ -244,6 +264,8 @@ class HistogramL2:
             GLOWS histogram L1B dataset, as produced by glows_l1b.py.
         pipeline_settings : PipelineSettings
             Pipeline settings object read from ancillary file.
+        calibration_dataset : xr.Dataset
+            The cps-to-Rayleigh calibration dataset needed for flux calculations.
 
     Attributes
     ----------
@@ -327,7 +349,12 @@ class HistogramL2:
     spin_axis_orientation_average: np.ndarray[np.double]
     bad_time_flag_occurrences: np.ndarray
 
-    def __init__(self, l1b_dataset: xr.Dataset, pipeline_settings: PipelineSettings):
+    def __init__(
+        self,
+        l1b_dataset: xr.Dataset,
+        pipeline_settings: PipelineSettings,
+        calibration_dataset: xr.Dataset,
+    ) -> None:
         """
         Given an L1B dataset, process data into an output HistogramL2 object.
 
@@ -337,6 +364,9 @@ class HistogramL2:
             GLOWS histogram L1B dataset, as produced by glows_l1b.py.
         pipeline_settings : PipelineSettings
             Pipeline settings object read from ancillary file.
+        calibration_dataset : xr.Dataset
+            cps-to-Rayleigh calibration dataset used for flux calculations.
+            coords: start_time_utc,  data_vars: cps_per_r
         """
         active_flags = np.array(pipeline_settings.active_bad_time_flags, dtype=float)
 
@@ -438,7 +468,17 @@ class HistogramL2:
             .data[np.newaxis, :]
         )
 
-        self.daily_lightcurve = DailyLightcurve(good_data, position_angle)
+        # Select calibration factor corresponding to the mid-epoch in the L1B data.
+        if len(good_data["epoch"].data) != 0:
+            calibration_factor = self.get_calibration_factor(
+                good_data["epoch"].data, calibration_dataset
+            )
+        else:
+            calibration_factor = None  # No good data available. Still proceed
+
+        self.daily_lightcurve = DailyLightcurve(
+            good_data, position_angle, calibration_factor
+        )
 
     def filter_bad_bins(self, histograms: NDArray, bin_exclusions: NDArray) -> NDArray:
         """
@@ -517,3 +557,56 @@ class HistogramL2:
         # doesn't move from the SPICE determined mounting angle.
         glows_mounting_azimuth, _ = get_instrument_mounting_az_el(SpiceFrame.IMAP_GLOWS)
         return (360.0 - glows_mounting_azimuth) % 360.0
+
+    @staticmethod
+    def get_calibration_factor(
+        epoch_values: np.ndarray, calibration_dataset: xr.Dataset
+    ) -> float:
+        """
+        Select calibration factor for an observational day.
+
+        The calibration factor is needed to compute flux in Rayleigh units.
+        There is a strong assumption that the calibration is constant for
+        a given observational day.
+
+        Parameters
+        ----------
+        epoch_values : np.ndarray
+            Array of epoch values from the L1B dataset, in TT J2000 nanoseconds.
+        calibration_dataset : xr.Dataset
+            Dataset containing calibration data with the following structure:
+                Coords: epoch (datetime64[s])
+                Dims: epoch, cps_per_r_dim_0, start_time_utc_dim_0
+                Data vars: "cps_per_r" and "start_time_utc" are 2D (epoch, *_dim_0)
+
+                Note: epoch and start_time_utc do not necessarily match in size or
+                      values
+                    - epoch contains timestamps in the calibration data up to a defined
+                      day buffer and start_time_utc are the timestamps for all the
+                      calibration data entries.
+                    - epoch is used for selecting the time block, and start_time_utc is
+                      used for selecting the calibration value within that block.
+
+        Returns
+        -------
+        float
+            The calibration factor needed to compute flux in Rayleigh units.
+        """
+        # Use the midpoint epoch for the observation day
+        mid_idx = len(epoch_values) // 2
+        mid_epoch_utc = et_to_datetime64(ttj2000ns_to_et(epoch_values[mid_idx].item()))
+
+        # Select calibration data before or equal to mid_epoch_utc using "pad" to find
+        # the nearest preceding entry in the calibration dataset's epoch
+        # coordinate which is in UTC datetime64 format.
+        cal_at_epoch = calibration_dataset.sel(epoch=mid_epoch_utc, method="pad")
+
+        # start_time_utc is a data variable with its own index dimension.
+        # Use searchsorted to find the last entry whose start_time_utc <= mid_epoch_utc.
+        start_times = np.array(
+            cal_at_epoch["start_time_utc"].values, dtype="datetime64[ns]"
+        )
+        nearest_idx = np.searchsorted(start_times, mid_epoch_utc, side="right") - 1
+
+        # Select the calibration value at the nearest index.
+        return float(cal_at_epoch["cps_per_r"].isel(cps_per_r_dim_0=nearest_idx))