imap-processing 0.18.0__py3-none-any.whl → 0.19.2__py3-none-any.whl

imap_processing/ultra/l2/ultra_l2.py

@@ -10,6 +10,7 @@ import xarray as xr
 from numpy.typing import NDArray
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.cdf.utils import load_cdf
 from imap_processing.ena_maps import ena_maps
 from imap_processing.ena_maps.utils.coordinates import CoordNames
 from imap_processing.ena_maps.utils.naming import (
@@ -17,10 +18,10 @@ from imap_processing.ena_maps.utils.naming import (
     MapDescriptor,
     ns_to_duration_months,
 )
+from imap_processing.quality_flags import ImapPSETUltraFlags
 from imap_processing.ultra.l1c.ultra_l1c_pset_bins import get_energy_delta_minus_plus
 
 logger = logging.getLogger(__name__)
-logger.info("Importing ultra_l2 module")
 
 # Default properties for the Ultra L2 map
 DEFAULT_ULTRA_L2_MAP_STRUCTURE: ena_maps.RectangularSkyMap | ena_maps.HealpixSkyMap = (
@@ -59,7 +60,15 @@ REQUIRED_L1C_VARIABLES_PULL = [
     "background_rates",
     "obs_date",
 ]
-
+# These variables are expected but not strictly required. In certain test scenarios,
+# they may be missing, in which case we will raise a warning and continue.
+# All psets must be consistent and either have these variables or not.
+EXPECTED_L1C_VARIABLES_PULL = [
+    "geometric_function",
+    "efficiency",
+    "scatter_theta",
+    "scatter_phi",
+]
 # These variables are projected to the map as the mean of pointing set pixels value,
 # weighted by that pointing set pixel's exposure and solid angle
 VARIABLES_TO_WEIGHT_BY_POINTING_SET_EXPOSURE_TIMES_SOLID_ANGLE = [
@@ -72,12 +81,9 @@ VARIABLES_TO_WEIGHT_BY_POINTING_SET_EXPOSURE_TIMES_SOLID_ANGLE = [
 # calculate ena_intensity and its statistical uncertainty
 # They will not be present in the final map
 VARIABLES_TO_DROP_AFTER_INTENSITY_CALCULATION = [
-    "counts",
-    "background_rates",
     "pointing_set_exposure_times_solid_angle",
     "num_pointing_set_pixel_members",
     "corrected_count_rate",
-    "obs_date_for_std",
     "obs_date_squared_for_std",
 ]
 
@@ -128,6 +134,8 @@ def get_variable_attributes_optional_energy_dependence(
         and (CoordNames.ENERGY_ULTRA_L1C.value not in variable_dims)
     ):
         variable_name = f"{variable_name}_energy_independent"
+    if variable_name == "counts":
+        variable_name = "ena_count"
 
     metadata = cdf_attrs.get_variable_attributes(
         variable_name=variable_name,
@@ -206,7 +214,7 @@ def generate_ultra_healpix_skymap(
     output_map_structure.values_to_push_project.extend(
         [
             "num_pointing_set_pixel_members",
-            "obs_date_for_std",
+            "obs_date_range",
             "obs_date_squared_for_std",
         ]
     )
@@ -236,6 +244,27 @@ def generate_ultra_healpix_skymap(
             f"PUSH Variables: {output_map_structure.values_to_push_project} \n"
             f"PULL Variables: {output_map_structure.values_to_pull_project}"
         )
+    # TODO remove this in the future once all test data includes these variables
+    # Add expected but not required variables to the pull projection list
+    # Log a warning if they are missing from any PSET but continue processing.
+    expected_present_vars = []
+    first_pset = (
+        load_cdf(ultra_l1c_psets[0])
+        if isinstance(ultra_l1c_psets[0], (str, Path))
+        else ultra_l1c_psets[0]
+    )
+    for var in EXPECTED_L1C_VARIABLES_PULL:
+        if var not in first_pset.variables:
+            logger.warning(
+                f"Expected variable {var} not found in the first L1C PSET. "
+                "This variable will not be projected to the map."
+            )
+        else:
+            expected_present_vars.append(var)
+
+    output_map_structure.values_to_pull_project = list(
+        set(output_map_structure.values_to_pull_project + expected_present_vars)
+    )
 
     all_pset_epochs = []
     for ultra_l1c_pset in ultra_l1c_psets:
@@ -249,9 +278,15 @@ def generate_ultra_healpix_skymap(
             "\nThese values will be pull projected: "
             f">> {output_map_structure.values_to_pull_project}",
         )
+        flags_1d = pointing_set.data["quality_flags"].isel(epoch=0)
+        # This is a good pixel mask where zero is when the earth is not in the FOV.
+        good_pixel_mask = (
+            (flags_1d & ImapPSETUltraFlags.EARTH_FOV.value) == 0
+        ).to_numpy()
 
+        # Only count the number of pointing set pixels which are not flagged.
         pointing_set.data["num_pointing_set_pixel_members"] = xr.DataArray(
-            np.ones(pointing_set.num_points, dtype=int),
+            good_pixel_mask.astype(int),
             dims=(CoordNames.HEALPIX_INDEX.value),
         )
 
@@ -262,11 +297,11 @@ def generate_ultra_healpix_skymap(
             fill_value=pointing_set.epoch,
             dtype=np.int64,
         )
-        pointing_set.data["obs_date_for_std"] = pointing_set.data["obs_date"].astype(
+        pointing_set.data["obs_date_range"] = pointing_set.data["obs_date"].astype(
             np.float64
         )
         pointing_set.data["obs_date_squared_for_std"] = (
-            pointing_set.data["obs_date_for_std"] ** 2
+            pointing_set.data["obs_date_range"] ** 2
         )
 
         # Add solid_angle * exposure of pointing set as data_var
@@ -277,15 +312,22 @@ def generate_ultra_healpix_skymap(
 
         # Initial processing for weighted quantities at PSET level
         # Weight the values by exposure and solid angle
+        # Ensure only valid pointing set pixels contribute to the weighted mean.
         pointing_set.data[
             VARIABLES_TO_WEIGHT_BY_POINTING_SET_EXPOSURE_TIMES_SOLID_ANGLE
-        ] *= pointing_set.data["pointing_set_exposure_times_solid_angle"]
+        ] = (
+            pointing_set.data[
+                VARIABLES_TO_WEIGHT_BY_POINTING_SET_EXPOSURE_TIMES_SOLID_ANGLE
+            ]
+            * pointing_set.data["pointing_set_exposure_times_solid_angle"]
+        ).where(good_pixel_mask)
 
         # Project values such as counts via the PUSH method
         skymap.project_pset_values_to_map(
             pointing_set=pointing_set,
             value_keys=output_map_structure.values_to_push_project,
             index_match_method=ena_maps.IndexMatchMethod.PUSH,
+            pset_valid_mask=good_pixel_mask,
         )
 
         # Project values such as exposure_factor via the PULL method
@@ -293,6 +335,7 @@ def generate_ultra_healpix_skymap(
             pointing_set=pointing_set,
             value_keys=output_map_structure.values_to_pull_project,
             index_match_method=ena_maps.IndexMatchMethod.PULL,
+            pset_valid_mask=good_pixel_mask,
         )
 
     # Subsequent processing for weighted quantities at SkyMap level
@@ -348,7 +391,7 @@ def generate_ultra_healpix_skymap(
                 )
                 - (
                     (
-                        skymap.data_1d["obs_date_for_std"]
+                        skymap.data_1d["obs_date_range"]
                         / (skymap.data_1d["num_pointing_set_pixel_members"])
                     )
                     ** 2

imap_processing/ultra/utils/ultra_l1_utils.py

@@ -1,6 +1,5 @@
 """Create dataset."""
 
-import numpy as np
 import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
@@ -32,7 +31,7 @@ def create_dataset(  # noqa: PLR0912
     cdf_manager.add_instrument_global_attrs("ultra")
     cdf_manager.add_instrument_variable_attrs("ultra", level)
 
-    # L1b extended spin, badtimes, and cullingmask data products
+    # L1b extended spin, badtimes, and goodtimes data products
     if "spin_number" in data_dict.keys():
         coords = {
             "spin_number": ("spin_number", data_dict["spin_number"]),
@@ -40,7 +39,6 @@ def create_dataset(  # noqa: PLR0912
                 "energy_bin_geometric_mean",
                 data_dict["energy_bin_geometric_mean"],
             ),
-            "epoch": ("spin_number", np.asarray(data_dict["epoch"])),
         }
         default_dimension = "spin_number"
     # L1c pset data products
@@ -49,6 +47,7 @@ def create_dataset(  # noqa: PLR0912
             "epoch": data_dict["epoch"],
             "pixel_index": data_dict["pixel_index"],
             "energy_bin_geometric_mean": data_dict["energy_bin_geometric_mean"],
+            "spin_phase_step": data_dict["spin_phase_step"],
         }
         default_dimension = "pixel_index"
     # L1b de data product
@@ -95,7 +94,13 @@ def create_dataset(  # noqa: PLR0912
 
     for key, data in data_dict.items():
         # Skip keys that are coordinates.
-        if key in ["epoch", "spin_number", "energy_bin_geometric_mean", "pixel_index"]:
+        if key in [
+            "epoch",
+            "spin_number",
+            "energy_bin_geometric_mean",
+            "pixel_index",
+            "spin_phase_step",
+        ]:
             continue
         elif key in velocity_keys:
             dataset[key] = xr.DataArray(
@@ -103,7 +108,12 @@ def create_dataset(  # noqa: PLR0912
                 dims=["epoch", "component"],
                 attrs=cdf_manager.get_variable_attributes(key, check_schema=False),
             )
-        elif key == "ena_rates_threshold":
+        elif key in [
+            "ena_rates_threshold",
+            "scatter_threshold",
+            "energy_delta_minus",
+            "energy_delta_plus",
+        ]:
             dataset[key] = xr.DataArray(
                 data,
                 dims=["energy_bin_geometric_mean"],
@@ -127,7 +137,7 @@ def create_dataset(  # noqa: PLR0912
                 dims=["energy_bin_geometric_mean", "spin_number"],
                 attrs=cdf_manager.get_variable_attributes(key, check_schema=False),
             )
-        elif key in {"latitude", "longitude", "exposure_factor"}:
+        elif key in {"quality_flags", "latitude", "longitude"}:
             dataset[key] = xr.DataArray(
                 data,
                 dims=["epoch", "pixel_index"],
@@ -136,12 +146,32 @@ def create_dataset(  # noqa: PLR0912
         elif key in {
             "counts",
             "background_rates",
+        }:
+            dataset[key] = xr.DataArray(
+                data,
+                dims=["epoch", "energy_bin_geometric_mean", "pixel_index"],
+                attrs=cdf_manager.get_variable_attributes(key, check_schema=False),
+            )
+        elif key in {
+            "exposure_factor",
             "helio_exposure_factor",
             "sensitivity",
+            "efficiency",
+            "geometric_function",
+            "scatter_theta",
+            "scatter_phi",
         }:
             dataset[key] = xr.DataArray(
                 data,
-                dims=["epoch", "energy_bin_geometric_mean", "pixel_index"],
+                dims=["energy_bin_geometric_mean", "pixel_index"],
+                attrs=cdf_manager.get_variable_attributes(key, check_schema=False),
+            )
+        elif key in {
+            "dead_time_ratio",
+        }:
+            dataset[key] = xr.DataArray(
+                data,
+                dims=["spin_phase_step"],
                 attrs=cdf_manager.get_variable_attributes(key, check_schema=False),
             )
         else:

imap_processing/utils.py

@@ -3,7 +3,6 @@
 import collections
 import logging
 from pathlib import Path
-from typing import Optional, Union
 
 import numpy as np
 import pandas as pd
@@ -141,7 +140,7 @@ def _get_minimum_numpy_datatype(  # noqa: PLR0912 - Too many branches
     name: str,
     definition: definitions.XtcePacketDefinition,
     use_derived_value: bool = True,
-) -> Optional[str]:
+) -> str | None:
     """
     Get the minimum datatype for a given variable.
 
@@ -209,8 +208,8 @@ def _get_minimum_numpy_datatype(  # noqa: PLR0912 - Too many branches
 
 
 def packet_file_to_datasets(
-    packet_file: Union[str, Path],
-    xtce_packet_definition: Union[str, Path],
+    packet_file: str | Path,
+    xtce_packet_definition: str | Path,
     use_derived_value: bool = False,
 ) -> dict[int, xr.Dataset]:
     """

{imap_processing-0.18.0.dist-info → imap_processing-0.19.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: imap-processing
-Version: 0.18.0
+Version: 0.19.2
 Summary: IMAP Science Operations Center Processing
 License: MIT
 Keywords: IMAP,SDC,SOC,Science Operations
@@ -28,7 +28,7 @@ Provides-Extra: map-visualization
 Provides-Extra: test
 Provides-Extra: tools
 Requires-Dist: astropy-healpix (>=1.0)
-Requires-Dist: cdflib (>=1.3.1,<2.0.0)
+Requires-Dist: cdflib (>=1.3.6,<2.0.0)
 Requires-Dist: healpy (>=1.18.0,<2.0.0) ; extra == "map-visualization"
 Requires-Dist: imap-data-access (>=0.32.0)
 Requires-Dist: mypy (==1.10.1) ; extra == "dev"