imap-processing 0.19.0__py3-none-any.whl → 0.19.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of imap-processing might be problematic. Click here for more details.

Files changed (73) hide show
  1. imap_processing/_version.py +2 -2
  2. imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml +6 -0
  3. imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +31 -894
  4. imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml +279 -255
  5. imap_processing/cdf/config/imap_enamaps_l2-common_variable_attrs.yaml +55 -0
  6. imap_processing/cdf/config/imap_enamaps_l2-healpix_variable_attrs.yaml +29 -0
  7. imap_processing/cdf/config/imap_enamaps_l2-rectangular_variable_attrs.yaml +32 -0
  8. imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml +3 -1
  9. imap_processing/cdf/config/imap_lo_global_cdf_attrs.yaml +5 -4
  10. imap_processing/cdf/config/imap_ultra_global_cdf_attrs.yaml +28 -16
  11. imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml +33 -31
  12. imap_processing/cdf/config/imap_ultra_l1c_variable_attrs.yaml +61 -1
  13. imap_processing/cli.py +62 -71
  14. imap_processing/codice/codice_l0.py +2 -1
  15. imap_processing/codice/codice_l1a.py +47 -49
  16. imap_processing/codice/codice_l1b.py +42 -32
  17. imap_processing/codice/codice_l2.py +105 -7
  18. imap_processing/codice/constants.py +50 -8
  19. imap_processing/codice/data/lo_stepping_values.csv +1 -1
  20. imap_processing/ena_maps/ena_maps.py +39 -18
  21. imap_processing/ena_maps/utils/corrections.py +291 -0
  22. imap_processing/ena_maps/utils/map_utils.py +20 -4
  23. imap_processing/glows/l1b/glows_l1b.py +38 -23
  24. imap_processing/glows/l1b/glows_l1b_data.py +10 -11
  25. imap_processing/hi/hi_l1c.py +4 -109
  26. imap_processing/hi/hi_l2.py +34 -23
  27. imap_processing/hi/utils.py +109 -0
  28. imap_processing/ialirt/l0/ialirt_spice.py +1 -1
  29. imap_processing/ialirt/l0/parse_mag.py +18 -4
  30. imap_processing/ialirt/l0/process_hit.py +9 -4
  31. imap_processing/ialirt/l0/process_swapi.py +9 -4
  32. imap_processing/ialirt/l0/process_swe.py +9 -4
  33. imap_processing/ialirt/utils/create_xarray.py +1 -1
  34. imap_processing/lo/ancillary_data/imap_lo_hydrogen-geometric-factor_v001.csv +75 -0
  35. imap_processing/lo/ancillary_data/imap_lo_oxygen-geometric-factor_v001.csv +75 -0
  36. imap_processing/lo/l1b/lo_l1b.py +90 -16
  37. imap_processing/lo/l1c/lo_l1c.py +164 -50
  38. imap_processing/lo/l2/lo_l2.py +941 -127
  39. imap_processing/mag/l1d/mag_l1d_data.py +36 -3
  40. imap_processing/mag/l2/mag_l2.py +2 -0
  41. imap_processing/mag/l2/mag_l2_data.py +4 -3
  42. imap_processing/quality_flags.py +14 -0
  43. imap_processing/spice/geometry.py +13 -8
  44. imap_processing/spice/pointing_frame.py +4 -2
  45. imap_processing/spice/repoint.py +49 -0
  46. imap_processing/ultra/constants.py +29 -0
  47. imap_processing/ultra/l0/decom_tools.py +58 -46
  48. imap_processing/ultra/l0/decom_ultra.py +21 -9
  49. imap_processing/ultra/l0/ultra_utils.py +4 -4
  50. imap_processing/ultra/l1b/badtimes.py +35 -11
  51. imap_processing/ultra/l1b/de.py +15 -9
  52. imap_processing/ultra/l1b/extendedspin.py +24 -12
  53. imap_processing/ultra/l1b/goodtimes.py +112 -0
  54. imap_processing/ultra/l1b/lookup_utils.py +1 -1
  55. imap_processing/ultra/l1b/ultra_l1b.py +7 -7
  56. imap_processing/ultra/l1b/ultra_l1b_culling.py +8 -4
  57. imap_processing/ultra/l1b/ultra_l1b_extended.py +79 -43
  58. imap_processing/ultra/l1c/helio_pset.py +68 -39
  59. imap_processing/ultra/l1c/l1c_lookup_utils.py +45 -12
  60. imap_processing/ultra/l1c/spacecraft_pset.py +81 -37
  61. imap_processing/ultra/l1c/ultra_l1c.py +27 -22
  62. imap_processing/ultra/l1c/ultra_l1c_culling.py +7 -0
  63. imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +41 -41
  64. imap_processing/ultra/l2/ultra_l2.py +75 -18
  65. imap_processing/ultra/utils/ultra_l1_utils.py +10 -5
  66. {imap_processing-0.19.0.dist-info → imap_processing-0.19.3.dist-info}/METADATA +2 -2
  67. {imap_processing-0.19.0.dist-info → imap_processing-0.19.3.dist-info}/RECORD +71 -69
  68. imap_processing/ultra/l1b/cullingmask.py +0 -90
  69. imap_processing/ultra/l1c/histogram.py +0 -36
  70. /imap_processing/glows/ancillary/{imap_glows_pipeline_settings_20250923_v002.json → imap_glows_pipeline-settings_20250923_v002.json} +0 -0
  71. {imap_processing-0.19.0.dist-info → imap_processing-0.19.3.dist-info}/LICENSE +0 -0
  72. {imap_processing-0.19.0.dist-info → imap_processing-0.19.3.dist-info}/WHEEL +0 -0
  73. {imap_processing-0.19.0.dist-info → imap_processing-0.19.3.dist-info}/entry_points.txt +0 -0
@@ -4,6 +4,7 @@ import astropy_healpix.healpy as hp
4
4
  import numpy as np
5
5
  from numpy.typing import NDArray
6
6
 
7
+ from imap_processing.quality_flags import ImapPSETUltraFlags
7
8
  from imap_processing.spice.geometry import (
8
9
  SpiceBody,
9
10
  SpiceFrame,
@@ -14,6 +15,7 @@ from imap_processing.spice.geometry import (
14
15
  def compute_culling_mask(
15
16
  et: NDArray,
16
17
  keepout_radius_km: float,
18
+ pset_quality_flags: NDArray,
17
19
  observer: SpiceBody = SpiceBody.EARTH,
18
20
  nside: int = 128,
19
21
  nested: bool = False,
@@ -27,6 +29,9 @@ def compute_culling_mask(
27
29
  Ephemeris times in TDB seconds past J2000.
28
30
  keepout_radius_km : float
29
31
  Radius (in km) within which HEALPix pixels will be excluded.
32
+ pset_quality_flags : NDArray,
33
+ Quality flag to set when HEALPIX pixels are within a
34
+ keep-out radius of the target body.
30
35
  observer : SpiceBody, optional
31
36
  Body from which IMAP is observed.
32
37
  nside : int, optional
@@ -81,5 +86,7 @@ def compute_culling_mask(
81
86
  # Exclude pixels within the keepout angle.
82
87
  # mask.shape = (len(et), npix)
83
88
  mask = sep_angle > keepout_angle[:, np.newaxis]
89
+ culled_any_time = np.any(~mask, axis=0) # shape: (npix,)
90
+ pset_quality_flags[culled_any_time] |= ImapPSETUltraFlags.EARTH_FOV.value
84
91
 
85
92
  return mask, unit_target_vecs
@@ -1,8 +1,9 @@
1
1
  """Module to create pointing sets."""
2
2
 
3
+ import logging
4
+
3
5
  import astropy_healpix.healpy as hp
4
6
  import numpy as np
5
- import pandas
6
7
  import xarray as xr
7
8
  from numpy.typing import NDArray
8
9
  from scipy import interpolate
@@ -13,6 +14,7 @@ from imap_processing.spice.geometry import (
13
14
  imap_state,
14
15
  )
15
16
  from imap_processing.spice.spin import get_spacecraft_spin_phase, get_spin_angle
17
+ from imap_processing.spice.time import ttj2000ns_to_met
16
18
  from imap_processing.ultra.constants import UltraConstants
17
19
  from imap_processing.ultra.l1b.lookup_utils import (
18
20
  get_geometric_factor,
@@ -31,6 +33,8 @@ from imap_processing.ultra.l1b.ultra_l1b_extended import (
31
33
  # TODO: add species binning.
32
34
  FILLVAL_FLOAT32 = -1.0e31
33
35
 
36
+ logger = logging.getLogger(__name__)
37
+
34
38
 
35
39
  def build_energy_bins() -> tuple[list[tuple[float, float]], np.ndarray, np.ndarray]:
36
40
  """
@@ -46,7 +50,7 @@ def build_energy_bins() -> tuple[list[tuple[float, float]], np.ndarray, np.ndarr
46
50
  Array of geometric means of energy bins.
47
51
  """
48
52
  # Create energy bins.
49
- energy_bin_edges = np.array(UltraConstants.CULLING_ENERGY_BIN_EDGES)
53
+ energy_bin_edges = np.array(UltraConstants.PSET_ENERGY_BIN_EDGES)
50
54
  energy_midpoints = (energy_bin_edges[:-1] + energy_bin_edges[1:]) / 2
51
55
 
52
56
  intervals = [
@@ -222,7 +226,6 @@ def get_deadtime_ratios(sectored_rates_ds: xr.Dataset) -> xr.DataArray:
222
226
  - sectored_rates_ds.stop_tn
223
227
  - sectored_rates_ds.stop_bn
224
228
  )
225
-
226
229
  corrected_valid_events = b * np.exp(1e-7 * 8 * coin_stop_nd)
227
230
 
228
231
  # Compute dead time ratio
@@ -252,21 +255,24 @@ def get_sectored_rates(rates_ds: xr.Dataset, params_ds: xr.Dataset) -> xr.Datase
252
255
 
253
256
  # This means that data was collected as a function of spin allowing for fine grained
254
257
  # rate analysis.
255
- sector_mode_start_inds = np.where(params_ds["imageratescadence"] == 3)[0]
258
+ # Only get unique combinations of epoch and imageratescadence
259
+ params = params_ds.groupby(["epoch", "imageratescadence"]).first()
260
+
261
+ sector_mode_start_inds = np.where(params["imageratescadence"] == 3)[0]
262
+ if len(sector_mode_start_inds) == 0:
263
+ raise ValueError("No sector mode data found in the parameters dataset.")
256
264
  # get the sector mode start and stop indices
257
265
  sector_mode_stop_inds = sector_mode_start_inds + 1
258
266
  # get the sector mode start and stop times
259
- mode_3_start = params_ds["epoch"].values[sector_mode_start_inds]
260
-
267
+ mode_3_start = params["epoch"].values[sector_mode_start_inds]
261
268
  # if the last mode is a sector mode, we can assume that the sector data goes through
262
269
  # the end of the dataset, so we append np.inf to the end of the last time range.
263
- if sector_mode_stop_inds[-1] == len(params_ds["epoch"]):
270
+ if sector_mode_stop_inds[-1] == len(params["epoch"]):
264
271
  mode_3_end = np.append(
265
- params_ds["epoch"].values[sector_mode_stop_inds[:-1]], np.inf
272
+ params["epoch"].values[sector_mode_stop_inds[:-1]], np.inf
266
273
  )
267
274
  else:
268
- mode_3_end = params_ds["epoch"].values[sector_mode_stop_inds]
269
-
275
+ mode_3_end = params["epoch"].values[sector_mode_stop_inds]
270
276
  # Build a list of conditions for each sector mode time range
271
277
  conditions = [
272
278
  (rates_ds["epoch"] >= start) & (rates_ds["epoch"] < end)
@@ -295,10 +301,9 @@ def get_deadtime_ratios_by_spin_phase(
295
301
  """
296
302
  deadtime_ratios = get_deadtime_ratios(sectored_rates)
297
303
  # Get the spin phase at the start of each sector rate measurement
304
+ met_times = ttj2000ns_to_met(sectored_rates.epoch.data)
298
305
  spin_phases = np.asarray(
299
- get_spin_angle(
300
- get_spacecraft_spin_phase(np.array(sectored_rates.epoch.data)), degrees=True
301
- )
306
+ get_spin_angle(get_spacecraft_spin_phase(met_times), degrees=True)
302
307
  )
303
308
  # Assume the sectored rate data is evenly spaced in time, and find the middle spin
304
309
  # phase value for each sector.
@@ -325,11 +330,16 @@ def get_deadtime_ratios_by_spin_phase(
325
330
  deadtime_by_spin_phase = deadtime_by_spin_phase.sortby("spin_phase")
326
331
  # Group by spin phase and calculate the median dead time ratio for each phase
327
332
  deadtime_medians = deadtime_by_spin_phase.groupby("spin_phase").median(skipna=True)
328
-
329
333
  if np.any(np.isnan(deadtime_medians["deadtime_ratio"].values)):
330
- raise ValueError(
331
- "Dead time ratios contain NaN values, cannot create interpolator."
334
+ if not np.any(np.isfinite(deadtime_medians["deadtime_ratio"].values)):
335
+ raise ValueError("All dead time ratios are NaN, cannot interpolate.")
336
+ logger.warning(
337
+ "Dead time ratios contain NaN values, filtering data to only include "
338
+ "finite values."
332
339
  )
340
+ deadtime_medians = deadtime_medians.where(
341
+ np.isfinite(deadtime_medians["deadtime_ratio"]), drop=True
342
+ )
333
343
  interpolator = interpolate.PchipInterpolator(
334
344
  deadtime_medians["spin_phase"].values, deadtime_medians["deadtime_ratio"].values
335
345
  )
@@ -340,19 +350,17 @@ def get_deadtime_ratios_by_spin_phase(
340
350
  return interpolator(nominal_spin_phases_1ms_res)
341
351
 
342
352
 
343
- def apply_deadtime_correction(
344
- exposure_pointing: pandas.DataFrame,
353
+ def calculate_exposure_time(
345
354
  deadtime_ratios: np.ndarray,
346
355
  pixels_below_scattering: list,
347
356
  boundary_scale_factors: NDArray,
357
+ n_pix: int,
348
358
  ) -> np.ndarray:
349
359
  """
350
360
  Adjust the exposure time at each pixel to account for dead time.
351
361
 
352
362
  Parameters
353
363
  ----------
354
- exposure_pointing : pandas.DataFrame
355
- Exposure data.
356
364
  deadtime_ratios : PchipInterpolator
357
365
  Interpolating function for dead time ratios.
358
366
  pixels_below_scattering : list
@@ -362,6 +370,8 @@ def apply_deadtime_correction(
362
370
  the FWHM scattering threshold.
363
371
  boundary_scale_factors : np.ndarray
364
372
  Boundary scale factors for each pixel at each spin phase.
373
+ n_pix : int
374
+ Number of HEALPix pixels.
365
375
 
366
376
  Returns
367
377
  -------
@@ -370,12 +380,8 @@ def apply_deadtime_correction(
370
380
  """
371
381
  # Get energy bin geometric means
372
382
  energy_bin_geometric_means = build_energy_bins()[2]
373
- # Exposure time should now be of shape (npix, energy)
374
- exposure_pointing = np.repeat(
375
- exposure_pointing.to_numpy()[np.newaxis, :],
376
- len(energy_bin_geometric_means),
377
- axis=0,
378
- )
383
+ # Exposure time should now be of shape (energy, npix)
384
+ exposure_pointing = np.zeros((len(energy_bin_geometric_means), n_pix))
379
385
  # nominal spin phase step.
380
386
  nominal_ms_step = 15 / len(pixels_below_scattering) # time step
381
387
  # Query the dead-time ratio and apply the nominal exposure time to pixels in the FOR
@@ -400,19 +406,17 @@ def apply_deadtime_correction(
400
406
 
401
407
 
402
408
  def get_spacecraft_exposure_times(
403
- constant_exposure: pandas.DataFrame,
404
409
  rates_dataset: xr.Dataset,
405
410
  params_dataset: xr.Dataset,
406
411
  pixels_below_scattering: list[list],
407
412
  boundary_scale_factors: NDArray,
413
+ n_pix: int,
408
414
  ) -> tuple[NDArray, NDArray]:
409
415
  """
410
416
  Compute exposure times for HEALPix pixels.
411
417
 
412
418
  Parameters
413
419
  ----------
414
- constant_exposure : pandas.DataFrame
415
- Exposure data.
416
420
  rates_dataset : xarray.Dataset
417
421
  Dataset containing image rates data.
418
422
  params_dataset : xarray.Dataset
@@ -424,6 +428,8 @@ def get_spacecraft_exposure_times(
424
428
  below the FWHM scattering threshold.
425
429
  boundary_scale_factors : np.ndarray
426
430
  Boundary scale factors for each pixel at each spin phase.
431
+ n_pix : int
432
+ Number of HEALPix pixels.
427
433
 
428
434
  Returns
429
435
  -------
@@ -438,14 +444,8 @@ def get_spacecraft_exposure_times(
438
444
  # universal pointing table here to determine actual number of spins
439
445
  sectored_rates = get_sectored_rates(rates_dataset, params_dataset)
440
446
  nominal_deadtime_ratios = get_deadtime_ratios_by_spin_phase(sectored_rates)
441
- exposure_pointing = (
442
- constant_exposure["Exposure Time"] * 5760
443
- ) # 5760 spins per pointing (for now)
444
- exposure_pointing_adjusted = apply_deadtime_correction(
445
- exposure_pointing,
446
- nominal_deadtime_ratios,
447
- pixels_below_scattering,
448
- boundary_scale_factors,
447
+ exposure_pointing_adjusted = calculate_exposure_time(
448
+ nominal_deadtime_ratios, pixels_below_scattering, boundary_scale_factors, n_pix
449
449
  )
450
450
  return exposure_pointing_adjusted, nominal_deadtime_ratios
451
451
 
@@ -664,7 +664,7 @@ def get_spacecraft_background_rates(
664
664
  sensor: str,
665
665
  ancillary_files: dict,
666
666
  energy_bin_edges: list[tuple[float, float]],
667
- cullingmask_spin_number: NDArray,
667
+ goodtimes_spin_number: NDArray,
668
668
  nside: int = 128,
669
669
  ) -> NDArray:
670
670
  """
@@ -680,9 +680,9 @@ def get_spacecraft_background_rates(
680
680
  Ancillary files containing the lookup tables.
681
681
  energy_bin_edges : list[tuple[float, float]]
682
682
  Energy bin edges.
683
- cullingmask_spin_number : NDArray
683
+ goodtimes_spin_number : NDArray
684
684
  Goodtime spins.
685
- Ex. imap_ultra_l1b_45sensor-cullingmask[0]["spin_number"]
685
+ Ex. imap_ultra_l1b_45sensor-goodtimes[0]["spin_number"]
686
686
  This is used to determine the number of pulses per spin.
687
687
  nside : int, optional
688
688
  The nside parameter of the Healpix tessellation (default is 128).
@@ -714,7 +714,7 @@ def get_spacecraft_background_rates(
714
714
  background_rates = np.zeros((len(energy_bin_edges), n_pix))
715
715
 
716
716
  # Only select pulses from goodtimes.
717
- goodtime_mask = np.isin(spin_number, cullingmask_spin_number)
717
+ goodtime_mask = np.isin(spin_number, goodtimes_spin_number)
718
718
  mean_start_pulses = np.mean(pulses.start_pulses[goodtime_mask])
719
719
  mean_stop_pulses = np.mean(pulses.stop_pulses[goodtime_mask])
720
720
  mean_coin_pulses = np.mean(pulses.coin_pulses[goodtime_mask])
@@ -10,6 +10,7 @@ import xarray as xr
10
10
  from numpy.typing import NDArray
11
11
 
12
12
  from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
13
+ from imap_processing.cdf.utils import load_cdf
13
14
  from imap_processing.ena_maps import ena_maps
14
15
  from imap_processing.ena_maps.utils.coordinates import CoordNames
15
16
  from imap_processing.ena_maps.utils.naming import (
@@ -17,6 +18,7 @@ from imap_processing.ena_maps.utils.naming import (
17
18
  MapDescriptor,
18
19
  ns_to_duration_months,
19
20
  )
21
+ from imap_processing.quality_flags import ImapPSETUltraFlags
20
22
  from imap_processing.ultra.l1c.ultra_l1c_pset_bins import get_energy_delta_minus_plus
21
23
 
22
24
  logger = logging.getLogger(__name__)
@@ -58,25 +60,34 @@ REQUIRED_L1C_VARIABLES_PULL = [
58
60
  "background_rates",
59
61
  "obs_date",
60
62
  ]
61
-
63
+ # These variables are expected but not strictly required. In certain test scenarios,
64
+ # they may be missing, in which case we will raise a warning and continue.
65
+ # All psets must be consistent and either have these variables or not.
66
+ EXPECTED_L1C_VARIABLES_PULL = [
67
+ "geometric_function",
68
+ "efficiency",
69
+ "scatter_theta",
70
+ "scatter_phi",
71
+ ]
62
72
  # These variables are projected to the map as the mean of pointing set pixels value,
63
73
  # weighted by that pointing set pixel's exposure and solid angle
64
74
  VARIABLES_TO_WEIGHT_BY_POINTING_SET_EXPOSURE_TIMES_SOLID_ANGLE = [
65
75
  "sensitivity",
66
76
  "background_rates",
67
77
  "obs_date",
78
+ "geometric_function",
79
+ "efficiency",
80
+ "scatter_theta",
81
+ "scatter_phi",
68
82
  ]
69
83
 
70
84
  # These variables are dropped after they are used to
71
85
  # calculate ena_intensity and its statistical uncertainty
72
86
  # They will not be present in the final map
73
87
  VARIABLES_TO_DROP_AFTER_INTENSITY_CALCULATION = [
74
- "counts",
75
- "background_rates",
76
88
  "pointing_set_exposure_times_solid_angle",
77
89
  "num_pointing_set_pixel_members",
78
90
  "corrected_count_rate",
79
- "obs_date_for_std",
80
91
  "obs_date_squared_for_std",
81
92
  ]
82
93
 
@@ -127,6 +138,8 @@ def get_variable_attributes_optional_energy_dependence(
127
138
  and (CoordNames.ENERGY_ULTRA_L1C.value not in variable_dims)
128
139
  ):
129
140
  variable_name = f"{variable_name}_energy_independent"
141
+ if variable_name == "counts":
142
+ variable_name = "ena_count"
130
143
 
131
144
  metadata = cdf_attrs.get_variable_attributes(
132
145
  variable_name=variable_name,
@@ -205,7 +218,7 @@ def generate_ultra_healpix_skymap(
205
218
  output_map_structure.values_to_push_project.extend(
206
219
  [
207
220
  "num_pointing_set_pixel_members",
208
- "obs_date_for_std",
221
+ "obs_date_range",
209
222
  "obs_date_squared_for_std",
210
223
  ]
211
224
  )
@@ -235,6 +248,27 @@ def generate_ultra_healpix_skymap(
235
248
  f"PUSH Variables: {output_map_structure.values_to_push_project} \n"
236
249
  f"PULL Variables: {output_map_structure.values_to_pull_project}"
237
250
  )
251
+ # TODO remove this in the future once all test data includes these variables
252
+ # Add expected but not required variables to the pull projection list
253
+ # Log a warning if they are missing from any PSET but continue processing.
254
+ expected_present_vars = []
255
+ first_pset = (
256
+ load_cdf(ultra_l1c_psets[0])
257
+ if isinstance(ultra_l1c_psets[0], (str, Path))
258
+ else ultra_l1c_psets[0]
259
+ )
260
+ for var in EXPECTED_L1C_VARIABLES_PULL:
261
+ if var not in first_pset.variables:
262
+ logger.warning(
263
+ f"Expected variable {var} not found in the first L1C PSET. "
264
+ "This variable will not be projected to the map."
265
+ )
266
+ else:
267
+ expected_present_vars.append(var)
268
+
269
+ output_map_structure.values_to_pull_project = list(
270
+ set(output_map_structure.values_to_pull_project + expected_present_vars)
271
+ )
238
272
 
239
273
  all_pset_epochs = []
240
274
  for ultra_l1c_pset in ultra_l1c_psets:
@@ -248,9 +282,15 @@ def generate_ultra_healpix_skymap(
248
282
  "\nThese values will be pull projected: "
249
283
  f">> {output_map_structure.values_to_pull_project}",
250
284
  )
285
+ flags_1d = pointing_set.data["quality_flags"].isel(epoch=0)
286
+ # This is a good pixel mask where zero is when the earth is not in the FOV.
287
+ good_pixel_mask = (
288
+ (flags_1d & ImapPSETUltraFlags.EARTH_FOV.value) == 0
289
+ ).to_numpy()
251
290
 
291
+ # Only count the number of pointing set pixels which are not flagged.
252
292
  pointing_set.data["num_pointing_set_pixel_members"] = xr.DataArray(
253
- np.ones(pointing_set.num_points, dtype=int),
293
+ good_pixel_mask.astype(int),
254
294
  dims=(CoordNames.HEALPIX_INDEX.value),
255
295
  )
256
296
 
@@ -261,11 +301,11 @@ def generate_ultra_healpix_skymap(
261
301
  fill_value=pointing_set.epoch,
262
302
  dtype=np.int64,
263
303
  )
264
- pointing_set.data["obs_date_for_std"] = pointing_set.data["obs_date"].astype(
304
+ pointing_set.data["obs_date_range"] = pointing_set.data["obs_date"].astype(
265
305
  np.float64
266
306
  )
267
307
  pointing_set.data["obs_date_squared_for_std"] = (
268
- pointing_set.data["obs_date_for_std"] ** 2
308
+ pointing_set.data["obs_date_range"] ** 2
269
309
  )
270
310
 
271
311
  # Add solid_angle * exposure of pointing set as data_var
@@ -274,17 +314,26 @@ def generate_ultra_healpix_skymap(
274
314
  pointing_set.data["exposure_factor"] * pointing_set.solid_angle
275
315
  )
276
316
 
317
+ # Get variables that should be weighted by exposure and solid angle
318
+ existing_vars_to_weight = []
319
+ for var in VARIABLES_TO_WEIGHT_BY_POINTING_SET_EXPOSURE_TIMES_SOLID_ANGLE:
320
+ if var in pointing_set.data:
321
+ existing_vars_to_weight.append(var)
322
+
277
323
  # Initial processing for weighted quantities at PSET level
278
324
  # Weight the values by exposure and solid angle
279
- pointing_set.data[
280
- VARIABLES_TO_WEIGHT_BY_POINTING_SET_EXPOSURE_TIMES_SOLID_ANGLE
281
- ] *= pointing_set.data["pointing_set_exposure_times_solid_angle"]
325
+ # Ensure only valid pointing set pixels contribute to the weighted mean.
326
+ pointing_set.data[existing_vars_to_weight] = (
327
+ pointing_set.data[existing_vars_to_weight]
328
+ * pointing_set.data["pointing_set_exposure_times_solid_angle"]
329
+ ).where(good_pixel_mask)
282
330
 
283
331
  # Project values such as counts via the PUSH method
284
332
  skymap.project_pset_values_to_map(
285
333
  pointing_set=pointing_set,
286
334
  value_keys=output_map_structure.values_to_push_project,
287
335
  index_match_method=ena_maps.IndexMatchMethod.PUSH,
336
+ pset_valid_mask=good_pixel_mask,
288
337
  )
289
338
 
290
339
  # Project values such as exposure_factor via the PULL method
@@ -292,12 +341,13 @@ def generate_ultra_healpix_skymap(
292
341
  pointing_set=pointing_set,
293
342
  value_keys=output_map_structure.values_to_pull_project,
294
343
  index_match_method=ena_maps.IndexMatchMethod.PULL,
344
+ pset_valid_mask=good_pixel_mask,
295
345
  )
296
346
 
297
347
  # Subsequent processing for weighted quantities at SkyMap level
298
- skymap.data_1d[VARIABLES_TO_WEIGHT_BY_POINTING_SET_EXPOSURE_TIMES_SOLID_ANGLE] /= (
299
- skymap.data_1d["pointing_set_exposure_times_solid_angle"]
300
- )
348
+ skymap.data_1d[existing_vars_to_weight] /= skymap.data_1d[
349
+ "pointing_set_exposure_times_solid_angle"
350
+ ]
301
351
 
302
352
  # Background rates must be scaled by the ratio of the solid angles of the
303
353
  # map pixel / pointing set pixel
@@ -347,7 +397,7 @@ def generate_ultra_healpix_skymap(
347
397
  )
348
398
  - (
349
399
  (
350
- skymap.data_1d["obs_date_for_std"]
400
+ skymap.data_1d["obs_date_range"]
351
401
  / (skymap.data_1d["num_pointing_set_pixel_members"])
352
402
  )
353
403
  ** 2
@@ -360,11 +410,10 @@ def generate_ultra_healpix_skymap(
360
410
  skymap.data_1d = skymap.data_1d.drop_vars(
361
411
  VARIABLES_TO_DROP_AFTER_INTENSITY_CALCULATION,
362
412
  )
363
-
364
413
  return skymap, np.array(all_pset_epochs)
365
414
 
366
415
 
367
- def ultra_l2(
416
+ def ultra_l2( # noqa: PLR0912
368
417
  data_dict: dict[str, xr.Dataset | str | Path],
369
418
  output_map_structure: (
370
419
  ena_maps.RectangularSkyMap | ena_maps.HealpixSkyMap
@@ -467,6 +516,7 @@ def ultra_l2(
467
516
  map_dataset = healpix_skymap.to_dataset()
468
517
  # Add attributes related to the map
469
518
  map_attrs = {
519
+ "HEALPix_solid_angle": str(healpix_skymap.solid_angle),
470
520
  "HEALPix_nside": str(output_map_structure.nside),
471
521
  "HEALPix_nest": str(output_map_structure.nested),
472
522
  }
@@ -541,6 +591,11 @@ def ultra_l2(
541
591
  # to "energy" for all instruments.
542
592
  map_dataset = map_dataset.rename({"energy_bin_geometric_mean": "energy"})
543
593
 
594
+ # Rename positional uncertainty variables if present
595
+ if "scatter_theta" in map_dataset and "scatter_phi" in map_dataset:
596
+ map_dataset = map_dataset.rename({"scatter_theta": "positional_uncert_theta"})
597
+ map_dataset = map_dataset.rename({"scatter_phi": "positional_uncert_phi"})
598
+
544
599
  # Add the defined attributes to the map's global attrs
545
600
  map_dataset.attrs.update(map_attrs)
546
601
 
@@ -606,6 +661,8 @@ def ultra_l2(
606
661
  )
607
662
  )
608
663
 
609
- # Adjust the dtype of obs_date to be int64
664
+ # Adjust the dtype of obs dates to be int64
610
665
  map_dataset["obs_date"] = map_dataset["obs_date"].astype(np.int64)
666
+ map_dataset["obs_date_range"] = map_dataset["obs_date_range"].astype(np.int64)
667
+
611
668
  return [map_dataset]
@@ -1,6 +1,5 @@
1
1
  """Create dataset."""
2
2
 
3
- import numpy as np
4
3
  import xarray as xr
5
4
 
6
5
  from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
@@ -32,7 +31,7 @@ def create_dataset( # noqa: PLR0912
32
31
  cdf_manager.add_instrument_global_attrs("ultra")
33
32
  cdf_manager.add_instrument_variable_attrs("ultra", level)
34
33
 
35
- # L1b extended spin, badtimes, and cullingmask data products
34
+ # L1b extended spin, badtimes, and goodtimes data products
36
35
  if "spin_number" in data_dict.keys():
37
36
  coords = {
38
37
  "spin_number": ("spin_number", data_dict["spin_number"]),
@@ -40,7 +39,6 @@ def create_dataset( # noqa: PLR0912
40
39
  "energy_bin_geometric_mean",
41
40
  data_dict["energy_bin_geometric_mean"],
42
41
  ),
43
- "epoch": ("spin_number", np.asarray(data_dict["epoch"])),
44
42
  }
45
43
  default_dimension = "spin_number"
46
44
  # L1c pset data products
@@ -110,7 +108,12 @@ def create_dataset( # noqa: PLR0912
110
108
  dims=["epoch", "component"],
111
109
  attrs=cdf_manager.get_variable_attributes(key, check_schema=False),
112
110
  )
113
- elif key == "ena_rates_threshold":
111
+ elif key in [
112
+ "ena_rates_threshold",
113
+ "scatter_threshold",
114
+ "energy_delta_minus",
115
+ "energy_delta_plus",
116
+ ]:
114
117
  dataset[key] = xr.DataArray(
115
118
  data,
116
119
  dims=["energy_bin_geometric_mean"],
@@ -134,7 +137,7 @@ def create_dataset( # noqa: PLR0912
134
137
  dims=["energy_bin_geometric_mean", "spin_number"],
135
138
  attrs=cdf_manager.get_variable_attributes(key, check_schema=False),
136
139
  )
137
- elif key in {"latitude", "longitude"}:
140
+ elif key in {"quality_flags", "latitude", "longitude"}:
138
141
  dataset[key] = xr.DataArray(
139
142
  data,
140
143
  dims=["epoch", "pixel_index"],
@@ -155,6 +158,8 @@ def create_dataset( # noqa: PLR0912
155
158
  "sensitivity",
156
159
  "efficiency",
157
160
  "geometric_function",
161
+ "scatter_theta",
162
+ "scatter_phi",
158
163
  }:
159
164
  dataset[key] = xr.DataArray(
160
165
  data,
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: imap-processing
3
- Version: 0.19.0
3
+ Version: 0.19.3
4
4
  Summary: IMAP Science Operations Center Processing
5
5
  License: MIT
6
6
  Keywords: IMAP,SDC,SOC,Science Operations
@@ -30,7 +30,7 @@ Provides-Extra: tools
30
30
  Requires-Dist: astropy-healpix (>=1.0)
31
31
  Requires-Dist: cdflib (>=1.3.6,<2.0.0)
32
32
  Requires-Dist: healpy (>=1.18.0,<2.0.0) ; extra == "map-visualization"
33
- Requires-Dist: imap-data-access (>=0.32.0)
33
+ Requires-Dist: imap-data-access (>=0.35.0)
34
34
  Requires-Dist: mypy (==1.10.1) ; extra == "dev"
35
35
  Requires-Dist: netcdf4 (>=1.7.2,<2.0.0) ; extra == "test"
36
36
  Requires-Dist: numpy (<=3)