imap-processing 0.17.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of imap-processing might be problematic. Click here for more details.

Files changed (141) hide show
  1. imap_processing/_version.py +2 -2
  2. imap_processing/ancillary/ancillary_dataset_combiner.py +161 -1
  3. imap_processing/ccsds/excel_to_xtce.py +12 -0
  4. imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml +6 -6
  5. imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +312 -274
  6. imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml +39 -28
  7. imap_processing/cdf/config/imap_codice_l2_variable_attrs.yaml +1048 -183
  8. imap_processing/cdf/config/imap_constant_attrs.yaml +4 -2
  9. imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml +12 -0
  10. imap_processing/cdf/config/imap_hi_global_cdf_attrs.yaml +5 -0
  11. imap_processing/cdf/config/imap_hit_global_cdf_attrs.yaml +10 -4
  12. imap_processing/cdf/config/imap_hit_l1a_variable_attrs.yaml +163 -100
  13. imap_processing/cdf/config/imap_hit_l2_variable_attrs.yaml +4 -4
  14. imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +97 -54
  15. imap_processing/cdf/config/imap_idex_l2a_variable_attrs.yaml +33 -4
  16. imap_processing/cdf/config/imap_idex_l2b_variable_attrs.yaml +44 -44
  17. imap_processing/cdf/config/imap_idex_l2c_variable_attrs.yaml +77 -61
  18. imap_processing/cdf/config/imap_lo_global_cdf_attrs.yaml +30 -0
  19. imap_processing/cdf/config/imap_lo_l1a_variable_attrs.yaml +4 -15
  20. imap_processing/cdf/config/imap_lo_l1c_variable_attrs.yaml +189 -98
  21. imap_processing/cdf/config/imap_mag_global_cdf_attrs.yaml +99 -2
  22. imap_processing/cdf/config/imap_mag_l1c_variable_attrs.yaml +24 -1
  23. imap_processing/cdf/config/imap_ultra_global_cdf_attrs.yaml +60 -0
  24. imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml +99 -11
  25. imap_processing/cdf/config/imap_ultra_l1c_variable_attrs.yaml +50 -7
  26. imap_processing/cli.py +121 -44
  27. imap_processing/codice/codice_l1a.py +165 -77
  28. imap_processing/codice/codice_l1b.py +1 -1
  29. imap_processing/codice/codice_l2.py +118 -19
  30. imap_processing/codice/constants.py +1217 -1089
  31. imap_processing/decom.py +1 -4
  32. imap_processing/ena_maps/ena_maps.py +32 -25
  33. imap_processing/ena_maps/utils/naming.py +8 -2
  34. imap_processing/glows/ancillary/imap_glows_exclusions-by-instr-team_20250923_v002.dat +10 -0
  35. imap_processing/glows/ancillary/imap_glows_map-of-excluded-regions_20250923_v002.dat +393 -0
  36. imap_processing/glows/ancillary/imap_glows_map-of-uv-sources_20250923_v002.dat +593 -0
  37. imap_processing/glows/ancillary/imap_glows_pipeline_settings_20250923_v002.json +54 -0
  38. imap_processing/glows/ancillary/imap_glows_suspected-transients_20250923_v002.dat +10 -0
  39. imap_processing/glows/l1b/glows_l1b.py +99 -9
  40. imap_processing/glows/l1b/glows_l1b_data.py +350 -38
  41. imap_processing/glows/l2/glows_l2.py +11 -0
  42. imap_processing/hi/hi_l1a.py +124 -3
  43. imap_processing/hi/hi_l1b.py +154 -71
  44. imap_processing/hi/hi_l2.py +84 -51
  45. imap_processing/hi/utils.py +153 -8
  46. imap_processing/hit/l0/constants.py +3 -0
  47. imap_processing/hit/l0/decom_hit.py +5 -8
  48. imap_processing/hit/l1a/hit_l1a.py +375 -45
  49. imap_processing/hit/l1b/constants.py +5 -0
  50. imap_processing/hit/l1b/hit_l1b.py +61 -131
  51. imap_processing/hit/l2/constants.py +1 -1
  52. imap_processing/hit/l2/hit_l2.py +10 -11
  53. imap_processing/ialirt/calculate_ingest.py +219 -0
  54. imap_processing/ialirt/constants.py +32 -1
  55. imap_processing/ialirt/generate_coverage.py +201 -0
  56. imap_processing/ialirt/l0/ialirt_spice.py +5 -2
  57. imap_processing/ialirt/l0/parse_mag.py +337 -29
  58. imap_processing/ialirt/l0/process_hit.py +5 -3
  59. imap_processing/ialirt/l0/process_swapi.py +41 -25
  60. imap_processing/ialirt/l0/process_swe.py +23 -7
  61. imap_processing/ialirt/process_ephemeris.py +70 -14
  62. imap_processing/ialirt/utils/constants.py +22 -16
  63. imap_processing/ialirt/utils/create_xarray.py +42 -19
  64. imap_processing/idex/idex_constants.py +1 -5
  65. imap_processing/idex/idex_l0.py +2 -2
  66. imap_processing/idex/idex_l1a.py +2 -3
  67. imap_processing/idex/idex_l1b.py +2 -3
  68. imap_processing/idex/idex_l2a.py +130 -4
  69. imap_processing/idex/idex_l2b.py +313 -119
  70. imap_processing/idex/idex_utils.py +1 -3
  71. imap_processing/lo/l0/lo_apid.py +1 -0
  72. imap_processing/lo/l0/lo_science.py +25 -24
  73. imap_processing/lo/l1a/lo_l1a.py +44 -0
  74. imap_processing/lo/l1b/lo_l1b.py +3 -3
  75. imap_processing/lo/l1c/lo_l1c.py +116 -50
  76. imap_processing/lo/l2/lo_l2.py +29 -29
  77. imap_processing/lo/lo_ancillary.py +55 -0
  78. imap_processing/lo/packet_definitions/lo_xtce.xml +5359 -106
  79. imap_processing/mag/constants.py +1 -0
  80. imap_processing/mag/l1a/mag_l1a.py +1 -0
  81. imap_processing/mag/l1a/mag_l1a_data.py +26 -0
  82. imap_processing/mag/l1b/mag_l1b.py +3 -2
  83. imap_processing/mag/l1c/interpolation_methods.py +14 -15
  84. imap_processing/mag/l1c/mag_l1c.py +23 -6
  85. imap_processing/mag/l1d/__init__.py +0 -0
  86. imap_processing/mag/l1d/mag_l1d.py +176 -0
  87. imap_processing/mag/l1d/mag_l1d_data.py +725 -0
  88. imap_processing/mag/l2/__init__.py +0 -0
  89. imap_processing/mag/l2/mag_l2.py +25 -20
  90. imap_processing/mag/l2/mag_l2_data.py +199 -130
  91. imap_processing/quality_flags.py +28 -2
  92. imap_processing/spice/geometry.py +101 -36
  93. imap_processing/spice/pointing_frame.py +1 -7
  94. imap_processing/spice/repoint.py +29 -2
  95. imap_processing/spice/spin.py +32 -8
  96. imap_processing/spice/time.py +60 -19
  97. imap_processing/swapi/l1/swapi_l1.py +10 -4
  98. imap_processing/swapi/l2/swapi_l2.py +66 -24
  99. imap_processing/swapi/swapi_utils.py +1 -1
  100. imap_processing/swe/l1b/swe_l1b.py +3 -6
  101. imap_processing/ultra/constants.py +28 -3
  102. imap_processing/ultra/l0/decom_tools.py +15 -8
  103. imap_processing/ultra/l0/decom_ultra.py +35 -11
  104. imap_processing/ultra/l0/ultra_utils.py +102 -12
  105. imap_processing/ultra/l1a/ultra_l1a.py +26 -6
  106. imap_processing/ultra/l1b/cullingmask.py +6 -3
  107. imap_processing/ultra/l1b/de.py +122 -26
  108. imap_processing/ultra/l1b/extendedspin.py +29 -2
  109. imap_processing/ultra/l1b/lookup_utils.py +424 -50
  110. imap_processing/ultra/l1b/quality_flag_filters.py +23 -0
  111. imap_processing/ultra/l1b/ultra_l1b_culling.py +356 -5
  112. imap_processing/ultra/l1b/ultra_l1b_extended.py +534 -90
  113. imap_processing/ultra/l1c/helio_pset.py +127 -7
  114. imap_processing/ultra/l1c/l1c_lookup_utils.py +256 -0
  115. imap_processing/ultra/l1c/spacecraft_pset.py +90 -15
  116. imap_processing/ultra/l1c/ultra_l1c.py +6 -0
  117. imap_processing/ultra/l1c/ultra_l1c_culling.py +85 -0
  118. imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +446 -341
  119. imap_processing/ultra/l2/ultra_l2.py +0 -1
  120. imap_processing/ultra/utils/ultra_l1_utils.py +40 -3
  121. imap_processing/utils.py +3 -4
  122. {imap_processing-0.17.0.dist-info → imap_processing-0.19.0.dist-info}/METADATA +3 -3
  123. {imap_processing-0.17.0.dist-info → imap_processing-0.19.0.dist-info}/RECORD +126 -126
  124. imap_processing/idex/idex_l2c.py +0 -250
  125. imap_processing/spice/kernels.py +0 -187
  126. imap_processing/ultra/lookup_tables/Angular_Profiles_FM45_LeftSlit.csv +0 -526
  127. imap_processing/ultra/lookup_tables/Angular_Profiles_FM45_RightSlit.csv +0 -526
  128. imap_processing/ultra/lookup_tables/Angular_Profiles_FM90_LeftSlit.csv +0 -526
  129. imap_processing/ultra/lookup_tables/Angular_Profiles_FM90_RightSlit.csv +0 -524
  130. imap_processing/ultra/lookup_tables/EgyNorm.mem.csv +0 -32769
  131. imap_processing/ultra/lookup_tables/FM45_Startup1_ULTRA_IMGPARAMS_20240719.csv +0 -2
  132. imap_processing/ultra/lookup_tables/FM90_Startup1_ULTRA_IMGPARAMS_20240719.csv +0 -2
  133. imap_processing/ultra/lookup_tables/dps_grid45_compressed.cdf +0 -0
  134. imap_processing/ultra/lookup_tables/ultra45_back-pos-luts.csv +0 -4097
  135. imap_processing/ultra/lookup_tables/ultra45_tdc_norm.csv +0 -2050
  136. imap_processing/ultra/lookup_tables/ultra90_back-pos-luts.csv +0 -4097
  137. imap_processing/ultra/lookup_tables/ultra90_tdc_norm.csv +0 -2050
  138. imap_processing/ultra/lookup_tables/yadjust.csv +0 -257
  139. {imap_processing-0.17.0.dist-info → imap_processing-0.19.0.dist-info}/LICENSE +0 -0
  140. {imap_processing-0.17.0.dist-info → imap_processing-0.19.0.dist-info}/WHEEL +0 -0
  141. {imap_processing-0.17.0.dist-info → imap_processing-0.19.0.dist-info}/entry_points.txt +0 -0
@@ -1,19 +1,45 @@
1
1
  """Calculate Pointing Set Grids."""
2
2
 
3
- import astropy_healpix.healpy as hp
3
+ import logging
4
+
4
5
  import numpy as np
6
+ import pandas as pd
5
7
  import xarray as xr
6
8
 
7
- from imap_processing.ultra.l1c.ultra_l1c_pset_bins import build_energy_bins
9
+ from imap_processing.spice.repoint import get_pointing_times
10
+ from imap_processing.spice.time import (
11
+ et_to_met,
12
+ met_to_ttj2000ns,
13
+ sct_to_et,
14
+ ttj2000ns_to_et,
15
+ )
16
+ from imap_processing.ultra.l1b.ultra_l1b_culling import get_de_rejection_mask
17
+ from imap_processing.ultra.l1c.l1c_lookup_utils import (
18
+ calculate_pixels_within_scattering_threshold,
19
+ get_spacecraft_pointing_lookup_tables,
20
+ )
21
+ from imap_processing.ultra.l1c.ultra_l1c_pset_bins import (
22
+ build_energy_bins,
23
+ get_efficiencies_and_geometric_function,
24
+ get_helio_adjusted_data,
25
+ get_spacecraft_exposure_times,
26
+ get_spacecraft_histogram,
27
+ )
8
28
  from imap_processing.ultra.utils.ultra_l1_utils import create_dataset
9
29
 
30
+ logger = logging.getLogger(__name__)
31
+
10
32
 
11
33
  def calculate_helio_pset(
12
34
  de_dataset: xr.Dataset,
13
35
  extendedspin_dataset: xr.Dataset,
14
36
  cullingmask_dataset: xr.Dataset,
37
+ rates_dataset: xr.Dataset,
38
+ params_dataset: xr.Dataset,
15
39
  name: str,
16
40
  ancillary_files: dict,
41
+ instrument_id: int,
42
+ species_id: int = 1,
17
43
  ) -> xr.Dataset:
18
44
  """
19
45
  Create dictionary with defined datatype for Pointing Set Grid Data.
@@ -26,27 +52,121 @@ def calculate_helio_pset(
26
52
  Dataset containing extendedspin data.
27
53
  cullingmask_dataset : xarray.Dataset
28
54
  Dataset containing cullingmask data.
55
+ rates_dataset : xarray.Dataset
56
+ Dataset containing image rates data.
57
+ params_dataset : xarray.Dataset
58
+ Dataset containing image parameters data.
29
59
  name : str
30
60
  Name of the dataset.
31
61
  ancillary_files : dict
32
62
  Ancillary files.
63
+ instrument_id : int
64
+ Instrument ID, either 45 or 90.
65
+ species_id : int
66
+ Species ID, default of 1 refers to Hydrogen.
33
67
 
34
68
  Returns
35
69
  -------
36
70
  dataset : xarray.Dataset
37
71
  Dataset containing the data.
38
72
  """
39
- # TODO: Fill in the rest of this later.
40
73
  pset_dict: dict[str, np.ndarray] = {}
41
- healpix = np.arange(hp.nside2npix(128))
42
- _, _, energy_bin_geometric_means = build_energy_bins()
74
+ # Select only the species we are interested in.
75
+ indices = np.where(de_dataset["species"].values == species_id)[0]
76
+ species_dataset = de_dataset.isel(epoch=indices)
77
+
78
+ rejected = get_de_rejection_mask(
79
+ species_dataset["quality_scattering"].values,
80
+ species_dataset["quality_outliers"].values,
81
+ )
82
+ de_dataset = species_dataset.isel(epoch=~rejected)
83
+
84
+ v_mag_helio_spacecraft = np.linalg.norm(
85
+ species_dataset["velocity_dps_helio"].values, axis=1
86
+ )
87
+ vhat_dps_helio = (
88
+ species_dataset["velocity_dps_helio"].values
89
+ / v_mag_helio_spacecraft[:, np.newaxis]
90
+ )
91
+ intervals, _, energy_bin_geometric_means = build_energy_bins()
92
+ counts, latitude, longitude, n_pix = get_spacecraft_histogram(
93
+ vhat_dps_helio,
94
+ species_dataset["energy_heliosphere"].values,
95
+ intervals,
96
+ nside=128,
97
+ )
98
+
99
+ healpix = np.arange(n_pix)
43
100
 
101
+ # Get lookup table for FOR indices by spin phase step
102
+ (
103
+ for_indices_by_spin_phase,
104
+ theta_vals,
105
+ phi_vals,
106
+ ra_and_dec,
107
+ boundary_scale_factors,
108
+ ) = get_spacecraft_pointing_lookup_tables(ancillary_files, instrument_id)
109
+ # Check that the number of rows in the lookup table matches the number of pixels
110
+ if for_indices_by_spin_phase.shape[0] != n_pix:
111
+ logger.warning(
112
+ "The lookup table is expected to have the same number of rows as "
113
+ "the number of HEALPix pixels."
114
+ )
115
+
116
+ pixels_below_scattering = calculate_pixels_within_scattering_threshold(
117
+ for_indices_by_spin_phase, theta_vals, phi_vals, ancillary_files, instrument_id
118
+ )
119
+ # Calculate exposure
120
+ constant_exposure = ancillary_files["l1c-90sensor-dps-exposure"]
121
+ df_exposure = pd.read_csv(constant_exposure)
122
+ exposure_time, deadtime_ratios = get_spacecraft_exposure_times(
123
+ df_exposure,
124
+ rates_dataset,
125
+ params_dataset,
126
+ pixels_below_scattering,
127
+ boundary_scale_factors,
128
+ )
129
+ # calculate efficiency and geometric function as a function of energy
130
+ efficiencies, geometric_function = get_efficiencies_and_geometric_function(
131
+ pixels_below_scattering,
132
+ boundary_scale_factors,
133
+ theta_vals,
134
+ phi_vals,
135
+ n_pix,
136
+ ancillary_files,
137
+ )
138
+ # Get midpoint timestamp for pointing.
139
+ # TODO remove sct_to_et conversion
140
+ pointing_start, pointing_stop = get_pointing_times(
141
+ et_to_met(sct_to_et(species_dataset["event_times"].data[0]))
142
+ )
143
+ mid_time = ttj2000ns_to_et(met_to_ttj2000ns((pointing_start + pointing_stop) / 2))
144
+ exposure_time, efficiency, geometric_function = get_helio_adjusted_data(
145
+ mid_time,
146
+ exposure_time,
147
+ geometric_function,
148
+ efficiencies,
149
+ ra_and_dec[:, 0],
150
+ ra_and_dec[:, 1],
151
+ )
152
+ sensitivity = efficiencies * geometric_function
153
+
154
+ # For ISTP, epoch should be the center of the time bin.
44
155
  pset_dict["epoch"] = de_dataset.epoch.data[:1].astype(np.int64)
45
- pset_dict["pixel_index"] = healpix
156
+ pset_dict["counts"] = counts[np.newaxis, ...]
157
+ pset_dict["latitude"] = latitude[np.newaxis, ...]
158
+ pset_dict["longitude"] = longitude[np.newaxis, ...]
46
159
  pset_dict["energy_bin_geometric_mean"] = energy_bin_geometric_means
47
- pset_dict["exposure_factor"] = np.zeros(len(healpix), dtype=np.uint8)[
160
+ pset_dict["helio_exposure_factor"] = exposure_time
161
+ pset_dict["pixel_index"] = healpix
162
+ pset_dict["energy_bin_delta"] = np.diff(intervals, axis=1).squeeze()[
48
163
  np.newaxis, ...
49
164
  ]
165
+ pset_dict["sensitivity"] = sensitivity
166
+ pset_dict["efficiency"] = efficiencies
167
+ pset_dict["geometric_function"] = geometric_function
168
+ pset_dict["dead_time_ratio"] = deadtime_ratios
169
+ pset_dict["spin_phase_step"] = np.arange(len(deadtime_ratios))
50
170
 
51
171
  dataset = create_dataset(pset_dict, name, "l1c")
52
172
 
@@ -0,0 +1,256 @@
1
+ """Contains tools for lookup tables for l1c."""
2
+
3
+ import logging
4
+
5
+ import numpy as np
6
+ import pandas as pd
7
+ from numpy._typing import NDArray
8
+
9
+ from imap_processing.ultra.l1b.lookup_utils import (
10
+ get_scattering_coefficients,
11
+ get_scattering_thresholds,
12
+ load_scattering_lookup_tables,
13
+ )
14
+ from imap_processing.ultra.l1c.ultra_l1c_pset_bins import build_energy_bins
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+
19
+ def mask_below_fwhm_scattering_threshold(
20
+ theta_coeffs: np.ndarray,
21
+ phi_coeffs: np.ndarray,
22
+ energy: np.ndarray,
23
+ scattering_thresholds: np.ndarray,
24
+ ) -> np.ndarray:
25
+ """
26
+ Determine indices of theta and phi values below the FWHM scattering threshold.
27
+
28
+ For each phi and theta, calculate the FWHM using the formula:
29
+ FWHM = A*E^g
30
+ If Phi FWHM or Theta FWHM > the scattering requirements from the table above,
31
+ mask the instrument frame pixel.
32
+
33
+ Parameters
34
+ ----------
35
+ theta_coeffs : NDArray
36
+ Coefficients for theta FWHM calculation (a and g) for each pixel.
37
+ phi_coeffs : NDArray
38
+ Coefficients for phi FWHM calculation (a and g) for each pixel.
39
+ energy : NDArray
40
+ Energy corresponding to each theta and phi val in keV.
41
+ scattering_thresholds : dict
42
+ Scattering thresholds corresponding to each energy.
43
+
44
+ Returns
45
+ -------
46
+ numpy.ndarray
47
+ Boolean array indicating indices below the scattering threshold.
48
+ """
49
+ # Calculate FWHM for all pixels and all energies
50
+ fwhm_theta = theta_coeffs[..., 0:1] * (
51
+ energy ** theta_coeffs[..., 1:2]
52
+ ) # (npix, energy.shape[1])
53
+ fwhm_phi = phi_coeffs[..., 0:1] * (
54
+ energy ** phi_coeffs[..., 1:2]
55
+ ) # (npix, energy.shape[1])
56
+
57
+ thresholds = scattering_thresholds[np.newaxis, :] # (1, energy.shape[1])
58
+
59
+ # Combine conditions for both theta and phi.
60
+ # shape = (npix, energy.shape[1])
61
+ return np.logical_and(fwhm_theta <= thresholds, fwhm_phi <= thresholds)
62
+
63
+
64
+ def calculate_pixels_within_scattering_threshold(
65
+ for_indices_by_spin_phase: np.ndarray,
66
+ theta_vals: np.ndarray,
67
+ phi_vals: np.ndarray,
68
+ ancillary_files: dict,
69
+ instrument_id: int,
70
+ ) -> list:
71
+ """
72
+ Calculate pixels within the FWHM scattering threshold for each spin phase step.
73
+
74
+ Parameters
75
+ ----------
76
+ for_indices_by_spin_phase : np.ndarray
77
+ A 2D boolean array where cols are spin phase steps are rows are HEALPix pixels.
78
+ True indicates pixels that are within the Field of Regard (FOR) at that
79
+ spin phase.
80
+ theta_vals : np.ndarray
81
+ A 2D array of theta values for each HEALPix pixel at each spin phase step.
82
+ phi_vals : np.ndarray
83
+ A 2D array of phi values for each HEALPix pixel at each spin phase step.
84
+ ancillary_files : dict
85
+ Dictionary containing ancillary files.
86
+ instrument_id : int,
87
+ Instrument ID, either 45 or 90.
88
+
89
+ Returns
90
+ -------
91
+ pixels_below_scattering : list
92
+ A Nested list of arrays indicating pixels within the scattering threshold.
93
+ The outer list indicates spin phase steps, the middle list indicates energy
94
+ bins, and the inner arrays contain indices indicating pixels that are below
95
+ the FWHM scattering threshold.
96
+ """
97
+ # Load scattering coefficient lookup table
98
+ scattering_luts = load_scattering_lookup_tables(ancillary_files, instrument_id)
99
+ pixels_below_scattering = []
100
+ # Get energy bin geometric means
101
+ energy_bin_geometric_means = build_energy_bins()[2]
102
+ # Load scattering thresholds for the energy bin geometric means
103
+ scattering_thresholds_for_energy_mean = get_scattering_thresholds_for_energy(
104
+ energy_bin_geometric_means, ancillary_files
105
+ )
106
+ steps = for_indices_by_spin_phase.shape[1]
107
+ energies = energy_bin_geometric_means[np.newaxis, :]
108
+ # The "for_indices_by_spin_phase" lookup table contains the boolean values of each
109
+ # pixel at each spin phase step, indicating whether the pixel is inside the FOR.
110
+ # It starts at Spin-phase = 0, and increments in fine steps (1 ms), spinning the
111
+ # spacecraft in the despun frame. At each iteration, query for the pixels in the
112
+ # FOR, and calculate whether the FWHM value is below the threshold at the energy.
113
+ for i in range(steps):
114
+ # Calculate spin phase for the current iteration
115
+ for_inds = for_indices_by_spin_phase[:, i]
116
+
117
+ # Skip if no pixels in FOR
118
+ if not np.any(for_inds):
119
+ logger.info(f"No pixels found in FOR at spin phase step {i}")
120
+ pixels_below_scattering.append(
121
+ [
122
+ np.array([], dtype=int)
123
+ for _ in range(len(energy_bin_geometric_means))
124
+ ]
125
+ )
126
+ continue
127
+ # Using the lookup table, get the indices of the pixels inside the FOR at
128
+ # the current spin phase step.
129
+ theta = theta_vals[for_inds, i]
130
+ phi = phi_vals[for_inds, i]
131
+ theta_coeffs, phi_coeffs = get_scattering_coefficients(
132
+ theta, phi, lookup_tables=scattering_luts
133
+ )
134
+ # Get a mask for pixels below the FWHM scattering threshold
135
+ scattering_mask = mask_below_fwhm_scattering_threshold(
136
+ theta_coeffs,
137
+ phi_coeffs,
138
+ energies,
139
+ scattering_thresholds=scattering_thresholds_for_energy_mean,
140
+ )
141
+ # Extract pixel indices for each energy
142
+ for_pixel_indices = np.where(for_inds)[0]
143
+ pixels_below_scattering_for_energy = []
144
+
145
+ for energy_idx in range(len(energy_bin_geometric_means)):
146
+ valid_pixels = scattering_mask[:, energy_idx]
147
+ pixels_below_scattering_for_energy.append(for_pixel_indices[valid_pixels])
148
+
149
+ pixels_below_scattering.append(pixels_below_scattering_for_energy)
150
+
151
+ return pixels_below_scattering
152
+
153
+
154
+ def get_spacecraft_pointing_lookup_tables(
155
+ ancillary_files: dict, instrument_id: int
156
+ ) -> tuple[NDArray, NDArray, NDArray, NDArray, NDArray]:
157
+ """
158
+ Get indices of pixels in the nominal FOR as a function of spin phase.
159
+
160
+ This function also returns the theta / phi values in the instrument frame per spin
161
+ phase, right ascension / declination values in the SC frame, and boundary scale
162
+ factors for each pixel at each spin phase.
163
+
164
+ Parameters
165
+ ----------
166
+ ancillary_files : dict[Path]
167
+ Ancillary files.
168
+ instrument_id : int
169
+ Instrument ID, either 45 or 90.
170
+
171
+ Returns
172
+ -------
173
+ for_indices_by_spin_phase : NDArray
174
+ A 2D boolean array of shape (npix, n_spin_phase_steps).
175
+ True indicates pixels that are within the Field of Regard (FOR) at that
176
+ spin phase.
177
+ theta_vals : NDArray
178
+ A 2D array of theta values for each HEALPix pixel at each spin phase step.
179
+ phi_vals : NDArray
180
+ A 2D array of phi values for each HEALPix pixel at each spin phase step.
181
+ ra_and_dec : NDArray
182
+ A 2D array of right ascension and declination values for each HEALPix pixel.
183
+ boundary_scale_factors : NDArray
184
+ A 2D array of boundary scale factors for each HEALPix pixel at each spin phase
185
+ step.
186
+ """
187
+ theta_descriptor = f"l1c-{instrument_id}sensor-sc-pointing-theta-n32"
188
+ phi_descriptor = f"l1c-{instrument_id}sensor-sc-pointing-phi-n32"
189
+ index_descriptor = f"l1c-{instrument_id}sensor-sc-pointing-index-n32"
190
+ bsf_descriptor = f"l1c-{instrument_id}sensor-sc-pointing-bsf-n32"
191
+
192
+ theta_vals = pd.read_csv(
193
+ ancillary_files[theta_descriptor], header=None, skiprows=1
194
+ ).to_numpy(dtype=float)[:, 2:]
195
+ phi_vals = pd.read_csv(
196
+ ancillary_files[phi_descriptor], header=None, skiprows=1
197
+ ).to_numpy(dtype=float)[:, 2:]
198
+ index_grid = pd.read_csv(
199
+ ancillary_files[index_descriptor], header=None, skiprows=1
200
+ ).to_numpy(dtype=float)
201
+ boundary_scale_factors = pd.read_csv(
202
+ ancillary_files[bsf_descriptor], header=None, skiprows=1
203
+ ).to_numpy(dtype=float)[:, 2:]
204
+
205
+ ra_and_dec = index_grid[:, :2] # Shape (npix, 2)
206
+ # This array indicates whether each pixel is in the nominal FOR at each spin phase
207
+ # step (15000 steps for a full rotation with 1 ms resolution).
208
+ for_indices_by_spin_phase = np.nan_to_num(index_grid[:, 2:], nan=0).astype(
209
+ bool
210
+ ) # Shape (npix, 15000)
211
+ return (
212
+ for_indices_by_spin_phase,
213
+ theta_vals,
214
+ phi_vals,
215
+ ra_and_dec,
216
+ boundary_scale_factors,
217
+ )
218
+
219
+
220
+ def get_scattering_thresholds_for_energy(
221
+ energy: np.ndarray, ancillary_files: dict
222
+ ) -> np.ndarray:
223
+ """
224
+ Find the scattering thresholds for each energy bin.
225
+
226
+ Parameters
227
+ ----------
228
+ energy : np.ndarray
229
+ Array of energy values in keV.
230
+ ancillary_files : dict
231
+ Dictionary containing ancillary files.
232
+
233
+ Returns
234
+ -------
235
+ np.ndarray
236
+ Array of scattering thresholds for each energy bin.
237
+ """
238
+ scattering_thresholds = get_scattering_thresholds(ancillary_files)
239
+ # Get thresholds for all energies
240
+ thresholds = []
241
+ for e in energy:
242
+ try:
243
+ threshold = next(
244
+ threshold
245
+ for energy_range, threshold in scattering_thresholds.items()
246
+ if energy_range[0] <= e < energy_range[1]
247
+ )
248
+ except StopIteration:
249
+ logger.warning(
250
+ f"Energy {e} keV is out of bounds for scattering thresholds. Using"
251
+ f" zero for as threshold."
252
+ )
253
+
254
+ threshold = 0
255
+ thresholds.append(threshold)
256
+ return np.array(thresholds)
@@ -1,25 +1,39 @@
1
1
  """Calculate Pointing Set Grids."""
2
2
 
3
+ import logging
4
+
3
5
  import numpy as np
4
6
  import pandas as pd
5
7
  import xarray as xr
6
8
 
9
+ from imap_processing.cdf.utils import parse_filename_like
10
+ from imap_processing.ultra.l1b.ultra_l1b_culling import get_de_rejection_mask
11
+ from imap_processing.ultra.l1c.l1c_lookup_utils import (
12
+ calculate_pixels_within_scattering_threshold,
13
+ get_spacecraft_pointing_lookup_tables,
14
+ )
7
15
  from imap_processing.ultra.l1c.ultra_l1c_pset_bins import (
8
16
  build_energy_bins,
17
+ get_efficiencies_and_geometric_function,
9
18
  get_spacecraft_background_rates,
10
19
  get_spacecraft_exposure_times,
11
20
  get_spacecraft_histogram,
12
- interpolate_sensitivity,
13
21
  )
14
22
  from imap_processing.ultra.utils.ultra_l1_utils import create_dataset
15
23
 
24
+ logger = logging.getLogger(__name__)
25
+
16
26
 
17
27
  def calculate_spacecraft_pset(
18
28
  de_dataset: xr.Dataset,
19
29
  extendedspin_dataset: xr.Dataset,
20
30
  cullingmask_dataset: xr.Dataset,
31
+ rates_dataset: xr.Dataset,
32
+ params_dataset: xr.Dataset,
21
33
  name: str,
22
34
  ancillary_files: dict,
35
+ instrument_id: int,
36
+ species_id: int = 1,
23
37
  ) -> xr.Dataset:
24
38
  """
25
39
  Create dictionary with defined datatype for Pointing Set Grid Data.
@@ -32,10 +46,18 @@ def calculate_spacecraft_pset(
32
46
  Dataset containing extendedspin data.
33
47
  cullingmask_dataset : xarray.Dataset
34
48
  Dataset containing cullingmask data.
49
+ rates_dataset : xarray.Dataset
50
+ Dataset containing image rates data.
51
+ params_dataset : xarray.Dataset
52
+ Dataset containing image parameters data.
35
53
  name : str
36
54
  Name of the dataset.
37
55
  ancillary_files : dict
38
56
  Ancillary files.
57
+ instrument_id : int
58
+ Instrument ID, either 45 or 90.
59
+ species_id : int
60
+ Species ID, default of 1 refers to Hydrogen.
39
61
 
40
62
  Returns
41
63
  -------
@@ -43,35 +65,83 @@ def calculate_spacecraft_pset(
43
65
  Dataset containing the data.
44
66
  """
45
67
  pset_dict: dict[str, np.ndarray] = {}
68
+ sensor = parse_filename_like(name)["sensor"][0:2]
69
+ # Select only the species we are interested in.
70
+ indices = np.where(de_dataset["species"].values == species_id)[0]
71
+ species_dataset = de_dataset.isel(epoch=indices)
72
+
73
+ # Before we use the de_dataset to calculate the pointing set grid we need to filter.
74
+ rejected = get_de_rejection_mask(
75
+ species_dataset["quality_scattering"].values,
76
+ species_dataset["quality_outliers"].values,
77
+ )
78
+ species_dataset = species_dataset.isel(epoch=~rejected)
46
79
 
47
- v_mag_dps_spacecraft = np.linalg.norm(de_dataset["velocity_dps_sc"].values, axis=1)
80
+ v_mag_dps_spacecraft = np.linalg.norm(
81
+ species_dataset["velocity_dps_sc"].values, axis=1
82
+ )
48
83
  vhat_dps_spacecraft = (
49
- de_dataset["velocity_dps_sc"].values / v_mag_dps_spacecraft[:, np.newaxis]
84
+ species_dataset["velocity_dps_sc"].values / v_mag_dps_spacecraft[:, np.newaxis]
50
85
  )
51
86
 
52
87
  intervals, _, energy_bin_geometric_means = build_energy_bins()
53
88
  counts, latitude, longitude, n_pix = get_spacecraft_histogram(
54
89
  vhat_dps_spacecraft,
55
- de_dataset["energy_spacecraft"].values,
90
+ species_dataset["energy_spacecraft"].values,
56
91
  intervals,
57
92
  nside=128,
58
93
  )
59
94
  healpix = np.arange(n_pix)
60
95
 
61
- # calculate background rates
62
- background_rates = get_spacecraft_background_rates()
96
+ # Get lookup table for FOR indices by spin phase step
97
+ (
98
+ for_indices_by_spin_phase,
99
+ theta_vals,
100
+ phi_vals,
101
+ ra_and_dec,
102
+ boundary_scale_factors,
103
+ ) = get_spacecraft_pointing_lookup_tables(ancillary_files, instrument_id)
104
+ # Check that the number of rows in the lookup table matches the number of pixels
105
+ if for_indices_by_spin_phase.shape[0] != n_pix:
106
+ logger.warning(
107
+ "The lookup table is expected to have the same number of rows as "
108
+ "the number of HEALPix pixels."
109
+ )
63
110
 
64
- efficiencies = ancillary_files["l1c-90sensor-efficiencies"]
65
- geometric_function = ancillary_files["l1c-90sensor-gf"]
66
-
67
- df_efficiencies = pd.read_csv(efficiencies)
68
- df_geometric_function = pd.read_csv(geometric_function)
69
- sensitivity = interpolate_sensitivity(df_efficiencies, df_geometric_function)
111
+ pixels_below_scattering = calculate_pixels_within_scattering_threshold(
112
+ for_indices_by_spin_phase, theta_vals, phi_vals, ancillary_files, instrument_id
113
+ )
114
+ # calculate efficiency and geometric function as a function of energy
115
+ efficiencies, geometric_function = get_efficiencies_and_geometric_function(
116
+ pixels_below_scattering,
117
+ boundary_scale_factors,
118
+ theta_vals,
119
+ phi_vals,
120
+ n_pix,
121
+ ancillary_files,
122
+ )
123
+ sensitivity = efficiencies * geometric_function
70
124
 
71
125
  # Calculate exposure
72
126
  constant_exposure = ancillary_files["l1c-90sensor-dps-exposure"]
73
127
  df_exposure = pd.read_csv(constant_exposure)
74
- exposure_pointing = get_spacecraft_exposure_times(df_exposure)
128
+
129
+ exposure_pointing, deadtime_ratios = get_spacecraft_exposure_times(
130
+ df_exposure,
131
+ rates_dataset,
132
+ params_dataset,
133
+ pixels_below_scattering,
134
+ boundary_scale_factors,
135
+ )
136
+
137
+ # Calculate background rates
138
+ background_rates = get_spacecraft_background_rates(
139
+ rates_dataset,
140
+ sensor,
141
+ ancillary_files,
142
+ intervals,
143
+ cullingmask_dataset["spin_number"].values,
144
+ )
75
145
 
76
146
  # For ISTP, epoch should be the center of the time bin.
77
147
  pset_dict["epoch"] = de_dataset.epoch.data[:1].astype(np.int64)
@@ -80,12 +150,17 @@ def calculate_spacecraft_pset(
80
150
  pset_dict["longitude"] = longitude[np.newaxis, ...]
81
151
  pset_dict["energy_bin_geometric_mean"] = energy_bin_geometric_means
82
152
  pset_dict["background_rates"] = background_rates[np.newaxis, ...]
83
- pset_dict["exposure_factor"] = exposure_pointing.to_numpy()[np.newaxis, ...]
153
+ pset_dict["exposure_factor"] = exposure_pointing
84
154
  pset_dict["pixel_index"] = healpix
85
155
  pset_dict["energy_bin_delta"] = np.diff(intervals, axis=1).squeeze()[
86
156
  np.newaxis, ...
87
157
  ]
88
- pset_dict["sensitivity"] = sensitivity[np.newaxis, ...]
158
+
159
+ pset_dict["sensitivity"] = sensitivity
160
+ pset_dict["efficiency"] = efficiencies
161
+ pset_dict["geometric_function"] = geometric_function
162
+ pset_dict["dead_time_ratio"] = deadtime_ratios
163
+ pset_dict["spin_phase_step"] = np.arange(len(deadtime_ratios))
89
164
 
90
165
  dataset = create_dataset(pset_dict, name, "l1c")
91
166
 
@@ -50,8 +50,11 @@ def ultra_l1c(
50
50
  data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"],
51
51
  data_dict[f"imap_ultra_l1b_{instrument_id}sensor-extendedspin"],
52
52
  data_dict[f"imap_ultra_l1b_{instrument_id}sensor-cullingmask"],
53
+ data_dict[f"imap_ultra_l1a_{instrument_id}sensor-rates"],
54
+ data_dict[f"imap_ultra_l1a_{instrument_id}sensor-params"],
53
55
  f"imap_ultra_l1c_{instrument_id}sensor-heliopset",
54
56
  ancillary_files,
57
+ instrument_id,
55
58
  )
56
59
  output_datasets = [helio_pset]
57
60
  elif (
@@ -63,8 +66,11 @@ def ultra_l1c(
63
66
  data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"],
64
67
  data_dict[f"imap_ultra_l1b_{instrument_id}sensor-extendedspin"],
65
68
  data_dict[f"imap_ultra_l1b_{instrument_id}sensor-cullingmask"],
69
+ data_dict[f"imap_ultra_l1a_{instrument_id}sensor-rates"],
70
+ data_dict[f"imap_ultra_l1a_{instrument_id}sensor-params"],
66
71
  f"imap_ultra_l1c_{instrument_id}sensor-spacecraftpset",
67
72
  ancillary_files,
73
+ instrument_id,
68
74
  )
69
75
  output_datasets = [spacecraft_pset]
70
76
  if not output_datasets: