dkist-processing-visp 4.0.0__py3-none-any.whl → 5.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. dkist_processing_visp/models/constants.py +50 -9
  2. dkist_processing_visp/models/fits_access.py +5 -1
  3. dkist_processing_visp/models/metric_code.py +10 -0
  4. dkist_processing_visp/models/parameters.py +128 -19
  5. dkist_processing_visp/parsers/spectrograph_configuration.py +75 -0
  6. dkist_processing_visp/parsers/visp_l0_fits_access.py +6 -0
  7. dkist_processing_visp/tasks/geometric.py +115 -7
  8. dkist_processing_visp/tasks/l1_output_data.py +202 -0
  9. dkist_processing_visp/tasks/lamp.py +50 -91
  10. dkist_processing_visp/tasks/parse.py +19 -0
  11. dkist_processing_visp/tasks/science.py +14 -14
  12. dkist_processing_visp/tasks/solar.py +894 -451
  13. dkist_processing_visp/tasks/visp_base.py +1 -0
  14. dkist_processing_visp/tests/conftest.py +98 -35
  15. dkist_processing_visp/tests/header_models.py +71 -20
  16. dkist_processing_visp/tests/local_trial_workflows/local_trial_helpers.py +25 -1
  17. dkist_processing_visp/tests/test_assemble_quality.py +89 -4
  18. dkist_processing_visp/tests/test_geometric.py +40 -0
  19. dkist_processing_visp/tests/test_instrument_polarization.py +2 -1
  20. dkist_processing_visp/tests/test_lamp.py +17 -22
  21. dkist_processing_visp/tests/test_parameters.py +120 -18
  22. dkist_processing_visp/tests/test_parse.py +73 -1
  23. dkist_processing_visp/tests/test_science.py +5 -6
  24. dkist_processing_visp/tests/test_solar.py +319 -102
  25. dkist_processing_visp/tests/test_visp_constants.py +35 -6
  26. {dkist_processing_visp-4.0.0.dist-info → dkist_processing_visp-5.0.0.dist-info}/METADATA +40 -37
  27. {dkist_processing_visp-4.0.0.dist-info → dkist_processing_visp-5.0.0.dist-info}/RECORD +31 -30
  28. docs/conf.py +4 -1
  29. docs/gain_correction.rst +50 -42
  30. dkist_processing_visp/tasks/mixin/line_zones.py +0 -116
  31. {dkist_processing_visp-4.0.0.dist-info → dkist_processing_visp-5.0.0.dist-info}/WHEEL +0 -0
  32. {dkist_processing_visp-4.0.0.dist-info → dkist_processing_visp-5.0.0.dist-info}/top_level.txt +0 -0
@@ -1,26 +1,124 @@
1
1
  """ViSP solar calibration task. See :doc:`this page </gain_correction>` for more information."""
2
2
 
3
+ from __future__ import annotations
4
+
5
+ from functools import partial
6
+ from typing import Callable
7
+
8
+ import astropy.units as u
3
9
  import numpy as np
4
10
  import scipy.ndimage as spnd
5
- import scipy.optimize as spo
11
+ from astropy.time import Time
12
+ from astropy.units import Quantity
13
+ from astropy.wcs import WCS
14
+ from dkist_processing_common.codecs.asdf import asdf_encoder
6
15
  from dkist_processing_common.codecs.fits import fits_access_decoder
7
16
  from dkist_processing_common.codecs.fits import fits_array_decoder
8
17
  from dkist_processing_common.codecs.fits import fits_array_encoder
18
+ from dkist_processing_common.models.dkist_location import location_of_dkist
9
19
  from dkist_processing_common.models.task_name import TaskName
10
20
  from dkist_processing_common.tasks.mixin.quality import QualityMixin
11
21
  from dkist_processing_math.arithmetic import divide_arrays_by_array
12
22
  from dkist_processing_math.arithmetic import subtract_array_from_arrays
13
23
  from dkist_processing_math.statistics import average_numpy_arrays
14
24
  from dkist_service_configuration.logging import logger
15
-
25
+ from lmfit.parameter import Parameters
26
+ from sklearn.linear_model import RANSACRegressor
27
+ from sklearn.pipeline import Pipeline
28
+ from sklearn.pipeline import make_pipeline
29
+ from sklearn.preprocessing import PolynomialFeatures
30
+ from sklearn.preprocessing import RobustScaler
31
+ from solar_wavelength_calibration import AngleBoundRange
32
+ from solar_wavelength_calibration import Atlas
33
+ from solar_wavelength_calibration import BoundsModel
34
+ from solar_wavelength_calibration import DispersionBoundRange
35
+ from solar_wavelength_calibration import FitFlagsModel
36
+ from solar_wavelength_calibration import LengthBoundRange
37
+ from solar_wavelength_calibration import UnitlessBoundRange
38
+ from solar_wavelength_calibration import WavelengthCalibrationFitter
39
+ from solar_wavelength_calibration import WavelengthCalibrationParameters
40
+ from solar_wavelength_calibration.fitter.wavelength_fitter import FitResult
41
+ from solar_wavelength_calibration.fitter.wavelength_fitter import WavelengthParameters
42
+ from solar_wavelength_calibration.fitter.wavelength_fitter import calculate_initial_crval_guess
43
+ from sunpy.coordinates import HeliocentricInertial
44
+
45
+ from dkist_processing_visp.models.metric_code import VispMetricCode
16
46
  from dkist_processing_visp.models.tags import VispTag
17
47
  from dkist_processing_visp.parsers.visp_l0_fits_access import VispL0FitsAccess
18
48
  from dkist_processing_visp.tasks.mixin.beam_access import BeamAccessMixin
19
49
  from dkist_processing_visp.tasks.mixin.corrections import CorrectionsMixin
20
- from dkist_processing_visp.tasks.mixin.line_zones import LineZonesMixin
21
50
  from dkist_processing_visp.tasks.visp_base import VispTaskBase
22
51
 
23
- __all__ = ["SolarCalibration"]
52
+ __all__ = [
53
+ "SolarCalibration",
54
+ "WavelengthCalibrationParametersWithContinuum",
55
+ "polynomial_continuum_model",
56
+ "compute_order",
57
+ "compute_initial_dispersion",
58
+ "compute_doppler_velocity",
59
+ "compute_input_wavelength_vector",
60
+ ]
61
+
62
+
63
+ class WavelengthCalibrationParametersWithContinuum(WavelengthCalibrationParameters):
64
+ """
65
+ Subclass of `~solar_wavelength_calibration.WavelengthCalibrationParameters` that adds a polynomial continuum parameterization.
66
+
67
+ The order of the continuum polynomial is set with the new ``continuum_poly_fit_oder`` model field. The `lmfit_parameters`
68
+ property now adds a set of parameters that represent the polynomial coefficients.
69
+ """
70
+
71
+ continuum_poly_fit_order: int
72
+ normalized_abscissa: np.ndarray
73
+ zeroth_order_continuum_coefficient: float
74
+
75
+ @property
76
+ def continuum_function(self) -> Callable[[np.ndarray, Parameters], np.ndarray]:
77
+ """Return a partial function of `polynomial_continuum_model` pre-loaded with the fit order and normalized abscissa."""
78
+ return partial(
79
+ polynomial_continuum_model,
80
+ fit_order=self.continuum_poly_fit_order,
81
+ abscissa=self.normalized_abscissa,
82
+ )
83
+
84
+ @property
85
+ def lmfit_parameters(self) -> Parameters:
86
+ """
87
+ Add continuum polynomial coefficient parameters to the standard `~solar_wavelength_calibration.WavelengthCalibrationParameters.lmfit_parameters`.
88
+
89
+ Each coefficient gets its own parameter called ``poly_coeff_{o:02n}``. The 0th order (i.e., constant continuum
90
+ level) coefficient has bounds [0.7, 1.3] and all higher-order coefficients have bounds [-1, 1].
91
+ """
92
+ # NOTE: We set `vary=True` because otherwise we just wouldn't use this class
93
+ # We set the bounds here because it's easier that defining a custom `BoundsModel` class that can
94
+ # dynamically create the required number of "poly_coeff_{o:02n}" fields. Sorry if this bites you!
95
+ params = super().lmfit_parameters
96
+ for o in range(self.continuum_poly_fit_order + 1):
97
+ # `np.polyval` uses its input coefficient list "backwards", so `poly_coeff_{self.continuum_poly_fit_order}`
98
+ # is the 0th order polynomial term.
99
+ params.add(
100
+ f"poly_coeff_{o:02n}",
101
+ vary=True,
102
+ value=(
103
+ self.zeroth_order_continuum_coefficient
104
+ if o == self.continuum_poly_fit_order
105
+ else 0
106
+ ),
107
+ min=-1,
108
+ max=1,
109
+ )
110
+
111
+ params[f"poly_coeff_{self.continuum_poly_fit_order:02n}"].min = (
112
+ self.zeroth_order_continuum_coefficient * 0.5
113
+ )
114
+ params[f"poly_coeff_{self.continuum_poly_fit_order:02n}"].max = (
115
+ self.zeroth_order_continuum_coefficient * 1.5
116
+ )
117
+
118
+ # Remove the default continuum parameterization
119
+ del params["continuum_level"]
120
+
121
+ return params
24
122
 
25
123
 
26
124
  class SolarCalibration(
@@ -28,18 +126,19 @@ class SolarCalibration(
28
126
  BeamAccessMixin,
29
127
  CorrectionsMixin,
30
128
  QualityMixin,
31
- LineZonesMixin,
32
129
  ):
33
130
  """
34
131
  Task class for generating Solar Gain images for each beam/modstate.
35
132
 
36
133
  Parameters
37
134
  ----------
38
- recipe_run_id : int
135
+ recipe_run_id
39
136
  id of the recipe run used to identify the workflow run this task is part of
40
- workflow_name : str
137
+
138
+ workflow_name
41
139
  name of the workflow to which this instance of the task belongs
42
- workflow_version : str
140
+
141
+ workflow_version
43
142
  version of the workflow to which this instance of the task belongs
44
143
 
45
144
  """
@@ -50,121 +149,160 @@ class SolarCalibration(
50
149
  """
51
150
  For each beam.
52
151
 
53
- For each modstate:
54
- - Do dark, background, lamp, and geometric corrections
55
- - Compute the characteristic spectra
56
- - Re-apply the spectral curvature to the characteristic spectra
57
- - Re-apply angle and state offset distortions to the characteristic spectra
58
- - Remove the distorted characteristic solar spectra from the original spectra
59
- - Write master solar gain
60
-
61
- Returns
62
- -------
63
- None
64
-
152
+ #. Do dark, background, lamp, and geometric corrections
153
+ #. Compute an initial separation of low-order spectral vignette signal from the true solar spectrum
154
+ #. Compute a 2D vignette signal by fitting solar spectrum residuals along the slit
155
+ #. Remove the 2D vignette signal from the averaged gain array
156
+ #. Compute the characteristic spectra
157
+ #. Re-apply the spectral curvature to the characteristic spectra
158
+ #. Re-apply angle and state offset distortions to the characteristic spectra
159
+ #. Remove the distorted characteristic solar spectra from the original, dark-corrected spectra
160
+ #. Write final gain to disk
65
161
  """
66
162
  for beam in range(1, self.constants.num_beams + 1):
67
163
 
68
- pre_equalized_gain_dict = dict()
69
-
70
- for modstate in range(1, self.constants.num_modstates + 1):
71
- apm_str = f"{beam = } and {modstate = }"
72
- with self.telemetry_span(f"Initial corrections for {apm_str}"):
73
- self.do_initial_corrections(beam=beam, modstate=modstate)
74
-
75
- with self.telemetry_span(f"Computing characteristic spectra for {apm_str}"):
76
- char_spec = self.compute_characteristic_spectra(beam=beam, modstate=modstate)
77
- self.write(
78
- data=char_spec,
79
- encoder=fits_array_encoder,
80
- tags=[VispTag.debug(), VispTag.frame()],
81
- relative_path=f"DEBUG_SC_CHAR_SPEC_BEAM_{beam}_MODSTATE_{modstate}.dat",
82
- overwrite=True,
83
- )
164
+ apm_str = f"{beam = }"
165
+ with self.telemetry_span(f"Initial corrections for {apm_str}"):
166
+ self.do_initial_corrections(beam=beam)
84
167
 
85
- with self.telemetry_span(f"Re-distorting characteristic spectra for {apm_str}"):
86
- spec_shift = next(
87
- self.read(
88
- tags=[
89
- VispTag.intermediate_frame(beam=beam),
90
- VispTag.task_geometric_spectral_shifts(),
91
- ],
92
- decoder=fits_array_decoder,
93
- )
94
- )
95
- redistorted_char_spec = next(
96
- self.corrections_remove_spec_geometry(
97
- arrays=char_spec, spec_shift=-1 * spec_shift
98
- )
99
- )
100
- self.write(
101
- data=redistorted_char_spec,
102
- encoder=fits_array_encoder,
103
- tags=[VispTag.debug(), VispTag.frame()],
104
- relative_path=f"DEBUG_SC_CHAR_DISTORT_BEAM_{beam}_MODSTATE_{modstate}.dat",
105
- overwrite=True,
106
- )
168
+ with self.telemetry_span(f"Fit atlas with continuum for {apm_str}"):
169
+ representative_spectrum = self.get_representative_spectrum(beam)
107
170
 
108
- with self.telemetry_span(f"Re-shifting characteristic spectra for {apm_str}"):
109
- reshifted_char_spec = self.distort_characteristic_spectra(
110
- char_spec=redistorted_char_spec, beam=beam, modstate=modstate
111
- )
112
- self.write(
113
- data=reshifted_char_spec,
114
- encoder=fits_array_encoder,
171
+ self.write(
172
+ data=representative_spectrum,
173
+ tags=[VispTag.debug(), VispTag.beam(beam), VispTag.task("REP_SPEC")],
174
+ encoder=fits_array_encoder,
175
+ )
176
+
177
+ logger.info("Deriving values from instrument configuration")
178
+ order = compute_order(
179
+ central_wavelength=self.constants.wavelength * u.nm,
180
+ incident_light_angle=self.constants.incident_light_angle_deg,
181
+ reflected_light_angle=self.constants.reflected_light_angle_deg,
182
+ grating_constant=self.constants.grating_constant_inverse_mm,
183
+ )
184
+
185
+ initial_dispersion = compute_initial_dispersion(
186
+ central_wavelength=self.constants.wavelength * u.nm,
187
+ incident_light_angle=self.constants.incident_light_angle_deg,
188
+ reflected_light_angle=self.constants.reflected_light_angle_deg,
189
+ lens_parameters=self.parameters.wavecal_camera_lens_parameters,
190
+ pixel_pitch=self.parameters.wavecal_pixel_pitch_micron_per_pix,
191
+ )
192
+
193
+ doppler_velocity = compute_doppler_velocity(
194
+ time_of_observation=self.constants.solar_gain_ip_start_time
195
+ )
196
+
197
+ self._log_wavecal_parameters(
198
+ dispersion=initial_dispersion, order=order, doppler_velocity=doppler_velocity
199
+ )
200
+
201
+ fit_result = self.fit_initial_vignette(
202
+ representative_spectrum=representative_spectrum,
203
+ dispersion=initial_dispersion,
204
+ spectral_order=order,
205
+ doppler_velocity=doppler_velocity,
206
+ )
207
+
208
+ first_vignette_estimation = self.compute_initial_vignette_estimation(
209
+ beam=beam,
210
+ representative_spectrum=representative_spectrum,
211
+ continuum=fit_result.best_fit_continuum,
212
+ )
213
+
214
+ self.write(
215
+ data=first_vignette_estimation,
216
+ tags=[VispTag.debug(), VispTag.beam(beam), VispTag.task("FIRST_VIGNETTE_EST")],
217
+ encoder=fits_array_encoder,
218
+ )
219
+
220
+ with self.telemetry_span(f"Estimate 2D vignetting signature for {apm_str}"):
221
+ final_vignette = self.compute_final_vignette_estimate(
222
+ init_vignette_correction=first_vignette_estimation
223
+ )
224
+ vignette_corrected_gain = self.geo_corrected_beam_data(beam=beam) / final_vignette
225
+
226
+ self.write(
227
+ data=final_vignette,
228
+ tags=[VispTag.debug(), VispTag.beam(beam), VispTag.task("FINAL_VIGNETTE")],
229
+ encoder=fits_array_encoder,
230
+ )
231
+ self.write(
232
+ data=vignette_corrected_gain,
233
+ tags=[VispTag.debug(), VispTag.beam(beam), VispTag.task("VIGNETTE_CORR")],
234
+ encoder=fits_array_encoder,
235
+ )
236
+
237
+ with self.telemetry_span(f"Save vignette quality metrics for {apm_str}"):
238
+ self.record_vignette_quality_metrics(
239
+ beam=beam,
240
+ representative_spectrum=representative_spectrum,
241
+ fit_result=fit_result,
242
+ vignette_corrected_gain=vignette_corrected_gain,
243
+ )
244
+
245
+ with self.telemetry_span(f"Compute characteristic spectra for {apm_str}"):
246
+ char_spec = self.compute_characteristic_spectra(vignette_corrected_gain)
247
+ self.write(
248
+ data=char_spec,
249
+ encoder=fits_array_encoder,
250
+ tags=[VispTag.debug(), VispTag.frame()],
251
+ relative_path=f"DEBUG_SC_CHAR_SPEC_BEAM_{beam}.dat",
252
+ overwrite=True,
253
+ )
254
+
255
+ with self.telemetry_span(
256
+ f"Applying spectral shifts to characteristic spectra for {apm_str}"
257
+ ):
258
+ spec_shift = next(
259
+ self.read(
115
260
  tags=[
116
- VispTag.beam(beam),
117
- VispTag.modstate(modstate),
118
- VispTag.task("CHAR_SPEC_DISTORT_SHIFT"),
261
+ VispTag.intermediate_frame(beam=beam),
262
+ VispTag.task_geometric_spectral_shifts(),
119
263
  ],
120
- relative_path=f"DEBUG_SC_CHAR_SPEC_DISTORT_SHIFT_BEAM_{beam}_MODSTATE_{modstate}.dat",
121
- overwrite=True,
122
- )
123
-
124
- with self.telemetry_span(f"Refining characteristic spectral shifts for {apm_str}"):
125
- refined_char_spec = self.refine_gain_shifts(
126
- char_spec=reshifted_char_spec, beam=beam, modstate=modstate
127
- )
128
- self.write(
129
- data=refined_char_spec,
130
- encoder=fits_array_encoder,
131
- tags=[VispTag.debug(), VispTag.frame()],
132
- relative_path=f"DEBUG_SC_CHAR_SPEC_REFINE_BEAM_{beam}_MODSTATE_{modstate}.dat",
133
- overwrite=True,
264
+ decoder=fits_array_decoder,
134
265
  )
135
-
136
- with self.telemetry_span(f"Removing solar signal from {apm_str}"):
137
- gain = self.remove_solar_signal(
138
- char_solar_spectra=refined_char_spec, beam=beam, modstate=modstate
266
+ )
267
+ redistorted_char_spec = next(
268
+ self.corrections_remove_spec_geometry(
269
+ arrays=char_spec, spec_shift=-1 * spec_shift
139
270
  )
271
+ )
272
+ self.write(
273
+ data=redistorted_char_spec,
274
+ encoder=fits_array_encoder,
275
+ tags=[VispTag.debug(), VispTag.frame()],
276
+ relative_path=f"DEBUG_SC_CHAR_DISTORT_BEAM_{beam}.dat",
277
+ overwrite=True,
278
+ )
140
279
 
141
- with self.telemetry_span(f"Masking hairlines from {apm_str}"):
142
- gain = self.corrections_mask_hairlines(gain)
143
-
280
+ with self.telemetry_span(f"Re-distorting characteristic spectra for {apm_str}"):
281
+ reshifted_char_spec = self.distort_characteristic_spectra(
282
+ char_spec=redistorted_char_spec, beam=beam
283
+ )
144
284
  self.write(
145
- data=gain,
285
+ data=reshifted_char_spec,
146
286
  encoder=fits_array_encoder,
147
287
  tags=[
148
- VispTag.debug(),
149
- VispTag.frame(),
288
+ VispTag.beam(beam),
289
+ VispTag.task("CHAR_SPEC_DISTORT_SHIFT"),
150
290
  ],
151
- relative_path=f"DEBUG_SC_PRE_EQ_SOLAR_GAIN_BEAM_{beam}_MODSTATE_{modstate}.dat",
291
+ relative_path=f"DEBUG_SC_CHAR_SPEC_DISTORT_SHIFT_BEAM_{beam}.dat",
152
292
  overwrite=True,
153
293
  )
154
294
 
155
- pre_equalized_gain_dict[modstate] = gain
295
+ with self.telemetry_span(f"Removing solar signal from {apm_str}"):
296
+ gain = self.remove_solar_signal(char_solar_spectra=reshifted_char_spec, beam=beam)
156
297
 
157
- with self.telemetry_span(f"Equalizing modstates for {beam = }"):
158
- equalized_gain_dict = self.equalize_modstates(pre_equalized_gain_dict)
298
+ with self.telemetry_span(f"Masking hairlines from {apm_str}"):
299
+ final_gain = self.corrections_mask_hairlines(gain)
159
300
 
160
- for modstate in range(1, self.constants.num_modstates + 1):
161
-
162
- final_gain = equalized_gain_dict[modstate]
163
-
164
- with self.telemetry_span(f"Writing solar gain for {beam = } and {modstate = }"):
165
- self.write_solar_gain_calibration(
166
- gain_array=final_gain, beam=beam, modstate=modstate
167
- )
301
+ with self.telemetry_span(f"Writing solar gain for {beam = }"):
302
+ self.write_solar_gain_calibration(
303
+ gain_array=final_gain,
304
+ beam=beam,
305
+ )
168
306
 
169
307
  with self.telemetry_span("Computing and logging quality metrics"):
170
308
  no_of_raw_solar_frames: int = self.scratch.count_all(
@@ -179,84 +317,36 @@ class SolarCalibration(
179
317
  task_type=TaskName.solar_gain.value, total_frames=no_of_raw_solar_frames
180
318
  )
181
319
 
182
- def unshifted_geo_corrected_modstate_data(self, beam: int, modstate: int) -> np.ndarray:
183
- """
184
- Array for a single beam/modstate that has dark, lamp, angle, and state offset corrections.
185
-
186
- Parameters
187
- ----------
188
- beam : int
189
- The beam number for this array
190
-
191
- modstate : int
192
- The modulator state for this array
193
-
194
-
195
- Returns
196
- -------
197
- np.ndarray
198
- Array with dark signal, lamp signal, angle and state offset removed
199
-
200
- """
201
- tags = [
202
- VispTag.intermediate_frame(beam=beam, modstate=modstate),
203
- VispTag.task("SC_GEO_NOSHIFT"),
204
- ]
205
- array_generator = self.read(tags=tags, decoder=fits_array_decoder)
206
- return next(array_generator)
207
-
208
- def geo_corrected_modstate_data(self, beam: int, modstate: int) -> np.ndarray:
320
+ def geo_corrected_beam_data(self, beam: int) -> np.ndarray:
209
321
  """
210
- Array for a single beam/modstate that has dark, lamp, and ALL of the geometric corrects.
322
+ Array for a single beam that has dark, lamp, and ALL of the geometric corrects.
211
323
 
212
324
  Parameters
213
325
  ----------
214
- beam : int
326
+ beam
215
327
  The beam number for this array
216
328
 
217
- modstate : int
218
- The modulator state for this array
219
-
220
-
221
329
  Returns
222
330
  -------
223
- np.ndarray
331
+ Fully corrected array
224
332
  Array with dark signal, and lamp signal removed, and all geometric corrections made
225
333
  """
226
334
  tags = [
227
- VispTag.intermediate_frame(beam=beam, modstate=modstate),
335
+ VispTag.intermediate_frame(beam=beam),
228
336
  VispTag.task("SC_GEO_ALL"),
229
337
  ]
230
338
  array_generator = self.read(tags=tags, decoder=fits_array_decoder)
231
339
  return next(array_generator)
232
340
 
233
- def lamp_corrected_modstate_data(self, beam: int, modstate: int) -> np.ndarray:
234
- """
235
- Array for a single beam/modstate that has dark, background, and lamp gain applied.
236
-
237
- This is used to refine the final shifts in the re-distorted characteristic spectra. Having the lamp gain applied
238
- removes large optical features that would otherwise pollute the match to the characteristic spectra (which has
239
- no optical features).
240
- """
241
- tags = [
242
- VispTag.intermediate_frame(beam=beam, modstate=modstate),
243
- VispTag.task("SC_LAMP_CORR"),
244
- ]
245
- array_generator = self.read(tags=tags, decoder=fits_array_decoder)
246
- return next(array_generator)
247
-
248
- def bg_corrected_modstate_data(self, beam: int, modstate: int) -> np.ndarray:
341
+ def bg_corrected_beam_data(self, beam: int) -> np.ndarray:
249
342
  """
250
- Array for a single beam/modstate that has only has dark and background corrects applied.
343
+ Array for a single beam that has only has dark and background corrects applied.
251
344
 
252
345
  Parameters
253
346
  ----------
254
- beam : int
347
+ beam
255
348
  The beam number for this array
256
349
 
257
- modstate : int
258
- The modulator state for this array
259
-
260
350
 
261
351
  Returns
262
352
  -------
@@ -264,37 +354,25 @@ class SolarCalibration(
264
354
  Array with dark and background signals removed
265
355
  """
266
356
  tags = [
267
- VispTag.intermediate_frame(beam=beam, modstate=modstate),
357
+ VispTag.intermediate_frame(beam=beam),
268
358
  VispTag.task("SC_BG_ONLY"),
269
359
  ]
270
360
  array_generator = self.read(tags=tags, decoder=fits_array_decoder)
271
361
  return next(array_generator)
272
362
 
273
- def do_initial_corrections(self, beam: int, modstate: int) -> None:
363
+ def do_initial_corrections(self, beam: int) -> None:
274
364
  """
275
365
  Do dark, lamp, and geometric corrections for all data that will be used.
276
366
 
277
- At two intermediate points the current arrays are saved because they'll be needed by various helpers:
278
-
279
- SC_BG_ONLY - The solar gain arrays with only a dark and background correction.
280
-
281
- SC_GEO_NOSHIFT - The solar gain arrays after dark, lamp, angle, and state offset correction. In other words,
282
- they do not have spectral curvature removed. These are used to reshift the characteristic
283
- spectra to the original spectral curvature.
367
+ We also save an intermediate product of the average solar gain array with only dark and background corrections.
368
+ This array will later be used to produce the final solar gain. It's task tag is SC_BG_ONLY.
284
369
 
285
370
  Parameters
286
371
  ----------
287
- beam : int
372
+ beam
288
373
  The beam number for this array
289
-
290
- modstate : int
291
- The modulator state for this array
292
-
293
-
294
- Returns
295
- -------
296
- None
297
374
  """
375
+ all_exp_time_arrays = []
298
376
  for readout_exp_time in self.constants.solar_readout_exp_times:
299
377
  dark_array = next(
300
378
  self.read(
@@ -314,15 +392,12 @@ class SolarCalibration(
314
392
  )
315
393
  )
316
394
 
317
- logger.info(
318
- f"Doing dark, background, lamp, and geo corrections for {beam=} and {modstate=}"
319
- )
395
+ logger.info(f"Doing dark, background, lamp, and geo corrections for {beam=}")
320
396
  ## Load frames
321
397
  tags = [
322
398
  VispTag.input(),
323
399
  VispTag.frame(),
324
400
  VispTag.task_solar_gain(),
325
- VispTag.modstate(modstate),
326
401
  VispTag.readout_exp_time(readout_exp_time),
327
402
  ]
328
403
  input_solar_gain_objs = self.read(
@@ -334,140 +409,120 @@ class SolarCalibration(
334
409
  for o in input_solar_gain_objs
335
410
  )
336
411
 
337
- ## Average
338
412
  avg_solar_array = average_numpy_arrays(readout_normalized_arrays)
339
413
 
340
- ## Dark correction
341
414
  dark_corrected_solar_array = subtract_array_from_arrays(
342
415
  arrays=avg_solar_array, array_to_subtract=dark_array
343
416
  )
344
417
 
345
- ## Residual background correction
346
418
  background_corrected_solar_array = next(
347
419
  subtract_array_from_arrays(dark_corrected_solar_array, background_array)
348
420
  )
349
421
 
350
- # Save the only-dark-corr because this will be used to make the final Solar Gain object
351
- self.write(
352
- data=background_corrected_solar_array,
353
- tags=[
354
- VispTag.intermediate_frame(beam=beam, modstate=modstate),
355
- VispTag.task("SC_BG_ONLY"),
356
- ],
357
- encoder=fits_array_encoder,
358
- )
422
+ all_exp_time_arrays.append(background_corrected_solar_array)
359
423
 
360
- ## Lamp correction
361
- lamp_array = next(
362
- self.read(
363
- tags=[
364
- VispTag.intermediate_frame(beam=beam, modstate=modstate),
365
- VispTag.task_lamp_gain(),
366
- ],
367
- decoder=fits_array_decoder,
368
- )
369
- )
370
- lamp_corrected_solar_array = next(
371
- divide_arrays_by_array(
372
- arrays=background_corrected_solar_array, array_to_divide_by=lamp_array
373
- )
374
- )
424
+ avg_dark_bg_corrected_array = average_numpy_arrays(all_exp_time_arrays)
375
425
 
376
- self.write(
377
- data=lamp_corrected_solar_array,
426
+ # Save the only-dark-corr because this will be used to make the final Solar Gain object
427
+ self.write(
428
+ data=avg_dark_bg_corrected_array,
429
+ tags=[
430
+ VispTag.intermediate_frame(beam=beam),
431
+ VispTag.task("SC_BG_ONLY"),
432
+ ],
433
+ encoder=fits_array_encoder,
434
+ )
435
+
436
+ ## Lamp correction
437
+ lamp_array = next(
438
+ self.read(
378
439
  tags=[
379
- VispTag.intermediate_frame(beam=beam, modstate=modstate),
380
- VispTag.task("SC_LAMP_CORR"),
440
+ VispTag.intermediate_frame(beam=beam),
441
+ VispTag.task_lamp_gain(),
381
442
  ],
382
- encoder=fits_array_encoder,
383
- )
384
-
385
- ## Geo correction
386
- angle_array = next(
387
- self.read(
388
- tags=[VispTag.intermediate_frame(beam=beam), VispTag.task_geometric_angle()],
389
- decoder=fits_array_decoder,
390
- )
391
- )
392
- angle = angle_array[0]
393
- state_offset = next(
394
- self.read(
395
- tags=[
396
- VispTag.intermediate_frame(beam=beam, modstate=modstate),
397
- VispTag.task_geometric_offset(),
398
- ],
399
- decoder=fits_array_decoder,
400
- )
443
+ decoder=fits_array_decoder,
401
444
  )
402
- spec_shift = next(
403
- self.read(
404
- tags=[
405
- VispTag.intermediate_frame(beam=beam),
406
- VispTag.task_geometric_spectral_shifts(),
407
- ],
408
- decoder=fits_array_decoder,
409
- )
445
+ )
446
+ lamp_corrected_solar_array = next(
447
+ divide_arrays_by_array(
448
+ arrays=avg_dark_bg_corrected_array, array_to_divide_by=lamp_array
410
449
  )
450
+ )
411
451
 
412
- geo_corrected_array = next(
413
- self.corrections_correct_geometry(lamp_corrected_solar_array, state_offset, angle)
414
- )
415
- # We need unshifted, but geo-corrected arrays for reshifting and normalization
416
- self.write(
417
- data=geo_corrected_array,
418
- tags=[
419
- VispTag.intermediate_frame(beam=beam, modstate=modstate),
420
- VispTag.task("SC_GEO_NOSHIFT"),
421
- ],
422
- encoder=fits_array_encoder,
423
- )
452
+ self.write(
453
+ data=lamp_corrected_solar_array,
454
+ tags=[
455
+ VispTag.debug(),
456
+ VispTag.beam(beam),
457
+ VispTag.task("SC_LAMP_CORR"),
458
+ ],
459
+ encoder=fits_array_encoder,
460
+ )
424
461
 
425
- # Now finish the spectral shift correction
426
- spectral_corrected_array = next(
427
- self.corrections_remove_spec_geometry(geo_corrected_array, spec_shift)
462
+ ## Geo correction
463
+ angle_array = next(
464
+ self.read(
465
+ tags=[VispTag.intermediate_frame(beam=beam), VispTag.task_geometric_angle()],
466
+ decoder=fits_array_decoder,
428
467
  )
429
- self.write(
430
- data=spectral_corrected_array,
468
+ )
469
+ angle = angle_array[0]
470
+
471
+ # Because we have averaged over all modstates, the per-modestate offsets don't matter. Also, we're going to
472
+ # undo these corrections to make the final gain, so the beam2 -> beam1 offset doesn't matter either.
473
+ state_offset = np.array([0.0, 0.0])
474
+ spec_shift = next(
475
+ self.read(
431
476
  tags=[
432
- VispTag.intermediate_frame(beam=beam, modstate=modstate),
433
- VispTag.task("SC_GEO_ALL"),
477
+ VispTag.intermediate_frame(beam=beam),
478
+ VispTag.task_geometric_spectral_shifts(),
434
479
  ],
435
- encoder=fits_array_encoder,
480
+ decoder=fits_array_decoder,
436
481
  )
482
+ )
437
483
 
438
- def compute_characteristic_spectra(self, beam: int, modstate: int) -> np.ndarray:
439
- """
440
- Compute the 2D characteristic spectra via a Gaussian smooth in the spatial dimension.
441
-
442
- A 2D characteristic spectra is needed because the line shape varys along the slit to the degree that a
443
- single, 1D characteristic spectrum will not fully remove the solar lines for all positions in the final gain.
444
-
445
- In this step we also normalize each spatial position to its median value. This removes low-order gradients in
446
- the spatial direction that are known to be caused by imperfect illumination of the Lamp gains (which were used
447
- to correct the data that will become the characteristic spectra).
448
-
449
- Parameters
450
- ----------
451
- beam : int
452
- The beam number for this array
484
+ geo_corrected_array = next(
485
+ self.corrections_correct_geometry(lamp_corrected_solar_array, state_offset, angle)
486
+ )
487
+ self.write(
488
+ data=geo_corrected_array,
489
+ tags=[
490
+ VispTag.debug(),
491
+ VispTag.beam(beam),
492
+ VispTag.task("SC_GEO_NOSHIFT"),
493
+ ],
494
+ encoder=fits_array_encoder,
495
+ )
453
496
 
454
- modstate : int
455
- The modulator state for this array
497
+ # Now finish the spectral shift correction
498
+ spectral_corrected_array = next(
499
+ self.corrections_remove_spec_geometry(geo_corrected_array, spec_shift)
500
+ )
501
+ self.write(
502
+ data=spectral_corrected_array,
503
+ tags=[
504
+ VispTag.intermediate_frame(beam=beam),
505
+ VispTag.task("SC_GEO_ALL"),
506
+ ],
507
+ encoder=fits_array_encoder,
508
+ )
456
509
 
510
+ def get_representative_spectrum(self, beam: int) -> np.ndarray:
511
+ """
512
+ Compute a representative spectrum that will be used for solar atlas and continuum fitting.
457
513
 
458
- Returns
459
- -------
460
- np.ndarray
461
- Characteristic spectra array
514
+ The spectrum is the spatial median of the lamp-and-spectral-shift corrected solar gain image for this beam.
515
+ Prior to computing the spatial median each spatial pixel is normalized by its continuum level so that variations
516
+ in overall scaling as a function of slit position don't skew the median line shapes. The continuum for each
517
+ spatial pixel is estimated from a percentage of the CDF; this percentage is a pipeline parameter.
462
518
  """
463
- spectral_avg_window = self.parameters.solar_spectral_avg_window
464
519
  normalization_percentile = (
465
520
  self.parameters.solar_characteristic_spatial_normalization_percentile
466
521
  )
467
522
  logger.info(
468
- f"Computing characteristic spectra for {beam = } and {modstate = } with {spectral_avg_window = } and {normalization_percentile = }"
523
+ f"Computing representative spectra for {beam = } with {normalization_percentile = }"
469
524
  )
470
- full_spectra = self.geo_corrected_modstate_data(beam=beam, modstate=modstate)
525
+ full_spectra = self.geo_corrected_beam_data(beam=beam)
471
526
 
472
527
  full_spectra = self.corrections_mask_hairlines(full_spectra)
473
528
  # Normalize each spatial pixel by its own percentile. This removes large spatial gradients that are not solar
@@ -476,86 +531,315 @@ class SolarCalibration(
476
531
  full_spectra, normalization_percentile, axis=0
477
532
  )
478
533
 
479
- # size = (1, window) means don't smooth in the spectra dimension
480
- char_spec = spnd.median_filter(normed_spectra, size=(1, spectral_avg_window))
534
+ representative_spectrum = np.nanmedian(normed_spectra, axis=1)
481
535
 
482
- return char_spec
536
+ return representative_spectrum
483
537
 
484
- def refine_gain_shifts(self, char_spec: np.ndarray, beam: int, modstate: int) -> np.ndarray:
538
+ def fit_initial_vignette(
539
+ self,
540
+ representative_spectrum: np.ndarray,
541
+ dispersion: Quantity,
542
+ spectral_order: int,
543
+ doppler_velocity: Quantity,
544
+ ) -> FitResult:
485
545
  """
486
- Refine the spectral shifts when matching characteristic spectra to the rectified input spectra.
546
+ Fit a global continuum to a single, representative spectrum.
487
547
 
488
- An important detail of this functino is that the goodness of fit metric is the final gain image (i.e., raw
489
- input with solar spectrum removed). We minimize the residuals in the gain image.
548
+ The representative spectrum's continuum is estimated by fitting the spectrum to a solar atlas. The continuum
549
+ is parameterized with a polynomial as seen in `WavelengthCalibrationParametersWithContinuum` and
550
+ `polynomial_continuum_model`.
551
+ """
552
+ atlas = Atlas(self.parameters.wavecal_atlas_download_config)
553
+
554
+ logger.info("Initializing wavecal fit parameters")
555
+ init_parameters = self.initialize_starting_fit_parameters(
556
+ representative_spectrum=representative_spectrum,
557
+ dispersion=dispersion,
558
+ spectral_order=spectral_order,
559
+ doppler_velocity=doppler_velocity,
560
+ atlas=atlas,
561
+ )
490
562
 
491
- Parameters
492
- ----------
493
- char_spec : np.ndarray
494
- Computed characteristic spectra
563
+ fitter = WavelengthCalibrationFitter(input_parameters=init_parameters, atlas=atlas)
564
+ with self.telemetry_span("Fit atlas and continuum"):
565
+ extra_kwargs = self.parameters.solar_vignette_wavecal_fit_kwargs
566
+ logger.info(f"Calling fitter with extra kwargs: {extra_kwargs}")
567
+ fit_result = fitter(
568
+ input_spectrum=representative_spectrum,
569
+ **extra_kwargs,
570
+ )
495
571
 
496
- beam : int
497
- The beam number for this array
572
+ return fit_result
498
573
 
499
- modstate : int
500
- The modulator state for this array
574
+ def compute_initial_vignette_estimation(
575
+ self, beam: int, representative_spectrum: np.ndarray, continuum: np.ndarray
576
+ ) -> np.ndarray:
577
+ """
578
+ Compute the initial, 1D estimate of the spectral vignette signal.
579
+
580
+ The continuum is first removed from the representative spectrum to make the first guess at a vignette-corrected
581
+ solar spectrum. Then this solar spectrum is divided from the 2D lamp-and-spectral-shift corrected solar gain data.
582
+
583
+ Because the continuum was fit for a single spectrum, the result is a 2D array that contains the *spatial*
584
+ variation in the vignetting signal.
585
+ """
586
+ first_vignette_array = (
587
+ self.geo_corrected_beam_data(beam=beam) / (representative_spectrum / continuum)[:, None]
588
+ )
589
+ return first_vignette_array
590
+
591
+ def initialize_starting_fit_parameters(
592
+ self,
593
+ representative_spectrum: np.ndarray,
594
+ dispersion: Quantity,
595
+ spectral_order: int,
596
+ doppler_velocity: Quantity,
597
+ atlas: Atlas,
598
+ ) -> WavelengthCalibrationParametersWithContinuum:
599
+ """
600
+ Construct a `WavelengthCalibrationParametersWithContinuum` object containing initial guesses, fit flags, and bounds.
601
+
602
+ A rough estimate of the initial CRVAL value is made using `solar_wavelength_calibration.calculate_initial_crval_guess`.
603
+ """
604
+ num_wave = representative_spectrum.size
605
+ normalized_abscissa = np.linspace(-1, 1, num_wave)
606
+
607
+ logger.info("Computing input wavelength vector")
608
+ input_wavelength_vector = compute_input_wavelength_vector(
609
+ central_wavelength=self.constants.wavelength * u.nm,
610
+ dispersion=dispersion,
611
+ grating_constant=self.constants.grating_constant_inverse_mm,
612
+ order=spectral_order,
613
+ incident_light_angle=self.constants.incident_light_angle_deg,
614
+ num_spec_px=representative_spectrum.size,
615
+ )
616
+ wavelength_range = input_wavelength_vector.max() - input_wavelength_vector.min()
617
+
618
+ logger.info("Computing initial CRVAL guess")
619
+ crval_init = calculate_initial_crval_guess(
620
+ input_wavelength_vector=input_wavelength_vector,
621
+ input_spectrum=representative_spectrum,
622
+ atlas=atlas,
623
+ negative_limit=-wavelength_range / 2,
624
+ positive_limit=wavelength_range / 2,
625
+ num_steps=500,
626
+ normalization_percentile=self.parameters.wavecal_init_crval_guess_normalization_percentile,
627
+ )
628
+
629
+ logger.info(f"{crval_init = !s}")
630
+
631
+ fit_flags = FitFlagsModel(
632
+ continuum_level=False, # Because we're fitting the continuum with a function
633
+ incident_light_angle=False,
634
+ straylight_fraction=True,
635
+ resolving_power=True,
636
+ opacity_factor=True,
637
+ )
638
+
639
+ incident_light_angle = self.constants.incident_light_angle_deg
640
+ grating_constant = self.constants.grating_constant_inverse_mm
641
+ resolving_power = self.parameters.wavecal_init_resolving_power
642
+ opacity_factor = self.parameters.wavecal_init_opacity_factor
643
+ straylight_faction = self.parameters.wavecal_init_straylight_fraction
644
+ relative_atlas_scaling = self.estimate_relative_continuum_level(
645
+ crval_init=crval_init,
646
+ wavelength_range=wavelength_range,
647
+ atlas=atlas,
648
+ representative_spectrum=representative_spectrum,
649
+ )
650
+ logger.info(f"0th order coefficient initial guess: {relative_atlas_scaling}")
651
+
652
+ wavelength_search_width = dispersion * self.parameters.solar_vignette_crval_bounds_px
653
+ bounds = BoundsModel(
654
+ crval=LengthBoundRange(
655
+ min=crval_init - wavelength_search_width, max=crval_init + wavelength_search_width
656
+ ),
657
+ dispersion=DispersionBoundRange(
658
+ min=dispersion * (1 - self.parameters.solar_vignette_dispersion_bounds_fraction),
659
+ max=dispersion * (1 + self.parameters.solar_vignette_dispersion_bounds_fraction),
660
+ ),
661
+ incident_light_angle=AngleBoundRange(
662
+ min=incident_light_angle - 1 * u.deg, max=incident_light_angle + 1 * u.deg
663
+ ),
664
+ resolving_power=UnitlessBoundRange(min=1e5, max=5e5),
665
+ opacity_factor=UnitlessBoundRange(min=1.0, max=10),
666
+ straylight_fraction=UnitlessBoundRange(min=0.0, max=0.8),
667
+ )
668
+
669
+ init_params = WavelengthCalibrationParametersWithContinuum(
670
+ crval=crval_init,
671
+ dispersion=dispersion,
672
+ incident_light_angle=incident_light_angle,
673
+ grating_constant=grating_constant,
674
+ doppler_velocity=doppler_velocity,
675
+ order=spectral_order,
676
+ continuum_level=1,
677
+ resolving_power=resolving_power,
678
+ opacity_factor=opacity_factor,
679
+ straylight_fraction=straylight_faction,
680
+ fit_flags=fit_flags,
681
+ bounds=bounds,
682
+ continuum_poly_fit_order=self.parameters.solar_vignette_initial_continuum_poly_fit_order,
683
+ normalized_abscissa=normalized_abscissa,
684
+ zeroth_order_continuum_coefficient=relative_atlas_scaling,
685
+ )
501
686
 
687
+ return init_params
688
+
689
+ def estimate_relative_continuum_level(
690
+ self,
691
+ *,
692
+ crval_init: Quantity,
693
+ wavelength_range: Quantity,
694
+ atlas: Atlas,
695
+ representative_spectrum: np.ndarray,
696
+ ) -> float:
697
+ """
698
+ Estimate the multiplicative scaling between the representative spectrum and atlas solar transmission.
699
+
700
+ This scaling is used to set the initial guess of 0th-order polynomial fit coefficient. We estimate the scaling
701
+ factor by comparing the values of the two spectra at a given percent of the CDF. This percent is taken from
702
+ the `~dkist_processing_visp.models.parameters.VispParameters.wavecal_init_crval_guess_normalization_percentile`
703
+ pipeline parameter.
704
+ """
705
+ wave_min = crval_init - wavelength_range / 2
706
+ wave_max = crval_init + wavelength_range / 2
707
+
708
+ atlas_idx = np.where(
709
+ (atlas.solar_atlas_wavelength >= wave_min) & (atlas.solar_atlas_wavelength <= wave_max)
710
+ )
711
+ atlas_norm = np.nanpercentile(
712
+ atlas.solar_atlas_transmission[atlas_idx],
713
+ self.parameters.wavecal_init_crval_guess_normalization_percentile,
714
+ )
715
+ spec_norm = np.nanpercentile(
716
+ representative_spectrum,
717
+ self.parameters.wavecal_init_crval_guess_normalization_percentile,
718
+ )
719
+
720
+ return spec_norm / atlas_norm
721
+
722
+ def compute_final_vignette_estimate(self, init_vignette_correction: np.ndarray) -> np.ndarray:
723
+ """
724
+ Fit the spectral shape of continuum residuals for each spatial pixel.
725
+
726
+ The vignette estimation produced by `compute_initial_vignette_estimation` used a single continuum function for
727
+ the entire slit. This method fits the continuum for all pixels along the slit to build up a fully 2D estimate
728
+ of the vignette signal.
729
+
730
+ Each spatial pixel is fit separately with a polynomial via a RANSAC estimator.
731
+ """
732
+ model = self.build_RANSAC_model()
733
+ num_spectral, num_spatial = init_vignette_correction.shape
734
+ abscissa = np.arange(num_spectral)[
735
+ :, None
736
+ ] # Add extra dimension because sklearn requires it
737
+
738
+ logger.info(f"Fitting spectral vignetting for {num_spatial} spatial pixels")
739
+ final_vignette = np.zeros_like(init_vignette_correction)
740
+ for spatial_px in range(num_spatial):
741
+ finite_idx = np.isfinite(init_vignette_correction[:, spatial_px])
742
+ model.fit(abscissa[finite_idx, :], init_vignette_correction[finite_idx, spatial_px])
743
+ final_vignette[:, spatial_px] = model.predict(abscissa)
744
+
745
+ return final_vignette
746
+
747
+ def build_RANSAC_model(self) -> Pipeline:
748
+ """
749
+ Build a scikit-learn pipeline from a set of estimators.
750
+
751
+ Namely, construct a `~sklearn.pipeline.Pipeline` built from
752
+
753
+ `~sklearn.preprocessing.PolynomialFeatures` -> `~sklearn.preprocessing.RobustScaler` -> `~sklearn.linear_model.RANSACRegressor`
754
+ """
755
+ fit_order = self.parameters.solar_vignette_spectral_poly_fit_order
756
+ min_samples = self.parameters.solar_vignette_min_samples
757
+ logger.info(f"Building RANSAC model with {fit_order = } and {min_samples = }")
758
+
759
+ # PolynomialFeatures casts the pipeline as a polynomial fit
760
+ # see https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.PolynomialFeatures.html#sklearn.preprocessing.PolynomialFeatures
761
+ poly_feature = PolynomialFeatures(degree=fit_order)
762
+
763
+ # RobustScaler is a scale factor that is robust to outliers
764
+ # see https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.RobustScaler.html#sklearn.preprocessing.RobustScaler
765
+ scaler = RobustScaler()
766
+
767
+ # The RANSAC regressor iteratively sub-samples the input data and constructs a model with this sub-sample.
768
+ # The method used allows it to be robust to outliers.
769
+ # see https://scikit-learn.org/stable/modules/linear_model.html#ransac-regression
770
+ RANSAC = RANSACRegressor(min_samples=min_samples)
771
+
772
+ return make_pipeline(poly_feature, scaler, RANSAC)
773
+
774
+ def compute_characteristic_spectra(self, vignette_corrected_data: np.ndarray) -> np.ndarray:
775
+ """
776
+ Compute the 2D characteristic spectra via a median smooth in the spatial dimension.
777
+
778
+ A 2D characteristic spectra is needed because the line shape varys along the slit to the degree that a
779
+ single, 1D characteristic spectrum will not fully remove the solar lines for all positions in the final gain.
780
+
781
+ In this step we also normalize each spatial position by its continuum value. This removes low-order gradients in
782
+ the spatial direction that are known to be caused by imperfect illumination of the Lamp gains (which were used
783
+ to correct the data that will become the characteristic spectra).
784
+
785
+ Parameters
786
+ ----------
787
+ beam
788
+ The beam number for this array
502
789
 
503
790
  Returns
504
791
  -------
505
- np.ndarray
506
- Characteristic spectra array with refined spectral shifts
507
- """
508
- # Grab rectified input spectra that will be the shift target
509
- target_spectra = self.lamp_corrected_modstate_data(beam=beam, modstate=modstate)
510
- num_spec = target_spectra.shape[1]
511
-
512
- logger.info(f"Computing line zones for {beam=} and {modstate=}")
513
- zone_kwargs = {
514
- "prominence": self.parameters.solar_zone_prominence,
515
- "width": self.parameters.solar_zone_width,
516
- "bg_order": self.parameters.solar_zone_bg_order,
517
- "normalization_percentile": self.parameters.solar_zone_normalization_percentile,
518
- "rel_height": self.parameters.solar_zone_rel_height,
519
- }
520
- zones = self.compute_line_zones(char_spec, **zone_kwargs)
521
- logger.info(f"Found {zones=} for {beam=} and {modstate=}")
522
- if len(zones) == 0:
523
- raise ValueError(f"No zones found for {beam=} and {modstate=}")
792
+ char_spec
793
+ 2D characteristic spectra
794
+ """
795
+ spatial_median_window = self.parameters.solar_spatial_median_filter_width_px
796
+ normalization_percentile = (
797
+ self.parameters.solar_characteristic_spatial_normalization_percentile
798
+ )
799
+ logger.info(
800
+ f"Computing characteristic spectra for {spatial_median_window = } and {normalization_percentile = }"
801
+ )
802
+
803
+ masked_spectra = self.corrections_mask_hairlines(vignette_corrected_data)
804
+ # Normalize each spatial pixel by its own percentile. This removes large spatial gradients that are not solar
805
+ # signal.
806
+ normed_spectra = masked_spectra / np.nanpercentile(
807
+ masked_spectra, normalization_percentile, axis=0
808
+ )
524
809
 
525
- reshift_char_spec = np.zeros(char_spec.shape)
526
- logger.info(f"Refining shifts for {beam=} and {modstate=}")
527
- for i in range(num_spec):
528
- ref_spec = target_spectra[:, i] / np.nanmedian(target_spectra[:, i])
529
- spec = char_spec[:, i] / np.nanmedian(char_spec[:, i])
530
- shift = SolarCalibration.refine_shift(spec, ref_spec, zones=zones, x_init=0.0)
531
- reshift_char_spec[:, i] = spnd.shift(char_spec[:, i], shift, mode="reflect")
810
+ # size = (1, window) means don't smooth in the spectral dimension
811
+ char_spec = spnd.median_filter(normed_spectra, size=(1, spatial_median_window))
532
812
 
533
- return reshift_char_spec
813
+ normed_char_spec = char_spec / np.nanpercentile(char_spec, normalization_percentile, axis=0)
814
+
815
+ return normed_char_spec
534
816
 
535
817
  def distort_characteristic_spectra(
536
- self, char_spec: np.ndarray, beam: int, modstate: int
818
+ self,
819
+ char_spec: np.ndarray,
820
+ beam: int,
537
821
  ) -> np.ndarray:
538
822
  """
539
823
  Re-apply angle and state offset distortions to the characteristic spectra.
540
824
 
541
825
  Parameters
542
826
  ----------
543
- char_spec : np.ndarray
827
+ char_spec
544
828
  Computed characteristic spectra
545
829
 
546
- beam : int
830
+ beam
547
831
  The beam number for this array
548
832
 
549
- modstate : int
833
+ modstate
550
834
  The modulator state for this array
551
835
 
552
836
 
553
837
  Returns
554
838
  -------
555
- np.ndarray
839
+ distorted characteristic array
556
840
  Characteristic spectra array with angle and offset distortions re-applied
557
841
  """
558
- logger.info(f"Re-distorting characteristic spectra for {beam=} and {modstate=}")
842
+ logger.info(f"Re-distorting characteristic spectra for {beam=}")
559
843
  angle_array = next(
560
844
  self.read(
561
845
  tags=[VispTag.intermediate_frame(beam=beam), VispTag.task_geometric_angle()],
@@ -563,15 +847,9 @@ class SolarCalibration(
563
847
  )
564
848
  )
565
849
  angle = angle_array[0]
566
- state_offset = next(
567
- self.read(
568
- tags=[
569
- VispTag.intermediate_frame(beam=beam, modstate=modstate),
570
- VispTag.task_geometric_offset(),
571
- ],
572
- decoder=fits_array_decoder,
573
- )
574
- )
850
+
851
+ # See comment in `do_initial_corrections`; we don't care about the state_offset when making the solar gain
852
+ state_offset = np.array([0.0, 0.0])
575
853
 
576
854
  distorted_spec = next(
577
855
  self.corrections_correct_geometry(char_spec, -1 * state_offset, -1 * angle)
@@ -579,164 +857,329 @@ class SolarCalibration(
579
857
 
580
858
  return distorted_spec
581
859
 
582
- def equalize_modstates(self, array_dict: dict[int, np.ndarray]) -> dict[int, np.ndarray]:
583
- """Adjust the flux of all modstates for a single beam so they all have the same median.
584
-
585
- That median is the global median over all modstates.
586
- """
587
- global_median_array = np.nanmedian(list(array_dict.values()), axis=0)
588
- output_dict = dict()
589
-
590
- for modstate, array in array_dict.items():
591
- correction = np.nanmedian(global_median_array / array)
592
- logger.info(f"Equalization correction for {modstate = } is {correction:.6f}")
593
- final_gain = array * correction
594
- output_dict[modstate] = final_gain
595
-
596
- return output_dict
597
-
598
860
  def remove_solar_signal(
599
- self, char_solar_spectra: np.ndarray, beam: int, modstate: int
861
+ self,
862
+ char_solar_spectra: np.ndarray,
863
+ beam: int,
600
864
  ) -> np.ndarray:
601
865
  """
602
866
  Remove the distorted characteristic solar spectra from the original spectra.
603
867
 
604
868
  Parameters
605
869
  ----------
606
- char_solar_spectra : np.ndarray
870
+ char_solar_spectra
607
871
  Characteristic solar spectra
608
872
 
609
- beam : int
873
+ beam
610
874
  The beam number for this array
611
875
 
612
- modstate : int
613
- The modulator state for this array
614
-
615
-
616
876
  Returns
617
877
  -------
618
- np.ndarray
878
+ final gain
619
879
  Original spectral array with characteristic solar spectra removed
620
-
621
880
  """
622
- logger.info(f"Removing characteristic solar spectra from {beam=} and {modstate=}")
623
- input_gain = self.bg_corrected_modstate_data(beam=beam, modstate=modstate)
881
+ logger.info(f"Removing characteristic solar spectra from {beam=}")
882
+ input_gain = self.bg_corrected_beam_data(beam=beam)
624
883
 
625
884
  final_gain = input_gain / char_solar_spectra
626
885
 
627
886
  return final_gain
628
887
 
629
888
  def write_solar_gain_calibration(
630
- self, gain_array: np.ndarray, beam: int, modstate: int
889
+ self,
890
+ gain_array: np.ndarray,
891
+ beam: int,
631
892
  ) -> None:
632
893
  """
633
- Write a solar gain array for a single beam and modstate.
894
+ Write a solar gain array for a single beam.
634
895
 
635
896
  Parameters
636
897
  ----------
637
- gain_array: np.ndarray
898
+ gain_array
638
899
  Solar gain array
639
900
 
640
- beam : int
901
+ beam
641
902
  The beam number for this array
642
-
643
- modstate : int
644
- The modulator state for this array
645
-
646
-
647
- Returns
648
- -------
649
- None
650
903
  """
651
- logger.info(f"Writing final SolarGain for {beam=} and {modstate=}")
904
+ logger.info(f"Writing final SolarGain for {beam = }")
652
905
  self.write(
653
906
  data=gain_array,
654
907
  tags=[
655
- VispTag.intermediate_frame(beam=beam, modstate=modstate),
908
+ VispTag.intermediate_frame(beam=beam),
656
909
  VispTag.task_solar_gain(),
657
910
  ],
658
911
  encoder=fits_array_encoder,
659
912
  )
660
913
 
661
- @staticmethod
662
- def refine_shift(
663
- spec: np.ndarray, target_spec: np.ndarray, zones: list[tuple[int, int]], x_init: float
664
- ) -> float:
914
+ def record_vignette_quality_metrics(
915
+ self,
916
+ beam: int,
917
+ representative_spectrum: np.ndarray,
918
+ fit_result: FitResult,
919
+ vignette_corrected_gain: np.ndarray,
920
+ ) -> None:
665
921
  """
666
- Refine the shift for a single spatial position back to the rectified input spectra.
922
+ Save vignette fit results to disk for later quality metric building.
667
923
 
668
- Line zones are used to increase the SNR of the chisq and the final shift is the mean of the shifts computed
669
- for each zone.
924
+ We save the atlas fit of the initial vignette estimation and the global solar spectrum derived from the
925
+ vignette-corrected 2D gain images.
926
+ """
927
+ logger.info(f"Recording initial vignette quality data for {beam = }")
928
+ first_vignette_quality_outputs = {
929
+ "output_wave_vec": fit_result.best_fit_wavelength_vector,
930
+ "input_spectrum": representative_spectrum,
931
+ "best_fit_atlas": fit_result.best_fit_atlas,
932
+ "best_fit_continuum": fit_result.best_fit_continuum,
933
+ "residuals": fit_result.minimizer_result.residual,
934
+ }
935
+ self.write(
936
+ data=first_vignette_quality_outputs,
937
+ tags=[VispTag.quality(VispMetricCode.solar_first_vignette), VispTag.beam(beam)],
938
+ encoder=asdf_encoder,
939
+ )
670
940
 
671
- Parameters
672
- ----------
673
- spec : np.ndarray
674
- The 1D spectrum to shift back
941
+ logger.info(f"Recording final vignette-correced gain quality data for {beam = }")
942
+ global_median = np.nanmedian(vignette_corrected_gain, axis=1)
943
+ low_deviation = np.nanpercentile(vignette_corrected_gain, 5, axis=1)
944
+ high_deviation = np.nanpercentile(vignette_corrected_gain, 95, axis=1)
945
+ final_correction_quality_outputs = {
946
+ "output_wave_vec": fit_result.best_fit_wavelength_vector,
947
+ "median_spec": global_median,
948
+ "low_deviation": low_deviation,
949
+ "high_deviation": high_deviation,
950
+ }
951
+ self.write(
952
+ data=final_correction_quality_outputs,
953
+ tags=[VispTag.quality(VispMetricCode.solar_final_vignette), VispTag.beam(beam)],
954
+ encoder=asdf_encoder,
955
+ )
675
956
 
676
- target_spec : np.ndarray
677
- The reference spectrum. This should be the un-shifted, raw spectrum at the same position as `spec`
957
+ def _log_wavecal_parameters(
958
+ self, dispersion: Quantity, order: int, doppler_velocity: Quantity
959
+ ) -> None:
960
+ """Log initial guess and instrument-derived wavecal parameters."""
961
+ logger.info(f"central wavelength = {self.constants.wavelength * u.nm !s}")
962
+ logger.info(f"{dispersion = !s}")
963
+ logger.info(f"{order = }")
964
+ logger.info(f"grating constant = {self.constants.grating_constant_inverse_mm !s}")
965
+ logger.info(f"incident light angle = {self.constants.incident_light_angle_deg !s}")
966
+ logger.info(f"reflected light angle = {self.constants.reflected_light_angle_deg !s}")
967
+ logger.info(f"{doppler_velocity = !s}")
968
+ logger.info(f"pixel pitch = {self.parameters.wavecal_pixel_pitch_micron_per_pix !s}")
969
+ logger.info(f"solar ip start time = {self.constants.solar_gain_ip_start_time}")
970
+
971
+
972
+ def polynomial_continuum_model(
973
+ wavelength_vector: np.ndarray, fit_parameters: Parameters, fit_order: int, abscissa: np.ndarray
974
+ ) -> np.ndarray:
975
+ """
976
+ Parameterize the continuum as a polynomial.
678
977
 
679
- zones : List
680
- List of zone borders (in px coords)
978
+ Polynomial coefficients are taken from the input ``fit_parameters`` object. The ``wavelength_vector`` argument is not
979
+ used by required to conform to the signature expected by `WavelengthCalibrationFitter`.
681
980
 
682
- x_init: float
683
- Initial guess for the shift. This is used to shift the zones so it needs to be pretty good, but not perfect.
981
+ Parameters
982
+ ----------
983
+ wavelength_vector
984
+ Unused, but required by `WavelengthCalibrationFitter`
684
985
 
685
- Returns
686
- -------
687
- float
688
- The shift value
689
- """
690
- shifts = np.zeros(len(zones))
691
- for i, z in enumerate(zones):
692
- if z[1] + int(x_init) >= spec.size:
693
- logger.info(f"Ignoring zone {z} with init {x_init} because it's out of range")
694
- continue
695
- idx = np.arange(z[0], z[1]) + int(x_init)
696
- shift = spo.minimize(
697
- SolarCalibration.shift_func,
698
- np.array([x_init]),
699
- args=(target_spec, spec, idx),
700
- method="nelder-mead",
701
- ).x[0]
702
- shifts[i] = shift
703
-
704
- return np.nanmedian(shifts)
705
-
706
- @staticmethod
707
- def shift_func(
708
- par: list[float], ref_spec: np.ndarray, spec: np.ndarray, idx: np.ndarray
709
- ) -> float:
710
- """
711
- Non-chisq based goodness of fit calculator for computing spectral shifts.
986
+ fit_parameters
987
+ Object containing the current fit parameters
712
988
 
713
- Instead of chisq, the metric approximates the final Gain image.
989
+ fit_order
990
+ Polynomial order. This needs to match the number of polynomial coefficients in the ``fit_parameters`` argument.
991
+ Specifically, this method expects parameters called `[f"poly_coeff_{i:02n}" for i in range(fit_order + 1)]`.
714
992
 
715
- Parameters
716
- ----------
717
- par : List
718
- List of parameters for minimization
993
+ abscissa
994
+ Array of values used to compute the continuum. For the fit to be independent of wavelength these values should
995
+ be in the range [-1, 1].
996
+ """
997
+ coeffs = [fit_parameters[f"poly_coeff_{i:02n}"].value for i in range(fit_order + 1)]
998
+ return np.polyval(coeffs, abscissa)
719
999
 
720
- ref_spec : np.ndarray
721
- Reference spectra
722
1000
 
723
- spec : np.ndarray
724
- Data
1001
+ ##################################################################################
1002
+ # TODO: The following definitions should go in the wavecal module once it exists #
1003
+ ##################################################################################
725
1004
 
726
- idx : np.ndarray
727
- Range of wavelength pixels that will be compared in fit
728
1005
 
729
- Returns
730
- -------
731
- float
732
- Goodness of fit metric
733
-
734
- """
735
- shift = par[0]
736
- shifted_spec = spnd.shift(spec, shift, mode="constant", cval=np.nan)
737
- final_gain = (ref_spec / shifted_spec)[idx]
738
- slope = (final_gain[-1] - final_gain[0]) / final_gain.size
739
- bg = slope * np.arange(final_gain.size) + final_gain[0]
740
- subbed_gain = np.abs((final_gain) - bg)
741
- fit_metric = np.nansum(subbed_gain[np.isfinite(subbed_gain)])
742
- return fit_metric
1006
+ def compute_order(
1007
+ central_wavelength: Quantity,
1008
+ incident_light_angle: Quantity,
1009
+ reflected_light_angle: Quantity,
1010
+ grating_constant: Quantity,
1011
+ ) -> int:
1012
+ r"""
1013
+ Compute the spectral order from the spectrograph setup.
1014
+
1015
+ From the grating equation, the spectral order, :math:`m`:, is
1016
+
1017
+ .. math::
1018
+ m = \frac{\sin\alpha + \sin\beta}{G \lambda}
1019
+
1020
+ where :math:`\alpha` and :math:`\beta` are the incident and reflected light angles, respectively, :math:`G` is the
1021
+ grating constant (lines per mm), and :math:`\lambda` is the central wavelength. All of these values come from the
1022
+ input headers.
1023
+
1024
+ Parameters
1025
+ ----------
1026
+ central_wavelength
1027
+ Wavelength of the center of the spectral window.
1028
+
1029
+ incident_light_angle
1030
+ Angle of light incident to the spectrograph grating. Often called :math:`\alpha`.
1031
+
1032
+ reflected_light_angle
1033
+ Angle of light reflected from spectrograph grating. Often called :math:`\beta`.
1034
+
1035
+ grating_constant
1036
+ Grating constant of the spectrograph grating [lines per mm]
1037
+
1038
+ Returns
1039
+ -------
1040
+ spectral_order
1041
+ The order of the given spectrograph configuration
1042
+ """
1043
+ return int(
1044
+ (np.sin(incident_light_angle) + np.sin(reflected_light_angle))
1045
+ / (grating_constant * central_wavelength)
1046
+ )
1047
+
1048
+
1049
+ def compute_initial_dispersion(
1050
+ central_wavelength: Quantity,
1051
+ incident_light_angle: Quantity,
1052
+ reflected_light_angle: Quantity,
1053
+ lens_parameters: list[Quantity],
1054
+ pixel_pitch: Quantity,
1055
+ ) -> Quantity:
1056
+ r"""
1057
+ Compute the dispersion (:math:`d\,\lambda/d\, px`) given the spectrograph setup.
1058
+
1059
+ The dispersion is given via
1060
+
1061
+ .. math::
1062
+ d\,\lambda / d\, px = \frac{p \lambda_0 \cos\beta}{f (\sin\alpha + \sin\beta)}
1063
+
1064
+ where :math:`p` is the pixel pitch (microns per pix), :math:`\lambda_0` is the central wavelength, :math:`f` is the
1065
+ camera focal length, and :math:`\alpha` and :math:`\beta` are the incident and reflected light angles, respectively.
1066
+ :math:`\lambda_0`, :math:`\alpha`, and :math:`\beta` are taken from input headers, while :math:`f` and :math:`p` are
1067
+ pipeline parameters.
1068
+
1069
+ Parameters
1070
+ ----------
1071
+ central_wavelength
1072
+ Wavelength of the center of the spectral window.
1073
+
1074
+ incident_light_angle
1075
+ Angle of light incident to the spectrograph grating. Often called :math:`\alpha`.
1076
+
1077
+ reflected_light_angle
1078
+ Angle of light reflected from spectrograph grating. Often called :math:`\beta`.
1079
+
1080
+ lens_parameters
1081
+ Parameterization of lense focal length as zero, first, and second orders of wavelength. If the total focal
1082
+ length of the lens is :math:`f = a_0 + a_1\lambda + a_2\lambda^2` then this list is :math:`[a_0, a_1, a_2]`.
1083
+
1084
+ pixel_pitch
1085
+ The physical size of a single pixel
1086
+
1087
+ Returns
1088
+ -------
1089
+ dispersion
1090
+ The computed dispersion in units of nm / px
1091
+ """
1092
+ camera_focal_length = lens_parameters[0] + central_wavelength * (
1093
+ lens_parameters[1] + central_wavelength * lens_parameters[2]
1094
+ )
1095
+ logger.info(f"{camera_focal_length = !s}")
1096
+
1097
+ linear_dispersion = (
1098
+ camera_focal_length
1099
+ * (np.sin(incident_light_angle) + np.sin(reflected_light_angle))
1100
+ / (np.cos(reflected_light_angle) * central_wavelength)
1101
+ )
1102
+
1103
+ dispersion = pixel_pitch / linear_dispersion
1104
+
1105
+ return dispersion.to(u.nm / u.pix)
1106
+
1107
+
1108
+ def compute_doppler_velocity(time_of_observation: str) -> Quantity:
1109
+ """Find the speed at which DKIST is moving relative to the Sun's center.
1110
+
1111
+ Positive values refer to when DKIST is moving away from the sun.
1112
+
1113
+ Parameters
1114
+ ----------
1115
+ time_of_observation
1116
+ Time at which to compute the relative Dopper velocity. Any string that can be parsed by `astropy.time.Time` is
1117
+ acceptable.
1118
+
1119
+ Returns
1120
+ -------
1121
+ doppler_velocity
1122
+ The relative velocity between observer and the Sun with units of km / s
1123
+ """
1124
+ coord = location_of_dkist.get_gcrs(obstime=Time(time_of_observation))
1125
+ heliocentric_coord = coord.transform_to(HeliocentricInertial(obstime=Time(time_of_observation)))
1126
+ obs_vr_kms = heliocentric_coord.d_distance
1127
+ return obs_vr_kms
1128
+
1129
+
1130
+ def compute_input_wavelength_vector(
1131
+ *,
1132
+ central_wavelength: Quantity,
1133
+ dispersion: Quantity,
1134
+ grating_constant: Quantity,
1135
+ order: int,
1136
+ incident_light_angle: Quantity,
1137
+ num_spec_px: int,
1138
+ ) -> u.Quantity:
1139
+ r"""
1140
+ Compute a wavelength vector based on information about the spectrograph setup.
1141
+
1142
+ The parameterization of the grating equation is via `astropy.wcs.WCS`, which follows section 5 of
1143
+ `Greisen et al (2006) <https://ui.adsabs.harvard.edu/abs/2006A%26A...446..747G/abstract>`_.
1144
+
1145
+ Parameters
1146
+ ----------
1147
+ central_wavelength
1148
+ Wavelength at the center of the spectral window. This function forces the value of the output vector to be
1149
+ ``central_wavelength`` at index ``num_spec // 2 + 1``.
1150
+
1151
+ dispersion
1152
+ Spectrograph dispersion [nm / px]
1153
+
1154
+ grating_constant
1155
+ Grating constant of the spectrograph grating [lines per mm]
1156
+
1157
+ order
1158
+ Spectrograph order
1159
+
1160
+ incident_light_angle
1161
+ Angle of light incident to the spectrograph grating. Often called :math:`\alpha`.
1162
+
1163
+ num_spec_px
1164
+ The length of the output wavelength vector. Defines size and physical limits of the output.
1165
+
1166
+ Returns
1167
+ -------
1168
+ wave_vec
1169
+ 1D array of length ``num_spec`` containing the wavelength values described by the input WCS parameterization.
1170
+ The units of this array will be nanometers.
1171
+ """
1172
+ wavelength_parameters = WavelengthParameters(
1173
+ crpix=num_spec_px // 2 + 1,
1174
+ crval=central_wavelength.to_value(u.nm),
1175
+ dispersion=dispersion.to_value(u.nm / u.pix),
1176
+ grating_constant=grating_constant.to_value(1 / u.mm),
1177
+ order=order,
1178
+ incident_light_angle=incident_light_angle.to_value(u.deg),
1179
+ cunit="nm",
1180
+ )
1181
+ header = wavelength_parameters.to_header(axis_num=1)
1182
+ wcs = WCS(header)
1183
+ input_wavelength_vector = wcs.spectral.pixel_to_world(np.arange(num_spec_px)).to(u.nm)
1184
+
1185
+ return input_wavelength_vector