dkist-processing-trend 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. changelog/.gitempty +0 -0
  2. dkist_processing_trend/__init__.py +10 -0
  3. dkist_processing_trend/config.py +11 -0
  4. dkist_processing_trend/models/__init__.py +1 -0
  5. dkist_processing_trend/models/constants.py +143 -0
  6. dkist_processing_trend/models/fit_options.py +15 -0
  7. dkist_processing_trend/models/fits_access.py +65 -0
  8. dkist_processing_trend/models/instrument.py +35 -0
  9. dkist_processing_trend/models/instrument_options.py +35 -0
  10. dkist_processing_trend/models/parameters.py +134 -0
  11. dkist_processing_trend/models/tags.py +117 -0
  12. dkist_processing_trend/models/task_name.py +20 -0
  13. dkist_processing_trend/parsers/__init__.py +1 -0
  14. dkist_processing_trend/parsers/arm_id.py +103 -0
  15. dkist_processing_trend/parsers/instrument_unique_bud.py +40 -0
  16. dkist_processing_trend/parsers/time.py +27 -0
  17. dkist_processing_trend/parsers/trend_l0_fits_access.py +123 -0
  18. dkist_processing_trend/tasks/__init__.py +27 -0
  19. dkist_processing_trend/tasks/arm_task_factory.py +57 -0
  20. dkist_processing_trend/tasks/dark.py +56 -0
  21. dkist_processing_trend/tasks/gain.py +74 -0
  22. dkist_processing_trend/tasks/initialize_arm_tasks.py +50 -0
  23. dkist_processing_trend/tasks/parse.py +169 -0
  24. dkist_processing_trend/tasks/prepare_fit_data_base.py +257 -0
  25. dkist_processing_trend/tasks/run_pac_fitter.py +392 -0
  26. dkist_processing_trend/tasks/trend_base.py +97 -0
  27. dkist_processing_trend/tasks/trend_output_data.py +167 -0
  28. dkist_processing_trend/tasks/visp/__init__.py +6 -0
  29. dkist_processing_trend/tasks/visp/visp_dmpd.py +410 -0
  30. dkist_processing_trend/tasks/visp/visp_extract_beam.py +14 -0
  31. dkist_processing_trend/tasks/visp/visp_geometric.py +260 -0
  32. dkist_processing_trend/tasks/visp/visp_prep_fit_data.py +162 -0
  33. dkist_processing_trend/tasks/visp/visp_process_demod.py +236 -0
  34. dkist_processing_trend/tasks/write_trend.py +663 -0
  35. dkist_processing_trend/tests/__init__.py +1 -0
  36. dkist_processing_trend/tests/conftest.py +718 -0
  37. dkist_processing_trend/tests/local_trial_workflows/__init__.py +0 -0
  38. dkist_processing_trend/tests/local_trial_workflows/l0_to_trend_visp_polcal.py +294 -0
  39. dkist_processing_trend/tests/local_trial_workflows/local_trial_helpers.py +488 -0
  40. dkist_processing_trend/tests/test_arm_task_factory.py +82 -0
  41. dkist_processing_trend/tests/test_base_tasks.py +86 -0
  42. dkist_processing_trend/tests/test_constants.py +120 -0
  43. dkist_processing_trend/tests/test_dark.py +97 -0
  44. dkist_processing_trend/tests/test_gain.py +135 -0
  45. dkist_processing_trend/tests/test_parameters.py +149 -0
  46. dkist_processing_trend/tests/test_parse.py +276 -0
  47. dkist_processing_trend/tests/test_prep_fit_data_base.py +233 -0
  48. dkist_processing_trend/tests/test_publish_catalog_messages.py +45 -0
  49. dkist_processing_trend/tests/test_run_pac_fitter.py +371 -0
  50. dkist_processing_trend/tests/test_stems.py +75 -0
  51. dkist_processing_trend/tests/test_transfer_output_data.py +76 -0
  52. dkist_processing_trend/tests/test_trend_fits_access.py +173 -0
  53. dkist_processing_trend/tests/test_visp.py +874 -0
  54. dkist_processing_trend/tests/test_workflows.py +10 -0
  55. dkist_processing_trend/tests/test_write_trend.py +460 -0
  56. dkist_processing_trend/workflows/__init__.py +3 -0
  57. dkist_processing_trend/workflows/visp.py +58 -0
  58. dkist_processing_trend-0.1.0.dist-info/METADATA +549 -0
  59. dkist_processing_trend-0.1.0.dist-info/RECORD +66 -0
  60. dkist_processing_trend-0.1.0.dist-info/WHEEL +5 -0
  61. dkist_processing_trend-0.1.0.dist-info/top_level.txt +3 -0
  62. docs/conf.py +57 -0
  63. docs/index.rst +10 -0
  64. docs/l0_to_trend_visp_polcal.rst +4 -0
  65. docs/landing_page.rst +11 -0
  66. docs/requirements_table.rst +8 -0
@@ -0,0 +1,410 @@
1
+ """Task for producing ViSP Demodulated PolcalData (DMPD)."""
2
+
3
+ from dataclasses import dataclass
4
+ from functools import cached_property
5
+
6
+ import numpy as np
7
+ from dkist_processing_common.codecs.asdf import asdf_decoder
8
+ from dkist_processing_common.codecs.fits import fits_array_decoder
9
+ from dkist_processing_common.codecs.fits import fits_array_encoder
10
+ from dkist_processing_math.arithmetic import divide_arrays_by_array
11
+ from dkist_processing_math.arithmetic import subtract_array_from_arrays
12
+ from dkist_processing_math.linear_algebra import nd_left_matrix_multiply
13
+ from dkist_processing_math.statistics import average_numpy_arrays
14
+ from dkist_service_configuration.logging import logger
15
+ from scipy.ndimage import affine_transform
16
+
17
+ from dkist_processing_trend.models.fit_options import FitOptions
18
+ from dkist_processing_trend.models.instrument_options import VispInstrumentOptions
19
+ from dkist_processing_trend.models.tags import TrendTag
20
+ from dkist_processing_trend.tasks.trend_base import TrendArmTaskBase
21
+ from dkist_processing_trend.tasks.visp.visp_extract_beam import extract_visp_beam
22
+
23
+ __all__ = ["VispDemodulatedPolcalData"]
24
+
25
+
26
+ @dataclass
27
+ class VispCalibrationCollection:
28
+ """Dataclass to hold all calibration objects needed for ViSP "science" calibration."""
29
+
30
+ dark: np.ndarray
31
+ gain: np.ndarray
32
+ angle: dict
33
+ shift: dict
34
+ demod_matrices: dict
35
+
36
+ @cached_property
37
+ def beams_overlap_slice(self) -> tuple[slice, slice]:
38
+ """
39
+ Compute array slices that will extract the largest region with overlap from both beams.
40
+
41
+ This is done by considering the shifts computed by the GeometricCalibration task. Any sub-pixel
42
+ overlaps are rounded to the next integer that still guarantees overlap.
43
+
44
+ When "start pixels" are mentioned, those are pixels being counted from zero on a given axis in the positive direction.
45
+ When "end pixels" are mentioned, those are pixels being counted from the end of a given axis in the negative direction.
46
+ """
47
+ logger.info("Computing beam overlap slices")
48
+ # This will be a flat list of (x, y) pairs for both beams
49
+ flat_shifts = list(self.shift.values())
50
+ # Split out into an x list and a y list
51
+ all_x_shifts, all_y_shifts = zip(*flat_shifts)
52
+ all_x_shifts = np.array(all_x_shifts)
53
+ all_y_shifts = np.array(all_y_shifts)
54
+
55
+ logger.info(f"All x shifts: {all_x_shifts}")
56
+ logger.info(f"All y shifts: {all_y_shifts}")
57
+
58
+ # The amount we need to "slice in" from the start of the array is equivalent to the absolute value of the most negative shift.
59
+ # The call to `np.ceil` ensures that the integer rounding doesn't allow non-overlap regions to leak in.
60
+ start_pixels_to_slice_x = int(np.ceil(abs(np.min(all_x_shifts))))
61
+ start_pixels_to_slice_y = int(np.ceil(abs(np.min(all_y_shifts))))
62
+
63
+ # The amount we need to "chop off" the end of the array is the most positive shift.
64
+ #
65
+ # Here we rely on the fact that the fiducial array's shift is *always* (0, 0)
66
+ # (see `geometric.compute_modstate_offset`). Thus, if there are no negative shifts then the following lines
67
+ # will result in None. This is required for slicing because array[x:0] is no good. So if the max is 0 then we
68
+ # end up with array[x:None] which goes all the way to the end of the array.
69
+ #
70
+ # The call to `np.ceil` ensures that the integer rounding doesn't allow non-overlap regions to leak in.
71
+ # (because more negative slices will cut out more data).
72
+ end_pixels_to_slice_x = int(np.ceil(np.max(all_x_shifts))) or None
73
+ end_pixels_to_slice_y = int(np.ceil(np.max(all_y_shifts))) or None
74
+
75
+ # As the pixels to remove from the end of axes is given as a positive number, we need to make it negative for slicing.
76
+ if end_pixels_to_slice_x is not None:
77
+ end_pixels_to_slice_x *= -1
78
+
79
+ if end_pixels_to_slice_y is not None:
80
+ end_pixels_to_slice_y *= -1
81
+
82
+ # Construct the slices
83
+ x_slice = slice(start_pixels_to_slice_x, end_pixels_to_slice_x)
84
+ y_slice = slice(start_pixels_to_slice_y, end_pixels_to_slice_y)
85
+
86
+ logger.info(f"Trimming non-overlapping beam edges by ({x_slice}, {y_slice})")
87
+ return x_slice, y_slice
88
+
89
+
90
+ class VispDemodulatedPolcalData(TrendArmTaskBase):
91
+ """
92
+ Task class for producing demodulated polcal data (DMPD) for ViSP.
93
+
94
+ Demodulated polcal data is essentially a "science" reduction of the input POLCAL data.
95
+
96
+ Parameters
97
+ ----------
98
+ arm_id
99
+ id of the instrument arm to operate on
100
+
101
+ recipe_run_id
102
+ id of the recipe run used to identify the workflow run this task is part of
103
+
104
+ workflow_name
105
+ name of the workflow to which this instance of the task belongs
106
+
107
+ workflow_version
108
+ version of the workflow to which this instance of the task belongs
109
+ """
110
+
111
+ record_provenance = True
112
+
113
+ def run(self):
114
+ """
115
+ Loop over all PAC and instrument options and produce a set of demodulated polcal data for each.
116
+
117
+ The "set" includes data for each beam prior to combination and then a final array for the combined data.
118
+
119
+ Generally, the algorithm for each set of PAC and instrument options is:
120
+
121
+ #. Collect all calibration objects needed (dark, gain, geometric, demodulation matrices)
122
+ #. Apply dark and gain corrections
123
+ #. Split by beam and demodulated
124
+ #. Apply geometric corrections
125
+ #. Combined beams
126
+ #. Save each beam's output and the combined output
127
+ """
128
+ for fit_options in self.parameters.fit_options_list:
129
+ for inst_options in self.parameters.instrument_processing_options:
130
+ log_str = f"fit option {fit_options.name} and instrument option {inst_options.name}"
131
+
132
+ logger.info(f"Collecting calibration objects for {log_str}")
133
+ calibrations = self.collect_calibration_objects(
134
+ fit_options=fit_options, instrument_options=inst_options
135
+ )
136
+ beam_array = extract_visp_beam(
137
+ calibrations.gain, beam=1, beam_border=self.parameters.visp_beam_border
138
+ )
139
+ beam_shape = beam_array.shape
140
+ x_slice, y_slice = calibrations.beams_overlap_slice
141
+ cut_shape = beam_array[x_slice, y_slice].shape
142
+
143
+ with self.telemetry_span(f"Processing {self.constants.num_cs_steps} CS steps"):
144
+ beam1_full_dmpd = np.full(
145
+ (1, self.constants.num_cs_steps, 4, *cut_shape), np.nan, dtype=np.float32
146
+ )
147
+ beam2_full_dmpd = np.full_like(beam1_full_dmpd, np.nan)
148
+ combined_full_dmpd = np.full_like(beam1_full_dmpd, np.nan)
149
+ for cs_step in range(self.constants.num_cs_steps):
150
+ logger.info(
151
+ f"Processing demodulated polcal data for {cs_step = } with {log_str}"
152
+ )
153
+ beam1_dmpd, beam2_dmpd, combined_dmpd = self.process_single_cs_step(
154
+ cs_step=cs_step, calibrations=calibrations, beam_shape=beam_shape
155
+ )
156
+
157
+ # np.moveaxis here because the stokes axis is the last axis in `self.process_single_cs_step`
158
+ # (to make math easier), but we want the DMPD outputs to have stokes before the array axes.
159
+ beam1_full_dmpd[0, cs_step, :, :, :] = np.moveaxis(beam1_dmpd, -1, 0)
160
+ beam2_full_dmpd[0, cs_step, :, :, :] = np.moveaxis(beam2_dmpd, -1, 0)
161
+ combined_full_dmpd[0, cs_step, :, :, :] = np.moveaxis(combined_dmpd, -1, 0)
162
+
163
+ self.write_dmpd(
164
+ beam1_full_dmpd,
165
+ beam=1,
166
+ fit_options=fit_options,
167
+ instrument_options=inst_options,
168
+ )
169
+ del beam1_full_dmpd
170
+
171
+ self.write_dmpd(
172
+ beam2_full_dmpd,
173
+ beam=2,
174
+ fit_options=fit_options,
175
+ instrument_options=inst_options,
176
+ )
177
+ del beam2_full_dmpd
178
+
179
+ self.write_dmpd(
180
+ combined_full_dmpd,
181
+ fit_options=fit_options,
182
+ instrument_options=inst_options,
183
+ beam=None,
184
+ )
185
+ del combined_dmpd
186
+
187
+ def collect_calibration_objects(
188
+ self, fit_options: FitOptions, instrument_options: VispInstrumentOptions
189
+ ) -> VispCalibrationCollection:
190
+ """Collect the INTERMEDIATE calibration objects needed to correct demodulated polcal data."""
191
+ angle_dict = dict()
192
+ shift_dict = dict()
193
+ demod_dict = dict()
194
+
195
+ base_tags = [TrendTag.intermediate(), TrendTag.arm_id(self.arm_id)]
196
+
197
+ # Dark
198
+ ######
199
+ full_dark_array = next(
200
+ self.read(
201
+ tags=base_tags + [TrendTag.frame(), TrendTag.task_dark()],
202
+ decoder=fits_array_decoder,
203
+ )
204
+ )
205
+
206
+ # Gain
207
+ #######
208
+ full_gain_array = next(
209
+ self.read(
210
+ tags=base_tags + [TrendTag.frame(), TrendTag.task_gain()],
211
+ decoder=fits_array_decoder,
212
+ )
213
+ )
214
+
215
+ for beam in [1, 2]:
216
+ beam_tags = base_tags + [TrendTag.beam(beam)]
217
+
218
+ # Geo
219
+ #######
220
+ geo_dict = next(
221
+ self.read(
222
+ tags=beam_tags + [TrendTag.task_visp_geometric_calibration()],
223
+ decoder=asdf_decoder,
224
+ )
225
+ )
226
+ angle_dict[TrendTag.beam(beam)] = geo_dict["angle_rad"]
227
+ shift_dict[TrendTag.beam(beam)] = geo_dict["shift"]
228
+
229
+ # Demod
230
+ #######
231
+ demod_matrices = next(
232
+ self.read(
233
+ tags=beam_tags
234
+ + [
235
+ TrendTag.pac_fit_options(fit_options.name),
236
+ TrendTag.instrument_processing_options(instrument_options.name),
237
+ TrendTag.task_processed_demodulation_matrices(),
238
+ ],
239
+ decoder=fits_array_decoder,
240
+ )
241
+ )
242
+ demod_dict[TrendTag.beam(beam)] = demod_matrices
243
+
244
+ return VispCalibrationCollection(
245
+ dark=full_dark_array,
246
+ gain=full_gain_array,
247
+ angle=angle_dict,
248
+ shift=shift_dict,
249
+ demod_matrices=demod_dict,
250
+ )
251
+
252
+ def process_single_cs_step(
253
+ self,
254
+ cs_step: int,
255
+ calibrations: VispCalibrationCollection,
256
+ beam_shape: tuple[int, int],
257
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
258
+ """
259
+ Fully process the data for a single CS step.
260
+
261
+ To simplify the code, the dark and gain corrections are done on the full-frame before beam extraction.
262
+ Demodulation and geometric corrections are applied on a per-beam basis.
263
+
264
+ All outputs are trimmed to only include the regions that exist in both beams.
265
+
266
+ Returns
267
+ -------
268
+ The corrected beam 1 array
269
+ The corrected beam 2 array
270
+ The corrected combined array
271
+ """
272
+ modstate_full_frame_data_list = []
273
+ for modstate in range(1, self.constants.num_modstates + 1):
274
+ tags = [
275
+ TrendTag.input(),
276
+ TrendTag.frame(),
277
+ TrendTag.arm_id(self.arm_id),
278
+ TrendTag.task_polcal(),
279
+ TrendTag.cs_step(cs_step),
280
+ TrendTag.modstate(modstate),
281
+ ]
282
+ input_arrays = self.read(tags=tags, decoder=fits_array_decoder)
283
+ avg_input = average_numpy_arrays(input_arrays)
284
+
285
+ dark_corrected = subtract_array_from_arrays(
286
+ arrays=avg_input, array_to_subtract=calibrations.dark
287
+ )
288
+ gain_corrected = next(
289
+ divide_arrays_by_array(arrays=dark_corrected, array_to_divide_by=calibrations.gain)
290
+ )
291
+
292
+ modstate_full_frame_data_list.append(gain_corrected)
293
+
294
+ final_beam_data_list = []
295
+ for beam in [1, 2]:
296
+ demod_matrices = calibrations.demod_matrices[TrendTag.beam(beam)]
297
+ beam_stack = np.full(beam_shape + (self.constants.num_modstates,), np.nan)
298
+ for modstate in range(1, self.constants.num_modstates + 1):
299
+ full_frame_data = modstate_full_frame_data_list[modstate - 1]
300
+ beam_data = extract_visp_beam(
301
+ full_frame_data, beam=beam, beam_border=self.parameters.visp_beam_border
302
+ )
303
+ beam_stack[:, :, modstate - 1] = beam_data
304
+
305
+ demodulated_data = nd_left_matrix_multiply(
306
+ vector_stack=beam_stack,
307
+ matrix_stack=demod_matrices,
308
+ )
309
+
310
+ geo_corrected = np.full_like(demodulated_data, np.nan)
311
+ for s in range(demodulated_data.shape[-1]):
312
+ geo_corrected[:, :, s] = self.correct_geometry(
313
+ demodulated_data[:, :, s],
314
+ angle=calibrations.angle[TrendTag.beam(beam)],
315
+ shift=calibrations.shift[TrendTag.beam(beam)],
316
+ )
317
+
318
+ x_slice, y_slice = calibrations.beams_overlap_slice
319
+ cut_array = geo_corrected[x_slice, y_slice, :].astype(np.float32)
320
+
321
+ final_beam_data_list.append(cut_array)
322
+
323
+ combined_data = self.combine_beams(
324
+ beam1_data=final_beam_data_list[0], beam2_data=final_beam_data_list[1]
325
+ ).astype(np.float32)
326
+
327
+ return final_beam_data_list[0], final_beam_data_list[1], combined_data
328
+
329
+ def combine_beams(self, beam1_data: np.ndarray, beam2_data: np.ndarray) -> np.ndarray:
330
+ r"""
331
+ Combine polarimetric beams so that polarization states are normalized by the intensity state (Stokes I).
332
+
333
+ In other words:
334
+
335
+ .. math::
336
+
337
+ I_{avg} & = (I_1 + I_2) / 2
338
+
339
+ Q_{avg} & = I_{avg} (Q_1 / I_1 + Q_2 / I_2) / 2
340
+
341
+ ...and the same for U and V
342
+ """
343
+ avg_data = np.zeros_like(beam1_data)
344
+
345
+ # Rely on the fact that the Stokes states are in order after demodulation
346
+ avg_I = (beam1_data[:, :, 0] + beam2_data[:, :, 0]) / 2.0
347
+ avg_data[:, :, 0] = avg_I
348
+
349
+ for stokes in range(1, 4):
350
+ beam1_norm = beam1_data[:, :, stokes] / beam1_data[:, :, 0]
351
+ beam2_norm = beam2_data[:, :, stokes] / beam2_data[:, :, 0]
352
+ avg_data[:, :, stokes] = avg_I * (beam1_norm + beam2_norm) / 2.0
353
+
354
+ return avg_data
355
+
356
+ def correct_geometry(
357
+ self, array: np.ndarray, angle: float, shift: tuple[float, float]
358
+ ) -> np.ndarray:
359
+ """
360
+ Rotate and shift an array.
361
+
362
+ Note that the input ``angle`` and ``shift`` arguments are the *amount to rotate/shift by*. They are passed straight
363
+ through to the affine transform.
364
+ """
365
+ array_shape = array.shape
366
+ cosine_angle = np.cos(angle)
367
+ sine_angle = np.sin(angle)
368
+ inverse_rotation_matrix = np.array(
369
+ [[cosine_angle, -sine_angle], [sine_angle, cosine_angle]],
370
+ dtype=np.float64,
371
+ )
372
+
373
+ rotation_center = 0.5 * (np.asarray(array_shape, dtype=np.float64) - 1.0)
374
+ transform_offset = rotation_center + shift - inverse_rotation_matrix @ rotation_center
375
+
376
+ corrected_array = affine_transform(
377
+ array,
378
+ matrix=inverse_rotation_matrix,
379
+ offset=transform_offset,
380
+ order=5,
381
+ mode="nearest",
382
+ cval=np.nan,
383
+ prefilter=True,
384
+ )
385
+ return corrected_array
386
+
387
+ def write_dmpd(
388
+ self,
389
+ array: np.ndarray,
390
+ beam: int | None,
391
+ fit_options: FitOptions,
392
+ instrument_options: VispInstrumentOptions,
393
+ ) -> None:
394
+ """
395
+ Write a demodulated polcal data output to scratch.
396
+
397
+ If ``beam`` is not `None` then the output will be tagged as a `~dkist_processing_trend.models.tags.TrendTag.task_single_beam_demodulated_polcal_data`,
398
+ otherwise the output is tagged with `~dkist_processing_trend.models.tags.TrendTag.task_demodulated_polcal_data`.
399
+ """
400
+ tags = [
401
+ TrendTag.intermediate(),
402
+ TrendTag.arm_id(self.arm_id),
403
+ TrendTag.pac_fit_options(fit_options.name),
404
+ TrendTag.instrument_processing_options(instrument_options.name),
405
+ ]
406
+ if beam is not None:
407
+ tags += [TrendTag.beam(beam), TrendTag.task_single_beam_demodulated_polcal_data()]
408
+ else:
409
+ tags.append(TrendTag.task_demodulated_polcal_data())
410
+ self.write(data=array, tags=tags, encoder=fits_array_encoder)
@@ -0,0 +1,14 @@
1
+ """Function to split apart the two ViSP beams."""
2
+
3
+ from typing import Literal
4
+
5
+ import numpy as np
6
+
7
+
8
+ def extract_visp_beam(array: np.ndarray, beam: Literal[1, 2], beam_border: int) -> np.ndarray:
9
+ """Extract a single beam from a raw, dual-beam array."""
10
+ if beam == 1:
11
+ return np.copy(array[:beam_border, ...])
12
+ if beam == 2:
13
+ return np.copy(array[beam_border:, ...][::-1, :])
14
+ raise ValueError(f"ViSP only has beams 1 and 2. Can't extract data for beam '{beam}'.")
@@ -0,0 +1,260 @@
1
+ """Task for computing the ViSP beam offset."""
2
+
3
+ from typing import Literal
4
+
5
+ import numpy as np
6
+ import peakutils as pku
7
+ import scipy.ndimage as spnd
8
+ from astropy.modeling import fitting
9
+ from astropy.modeling import models
10
+ from astropy.stats import sigma_clip
11
+ from dkist_processing_common.codecs.asdf import asdf_encoder
12
+ from dkist_processing_common.codecs.fits import fits_array_decoder
13
+ from dkist_processing_math.statistics import average_numpy_arrays
14
+ from dkist_service_configuration.logging import logger
15
+ from skimage.registration import phase_cross_correlation
16
+
17
+ from dkist_processing_trend.models.tags import TrendTag
18
+ from dkist_processing_trend.tasks.trend_base import TrendArmTaskBase
19
+ from dkist_processing_trend.tasks.visp.visp_extract_beam import extract_visp_beam
20
+
21
+ __all__ = ["VispGeometricCalibration"]
22
+
23
+
24
+ class VispGeometricCalibration(TrendArmTaskBase):
25
+ """
26
+ Compute the X/Y offset between the two ViSP beams.
27
+
28
+ Unlike the main L1 pipeline, this task does not treat each modstate separately and does not compute a rotational offset.
29
+ For the purposes of the Trend pipeline a simple X/Y shift is sufficient.
30
+
31
+ Parameters
32
+ ----------
33
+ arm_id
34
+ id of the instrument arm to operate on
35
+
36
+ recipe_run_id
37
+ id of the recipe run used to identify the workflow run this task is part of
38
+
39
+ workflow_name
40
+ name of the workflow to which this instance of the task belongs
41
+
42
+ workflow_version
43
+ version of the workflow to which this instance of the task belongs
44
+ """
45
+
46
+ record_provenance = True
47
+
48
+ def run(self):
49
+ """
50
+ Compute the X/Y shift between ViSP beam 1 and 2.
51
+
52
+ Algorithm:
53
+
54
+ #. Compute the average of all input POLCAL_GAIN frames (i.e., "clear" CS step)
55
+ #. Split this average into beams 1 and 2
56
+ #. Compute the rotation angle for each beam
57
+ #. Correct each beam for its measured rotation angle
58
+ #. Collapse each beam along both the spectral and spatial axis and use `~skimage.registration.phase_cross_correlation` to compute the shift along each axis
59
+ #. Write the shifts and angles to scratch
60
+
61
+ NOTE: The outputs of this task are angles and shifts that should be used *directly* with the geometric correction
62
+ method; no inverting/sign flipping is needed.
63
+ """
64
+ with self.telemetry_span("Compute average clear array"):
65
+ logger.info("Computing average clear array")
66
+ full_avg_array = self.compute_avg_clear_array()
67
+
68
+ logger.info("Extracting beams")
69
+ beam_border = self.parameters.visp_beam_border
70
+ beam1_array = extract_visp_beam(full_avg_array, beam=1, beam_border=beam_border)
71
+ beam2_array = extract_visp_beam(full_avg_array, beam=2, beam_border=beam_border)
72
+
73
+ with self.telemetry_span("Compute beam angles"):
74
+ logger.info("Computing beam angles")
75
+ beam1_angle_rad = self.compute_beam_angle(array=beam1_array)
76
+ beam1_angle_deg = np.rad2deg(beam1_angle_rad)
77
+ logger.info(f"Beam 1 angle = {beam1_angle_deg:.5f} deg")
78
+
79
+ beam2_angle_rad = self.compute_beam_angle(array=beam2_array)
80
+ beam2_angle_deg = np.rad2deg(beam2_angle_rad)
81
+ logger.info(f"Beam 2 angle = {beam2_angle_deg:.5f} deg")
82
+
83
+ with self.telemetry_span("Compute beam offset"):
84
+ logger.info("Computing beam offsets")
85
+
86
+ logger.info("Removing beam angles")
87
+ beam1_rotated = spnd.rotate(
88
+ beam1_array, angle=-beam1_angle_deg, reshape=False, cval=np.nanmedian(beam1_array)
89
+ )
90
+ beam2_rotated = spnd.rotate(
91
+ beam2_array, angle=-beam2_angle_deg, reshape=False, cval=np.nanmedian(beam2_array)
92
+ )
93
+
94
+ logger.info("Filtering beams")
95
+ beam1_filtered = self.high_pass_filter_array(beam1_rotated)
96
+ beam2_filtered = self.high_pass_filter_array(beam2_rotated)
97
+
98
+ shift = np.full(2, np.nan)
99
+ for i, (ax_name, ax) in enumerate(zip(["spectral", "spatial"], [0, 1])):
100
+ logger.info(f"Computing {ax_name} shift")
101
+ shift[i] = self.compute_single_axis_shift(
102
+ reference_array=beam1_filtered, target_array=beam2_filtered, axis=ax
103
+ )
104
+
105
+ logger.info(f"Measured {shift = }")
106
+
107
+ logger.info("Writing angles and shift to scratch")
108
+ beam1_geo_data = {"angle_rad": beam1_angle_rad, "shift": np.array([0, 0.0])}
109
+ self.write(
110
+ data=beam1_geo_data,
111
+ tags=[
112
+ TrendTag.intermediate(),
113
+ TrendTag.arm_id(self.arm_id),
114
+ TrendTag.beam(1),
115
+ TrendTag.task_visp_geometric_calibration(),
116
+ ],
117
+ encoder=asdf_encoder,
118
+ )
119
+
120
+ beam2_geo_data = {
121
+ "angle_rad": beam2_angle_rad,
122
+ "shift": -shift,
123
+ }
124
+ self.write(
125
+ data=beam2_geo_data,
126
+ tags=[
127
+ TrendTag.intermediate(),
128
+ TrendTag.arm_id(self.arm_id),
129
+ TrendTag.beam(2),
130
+ TrendTag.task_visp_geometric_calibration(),
131
+ ],
132
+ encoder=asdf_encoder,
133
+ )
134
+
135
+ def compute_avg_clear_array(self) -> np.ndarray:
136
+ """Compute the average of all "clear" CS step arrays."""
137
+ tags = [TrendTag.input(), TrendTag.arm_id(self.arm_id), TrendTag.task_polcal_gain()]
138
+ logger.info(f"Using {self.count(tags)} INPUT clear arrays.")
139
+ clear_arrays = self.read(tags=tags, decoder=fits_array_decoder)
140
+ avg_array = average_numpy_arrays(clear_arrays)
141
+ return avg_array
142
+
143
+ def compute_beam_angle(self, array: np.ndarray) -> float:
144
+ """
145
+ Compute the rotation angle of a single beam.
146
+
147
+ The angle is computed by measuring the slope of the slit hairlines; the arctan of the slope then gives the rotation angle.
148
+ To measure the slope of the hairlines, first the hairline signal is isolated with by smoothing the image in the spatial
149
+ direction (which will remove the hairlines) and then subtracting this smoothed array from the input image.
150
+ Once the hairlines have been isolated, the location of hairline center is measured with `peakutils <https://pypi.org/project/PeakUtils/>`_.
151
+ Finally, a line is fit to the the location of hairline center as a function of wavelength and the slope of this line is
152
+ used to find the angle.
153
+
154
+ Multiple hairlines (always 2, and sometimes more if there are reflections) have their angles computed separately
155
+ and then average together for the final result.
156
+
157
+ Returns
158
+ -------
159
+ The rotation angle of the given array, in radians.
160
+ """
161
+ no_hairline_array = spnd.median_filter(array, size=(1, 30))
162
+ hairline_only_array = (array - no_hairline_array) / no_hairline_array
163
+
164
+ # Turn low-signal hairlines into "emission" lines so peakutils can find them
165
+ hairline_only_array *= -1
166
+
167
+ hairline_1d = np.nanmedian(hairline_only_array, axis=0)
168
+ hairline_locs = pku.indexes(hairline_1d, thres=0.8)
169
+
170
+ num_wave, num_spat = array.shape
171
+ fit_centers = np.full((len(hairline_locs), num_wave), np.nan)
172
+ spectral_abscissa = np.arange(num_wave)
173
+ spatial_abscissa = np.arange(num_spat)
174
+ for i in range(num_wave):
175
+ try:
176
+ interp_center = pku.interpolate(
177
+ spatial_abscissa, hairline_only_array[i, :], ind=hairline_locs, width=10
178
+ )
179
+ fit_centers[:, i] = interp_center
180
+ except:
181
+ logger.info(f"Failed to fit hairline center for spectral pixel {i}. Ignoring.")
182
+
183
+ hairline_angles = []
184
+ for hairline in range(fit_centers.shape[0]):
185
+ fit_angle = self.fit_angle_from_slope(
186
+ abscissa=spectral_abscissa, centers=fit_centers[hairline]
187
+ )
188
+ hairline_angles.append(fit_angle)
189
+
190
+ return np.mean(hairline_angles)
191
+
192
+ @staticmethod
193
+ def fit_angle_from_slope(abscissa: np.ndarray, centers: np.ndarray, sigma: float = 3) -> float:
194
+ """
195
+ Fit a 1-degree polynomial (i.e., a line) to the hairline centers and compute an angle from the slope.
196
+
197
+ Outliers are iteratively removed using a sigma clipping algorithm as detailed here:
198
+ https://docs.astropy.org/en/stable/modeling/example-fitting-line.html#iterative-fitting-using-sigma-clipping
199
+
200
+ Returns
201
+ -------
202
+ The slope of the line, in radians.
203
+ """
204
+ line_fitter = fitting.LinearLSQFitter()
205
+ outlier_rejection_fitter = fitting.FittingWithOutlierRemoval(
206
+ fitter=line_fitter,
207
+ outlier_func=sigma_clip,
208
+ sigma=sigma,
209
+ cenfunc="median",
210
+ stdfunc="std",
211
+ )
212
+ linear_model = models.Linear1D()
213
+
214
+ fit_line = outlier_rejection_fitter(model=linear_model, x=abscissa, y=centers)[0]
215
+ angle_rad = np.arctan(fit_line.slope.value)
216
+ return angle_rad
217
+
218
+ @staticmethod
219
+ def high_pass_filter_array(array: np.ndarray) -> np.ndarray:
220
+ """
221
+ Remove low-frequency signal in an array by dividing it by a smoothed copy.
222
+
223
+ The smoothing is done with a Gaussian smoothing kernel. The resulting array has strong spectral features
224
+ and hairlines accentuated over the continuum.
225
+ """
226
+ # The sigma value is hardcoded in the ViSP L1 pipeline, too. It's valid for all ViSP data for all time.
227
+ return array / spnd.gaussian_filter(array, sigma=5)
228
+
229
+ def compute_single_axis_shift(
230
+ self, reference_array: np.ndarray, target_array: np.ndarray, axis: Literal[0, 1]
231
+ ) -> float:
232
+ """
233
+ Compute the offset between two arrays along a single axis.
234
+
235
+ ViSP data have strong, high-frequency signals along both of their axes; the solar spectrum and the slit hairlines.
236
+ This method isolates one of those signals by first computing a derivative along the axis of interest. Because
237
+ the two signals are orthogonal this derivative both accentuates the strong features along the axis of interest and
238
+ removes the signal in the orthogonal direction (e.g., the hairlines are constant along the spectral axis).
239
+
240
+ The resulting derivative is then collapsed along the orthogonal axis, via median, into a single vector, which
241
+ increases the signal to noise and simplifies the final correlation.
242
+
243
+ The single vectors from both arrays are then registered via `~skimage.registration.phase_cross_correlation` to
244
+ produce the final shift.
245
+ """
246
+ opposite_axis = abs(axis - 1)
247
+
248
+ reference_deriv = np.diff(reference_array, axis=axis)
249
+ reference_1d_signal = np.nanmedian(reference_deriv, axis=opposite_axis)
250
+
251
+ target_deriv = np.diff(target_array, axis=axis)
252
+ target_1d_signal = np.nanmedian(target_deriv, axis=opposite_axis)
253
+
254
+ shift = phase_cross_correlation(
255
+ reference_image=reference_1d_signal,
256
+ moving_image=target_1d_signal,
257
+ upsample_factor=self.parameters.visp_geo_upsample_factor,
258
+ normalization=None,
259
+ )[0]
260
+ return shift.item(0)