dkist-processing-trend 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- changelog/.gitempty +0 -0
- dkist_processing_trend/__init__.py +10 -0
- dkist_processing_trend/config.py +11 -0
- dkist_processing_trend/models/__init__.py +1 -0
- dkist_processing_trend/models/constants.py +143 -0
- dkist_processing_trend/models/fit_options.py +15 -0
- dkist_processing_trend/models/fits_access.py +65 -0
- dkist_processing_trend/models/instrument.py +35 -0
- dkist_processing_trend/models/instrument_options.py +35 -0
- dkist_processing_trend/models/parameters.py +134 -0
- dkist_processing_trend/models/tags.py +117 -0
- dkist_processing_trend/models/task_name.py +20 -0
- dkist_processing_trend/parsers/__init__.py +1 -0
- dkist_processing_trend/parsers/arm_id.py +103 -0
- dkist_processing_trend/parsers/instrument_unique_bud.py +40 -0
- dkist_processing_trend/parsers/time.py +27 -0
- dkist_processing_trend/parsers/trend_l0_fits_access.py +123 -0
- dkist_processing_trend/tasks/__init__.py +27 -0
- dkist_processing_trend/tasks/arm_task_factory.py +57 -0
- dkist_processing_trend/tasks/dark.py +56 -0
- dkist_processing_trend/tasks/gain.py +74 -0
- dkist_processing_trend/tasks/initialize_arm_tasks.py +50 -0
- dkist_processing_trend/tasks/parse.py +169 -0
- dkist_processing_trend/tasks/prepare_fit_data_base.py +257 -0
- dkist_processing_trend/tasks/run_pac_fitter.py +392 -0
- dkist_processing_trend/tasks/trend_base.py +97 -0
- dkist_processing_trend/tasks/trend_output_data.py +167 -0
- dkist_processing_trend/tasks/visp/__init__.py +6 -0
- dkist_processing_trend/tasks/visp/visp_dmpd.py +410 -0
- dkist_processing_trend/tasks/visp/visp_extract_beam.py +14 -0
- dkist_processing_trend/tasks/visp/visp_geometric.py +260 -0
- dkist_processing_trend/tasks/visp/visp_prep_fit_data.py +162 -0
- dkist_processing_trend/tasks/visp/visp_process_demod.py +236 -0
- dkist_processing_trend/tasks/write_trend.py +663 -0
- dkist_processing_trend/tests/__init__.py +1 -0
- dkist_processing_trend/tests/conftest.py +718 -0
- dkist_processing_trend/tests/local_trial_workflows/__init__.py +0 -0
- dkist_processing_trend/tests/local_trial_workflows/l0_to_trend_visp_polcal.py +294 -0
- dkist_processing_trend/tests/local_trial_workflows/local_trial_helpers.py +488 -0
- dkist_processing_trend/tests/test_arm_task_factory.py +82 -0
- dkist_processing_trend/tests/test_base_tasks.py +86 -0
- dkist_processing_trend/tests/test_constants.py +120 -0
- dkist_processing_trend/tests/test_dark.py +97 -0
- dkist_processing_trend/tests/test_gain.py +135 -0
- dkist_processing_trend/tests/test_parameters.py +149 -0
- dkist_processing_trend/tests/test_parse.py +276 -0
- dkist_processing_trend/tests/test_prep_fit_data_base.py +233 -0
- dkist_processing_trend/tests/test_publish_catalog_messages.py +45 -0
- dkist_processing_trend/tests/test_run_pac_fitter.py +371 -0
- dkist_processing_trend/tests/test_stems.py +75 -0
- dkist_processing_trend/tests/test_transfer_output_data.py +76 -0
- dkist_processing_trend/tests/test_trend_fits_access.py +173 -0
- dkist_processing_trend/tests/test_visp.py +874 -0
- dkist_processing_trend/tests/test_workflows.py +10 -0
- dkist_processing_trend/tests/test_write_trend.py +460 -0
- dkist_processing_trend/workflows/__init__.py +3 -0
- dkist_processing_trend/workflows/visp.py +58 -0
- dkist_processing_trend-0.1.0.dist-info/METADATA +549 -0
- dkist_processing_trend-0.1.0.dist-info/RECORD +66 -0
- dkist_processing_trend-0.1.0.dist-info/WHEEL +5 -0
- dkist_processing_trend-0.1.0.dist-info/top_level.txt +3 -0
- docs/conf.py +57 -0
- docs/index.rst +10 -0
- docs/l0_to_trend_visp_polcal.rst +4 -0
- docs/landing_page.rst +11 -0
- docs/requirements_table.rst +8 -0
|
@@ -0,0 +1,410 @@
|
|
|
1
|
+
"""Task for producing ViSP Demodulated PolcalData (DMPD)."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from functools import cached_property
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
from dkist_processing_common.codecs.asdf import asdf_decoder
|
|
8
|
+
from dkist_processing_common.codecs.fits import fits_array_decoder
|
|
9
|
+
from dkist_processing_common.codecs.fits import fits_array_encoder
|
|
10
|
+
from dkist_processing_math.arithmetic import divide_arrays_by_array
|
|
11
|
+
from dkist_processing_math.arithmetic import subtract_array_from_arrays
|
|
12
|
+
from dkist_processing_math.linear_algebra import nd_left_matrix_multiply
|
|
13
|
+
from dkist_processing_math.statistics import average_numpy_arrays
|
|
14
|
+
from dkist_service_configuration.logging import logger
|
|
15
|
+
from scipy.ndimage import affine_transform
|
|
16
|
+
|
|
17
|
+
from dkist_processing_trend.models.fit_options import FitOptions
|
|
18
|
+
from dkist_processing_trend.models.instrument_options import VispInstrumentOptions
|
|
19
|
+
from dkist_processing_trend.models.tags import TrendTag
|
|
20
|
+
from dkist_processing_trend.tasks.trend_base import TrendArmTaskBase
|
|
21
|
+
from dkist_processing_trend.tasks.visp.visp_extract_beam import extract_visp_beam
|
|
22
|
+
|
|
23
|
+
__all__ = ["VispDemodulatedPolcalData"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class VispCalibrationCollection:
|
|
28
|
+
"""Dataclass to hold all calibration objects needed for ViSP "science" calibration."""
|
|
29
|
+
|
|
30
|
+
dark: np.ndarray
|
|
31
|
+
gain: np.ndarray
|
|
32
|
+
angle: dict
|
|
33
|
+
shift: dict
|
|
34
|
+
demod_matrices: dict
|
|
35
|
+
|
|
36
|
+
@cached_property
|
|
37
|
+
def beams_overlap_slice(self) -> tuple[slice, slice]:
|
|
38
|
+
"""
|
|
39
|
+
Compute array slices that will extract the largest region with overlap from both beams.
|
|
40
|
+
|
|
41
|
+
This is done by considering the shifts computed by the GeometricCalibration task. Any sub-pixel
|
|
42
|
+
overlaps are rounded to the next integer that still guarantees overlap.
|
|
43
|
+
|
|
44
|
+
When "start pixels" are mentioned, those are pixels being counted from zero on a given axis in the positive direction.
|
|
45
|
+
When "end pixels" are mentioned, those are pixels being counted from the end of a given axis in the negative direction.
|
|
46
|
+
"""
|
|
47
|
+
logger.info("Computing beam overlap slices")
|
|
48
|
+
# This will be a flat list of (x, y) pairs for both beams
|
|
49
|
+
flat_shifts = list(self.shift.values())
|
|
50
|
+
# Split out into an x list and a y list
|
|
51
|
+
all_x_shifts, all_y_shifts = zip(*flat_shifts)
|
|
52
|
+
all_x_shifts = np.array(all_x_shifts)
|
|
53
|
+
all_y_shifts = np.array(all_y_shifts)
|
|
54
|
+
|
|
55
|
+
logger.info(f"All x shifts: {all_x_shifts}")
|
|
56
|
+
logger.info(f"All y shifts: {all_y_shifts}")
|
|
57
|
+
|
|
58
|
+
# The amount we need to "slice in" from the start of the array is equivalent to the absolute value of the most negative shift.
|
|
59
|
+
# The call to `np.ceil` ensures that the integer rounding doesn't allow non-overlap regions to leak in.
|
|
60
|
+
start_pixels_to_slice_x = int(np.ceil(abs(np.min(all_x_shifts))))
|
|
61
|
+
start_pixels_to_slice_y = int(np.ceil(abs(np.min(all_y_shifts))))
|
|
62
|
+
|
|
63
|
+
# The amount we need to "chop off" the end of the array is the most positive shift.
|
|
64
|
+
#
|
|
65
|
+
# Here we rely on the fact that the fiducial array's shift is *always* (0, 0)
|
|
66
|
+
# (see `geometric.compute_modstate_offset`). Thus, if there are no negative shifts then the following lines
|
|
67
|
+
# will result in None. This is required for slicing because array[x:0] is no good. So if the max is 0 then we
|
|
68
|
+
# end up with array[x:None] which goes all the way to the end of the array.
|
|
69
|
+
#
|
|
70
|
+
# The call to `np.ceil` ensures that the integer rounding doesn't allow non-overlap regions to leak in.
|
|
71
|
+
# (because more negative slices will cut out more data).
|
|
72
|
+
end_pixels_to_slice_x = int(np.ceil(np.max(all_x_shifts))) or None
|
|
73
|
+
end_pixels_to_slice_y = int(np.ceil(np.max(all_y_shifts))) or None
|
|
74
|
+
|
|
75
|
+
# As the pixels to remove from the end of axes is given as a positive number, we need to make it negative for slicing.
|
|
76
|
+
if end_pixels_to_slice_x is not None:
|
|
77
|
+
end_pixels_to_slice_x *= -1
|
|
78
|
+
|
|
79
|
+
if end_pixels_to_slice_y is not None:
|
|
80
|
+
end_pixels_to_slice_y *= -1
|
|
81
|
+
|
|
82
|
+
# Construct the slices
|
|
83
|
+
x_slice = slice(start_pixels_to_slice_x, end_pixels_to_slice_x)
|
|
84
|
+
y_slice = slice(start_pixels_to_slice_y, end_pixels_to_slice_y)
|
|
85
|
+
|
|
86
|
+
logger.info(f"Trimming non-overlapping beam edges by ({x_slice}, {y_slice})")
|
|
87
|
+
return x_slice, y_slice
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class VispDemodulatedPolcalData(TrendArmTaskBase):
|
|
91
|
+
"""
|
|
92
|
+
Task class for producing demodulated polcal data (DMPD) for ViSP.
|
|
93
|
+
|
|
94
|
+
Demodulated polcal data is essentially a "science" reduction of the input POLCAL data.
|
|
95
|
+
|
|
96
|
+
Parameters
|
|
97
|
+
----------
|
|
98
|
+
arm_id
|
|
99
|
+
id of the instrument arm to operate on
|
|
100
|
+
|
|
101
|
+
recipe_run_id
|
|
102
|
+
id of the recipe run used to identify the workflow run this task is part of
|
|
103
|
+
|
|
104
|
+
workflow_name
|
|
105
|
+
name of the workflow to which this instance of the task belongs
|
|
106
|
+
|
|
107
|
+
workflow_version
|
|
108
|
+
version of the workflow to which this instance of the task belongs
|
|
109
|
+
"""
|
|
110
|
+
|
|
111
|
+
record_provenance = True
|
|
112
|
+
|
|
113
|
+
def run(self):
|
|
114
|
+
"""
|
|
115
|
+
Loop over all PAC and instrument options and produce a set of demodulated polcal data for each.
|
|
116
|
+
|
|
117
|
+
The "set" includes data for each beam prior to combination and then a final array for the combined data.
|
|
118
|
+
|
|
119
|
+
Generally, the algorithm for each set of PAC and instrument options is:
|
|
120
|
+
|
|
121
|
+
#. Collect all calibration objects needed (dark, gain, geometric, demodulation matrices)
|
|
122
|
+
#. Apply dark and gain corrections
|
|
123
|
+
#. Split by beam and demodulated
|
|
124
|
+
#. Apply geometric corrections
|
|
125
|
+
#. Combined beams
|
|
126
|
+
#. Save each beam's output and the combined output
|
|
127
|
+
"""
|
|
128
|
+
for fit_options in self.parameters.fit_options_list:
|
|
129
|
+
for inst_options in self.parameters.instrument_processing_options:
|
|
130
|
+
log_str = f"fit option {fit_options.name} and instrument option {inst_options.name}"
|
|
131
|
+
|
|
132
|
+
logger.info(f"Collecting calibration objects for {log_str}")
|
|
133
|
+
calibrations = self.collect_calibration_objects(
|
|
134
|
+
fit_options=fit_options, instrument_options=inst_options
|
|
135
|
+
)
|
|
136
|
+
beam_array = extract_visp_beam(
|
|
137
|
+
calibrations.gain, beam=1, beam_border=self.parameters.visp_beam_border
|
|
138
|
+
)
|
|
139
|
+
beam_shape = beam_array.shape
|
|
140
|
+
x_slice, y_slice = calibrations.beams_overlap_slice
|
|
141
|
+
cut_shape = beam_array[x_slice, y_slice].shape
|
|
142
|
+
|
|
143
|
+
with self.telemetry_span(f"Processing {self.constants.num_cs_steps} CS steps"):
|
|
144
|
+
beam1_full_dmpd = np.full(
|
|
145
|
+
(1, self.constants.num_cs_steps, 4, *cut_shape), np.nan, dtype=np.float32
|
|
146
|
+
)
|
|
147
|
+
beam2_full_dmpd = np.full_like(beam1_full_dmpd, np.nan)
|
|
148
|
+
combined_full_dmpd = np.full_like(beam1_full_dmpd, np.nan)
|
|
149
|
+
for cs_step in range(self.constants.num_cs_steps):
|
|
150
|
+
logger.info(
|
|
151
|
+
f"Processing demodulated polcal data for {cs_step = } with {log_str}"
|
|
152
|
+
)
|
|
153
|
+
beam1_dmpd, beam2_dmpd, combined_dmpd = self.process_single_cs_step(
|
|
154
|
+
cs_step=cs_step, calibrations=calibrations, beam_shape=beam_shape
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
# np.moveaxis here because the stokes axis is the last axis in `self.process_single_cs_step`
|
|
158
|
+
# (to make math easier), but we want the DMPD outputs to have stokes before the array axes.
|
|
159
|
+
beam1_full_dmpd[0, cs_step, :, :, :] = np.moveaxis(beam1_dmpd, -1, 0)
|
|
160
|
+
beam2_full_dmpd[0, cs_step, :, :, :] = np.moveaxis(beam2_dmpd, -1, 0)
|
|
161
|
+
combined_full_dmpd[0, cs_step, :, :, :] = np.moveaxis(combined_dmpd, -1, 0)
|
|
162
|
+
|
|
163
|
+
self.write_dmpd(
|
|
164
|
+
beam1_full_dmpd,
|
|
165
|
+
beam=1,
|
|
166
|
+
fit_options=fit_options,
|
|
167
|
+
instrument_options=inst_options,
|
|
168
|
+
)
|
|
169
|
+
del beam1_full_dmpd
|
|
170
|
+
|
|
171
|
+
self.write_dmpd(
|
|
172
|
+
beam2_full_dmpd,
|
|
173
|
+
beam=2,
|
|
174
|
+
fit_options=fit_options,
|
|
175
|
+
instrument_options=inst_options,
|
|
176
|
+
)
|
|
177
|
+
del beam2_full_dmpd
|
|
178
|
+
|
|
179
|
+
self.write_dmpd(
|
|
180
|
+
combined_full_dmpd,
|
|
181
|
+
fit_options=fit_options,
|
|
182
|
+
instrument_options=inst_options,
|
|
183
|
+
beam=None,
|
|
184
|
+
)
|
|
185
|
+
del combined_dmpd
|
|
186
|
+
|
|
187
|
+
def collect_calibration_objects(
|
|
188
|
+
self, fit_options: FitOptions, instrument_options: VispInstrumentOptions
|
|
189
|
+
) -> VispCalibrationCollection:
|
|
190
|
+
"""Collect the INTERMEDIATE calibration objects needed to correct demodulated polcal data."""
|
|
191
|
+
angle_dict = dict()
|
|
192
|
+
shift_dict = dict()
|
|
193
|
+
demod_dict = dict()
|
|
194
|
+
|
|
195
|
+
base_tags = [TrendTag.intermediate(), TrendTag.arm_id(self.arm_id)]
|
|
196
|
+
|
|
197
|
+
# Dark
|
|
198
|
+
######
|
|
199
|
+
full_dark_array = next(
|
|
200
|
+
self.read(
|
|
201
|
+
tags=base_tags + [TrendTag.frame(), TrendTag.task_dark()],
|
|
202
|
+
decoder=fits_array_decoder,
|
|
203
|
+
)
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
# Gain
|
|
207
|
+
#######
|
|
208
|
+
full_gain_array = next(
|
|
209
|
+
self.read(
|
|
210
|
+
tags=base_tags + [TrendTag.frame(), TrendTag.task_gain()],
|
|
211
|
+
decoder=fits_array_decoder,
|
|
212
|
+
)
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
for beam in [1, 2]:
|
|
216
|
+
beam_tags = base_tags + [TrendTag.beam(beam)]
|
|
217
|
+
|
|
218
|
+
# Geo
|
|
219
|
+
#######
|
|
220
|
+
geo_dict = next(
|
|
221
|
+
self.read(
|
|
222
|
+
tags=beam_tags + [TrendTag.task_visp_geometric_calibration()],
|
|
223
|
+
decoder=asdf_decoder,
|
|
224
|
+
)
|
|
225
|
+
)
|
|
226
|
+
angle_dict[TrendTag.beam(beam)] = geo_dict["angle_rad"]
|
|
227
|
+
shift_dict[TrendTag.beam(beam)] = geo_dict["shift"]
|
|
228
|
+
|
|
229
|
+
# Demod
|
|
230
|
+
#######
|
|
231
|
+
demod_matrices = next(
|
|
232
|
+
self.read(
|
|
233
|
+
tags=beam_tags
|
|
234
|
+
+ [
|
|
235
|
+
TrendTag.pac_fit_options(fit_options.name),
|
|
236
|
+
TrendTag.instrument_processing_options(instrument_options.name),
|
|
237
|
+
TrendTag.task_processed_demodulation_matrices(),
|
|
238
|
+
],
|
|
239
|
+
decoder=fits_array_decoder,
|
|
240
|
+
)
|
|
241
|
+
)
|
|
242
|
+
demod_dict[TrendTag.beam(beam)] = demod_matrices
|
|
243
|
+
|
|
244
|
+
return VispCalibrationCollection(
|
|
245
|
+
dark=full_dark_array,
|
|
246
|
+
gain=full_gain_array,
|
|
247
|
+
angle=angle_dict,
|
|
248
|
+
shift=shift_dict,
|
|
249
|
+
demod_matrices=demod_dict,
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
def process_single_cs_step(
|
|
253
|
+
self,
|
|
254
|
+
cs_step: int,
|
|
255
|
+
calibrations: VispCalibrationCollection,
|
|
256
|
+
beam_shape: tuple[int, int],
|
|
257
|
+
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
|
258
|
+
"""
|
|
259
|
+
Fully process the data for a single CS step.
|
|
260
|
+
|
|
261
|
+
To simplify the code, the dark and gain corrections are done on the full-frame before beam extraction.
|
|
262
|
+
Demodulation and geometric corrections are applied on a per-beam basis.
|
|
263
|
+
|
|
264
|
+
All outputs are trimmed to only include the regions that exist in both beams.
|
|
265
|
+
|
|
266
|
+
Returns
|
|
267
|
+
-------
|
|
268
|
+
The corrected beam 1 array
|
|
269
|
+
The corrected beam 2 array
|
|
270
|
+
The corrected combined array
|
|
271
|
+
"""
|
|
272
|
+
modstate_full_frame_data_list = []
|
|
273
|
+
for modstate in range(1, self.constants.num_modstates + 1):
|
|
274
|
+
tags = [
|
|
275
|
+
TrendTag.input(),
|
|
276
|
+
TrendTag.frame(),
|
|
277
|
+
TrendTag.arm_id(self.arm_id),
|
|
278
|
+
TrendTag.task_polcal(),
|
|
279
|
+
TrendTag.cs_step(cs_step),
|
|
280
|
+
TrendTag.modstate(modstate),
|
|
281
|
+
]
|
|
282
|
+
input_arrays = self.read(tags=tags, decoder=fits_array_decoder)
|
|
283
|
+
avg_input = average_numpy_arrays(input_arrays)
|
|
284
|
+
|
|
285
|
+
dark_corrected = subtract_array_from_arrays(
|
|
286
|
+
arrays=avg_input, array_to_subtract=calibrations.dark
|
|
287
|
+
)
|
|
288
|
+
gain_corrected = next(
|
|
289
|
+
divide_arrays_by_array(arrays=dark_corrected, array_to_divide_by=calibrations.gain)
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
modstate_full_frame_data_list.append(gain_corrected)
|
|
293
|
+
|
|
294
|
+
final_beam_data_list = []
|
|
295
|
+
for beam in [1, 2]:
|
|
296
|
+
demod_matrices = calibrations.demod_matrices[TrendTag.beam(beam)]
|
|
297
|
+
beam_stack = np.full(beam_shape + (self.constants.num_modstates,), np.nan)
|
|
298
|
+
for modstate in range(1, self.constants.num_modstates + 1):
|
|
299
|
+
full_frame_data = modstate_full_frame_data_list[modstate - 1]
|
|
300
|
+
beam_data = extract_visp_beam(
|
|
301
|
+
full_frame_data, beam=beam, beam_border=self.parameters.visp_beam_border
|
|
302
|
+
)
|
|
303
|
+
beam_stack[:, :, modstate - 1] = beam_data
|
|
304
|
+
|
|
305
|
+
demodulated_data = nd_left_matrix_multiply(
|
|
306
|
+
vector_stack=beam_stack,
|
|
307
|
+
matrix_stack=demod_matrices,
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
geo_corrected = np.full_like(demodulated_data, np.nan)
|
|
311
|
+
for s in range(demodulated_data.shape[-1]):
|
|
312
|
+
geo_corrected[:, :, s] = self.correct_geometry(
|
|
313
|
+
demodulated_data[:, :, s],
|
|
314
|
+
angle=calibrations.angle[TrendTag.beam(beam)],
|
|
315
|
+
shift=calibrations.shift[TrendTag.beam(beam)],
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
x_slice, y_slice = calibrations.beams_overlap_slice
|
|
319
|
+
cut_array = geo_corrected[x_slice, y_slice, :].astype(np.float32)
|
|
320
|
+
|
|
321
|
+
final_beam_data_list.append(cut_array)
|
|
322
|
+
|
|
323
|
+
combined_data = self.combine_beams(
|
|
324
|
+
beam1_data=final_beam_data_list[0], beam2_data=final_beam_data_list[1]
|
|
325
|
+
).astype(np.float32)
|
|
326
|
+
|
|
327
|
+
return final_beam_data_list[0], final_beam_data_list[1], combined_data
|
|
328
|
+
|
|
329
|
+
def combine_beams(self, beam1_data: np.ndarray, beam2_data: np.ndarray) -> np.ndarray:
|
|
330
|
+
r"""
|
|
331
|
+
Combine polarimetric beams so that polarization states are normalized by the intensity state (Stokes I).
|
|
332
|
+
|
|
333
|
+
In other words:
|
|
334
|
+
|
|
335
|
+
.. math::
|
|
336
|
+
|
|
337
|
+
I_{avg} & = (I_1 + I_2) / 2
|
|
338
|
+
|
|
339
|
+
Q_{avg} & = I_{avg} (Q_1 / I_1 + Q_2 / I_2) / 2
|
|
340
|
+
|
|
341
|
+
...and the same for U and V
|
|
342
|
+
"""
|
|
343
|
+
avg_data = np.zeros_like(beam1_data)
|
|
344
|
+
|
|
345
|
+
# Rely on the fact that the Stokes states are in order after demodulation
|
|
346
|
+
avg_I = (beam1_data[:, :, 0] + beam2_data[:, :, 0]) / 2.0
|
|
347
|
+
avg_data[:, :, 0] = avg_I
|
|
348
|
+
|
|
349
|
+
for stokes in range(1, 4):
|
|
350
|
+
beam1_norm = beam1_data[:, :, stokes] / beam1_data[:, :, 0]
|
|
351
|
+
beam2_norm = beam2_data[:, :, stokes] / beam2_data[:, :, 0]
|
|
352
|
+
avg_data[:, :, stokes] = avg_I * (beam1_norm + beam2_norm) / 2.0
|
|
353
|
+
|
|
354
|
+
return avg_data
|
|
355
|
+
|
|
356
|
+
def correct_geometry(
|
|
357
|
+
self, array: np.ndarray, angle: float, shift: tuple[float, float]
|
|
358
|
+
) -> np.ndarray:
|
|
359
|
+
"""
|
|
360
|
+
Rotate and shift an array.
|
|
361
|
+
|
|
362
|
+
Note that the input ``angle`` and ``shift`` arguments are the *amount to rotate/shift by*. They are passed straight
|
|
363
|
+
through to the affine transform.
|
|
364
|
+
"""
|
|
365
|
+
array_shape = array.shape
|
|
366
|
+
cosine_angle = np.cos(angle)
|
|
367
|
+
sine_angle = np.sin(angle)
|
|
368
|
+
inverse_rotation_matrix = np.array(
|
|
369
|
+
[[cosine_angle, -sine_angle], [sine_angle, cosine_angle]],
|
|
370
|
+
dtype=np.float64,
|
|
371
|
+
)
|
|
372
|
+
|
|
373
|
+
rotation_center = 0.5 * (np.asarray(array_shape, dtype=np.float64) - 1.0)
|
|
374
|
+
transform_offset = rotation_center + shift - inverse_rotation_matrix @ rotation_center
|
|
375
|
+
|
|
376
|
+
corrected_array = affine_transform(
|
|
377
|
+
array,
|
|
378
|
+
matrix=inverse_rotation_matrix,
|
|
379
|
+
offset=transform_offset,
|
|
380
|
+
order=5,
|
|
381
|
+
mode="nearest",
|
|
382
|
+
cval=np.nan,
|
|
383
|
+
prefilter=True,
|
|
384
|
+
)
|
|
385
|
+
return corrected_array
|
|
386
|
+
|
|
387
|
+
def write_dmpd(
|
|
388
|
+
self,
|
|
389
|
+
array: np.ndarray,
|
|
390
|
+
beam: int | None,
|
|
391
|
+
fit_options: FitOptions,
|
|
392
|
+
instrument_options: VispInstrumentOptions,
|
|
393
|
+
) -> None:
|
|
394
|
+
"""
|
|
395
|
+
Write a demodulated polcal data output to scratch.
|
|
396
|
+
|
|
397
|
+
If ``beam`` is not `None` then the output will be tagged as a `~dkist_processing_trend.models.tags.TrendTag.task_single_beam_demodulated_polcal_data`,
|
|
398
|
+
otherwise the output is tagged with `~dkist_processing_trend.models.tags.TrendTag.task_demodulated_polcal_data`.
|
|
399
|
+
"""
|
|
400
|
+
tags = [
|
|
401
|
+
TrendTag.intermediate(),
|
|
402
|
+
TrendTag.arm_id(self.arm_id),
|
|
403
|
+
TrendTag.pac_fit_options(fit_options.name),
|
|
404
|
+
TrendTag.instrument_processing_options(instrument_options.name),
|
|
405
|
+
]
|
|
406
|
+
if beam is not None:
|
|
407
|
+
tags += [TrendTag.beam(beam), TrendTag.task_single_beam_demodulated_polcal_data()]
|
|
408
|
+
else:
|
|
409
|
+
tags.append(TrendTag.task_demodulated_polcal_data())
|
|
410
|
+
self.write(data=array, tags=tags, encoder=fits_array_encoder)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Function to split apart the two ViSP beams."""
|
|
2
|
+
|
|
3
|
+
from typing import Literal
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def extract_visp_beam(array: np.ndarray, beam: Literal[1, 2], beam_border: int) -> np.ndarray:
|
|
9
|
+
"""Extract a single beam from a raw, dual-beam array."""
|
|
10
|
+
if beam == 1:
|
|
11
|
+
return np.copy(array[:beam_border, ...])
|
|
12
|
+
if beam == 2:
|
|
13
|
+
return np.copy(array[beam_border:, ...][::-1, :])
|
|
14
|
+
raise ValueError(f"ViSP only has beams 1 and 2. Can't extract data for beam '{beam}'.")
|
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
"""Task for computing the ViSP beam offset."""
|
|
2
|
+
|
|
3
|
+
from typing import Literal
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
import peakutils as pku
|
|
7
|
+
import scipy.ndimage as spnd
|
|
8
|
+
from astropy.modeling import fitting
|
|
9
|
+
from astropy.modeling import models
|
|
10
|
+
from astropy.stats import sigma_clip
|
|
11
|
+
from dkist_processing_common.codecs.asdf import asdf_encoder
|
|
12
|
+
from dkist_processing_common.codecs.fits import fits_array_decoder
|
|
13
|
+
from dkist_processing_math.statistics import average_numpy_arrays
|
|
14
|
+
from dkist_service_configuration.logging import logger
|
|
15
|
+
from skimage.registration import phase_cross_correlation
|
|
16
|
+
|
|
17
|
+
from dkist_processing_trend.models.tags import TrendTag
|
|
18
|
+
from dkist_processing_trend.tasks.trend_base import TrendArmTaskBase
|
|
19
|
+
from dkist_processing_trend.tasks.visp.visp_extract_beam import extract_visp_beam
|
|
20
|
+
|
|
21
|
+
__all__ = ["VispGeometricCalibration"]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class VispGeometricCalibration(TrendArmTaskBase):
|
|
25
|
+
"""
|
|
26
|
+
Compute the X/Y offset between the two ViSP beams.
|
|
27
|
+
|
|
28
|
+
Unlike the main L1 pipeline, this task does not treat each modstate separately and does not compute a rotational offset.
|
|
29
|
+
For the purposes of the Trend pipeline a simple X/Y shift is sufficient.
|
|
30
|
+
|
|
31
|
+
Parameters
|
|
32
|
+
----------
|
|
33
|
+
arm_id
|
|
34
|
+
id of the instrument arm to operate on
|
|
35
|
+
|
|
36
|
+
recipe_run_id
|
|
37
|
+
id of the recipe run used to identify the workflow run this task is part of
|
|
38
|
+
|
|
39
|
+
workflow_name
|
|
40
|
+
name of the workflow to which this instance of the task belongs
|
|
41
|
+
|
|
42
|
+
workflow_version
|
|
43
|
+
version of the workflow to which this instance of the task belongs
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
record_provenance = True
|
|
47
|
+
|
|
48
|
+
def run(self):
|
|
49
|
+
"""
|
|
50
|
+
Compute the X/Y shift between ViSP beam 1 and 2.
|
|
51
|
+
|
|
52
|
+
Algorithm:
|
|
53
|
+
|
|
54
|
+
#. Compute the average of all input POLCAL_GAIN frames (i.e., "clear" CS step)
|
|
55
|
+
#. Split this average into beams 1 and 2
|
|
56
|
+
#. Compute the rotation angle for each beam
|
|
57
|
+
#. Correct each beam for its measured rotation angle
|
|
58
|
+
#. Collapse each beam along both the spectral and spatial axis and use `~skimage.registration.phase_cross_correlation` to compute the shift along each axis
|
|
59
|
+
#. Write the shifts and angles to scratch
|
|
60
|
+
|
|
61
|
+
NOTE: The outputs of this task are angles and shifts that should be used *directly* with the geometric correction
|
|
62
|
+
method; no inverting/sign flipping is needed.
|
|
63
|
+
"""
|
|
64
|
+
with self.telemetry_span("Compute average clear array"):
|
|
65
|
+
logger.info("Computing average clear array")
|
|
66
|
+
full_avg_array = self.compute_avg_clear_array()
|
|
67
|
+
|
|
68
|
+
logger.info("Extracting beams")
|
|
69
|
+
beam_border = self.parameters.visp_beam_border
|
|
70
|
+
beam1_array = extract_visp_beam(full_avg_array, beam=1, beam_border=beam_border)
|
|
71
|
+
beam2_array = extract_visp_beam(full_avg_array, beam=2, beam_border=beam_border)
|
|
72
|
+
|
|
73
|
+
with self.telemetry_span("Compute beam angles"):
|
|
74
|
+
logger.info("Computing beam angles")
|
|
75
|
+
beam1_angle_rad = self.compute_beam_angle(array=beam1_array)
|
|
76
|
+
beam1_angle_deg = np.rad2deg(beam1_angle_rad)
|
|
77
|
+
logger.info(f"Beam 1 angle = {beam1_angle_deg:.5f} deg")
|
|
78
|
+
|
|
79
|
+
beam2_angle_rad = self.compute_beam_angle(array=beam2_array)
|
|
80
|
+
beam2_angle_deg = np.rad2deg(beam2_angle_rad)
|
|
81
|
+
logger.info(f"Beam 2 angle = {beam2_angle_deg:.5f} deg")
|
|
82
|
+
|
|
83
|
+
with self.telemetry_span("Compute beam offset"):
|
|
84
|
+
logger.info("Computing beam offsets")
|
|
85
|
+
|
|
86
|
+
logger.info("Removing beam angles")
|
|
87
|
+
beam1_rotated = spnd.rotate(
|
|
88
|
+
beam1_array, angle=-beam1_angle_deg, reshape=False, cval=np.nanmedian(beam1_array)
|
|
89
|
+
)
|
|
90
|
+
beam2_rotated = spnd.rotate(
|
|
91
|
+
beam2_array, angle=-beam2_angle_deg, reshape=False, cval=np.nanmedian(beam2_array)
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
logger.info("Filtering beams")
|
|
95
|
+
beam1_filtered = self.high_pass_filter_array(beam1_rotated)
|
|
96
|
+
beam2_filtered = self.high_pass_filter_array(beam2_rotated)
|
|
97
|
+
|
|
98
|
+
shift = np.full(2, np.nan)
|
|
99
|
+
for i, (ax_name, ax) in enumerate(zip(["spectral", "spatial"], [0, 1])):
|
|
100
|
+
logger.info(f"Computing {ax_name} shift")
|
|
101
|
+
shift[i] = self.compute_single_axis_shift(
|
|
102
|
+
reference_array=beam1_filtered, target_array=beam2_filtered, axis=ax
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
logger.info(f"Measured {shift = }")
|
|
106
|
+
|
|
107
|
+
logger.info("Writing angles and shift to scratch")
|
|
108
|
+
beam1_geo_data = {"angle_rad": beam1_angle_rad, "shift": np.array([0, 0.0])}
|
|
109
|
+
self.write(
|
|
110
|
+
data=beam1_geo_data,
|
|
111
|
+
tags=[
|
|
112
|
+
TrendTag.intermediate(),
|
|
113
|
+
TrendTag.arm_id(self.arm_id),
|
|
114
|
+
TrendTag.beam(1),
|
|
115
|
+
TrendTag.task_visp_geometric_calibration(),
|
|
116
|
+
],
|
|
117
|
+
encoder=asdf_encoder,
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
beam2_geo_data = {
|
|
121
|
+
"angle_rad": beam2_angle_rad,
|
|
122
|
+
"shift": -shift,
|
|
123
|
+
}
|
|
124
|
+
self.write(
|
|
125
|
+
data=beam2_geo_data,
|
|
126
|
+
tags=[
|
|
127
|
+
TrendTag.intermediate(),
|
|
128
|
+
TrendTag.arm_id(self.arm_id),
|
|
129
|
+
TrendTag.beam(2),
|
|
130
|
+
TrendTag.task_visp_geometric_calibration(),
|
|
131
|
+
],
|
|
132
|
+
encoder=asdf_encoder,
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
def compute_avg_clear_array(self) -> np.ndarray:
|
|
136
|
+
"""Compute the average of all "clear" CS step arrays."""
|
|
137
|
+
tags = [TrendTag.input(), TrendTag.arm_id(self.arm_id), TrendTag.task_polcal_gain()]
|
|
138
|
+
logger.info(f"Using {self.count(tags)} INPUT clear arrays.")
|
|
139
|
+
clear_arrays = self.read(tags=tags, decoder=fits_array_decoder)
|
|
140
|
+
avg_array = average_numpy_arrays(clear_arrays)
|
|
141
|
+
return avg_array
|
|
142
|
+
|
|
143
|
+
def compute_beam_angle(self, array: np.ndarray) -> float:
|
|
144
|
+
"""
|
|
145
|
+
Compute the rotation angle of a single beam.
|
|
146
|
+
|
|
147
|
+
The angle is computed by measuring the slope of the slit hairlines; the arctan of the slope then gives the rotation angle.
|
|
148
|
+
To measure the slope of the hairlines, first the hairline signal is isolated with by smoothing the image in the spatial
|
|
149
|
+
direction (which will remove the hairlines) and then subtracting this smoothed array from the input image.
|
|
150
|
+
Once the hairlines have been isolated, the location of hairline center is measured with `peakutils <https://pypi.org/project/PeakUtils/>`_.
|
|
151
|
+
Finally, a line is fit to the the location of hairline center as a function of wavelength and the slope of this line is
|
|
152
|
+
used to find the angle.
|
|
153
|
+
|
|
154
|
+
Multiple hairlines (always 2, and sometimes more if there are reflections) have their angles computed separately
|
|
155
|
+
and then average together for the final result.
|
|
156
|
+
|
|
157
|
+
Returns
|
|
158
|
+
-------
|
|
159
|
+
The rotation angle of the given array, in radians.
|
|
160
|
+
"""
|
|
161
|
+
no_hairline_array = spnd.median_filter(array, size=(1, 30))
|
|
162
|
+
hairline_only_array = (array - no_hairline_array) / no_hairline_array
|
|
163
|
+
|
|
164
|
+
# Turn low-signal hairlines into "emission" lines so peakutils can find them
|
|
165
|
+
hairline_only_array *= -1
|
|
166
|
+
|
|
167
|
+
hairline_1d = np.nanmedian(hairline_only_array, axis=0)
|
|
168
|
+
hairline_locs = pku.indexes(hairline_1d, thres=0.8)
|
|
169
|
+
|
|
170
|
+
num_wave, num_spat = array.shape
|
|
171
|
+
fit_centers = np.full((len(hairline_locs), num_wave), np.nan)
|
|
172
|
+
spectral_abscissa = np.arange(num_wave)
|
|
173
|
+
spatial_abscissa = np.arange(num_spat)
|
|
174
|
+
for i in range(num_wave):
|
|
175
|
+
try:
|
|
176
|
+
interp_center = pku.interpolate(
|
|
177
|
+
spatial_abscissa, hairline_only_array[i, :], ind=hairline_locs, width=10
|
|
178
|
+
)
|
|
179
|
+
fit_centers[:, i] = interp_center
|
|
180
|
+
except:
|
|
181
|
+
logger.info(f"Failed to fit hairline center for spectral pixel {i}. Ignoring.")
|
|
182
|
+
|
|
183
|
+
hairline_angles = []
|
|
184
|
+
for hairline in range(fit_centers.shape[0]):
|
|
185
|
+
fit_angle = self.fit_angle_from_slope(
|
|
186
|
+
abscissa=spectral_abscissa, centers=fit_centers[hairline]
|
|
187
|
+
)
|
|
188
|
+
hairline_angles.append(fit_angle)
|
|
189
|
+
|
|
190
|
+
return np.mean(hairline_angles)
|
|
191
|
+
|
|
192
|
+
@staticmethod
|
|
193
|
+
def fit_angle_from_slope(abscissa: np.ndarray, centers: np.ndarray, sigma: float = 3) -> float:
|
|
194
|
+
"""
|
|
195
|
+
Fit a 1-degree polynomial (i.e., a line) to the hairline centers and compute an angle from the slope.
|
|
196
|
+
|
|
197
|
+
Outliers are iteratively removed using a sigma clipping algorithm as detailed here:
|
|
198
|
+
https://docs.astropy.org/en/stable/modeling/example-fitting-line.html#iterative-fitting-using-sigma-clipping
|
|
199
|
+
|
|
200
|
+
Returns
|
|
201
|
+
-------
|
|
202
|
+
The slope of the line, in radians.
|
|
203
|
+
"""
|
|
204
|
+
line_fitter = fitting.LinearLSQFitter()
|
|
205
|
+
outlier_rejection_fitter = fitting.FittingWithOutlierRemoval(
|
|
206
|
+
fitter=line_fitter,
|
|
207
|
+
outlier_func=sigma_clip,
|
|
208
|
+
sigma=sigma,
|
|
209
|
+
cenfunc="median",
|
|
210
|
+
stdfunc="std",
|
|
211
|
+
)
|
|
212
|
+
linear_model = models.Linear1D()
|
|
213
|
+
|
|
214
|
+
fit_line = outlier_rejection_fitter(model=linear_model, x=abscissa, y=centers)[0]
|
|
215
|
+
angle_rad = np.arctan(fit_line.slope.value)
|
|
216
|
+
return angle_rad
|
|
217
|
+
|
|
218
|
+
@staticmethod
|
|
219
|
+
def high_pass_filter_array(array: np.ndarray) -> np.ndarray:
|
|
220
|
+
"""
|
|
221
|
+
Remove low-frequency signal in an array by dividing it by a smoothed copy.
|
|
222
|
+
|
|
223
|
+
The smoothing is done with a Gaussian smoothing kernel. The resulting array has strong spectral features
|
|
224
|
+
and hairlines accentuated over the continuum.
|
|
225
|
+
"""
|
|
226
|
+
# The sigma value is hardcoded in the ViSP L1 pipeline, too. It's valid for all ViSP data for all time.
|
|
227
|
+
return array / spnd.gaussian_filter(array, sigma=5)
|
|
228
|
+
|
|
229
|
+
def compute_single_axis_shift(
|
|
230
|
+
self, reference_array: np.ndarray, target_array: np.ndarray, axis: Literal[0, 1]
|
|
231
|
+
) -> float:
|
|
232
|
+
"""
|
|
233
|
+
Compute the offset between two arrays along a single axis.
|
|
234
|
+
|
|
235
|
+
ViSP data have strong, high-frequency signals along both of their axes; the solar spectrum and the slit hairlines.
|
|
236
|
+
This method isolates one of those signals by first computing a derivative along the axis of interest. Because
|
|
237
|
+
the two signals are orthogonal this derivative both accentuates the strong features along the axis of interest and
|
|
238
|
+
removes the signal in the orthogonal direction (e.g., the hairlines are constant along the spectral axis).
|
|
239
|
+
|
|
240
|
+
The resulting derivative is then collapsed along the orthogonal axis, via median, into a single vector, which
|
|
241
|
+
increases the signal to noise and simplifies the final correlation.
|
|
242
|
+
|
|
243
|
+
The single vectors from both arrays are then registered via `~skimage.registration.phase_cross_correlation` to
|
|
244
|
+
produce the final shift.
|
|
245
|
+
"""
|
|
246
|
+
opposite_axis = abs(axis - 1)
|
|
247
|
+
|
|
248
|
+
reference_deriv = np.diff(reference_array, axis=axis)
|
|
249
|
+
reference_1d_signal = np.nanmedian(reference_deriv, axis=opposite_axis)
|
|
250
|
+
|
|
251
|
+
target_deriv = np.diff(target_array, axis=axis)
|
|
252
|
+
target_1d_signal = np.nanmedian(target_deriv, axis=opposite_axis)
|
|
253
|
+
|
|
254
|
+
shift = phase_cross_correlation(
|
|
255
|
+
reference_image=reference_1d_signal,
|
|
256
|
+
moving_image=target_1d_signal,
|
|
257
|
+
upsample_factor=self.parameters.visp_geo_upsample_factor,
|
|
258
|
+
normalization=None,
|
|
259
|
+
)[0]
|
|
260
|
+
return shift.item(0)
|