dkist-processing-visp 3.3.0__py3-none-any.whl → 5.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dkist_processing_visp/__init__.py +1 -0
- dkist_processing_visp/config.py +1 -0
- dkist_processing_visp/models/constants.py +52 -21
- dkist_processing_visp/models/fits_access.py +20 -0
- dkist_processing_visp/models/metric_code.py +10 -0
- dkist_processing_visp/models/parameters.py +129 -19
- dkist_processing_visp/models/tags.py +1 -0
- dkist_processing_visp/models/task_name.py +1 -0
- dkist_processing_visp/parsers/map_repeats.py +1 -0
- dkist_processing_visp/parsers/modulator_states.py +1 -0
- dkist_processing_visp/parsers/polarimeter_mode.py +3 -1
- dkist_processing_visp/parsers/raster_step.py +4 -1
- dkist_processing_visp/parsers/spectrograph_configuration.py +75 -0
- dkist_processing_visp/parsers/time.py +15 -7
- dkist_processing_visp/parsers/visp_l0_fits_access.py +19 -8
- dkist_processing_visp/parsers/visp_l1_fits_access.py +1 -0
- dkist_processing_visp/tasks/__init__.py +1 -0
- dkist_processing_visp/tasks/assemble_movie.py +1 -0
- dkist_processing_visp/tasks/background_light.py +2 -1
- dkist_processing_visp/tasks/dark.py +5 -4
- dkist_processing_visp/tasks/geometric.py +132 -20
- dkist_processing_visp/tasks/instrument_polarization.py +13 -12
- dkist_processing_visp/tasks/l1_output_data.py +203 -0
- dkist_processing_visp/tasks/lamp.py +53 -93
- dkist_processing_visp/tasks/make_movie_frames.py +8 -6
- dkist_processing_visp/tasks/mixin/beam_access.py +1 -0
- dkist_processing_visp/tasks/mixin/corrections.py +54 -4
- dkist_processing_visp/tasks/mixin/downsample.py +1 -0
- dkist_processing_visp/tasks/parse.py +34 -4
- dkist_processing_visp/tasks/quality_metrics.py +5 -4
- dkist_processing_visp/tasks/science.py +126 -46
- dkist_processing_visp/tasks/solar.py +896 -456
- dkist_processing_visp/tasks/visp_base.py +2 -0
- dkist_processing_visp/tasks/write_l1.py +25 -5
- dkist_processing_visp/tests/conftest.py +99 -35
- dkist_processing_visp/tests/header_models.py +92 -20
- dkist_processing_visp/tests/local_trial_workflows/l0_cals_only.py +4 -23
- dkist_processing_visp/tests/local_trial_workflows/l0_polcals_as_science.py +421 -0
- dkist_processing_visp/tests/local_trial_workflows/l0_solar_gain_as_science.py +10 -29
- dkist_processing_visp/tests/local_trial_workflows/l0_to_l1.py +1 -21
- dkist_processing_visp/tests/local_trial_workflows/local_trial_helpers.py +98 -14
- dkist_processing_visp/tests/test_assemble_movie.py +2 -3
- dkist_processing_visp/tests/test_assemble_quality.py +89 -4
- dkist_processing_visp/tests/test_background_light.py +8 -5
- dkist_processing_visp/tests/test_dark.py +4 -3
- dkist_processing_visp/tests/test_fits_access.py +43 -0
- dkist_processing_visp/tests/test_geometric.py +45 -4
- dkist_processing_visp/tests/test_instrument_polarization.py +4 -3
- dkist_processing_visp/tests/test_lamp.py +22 -26
- dkist_processing_visp/tests/test_make_movie_frames.py +4 -4
- dkist_processing_visp/tests/test_map_repeats.py +3 -1
- dkist_processing_visp/tests/test_parameters.py +122 -21
- dkist_processing_visp/tests/test_parse.py +98 -14
- dkist_processing_visp/tests/test_quality.py +2 -3
- dkist_processing_visp/tests/test_science.py +113 -15
- dkist_processing_visp/tests/test_solar.py +318 -99
- dkist_processing_visp/tests/test_visp_constants.py +36 -8
- dkist_processing_visp/tests/test_workflows.py +1 -0
- dkist_processing_visp/tests/test_write_l1.py +17 -3
- dkist_processing_visp/workflows/__init__.py +1 -0
- dkist_processing_visp/workflows/l0_processing.py +8 -2
- dkist_processing_visp/workflows/trial_workflows.py +8 -2
- dkist_processing_visp-5.1.1.dist-info/METADATA +552 -0
- dkist_processing_visp-5.1.1.dist-info/RECORD +94 -0
- docs/conf.py +5 -1
- docs/gain_correction.rst +50 -42
- dkist_processing_visp/tasks/mixin/line_zones.py +0 -115
- dkist_processing_visp-3.3.0.dist-info/METADATA +0 -459
- dkist_processing_visp-3.3.0.dist-info/RECORD +0 -90
- {dkist_processing_visp-3.3.0.dist-info → dkist_processing_visp-5.1.1.dist-info}/WHEEL +0 -0
- {dkist_processing_visp-3.3.0.dist-info → dkist_processing_visp-5.1.1.dist-info}/top_level.txt +0 -0
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""ViSP background light calibration task."""
|
|
2
|
+
|
|
2
3
|
import gc
|
|
3
4
|
import time
|
|
4
5
|
|
|
@@ -114,7 +115,7 @@ class BackgroundLightCalibration(
|
|
|
114
115
|
del full_background_light
|
|
115
116
|
gc.collect()
|
|
116
117
|
|
|
117
|
-
with self.
|
|
118
|
+
with self.telemetry_span("Computing and logging quality metrics"):
|
|
118
119
|
self.quality_store_task_type_counts(
|
|
119
120
|
task_type=VispTaskName.background.value,
|
|
120
121
|
total_frames=num_used_polcal_files,
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""Visp dark task."""
|
|
2
|
+
|
|
2
3
|
from dkist_processing_common.codecs.fits import fits_access_decoder
|
|
3
4
|
from dkist_processing_common.codecs.fits import fits_array_encoder
|
|
4
5
|
from dkist_processing_common.models.task_name import TaskName
|
|
@@ -49,7 +50,7 @@ class DarkCalibration(VispTaskBase, BeamAccessMixin, QualityMixin):
|
|
|
49
50
|
required_readout_exp_times = list(self.constants.non_dark_task_readout_exp_times)
|
|
50
51
|
logger.info(f"{required_readout_exp_times = }")
|
|
51
52
|
|
|
52
|
-
with self.
|
|
53
|
+
with self.telemetry_span(
|
|
53
54
|
f"Calculating dark frames for {self.constants.num_beams} beams and "
|
|
54
55
|
f"{len(required_readout_exp_times)} readout exp times"
|
|
55
56
|
):
|
|
@@ -74,7 +75,7 @@ class DarkCalibration(VispTaskBase, BeamAccessMixin, QualityMixin):
|
|
|
74
75
|
fits_access_class=VispL0FitsAccess,
|
|
75
76
|
)
|
|
76
77
|
|
|
77
|
-
with self.
|
|
78
|
+
with self.telemetry_span(
|
|
78
79
|
f"Calculating dark for {readout_exp_time = } and {beam = }"
|
|
79
80
|
):
|
|
80
81
|
readout_normalized_arrays = (
|
|
@@ -83,7 +84,7 @@ class DarkCalibration(VispTaskBase, BeamAccessMixin, QualityMixin):
|
|
|
83
84
|
)
|
|
84
85
|
averaged_dark_array = average_numpy_arrays(readout_normalized_arrays)
|
|
85
86
|
|
|
86
|
-
with self.
|
|
87
|
+
with self.telemetry_span(f"Writing dark for {readout_exp_time = } {beam = }"):
|
|
87
88
|
self.write(
|
|
88
89
|
data=averaged_dark_array,
|
|
89
90
|
tags=VispTag.intermediate_frame_dark(
|
|
@@ -92,7 +93,7 @@ class DarkCalibration(VispTaskBase, BeamAccessMixin, QualityMixin):
|
|
|
92
93
|
encoder=fits_array_encoder,
|
|
93
94
|
)
|
|
94
95
|
|
|
95
|
-
with self.
|
|
96
|
+
with self.telemetry_span("Computing and logging quality metrics"):
|
|
96
97
|
no_of_raw_dark_frames: int = self.scratch.count_all(
|
|
97
98
|
tags=[
|
|
98
99
|
VispTag.input(),
|
|
@@ -3,12 +3,15 @@ Visp geometric calibration task.
|
|
|
3
3
|
|
|
4
4
|
See :doc:`this page </geometric>` for more information.
|
|
5
5
|
"""
|
|
6
|
+
|
|
6
7
|
from typing import Generator
|
|
7
8
|
|
|
8
9
|
import numpy as np
|
|
10
|
+
import peakutils
|
|
9
11
|
import peakutils as pku
|
|
10
12
|
import scipy.ndimage as spnd
|
|
11
13
|
import scipy.optimize as spo
|
|
14
|
+
import scipy.signal as sps
|
|
12
15
|
import skimage.exposure as skie
|
|
13
16
|
import skimage.metrics as skim
|
|
14
17
|
import skimage.morphology as skimo
|
|
@@ -32,7 +35,6 @@ from dkist_processing_visp.models.tags import VispTag
|
|
|
32
35
|
from dkist_processing_visp.parsers.visp_l0_fits_access import VispL0FitsAccess
|
|
33
36
|
from dkist_processing_visp.tasks.mixin.beam_access import BeamAccessMixin
|
|
34
37
|
from dkist_processing_visp.tasks.mixin.corrections import CorrectionsMixin
|
|
35
|
-
from dkist_processing_visp.tasks.mixin.line_zones import LineZonesMixin
|
|
36
38
|
from dkist_processing_visp.tasks.visp_base import VispTaskBase
|
|
37
39
|
|
|
38
40
|
__all__ = ["GeometricCalibration"]
|
|
@@ -43,7 +45,6 @@ class GeometricCalibration(
|
|
|
43
45
|
BeamAccessMixin,
|
|
44
46
|
CorrectionsMixin,
|
|
45
47
|
QualityMixin,
|
|
46
|
-
LineZonesMixin,
|
|
47
48
|
):
|
|
48
49
|
"""
|
|
49
50
|
Task class for computing the spectral geometry. Geometry is represented by three quantities.
|
|
@@ -85,15 +86,15 @@ class GeometricCalibration(
|
|
|
85
86
|
"""
|
|
86
87
|
# This lives outside the run() loops and has its own internal loops because the angle calculation
|
|
87
88
|
# only happens for a single modstate
|
|
88
|
-
with self.
|
|
89
|
+
with self.telemetry_span("Basic corrections"):
|
|
89
90
|
self.do_basic_corrections()
|
|
90
91
|
|
|
91
92
|
for beam in range(1, self.constants.num_beams + 1):
|
|
92
|
-
with self.
|
|
93
|
-
with self.
|
|
93
|
+
with self.telemetry_span(f"Generating geometric calibrations for {beam = }"):
|
|
94
|
+
with self.telemetry_span(f"Computing and writing angle for {beam = }"):
|
|
94
95
|
angle = self.compute_beam_angle(beam=beam)
|
|
95
96
|
if beam == 2:
|
|
96
|
-
with self.
|
|
97
|
+
with self.telemetry_span("Refining beam 2 angle"):
|
|
97
98
|
ang_corr = self.refine_beam2_angle(angle)
|
|
98
99
|
logger.info(f"Beam 2 angle refinement = {np.rad2deg(ang_corr)} deg")
|
|
99
100
|
angle += ang_corr
|
|
@@ -101,14 +102,12 @@ class GeometricCalibration(
|
|
|
101
102
|
self.write_angle(angle=angle, beam=beam)
|
|
102
103
|
|
|
103
104
|
for modstate in range(1, self.constants.num_modstates + 1):
|
|
104
|
-
with self.
|
|
105
|
-
f"Removing angle from {beam = } and {modstate = }"
|
|
106
|
-
):
|
|
105
|
+
with self.telemetry_span(f"Removing angle from {beam = } and {modstate = }"):
|
|
107
106
|
angle_corr_array = self.remove_beam_angle(
|
|
108
107
|
angle=angle, beam=beam, modstate=modstate
|
|
109
108
|
)
|
|
110
109
|
|
|
111
|
-
with self.
|
|
110
|
+
with self.telemetry_span(
|
|
112
111
|
f"Computing state offset for {beam = } and {modstate = }"
|
|
113
112
|
):
|
|
114
113
|
state_offset = self.compute_modstate_offset(
|
|
@@ -116,7 +115,7 @@ class GeometricCalibration(
|
|
|
116
115
|
)
|
|
117
116
|
self.write_state_offset(offset=state_offset, beam=beam, modstate=modstate)
|
|
118
117
|
|
|
119
|
-
with self.
|
|
118
|
+
with self.telemetry_span(
|
|
120
119
|
f"Removing state offset for {beam = } and {modstate = }"
|
|
121
120
|
):
|
|
122
121
|
self.remove_state_offset(
|
|
@@ -126,13 +125,13 @@ class GeometricCalibration(
|
|
|
126
125
|
modstate=modstate,
|
|
127
126
|
)
|
|
128
127
|
|
|
129
|
-
with self.
|
|
128
|
+
with self.telemetry_span(f"Computing spectral shifts for {beam = }"):
|
|
130
129
|
spec_shifts = self.compute_spectral_shifts(beam=beam)
|
|
131
130
|
|
|
132
|
-
with self.
|
|
131
|
+
with self.telemetry_span(f"Writing spectral shifts for {beam = }"):
|
|
133
132
|
self.write_spectral_shifts(shifts=spec_shifts, beam=beam)
|
|
134
133
|
|
|
135
|
-
with self.
|
|
134
|
+
with self.telemetry_span("Computing and logging quality metrics"):
|
|
136
135
|
no_of_raw_geo_frames: int = self.scratch.count_all(
|
|
137
136
|
tags=[
|
|
138
137
|
VispTag.input(),
|
|
@@ -275,7 +274,12 @@ class GeometricCalibration(
|
|
|
275
274
|
self.prep_input_solar_gain()
|
|
276
275
|
|
|
277
276
|
def prep_lamp_gain(self):
|
|
278
|
-
"""
|
|
277
|
+
"""
|
|
278
|
+
Create average, dark-corrected lamp gain images for each modstate from INPUT lamp gains.
|
|
279
|
+
|
|
280
|
+
This is different from the results of the `~dkist_processing_visp.tasks.lamp.LampCalibration` task because in
|
|
281
|
+
that task the hairlines are masked out, but here we *need* the hairlines to compute the rotation angle.
|
|
282
|
+
"""
|
|
279
283
|
for readout_exp_time in self.constants.lamp_readout_exp_times:
|
|
280
284
|
for beam in range(1, self.constants.num_beams + 1):
|
|
281
285
|
logger.info(
|
|
@@ -1075,11 +1079,11 @@ class GeometricCalibration(
|
|
|
1075
1079
|
`compute_single_state_offset`.
|
|
1076
1080
|
"""
|
|
1077
1081
|
zone_kwargs = {
|
|
1078
|
-
"prominence": self.parameters.
|
|
1079
|
-
"width": self.parameters.
|
|
1080
|
-
"bg_order": self.parameters.
|
|
1081
|
-
"normalization_percentile": self.parameters.
|
|
1082
|
-
"rel_height": self.parameters.
|
|
1082
|
+
"prominence": self.parameters.geo_zone_prominence,
|
|
1083
|
+
"width": self.parameters.geo_zone_width,
|
|
1084
|
+
"bg_order": self.parameters.geo_zone_bg_order,
|
|
1085
|
+
"normalization_percentile": self.parameters.geo_zone_normalization_percentile,
|
|
1086
|
+
"rel_height": self.parameters.geo_zone_rel_height,
|
|
1083
1087
|
}
|
|
1084
1088
|
zones = self.compute_line_zones(array, **zone_kwargs)
|
|
1085
1089
|
logger.info(f"Found {zones = }")
|
|
@@ -1089,6 +1093,114 @@ class GeometricCalibration(
|
|
|
1089
1093
|
|
|
1090
1094
|
return mask
|
|
1091
1095
|
|
|
1096
|
+
def compute_line_zones(
|
|
1097
|
+
self,
|
|
1098
|
+
spec_2d: np.ndarray,
|
|
1099
|
+
prominence: float = 0.2,
|
|
1100
|
+
width: float = 2,
|
|
1101
|
+
bg_order: int = 22,
|
|
1102
|
+
normalization_percentile: int = 99,
|
|
1103
|
+
rel_height: float = 0.97,
|
|
1104
|
+
) -> list[tuple[int, int]]:
|
|
1105
|
+
"""
|
|
1106
|
+
Identify spectral regions around strong spectra features.
|
|
1107
|
+
|
|
1108
|
+
Parameters
|
|
1109
|
+
----------
|
|
1110
|
+
spec_2d
|
|
1111
|
+
Data
|
|
1112
|
+
|
|
1113
|
+
prominence
|
|
1114
|
+
Zone prominence threshold used to identify strong spectral features
|
|
1115
|
+
|
|
1116
|
+
width
|
|
1117
|
+
Zone width
|
|
1118
|
+
|
|
1119
|
+
bg_order
|
|
1120
|
+
Order of polynomial fit used to remove continuum when identifying strong spectral features
|
|
1121
|
+
|
|
1122
|
+
normalization_percentile
|
|
1123
|
+
Compute this percentile of the data along a specified axis
|
|
1124
|
+
|
|
1125
|
+
rel_height
|
|
1126
|
+
The relative height at which the peak width is measured as a percentage of its prominence. E.g., 1.0 measures
|
|
1127
|
+
the peak width at the lowest contour line.
|
|
1128
|
+
|
|
1129
|
+
Returns
|
|
1130
|
+
-------
|
|
1131
|
+
regions
|
|
1132
|
+
List of indices defining the found spectral lines
|
|
1133
|
+
|
|
1134
|
+
"""
|
|
1135
|
+
logger.info(
|
|
1136
|
+
f"Finding zones using {prominence=}, {width=}, {bg_order=}, {normalization_percentile=}, and {rel_height=}"
|
|
1137
|
+
)
|
|
1138
|
+
# Compute average along slit to improve signal. Line smearing isn't important here
|
|
1139
|
+
avg_1d = np.mean(spec_2d, axis=1)
|
|
1140
|
+
|
|
1141
|
+
# Convert to an emission spectrum and remove baseline continuum so peakutils has an easier time
|
|
1142
|
+
em_spec = -1 * avg_1d + avg_1d.max()
|
|
1143
|
+
em_spec /= np.nanpercentile(em_spec, normalization_percentile)
|
|
1144
|
+
baseline = peakutils.baseline(em_spec, bg_order)
|
|
1145
|
+
em_spec -= baseline
|
|
1146
|
+
|
|
1147
|
+
# Find indices of peaks
|
|
1148
|
+
peak_idxs = sps.find_peaks(em_spec, prominence=prominence, width=width)[0]
|
|
1149
|
+
|
|
1150
|
+
# Find the rough width based only on the height of the peak
|
|
1151
|
+
# rips and lips are the right and left borders of the region around the peak
|
|
1152
|
+
_, _, rips, lips = sps.peak_widths(em_spec, peak_idxs, rel_height=rel_height)
|
|
1153
|
+
|
|
1154
|
+
# Convert to ints so they can be used as indices
|
|
1155
|
+
rips = np.floor(rips).astype(int)
|
|
1156
|
+
lips = np.ceil(lips).astype(int)
|
|
1157
|
+
|
|
1158
|
+
# Remove any regions that are contained within another region
|
|
1159
|
+
ranges_to_remove = self.identify_overlapping_zones(rips, lips)
|
|
1160
|
+
rips = np.delete(rips, ranges_to_remove)
|
|
1161
|
+
lips = np.delete(lips, ranges_to_remove)
|
|
1162
|
+
|
|
1163
|
+
return list(zip(rips, lips))
|
|
1164
|
+
|
|
1165
|
+
@staticmethod
|
|
1166
|
+
def identify_overlapping_zones(rips: np.ndarray, lips: np.ndarray) -> list[int]:
|
|
1167
|
+
"""
|
|
1168
|
+
Identify line zones that overlap with other zones. Any overlap greater than 1 pixel is flagged.
|
|
1169
|
+
|
|
1170
|
+
Parameters
|
|
1171
|
+
----------
|
|
1172
|
+
rips
|
|
1173
|
+
Right borders of the region around the peak
|
|
1174
|
+
|
|
1175
|
+
lips
|
|
1176
|
+
Left borders of the region around the peak
|
|
1177
|
+
|
|
1178
|
+
Returns
|
|
1179
|
+
-------
|
|
1180
|
+
overlapping regions
|
|
1181
|
+
List indices into the input arrays that represent an overlapped region that can be removed
|
|
1182
|
+
"""
|
|
1183
|
+
all_ranges = [np.arange(zmin, zmax) for zmin, zmax in zip(rips, lips)]
|
|
1184
|
+
ranges_to_remove = []
|
|
1185
|
+
for i in range(len(all_ranges)):
|
|
1186
|
+
target_range = all_ranges[i]
|
|
1187
|
+
for j in range(i + 1, len(all_ranges)):
|
|
1188
|
+
if (
|
|
1189
|
+
np.intersect1d(target_range, all_ranges[j]).size > 1
|
|
1190
|
+
): # Allow for a single overlap just to be nice
|
|
1191
|
+
if target_range.size > all_ranges[j].size:
|
|
1192
|
+
ranges_to_remove.append(j)
|
|
1193
|
+
logger.info(
|
|
1194
|
+
f"Zone ({all_ranges[j][0]}, {all_ranges[j][-1]}) inside zone ({target_range[0]}, {target_range[-1]})"
|
|
1195
|
+
)
|
|
1196
|
+
else:
|
|
1197
|
+
ranges_to_remove.append(i)
|
|
1198
|
+
logger.info(
|
|
1199
|
+
f"Zone ({target_range[0]}, {target_range[-1]}) inside zone ({all_ranges[j][0]}, {all_ranges[j][-1]})"
|
|
1200
|
+
)
|
|
1201
|
+
|
|
1202
|
+
return ranges_to_remove
|
|
1203
|
+
|
|
1092
1204
|
@staticmethod
|
|
1093
1205
|
def high_pass_filter_array(array: np.ndarray) -> np.ndarray:
|
|
1094
1206
|
"""Perform a simple high-pass filter to accentuate narrow features (hairlines and spectra)."""
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""ViSP instrument polarization task. See :doc:`this page </polarization_calibration>` for more information."""
|
|
2
|
+
|
|
2
3
|
from collections import defaultdict
|
|
3
4
|
|
|
4
5
|
import numpy as np
|
|
@@ -18,8 +19,8 @@ from dkist_processing_pac.input_data.drawer import Drawer
|
|
|
18
19
|
from dkist_processing_pac.input_data.dresser import Dresser
|
|
19
20
|
from dkist_service_configuration.logging import logger
|
|
20
21
|
from sklearn.linear_model import RANSACRegressor
|
|
21
|
-
from sklearn.pipeline import make_pipeline
|
|
22
22
|
from sklearn.pipeline import Pipeline
|
|
23
|
+
from sklearn.pipeline import make_pipeline
|
|
23
24
|
from sklearn.preprocessing import PolynomialFeatures
|
|
24
25
|
from sklearn.preprocessing import RobustScaler
|
|
25
26
|
|
|
@@ -88,22 +89,22 @@ class InstrumentPolarizationCalibration(
|
|
|
88
89
|
)
|
|
89
90
|
remove_I_trend = self.parameters.pac_remove_linear_I_trend
|
|
90
91
|
for beam in range(1, self.constants.num_beams + 1):
|
|
91
|
-
with self.
|
|
92
|
+
with self.telemetry_span("Generate polcal DARK frame"):
|
|
92
93
|
logger.info("Generating polcal dark frame")
|
|
93
94
|
self.generate_polcal_dark_calibration(
|
|
94
95
|
readout_exp_times=polcal_readout_exposure_times, beam=beam
|
|
95
96
|
)
|
|
96
97
|
|
|
97
|
-
with self.
|
|
98
|
+
with self.telemetry_span("Generate polcal GAIN frame"):
|
|
98
99
|
logger.info("Generating polcal gain frame")
|
|
99
100
|
self.generate_polcal_gain_calibration(
|
|
100
101
|
readout_exp_times=polcal_readout_exposure_times, beam=beam
|
|
101
102
|
)
|
|
102
103
|
|
|
103
|
-
with self.
|
|
104
|
+
with self.telemetry_span(f"Reducing CS steps for {beam = }"):
|
|
104
105
|
local_reduced_arrays, global_reduced_arrays = self.reduce_cs_steps(beam)
|
|
105
106
|
|
|
106
|
-
with self.
|
|
107
|
+
with self.telemetry_span(f"Fit CU parameters for {beam = }"):
|
|
107
108
|
local_dresser = Dresser()
|
|
108
109
|
local_dresser.add_drawer(
|
|
109
110
|
Drawer(local_reduced_arrays, remove_I_trend=remove_I_trend)
|
|
@@ -122,7 +123,7 @@ class InstrumentPolarizationCalibration(
|
|
|
122
123
|
fit_TM=False,
|
|
123
124
|
)
|
|
124
125
|
|
|
125
|
-
with self.
|
|
126
|
+
with self.telemetry_span(f"Resampling demodulation matrices for {beam = }"):
|
|
126
127
|
demod_matrices = pac_fitter.demodulation_matrices
|
|
127
128
|
|
|
128
129
|
self.write(
|
|
@@ -148,7 +149,7 @@ class InstrumentPolarizationCalibration(
|
|
|
148
149
|
demod_matrices = self.reshape_demod_matrices(smoothed_demod)
|
|
149
150
|
logger.info(f"Shape of resampled demodulation matrices: {demod_matrices.shape}")
|
|
150
151
|
|
|
151
|
-
with self.
|
|
152
|
+
with self.telemetry_span(f"Writing demodulation matrices for {beam = }"):
|
|
152
153
|
# Save the demod matrices as intermediate products
|
|
153
154
|
self.write(
|
|
154
155
|
data=demod_matrices,
|
|
@@ -159,10 +160,10 @@ class InstrumentPolarizationCalibration(
|
|
|
159
160
|
encoder=fits_array_encoder,
|
|
160
161
|
)
|
|
161
162
|
|
|
162
|
-
with self.
|
|
163
|
+
with self.telemetry_span("Computing and recording polcal quality metrics"):
|
|
163
164
|
self.record_polcal_quality_metrics(beam, polcal_fitter=pac_fitter)
|
|
164
165
|
|
|
165
|
-
with self.
|
|
166
|
+
with self.telemetry_span("Computing and logging quality metrics"):
|
|
166
167
|
no_of_raw_polcal_frames: int = self.scratch.count_all(
|
|
167
168
|
tags=[
|
|
168
169
|
VispTag.input(),
|
|
@@ -339,7 +340,7 @@ class InstrumentPolarizationCalibration(
|
|
|
339
340
|
)
|
|
340
341
|
avg_inst_pol_cal_array = average_numpy_arrays(readout_normalized_arrays)
|
|
341
342
|
|
|
342
|
-
with self.
|
|
343
|
+
with self.telemetry_span(f"Apply basic corrections for {apm_str}"):
|
|
343
344
|
dark_corrected_array = subtract_array_from_arrays(avg_inst_pol_cal_array, dark_array)
|
|
344
345
|
|
|
345
346
|
background_corrected_array = subtract_array_from_arrays(
|
|
@@ -368,7 +369,7 @@ class InstrumentPolarizationCalibration(
|
|
|
368
369
|
self.corrections_remove_spec_geometry(geo_corrected_array, spec_shift)
|
|
369
370
|
)
|
|
370
371
|
|
|
371
|
-
with self.
|
|
372
|
+
with self.telemetry_span(f"Extract macro pixels from {apm_str}"):
|
|
372
373
|
self.set_original_beam_size(gain_corrected_array)
|
|
373
374
|
filtered_array = self.corrections_mask_hairlines(spectral_corrected_array)
|
|
374
375
|
|
|
@@ -388,7 +389,7 @@ class InstrumentPolarizationCalibration(
|
|
|
388
389
|
# Add two dummy dimensions just to keep it 2D.
|
|
389
390
|
global_binned_array = np.nanmedian(filtered_array)[None, None]
|
|
390
391
|
|
|
391
|
-
with self.
|
|
392
|
+
with self.telemetry_span(f"Create reduced VispL0FitsAccess for {apm_str}"):
|
|
392
393
|
local_result = VispL0FitsAccess(
|
|
393
394
|
fits.ImageHDU(local_array, avg_inst_pol_cal_header),
|
|
394
395
|
auto_squeeze=False,
|
|
@@ -1,13 +1,216 @@
|
|
|
1
1
|
"""Subclass of AssembleQualityData that causes the correct polcal metrics to build."""
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
from dkist_processing_common.codecs.asdf import asdf_decoder
|
|
5
|
+
from dkist_processing_common.models.quality import Plot2D
|
|
6
|
+
from dkist_processing_common.models.quality import ReportMetric
|
|
7
|
+
from dkist_processing_common.models.quality import VerticalMultiPanePlot2D
|
|
2
8
|
from dkist_processing_common.tasks import AssembleQualityData
|
|
3
9
|
|
|
4
10
|
__all__ = ["VispAssembleQualityData"]
|
|
5
11
|
|
|
12
|
+
from dkist_processing_visp.models.constants import VispConstants
|
|
13
|
+
from dkist_processing_visp.models.metric_code import VispMetricCode
|
|
14
|
+
from dkist_processing_visp.models.tags import VispTag
|
|
15
|
+
|
|
6
16
|
|
|
7
17
|
class VispAssembleQualityData(AssembleQualityData):
|
|
8
18
|
"""Subclass just so that the polcal_label_list can be populated."""
|
|
9
19
|
|
|
20
|
+
constants: VispConstants
|
|
21
|
+
|
|
22
|
+
@property
|
|
23
|
+
def constants_model_class(self):
|
|
24
|
+
"""Get ViSP pipeline constants."""
|
|
25
|
+
return VispConstants
|
|
26
|
+
|
|
10
27
|
@property
|
|
11
28
|
def polcal_label_list(self) -> list[str]:
|
|
12
29
|
"""Return labels for beams 1 and 2."""
|
|
13
30
|
return ["Beam 1", "Beam 2"]
|
|
31
|
+
|
|
32
|
+
def quality_assemble_data(self, polcal_label_list: list[str] | None = None) -> list[dict]:
|
|
33
|
+
"""
|
|
34
|
+
Assemble the full quality report and insert ViSP-specific metrics.
|
|
35
|
+
|
|
36
|
+
We try to place the new metrics right before default polcal ones, if possible.
|
|
37
|
+
"""
|
|
38
|
+
vignette_metrics = []
|
|
39
|
+
for beam in range(1, self.constants.num_beams + 1):
|
|
40
|
+
vignette_metrics.append(self.build_first_vignette_metric(beam=beam))
|
|
41
|
+
vignette_metrics.append(self.build_final_vignette_metric(beam=beam))
|
|
42
|
+
|
|
43
|
+
report = super().quality_assemble_data(polcal_label_list=polcal_label_list)
|
|
44
|
+
|
|
45
|
+
# Look for the first "PolCal" metric
|
|
46
|
+
first_polcal_metric_index = 0
|
|
47
|
+
try:
|
|
48
|
+
while not report[first_polcal_metric_index]["name"].lower().startswith("polcal"):
|
|
49
|
+
first_polcal_metric_index += 1
|
|
50
|
+
except:
|
|
51
|
+
# Wasn't found for whatever reason. No big deal, just put the new metrics at the front of the list
|
|
52
|
+
first_polcal_metric_index = 0
|
|
53
|
+
|
|
54
|
+
final_report = (
|
|
55
|
+
report[:first_polcal_metric_index]
|
|
56
|
+
+ vignette_metrics
|
|
57
|
+
+ report[first_polcal_metric_index:]
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
return final_report
|
|
61
|
+
|
|
62
|
+
def build_first_vignette_metric(self, beam: int) -> dict:
|
|
63
|
+
"""Build a ReportMetric showing the initial atlas-with-continuum fit and residuals."""
|
|
64
|
+
data = next(
|
|
65
|
+
self.read(
|
|
66
|
+
tags=[VispTag.quality(VispMetricCode.solar_first_vignette), VispTag.beam(beam)],
|
|
67
|
+
decoder=asdf_decoder,
|
|
68
|
+
)
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
wave_vec = data["output_wave_vec"].tolist()
|
|
72
|
+
input_spectrum = data["input_spectrum"].tolist()
|
|
73
|
+
best_fit_atlas = data["best_fit_atlas"].tolist()
|
|
74
|
+
continuum = data["best_fit_continuum"].tolist()
|
|
75
|
+
residuals = data["residuals"].tolist()
|
|
76
|
+
|
|
77
|
+
fit_series = {
|
|
78
|
+
"Raw input spectrum": [wave_vec, input_spectrum],
|
|
79
|
+
"Best fit atlas": [wave_vec, best_fit_atlas],
|
|
80
|
+
"Best fit continuum": [wave_vec, continuum],
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
fit_plot_kwargs = {
|
|
84
|
+
"Raw input spectrum": {
|
|
85
|
+
"ls": "-",
|
|
86
|
+
"ms": 0,
|
|
87
|
+
"color": "#FAA61C",
|
|
88
|
+
"zorder": 2.0,
|
|
89
|
+
"lw": 4,
|
|
90
|
+
"alpha": 0.6,
|
|
91
|
+
},
|
|
92
|
+
"Best fit atlas": {"color": "k", "ls": "-", "ms": 0, "zorder": 2.1},
|
|
93
|
+
"Best fit continuum": {"ls": "-", "ms": 0, "color": "g", "zorder": 2.2},
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
fit_plot = Plot2D(
|
|
97
|
+
xlabel="Wavelength [nm]",
|
|
98
|
+
ylabel="Signal",
|
|
99
|
+
series_data=fit_series,
|
|
100
|
+
plot_kwargs=fit_plot_kwargs,
|
|
101
|
+
sort_series=False,
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
residuals_series = {"Residuals": [wave_vec, residuals]}
|
|
105
|
+
residuals_plot_kwargs = {"Residuals": {"ls": "-", "color": "k", "ms": 0}}
|
|
106
|
+
|
|
107
|
+
y_min = np.nanpercentile(residuals, 2)
|
|
108
|
+
y_max = np.nanpercentile(residuals, 98)
|
|
109
|
+
y_range = y_max - y_min
|
|
110
|
+
y_min -= 0.1 * y_range
|
|
111
|
+
y_max += 0.1 * y_range
|
|
112
|
+
residuals_plot = Plot2D(
|
|
113
|
+
xlabel="Wavelength [nm]",
|
|
114
|
+
ylabel=r"$\frac{\mathrm{Obs - Atlas}}{\mathrm{Obs}}$",
|
|
115
|
+
series_data=residuals_series,
|
|
116
|
+
plot_kwargs=residuals_plot_kwargs,
|
|
117
|
+
ylim=(y_min, y_max),
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
plot_list = [fit_plot, residuals_plot]
|
|
121
|
+
height_ratios = [1.5, 1.0]
|
|
122
|
+
|
|
123
|
+
full_plot = VerticalMultiPanePlot2D(
|
|
124
|
+
top_to_bottom_plot_list=plot_list,
|
|
125
|
+
match_x_axes=True,
|
|
126
|
+
no_gap=True,
|
|
127
|
+
top_to_bottom_height_ratios=height_ratios,
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
metric = ReportMetric(
|
|
131
|
+
name=f"Initial Vignette Estimation - Beam {beam}",
|
|
132
|
+
description="These plots show the solar atlas fit used to estimate the initial, 1D spectral vignette "
|
|
133
|
+
"present in solar gain frames. The vignette signature is taken to be the fit continuum shown.",
|
|
134
|
+
metric_code=VispMetricCode.solar_first_vignette,
|
|
135
|
+
facet=self._format_facet(f"Beam {beam}"),
|
|
136
|
+
multi_plot_data=full_plot,
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
return metric.model_dump()
|
|
140
|
+
|
|
141
|
+
def build_final_vignette_metric(self, beam: int) -> dict:
|
|
142
|
+
"""Build a ReportMetric showing the quality of the vignette correction on solar gain data."""
|
|
143
|
+
data = next(
|
|
144
|
+
self.read(
|
|
145
|
+
tags=[VispTag.quality(VispMetricCode.solar_final_vignette), VispTag.beam(beam)],
|
|
146
|
+
decoder=asdf_decoder,
|
|
147
|
+
)
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
wave_vec = data["output_wave_vec"].tolist()
|
|
151
|
+
median_spec = data["median_spec"].tolist()
|
|
152
|
+
low_deviation = data["low_deviation"]
|
|
153
|
+
high_deviation = data["high_deviation"]
|
|
154
|
+
diff = (high_deviation - low_deviation).tolist()
|
|
155
|
+
low_deviation = low_deviation.tolist()
|
|
156
|
+
high_deviation = high_deviation.tolist()
|
|
157
|
+
|
|
158
|
+
bounds_series = {
|
|
159
|
+
"Median solar signal": [wave_vec, median_spec],
|
|
160
|
+
"5th percentile bounds": [wave_vec, low_deviation],
|
|
161
|
+
"95th percentile bounds": [wave_vec, high_deviation],
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
bounds_plot_kwargs = {
|
|
165
|
+
"Median solar signal": {"ls": "-", "color": "k", "alpha": 0.8, "ms": 0, "zorder": 2.2},
|
|
166
|
+
"5th percentile bounds": {"color": "#1E317A", "ls": "-", "ms": 0, "zorder": 2.0},
|
|
167
|
+
"95th percentile bounds": {"ls": "-", "color": "#FAA61C", "ms": 0, "zorder": 2.1},
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
bounds_plot = Plot2D(
|
|
171
|
+
xlabel="Wavelength [nm]",
|
|
172
|
+
ylabel="Signal",
|
|
173
|
+
series_data=bounds_series,
|
|
174
|
+
plot_kwargs=bounds_plot_kwargs,
|
|
175
|
+
sort_series=False,
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
residuals_series = {"Residuals": [wave_vec, diff]}
|
|
179
|
+
residuals_plot_kwargs = {"Residuals": {"ls": "-", "color": "k", "ms": 0}}
|
|
180
|
+
|
|
181
|
+
y_min = np.nanpercentile(diff, 5)
|
|
182
|
+
y_max = np.nanpercentile(diff, 95)
|
|
183
|
+
y_range = y_max - y_min
|
|
184
|
+
y_min -= 0.1 * y_range
|
|
185
|
+
y_max += 0.1 * y_range
|
|
186
|
+
residuals_plot = Plot2D(
|
|
187
|
+
xlabel="Wavelength [nm]",
|
|
188
|
+
ylabel="95th - 5th percentile",
|
|
189
|
+
series_data=residuals_series,
|
|
190
|
+
plot_kwargs=residuals_plot_kwargs,
|
|
191
|
+
ylim=(y_min, y_max),
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
plot_list = [bounds_plot, residuals_plot]
|
|
195
|
+
height_ratios = [1.5, 1.0]
|
|
196
|
+
|
|
197
|
+
full_plot = VerticalMultiPanePlot2D(
|
|
198
|
+
top_to_bottom_plot_list=plot_list,
|
|
199
|
+
match_x_axes=True,
|
|
200
|
+
no_gap=True,
|
|
201
|
+
top_to_bottom_height_ratios=height_ratios,
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
metric = ReportMetric(
|
|
205
|
+
name=f"Final Vignette Estimation - Beam {beam}",
|
|
206
|
+
description="These plots show how well the full, 2D vignette signal was removed from solar gain frames. "
|
|
207
|
+
"The median solar signal shows a full spatial median of the vignette corrected solar gain; "
|
|
208
|
+
"this should be very close to the true solar spectrum incident on the DKIST optics. "
|
|
209
|
+
"The 5th and 9th percentile ranges show how stable this spectrum is along the spatial dimension "
|
|
210
|
+
"after removing the vignette signal.",
|
|
211
|
+
metric_code=VispMetricCode.solar_final_vignette,
|
|
212
|
+
facet=self._format_facet(f"Beam {beam}"),
|
|
213
|
+
multi_plot_data=full_plot,
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
return metric.model_dump()
|