dkist-processing-visp 3.3.0__py3-none-any.whl → 5.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. dkist_processing_visp/__init__.py +1 -0
  2. dkist_processing_visp/config.py +1 -0
  3. dkist_processing_visp/models/constants.py +52 -21
  4. dkist_processing_visp/models/fits_access.py +20 -0
  5. dkist_processing_visp/models/metric_code.py +10 -0
  6. dkist_processing_visp/models/parameters.py +129 -19
  7. dkist_processing_visp/models/tags.py +1 -0
  8. dkist_processing_visp/models/task_name.py +1 -0
  9. dkist_processing_visp/parsers/map_repeats.py +1 -0
  10. dkist_processing_visp/parsers/modulator_states.py +1 -0
  11. dkist_processing_visp/parsers/polarimeter_mode.py +3 -1
  12. dkist_processing_visp/parsers/raster_step.py +4 -1
  13. dkist_processing_visp/parsers/spectrograph_configuration.py +75 -0
  14. dkist_processing_visp/parsers/time.py +15 -7
  15. dkist_processing_visp/parsers/visp_l0_fits_access.py +19 -8
  16. dkist_processing_visp/parsers/visp_l1_fits_access.py +1 -0
  17. dkist_processing_visp/tasks/__init__.py +1 -0
  18. dkist_processing_visp/tasks/assemble_movie.py +1 -0
  19. dkist_processing_visp/tasks/background_light.py +2 -1
  20. dkist_processing_visp/tasks/dark.py +5 -4
  21. dkist_processing_visp/tasks/geometric.py +132 -20
  22. dkist_processing_visp/tasks/instrument_polarization.py +13 -12
  23. dkist_processing_visp/tasks/l1_output_data.py +203 -0
  24. dkist_processing_visp/tasks/lamp.py +53 -93
  25. dkist_processing_visp/tasks/make_movie_frames.py +8 -6
  26. dkist_processing_visp/tasks/mixin/beam_access.py +1 -0
  27. dkist_processing_visp/tasks/mixin/corrections.py +54 -4
  28. dkist_processing_visp/tasks/mixin/downsample.py +1 -0
  29. dkist_processing_visp/tasks/parse.py +34 -4
  30. dkist_processing_visp/tasks/quality_metrics.py +5 -4
  31. dkist_processing_visp/tasks/science.py +126 -46
  32. dkist_processing_visp/tasks/solar.py +896 -456
  33. dkist_processing_visp/tasks/visp_base.py +2 -0
  34. dkist_processing_visp/tasks/write_l1.py +25 -5
  35. dkist_processing_visp/tests/conftest.py +99 -35
  36. dkist_processing_visp/tests/header_models.py +92 -20
  37. dkist_processing_visp/tests/local_trial_workflows/l0_cals_only.py +4 -23
  38. dkist_processing_visp/tests/local_trial_workflows/l0_polcals_as_science.py +421 -0
  39. dkist_processing_visp/tests/local_trial_workflows/l0_solar_gain_as_science.py +10 -29
  40. dkist_processing_visp/tests/local_trial_workflows/l0_to_l1.py +1 -21
  41. dkist_processing_visp/tests/local_trial_workflows/local_trial_helpers.py +98 -14
  42. dkist_processing_visp/tests/test_assemble_movie.py +2 -3
  43. dkist_processing_visp/tests/test_assemble_quality.py +89 -4
  44. dkist_processing_visp/tests/test_background_light.py +8 -5
  45. dkist_processing_visp/tests/test_dark.py +4 -3
  46. dkist_processing_visp/tests/test_fits_access.py +43 -0
  47. dkist_processing_visp/tests/test_geometric.py +45 -4
  48. dkist_processing_visp/tests/test_instrument_polarization.py +4 -3
  49. dkist_processing_visp/tests/test_lamp.py +22 -26
  50. dkist_processing_visp/tests/test_make_movie_frames.py +4 -4
  51. dkist_processing_visp/tests/test_map_repeats.py +3 -1
  52. dkist_processing_visp/tests/test_parameters.py +122 -21
  53. dkist_processing_visp/tests/test_parse.py +98 -14
  54. dkist_processing_visp/tests/test_quality.py +2 -3
  55. dkist_processing_visp/tests/test_science.py +113 -15
  56. dkist_processing_visp/tests/test_solar.py +318 -99
  57. dkist_processing_visp/tests/test_visp_constants.py +36 -8
  58. dkist_processing_visp/tests/test_workflows.py +1 -0
  59. dkist_processing_visp/tests/test_write_l1.py +17 -3
  60. dkist_processing_visp/workflows/__init__.py +1 -0
  61. dkist_processing_visp/workflows/l0_processing.py +8 -2
  62. dkist_processing_visp/workflows/trial_workflows.py +8 -2
  63. dkist_processing_visp-5.1.1.dist-info/METADATA +552 -0
  64. dkist_processing_visp-5.1.1.dist-info/RECORD +94 -0
  65. docs/conf.py +5 -1
  66. docs/gain_correction.rst +50 -42
  67. dkist_processing_visp/tasks/mixin/line_zones.py +0 -115
  68. dkist_processing_visp-3.3.0.dist-info/METADATA +0 -459
  69. dkist_processing_visp-3.3.0.dist-info/RECORD +0 -90
  70. {dkist_processing_visp-3.3.0.dist-info → dkist_processing_visp-5.1.1.dist-info}/WHEEL +0 -0
  71. {dkist_processing_visp-3.3.0.dist-info → dkist_processing_visp-5.1.1.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,5 @@
1
1
  """ViSP-specific assemble movie task subclass."""
2
+
2
3
  from dkist_processing_common.tasks import AssembleMovie
3
4
  from PIL import ImageDraw
4
5
 
@@ -1,4 +1,5 @@
1
1
  """ViSP background light calibration task."""
2
+
2
3
  import gc
3
4
  import time
4
5
 
@@ -114,7 +115,7 @@ class BackgroundLightCalibration(
114
115
  del full_background_light
115
116
  gc.collect()
116
117
 
117
- with self.apm_processing_step("Computing and logging quality metrics"):
118
+ with self.telemetry_span("Computing and logging quality metrics"):
118
119
  self.quality_store_task_type_counts(
119
120
  task_type=VispTaskName.background.value,
120
121
  total_frames=num_used_polcal_files,
@@ -1,4 +1,5 @@
1
1
  """Visp dark task."""
2
+
2
3
  from dkist_processing_common.codecs.fits import fits_access_decoder
3
4
  from dkist_processing_common.codecs.fits import fits_array_encoder
4
5
  from dkist_processing_common.models.task_name import TaskName
@@ -49,7 +50,7 @@ class DarkCalibration(VispTaskBase, BeamAccessMixin, QualityMixin):
49
50
  required_readout_exp_times = list(self.constants.non_dark_task_readout_exp_times)
50
51
  logger.info(f"{required_readout_exp_times = }")
51
52
 
52
- with self.apm_task_step(
53
+ with self.telemetry_span(
53
54
  f"Calculating dark frames for {self.constants.num_beams} beams and "
54
55
  f"{len(required_readout_exp_times)} readout exp times"
55
56
  ):
@@ -74,7 +75,7 @@ class DarkCalibration(VispTaskBase, BeamAccessMixin, QualityMixin):
74
75
  fits_access_class=VispL0FitsAccess,
75
76
  )
76
77
 
77
- with self.apm_processing_step(
78
+ with self.telemetry_span(
78
79
  f"Calculating dark for {readout_exp_time = } and {beam = }"
79
80
  ):
80
81
  readout_normalized_arrays = (
@@ -83,7 +84,7 @@ class DarkCalibration(VispTaskBase, BeamAccessMixin, QualityMixin):
83
84
  )
84
85
  averaged_dark_array = average_numpy_arrays(readout_normalized_arrays)
85
86
 
86
- with self.apm_writing_step(f"Writing dark for {readout_exp_time = } {beam = }"):
87
+ with self.telemetry_span(f"Writing dark for {readout_exp_time = } {beam = }"):
87
88
  self.write(
88
89
  data=averaged_dark_array,
89
90
  tags=VispTag.intermediate_frame_dark(
@@ -92,7 +93,7 @@ class DarkCalibration(VispTaskBase, BeamAccessMixin, QualityMixin):
92
93
  encoder=fits_array_encoder,
93
94
  )
94
95
 
95
- with self.apm_processing_step("Computing and logging quality metrics"):
96
+ with self.telemetry_span("Computing and logging quality metrics"):
96
97
  no_of_raw_dark_frames: int = self.scratch.count_all(
97
98
  tags=[
98
99
  VispTag.input(),
@@ -3,12 +3,15 @@ Visp geometric calibration task.
3
3
 
4
4
  See :doc:`this page </geometric>` for more information.
5
5
  """
6
+
6
7
  from typing import Generator
7
8
 
8
9
  import numpy as np
10
+ import peakutils
9
11
  import peakutils as pku
10
12
  import scipy.ndimage as spnd
11
13
  import scipy.optimize as spo
14
+ import scipy.signal as sps
12
15
  import skimage.exposure as skie
13
16
  import skimage.metrics as skim
14
17
  import skimage.morphology as skimo
@@ -32,7 +35,6 @@ from dkist_processing_visp.models.tags import VispTag
32
35
  from dkist_processing_visp.parsers.visp_l0_fits_access import VispL0FitsAccess
33
36
  from dkist_processing_visp.tasks.mixin.beam_access import BeamAccessMixin
34
37
  from dkist_processing_visp.tasks.mixin.corrections import CorrectionsMixin
35
- from dkist_processing_visp.tasks.mixin.line_zones import LineZonesMixin
36
38
  from dkist_processing_visp.tasks.visp_base import VispTaskBase
37
39
 
38
40
  __all__ = ["GeometricCalibration"]
@@ -43,7 +45,6 @@ class GeometricCalibration(
43
45
  BeamAccessMixin,
44
46
  CorrectionsMixin,
45
47
  QualityMixin,
46
- LineZonesMixin,
47
48
  ):
48
49
  """
49
50
  Task class for computing the spectral geometry. Geometry is represented by three quantities.
@@ -85,15 +86,15 @@ class GeometricCalibration(
85
86
  """
86
87
  # This lives outside the run() loops and has its own internal loops because the angle calculation
87
88
  # only happens for a single modstate
88
- with self.apm_processing_step("Basic corrections"):
89
+ with self.telemetry_span("Basic corrections"):
89
90
  self.do_basic_corrections()
90
91
 
91
92
  for beam in range(1, self.constants.num_beams + 1):
92
- with self.apm_task_step(f"Generating geometric calibrations for {beam = }"):
93
- with self.apm_processing_step(f"Computing and writing angle for {beam = }"):
93
+ with self.telemetry_span(f"Generating geometric calibrations for {beam = }"):
94
+ with self.telemetry_span(f"Computing and writing angle for {beam = }"):
94
95
  angle = self.compute_beam_angle(beam=beam)
95
96
  if beam == 2:
96
- with self.apm_processing_step("Refining beam 2 angle"):
97
+ with self.telemetry_span("Refining beam 2 angle"):
97
98
  ang_corr = self.refine_beam2_angle(angle)
98
99
  logger.info(f"Beam 2 angle refinement = {np.rad2deg(ang_corr)} deg")
99
100
  angle += ang_corr
@@ -101,14 +102,12 @@ class GeometricCalibration(
101
102
  self.write_angle(angle=angle, beam=beam)
102
103
 
103
104
  for modstate in range(1, self.constants.num_modstates + 1):
104
- with self.apm_processing_step(
105
- f"Removing angle from {beam = } and {modstate = }"
106
- ):
105
+ with self.telemetry_span(f"Removing angle from {beam = } and {modstate = }"):
107
106
  angle_corr_array = self.remove_beam_angle(
108
107
  angle=angle, beam=beam, modstate=modstate
109
108
  )
110
109
 
111
- with self.apm_processing_step(
110
+ with self.telemetry_span(
112
111
  f"Computing state offset for {beam = } and {modstate = }"
113
112
  ):
114
113
  state_offset = self.compute_modstate_offset(
@@ -116,7 +115,7 @@ class GeometricCalibration(
116
115
  )
117
116
  self.write_state_offset(offset=state_offset, beam=beam, modstate=modstate)
118
117
 
119
- with self.apm_processing_step(
118
+ with self.telemetry_span(
120
119
  f"Removing state offset for {beam = } and {modstate = }"
121
120
  ):
122
121
  self.remove_state_offset(
@@ -126,13 +125,13 @@ class GeometricCalibration(
126
125
  modstate=modstate,
127
126
  )
128
127
 
129
- with self.apm_processing_step(f"Computing spectral shifts for {beam = }"):
128
+ with self.telemetry_span(f"Computing spectral shifts for {beam = }"):
130
129
  spec_shifts = self.compute_spectral_shifts(beam=beam)
131
130
 
132
- with self.apm_writing_step(f"Writing spectral shifts for {beam = }"):
131
+ with self.telemetry_span(f"Writing spectral shifts for {beam = }"):
133
132
  self.write_spectral_shifts(shifts=spec_shifts, beam=beam)
134
133
 
135
- with self.apm_processing_step("Computing and logging quality metrics"):
134
+ with self.telemetry_span("Computing and logging quality metrics"):
136
135
  no_of_raw_geo_frames: int = self.scratch.count_all(
137
136
  tags=[
138
137
  VispTag.input(),
@@ -275,7 +274,12 @@ class GeometricCalibration(
275
274
  self.prep_input_solar_gain()
276
275
 
277
276
  def prep_lamp_gain(self):
278
- """Apply dark corrections fo INPUT lamp frames."""
277
+ """
278
+ Create average, dark-corrected lamp gain images for each modstate from INPUT lamp gains.
279
+
280
+ This is different from the results of the `~dkist_processing_visp.tasks.lamp.LampCalibration` task because in
281
+ that task the hairlines are masked out, but here we *need* the hairlines to compute the rotation angle.
282
+ """
279
283
  for readout_exp_time in self.constants.lamp_readout_exp_times:
280
284
  for beam in range(1, self.constants.num_beams + 1):
281
285
  logger.info(
@@ -1075,11 +1079,11 @@ class GeometricCalibration(
1075
1079
  `compute_single_state_offset`.
1076
1080
  """
1077
1081
  zone_kwargs = {
1078
- "prominence": self.parameters.solar_zone_prominence,
1079
- "width": self.parameters.solar_zone_width,
1080
- "bg_order": self.parameters.solar_zone_bg_order,
1081
- "normalization_percentile": self.parameters.solar_zone_normalization_percentile,
1082
- "rel_height": self.parameters.solar_zone_rel_height,
1082
+ "prominence": self.parameters.geo_zone_prominence,
1083
+ "width": self.parameters.geo_zone_width,
1084
+ "bg_order": self.parameters.geo_zone_bg_order,
1085
+ "normalization_percentile": self.parameters.geo_zone_normalization_percentile,
1086
+ "rel_height": self.parameters.geo_zone_rel_height,
1083
1087
  }
1084
1088
  zones = self.compute_line_zones(array, **zone_kwargs)
1085
1089
  logger.info(f"Found {zones = }")
@@ -1089,6 +1093,114 @@ class GeometricCalibration(
1089
1093
 
1090
1094
  return mask
1091
1095
 
1096
+ def compute_line_zones(
1097
+ self,
1098
+ spec_2d: np.ndarray,
1099
+ prominence: float = 0.2,
1100
+ width: float = 2,
1101
+ bg_order: int = 22,
1102
+ normalization_percentile: int = 99,
1103
+ rel_height: float = 0.97,
1104
+ ) -> list[tuple[int, int]]:
1105
+ """
1106
+ Identify spectral regions around strong spectra features.
1107
+
1108
+ Parameters
1109
+ ----------
1110
+ spec_2d
1111
+ Data
1112
+
1113
+ prominence
1114
+ Zone prominence threshold used to identify strong spectral features
1115
+
1116
+ width
1117
+ Zone width
1118
+
1119
+ bg_order
1120
+ Order of polynomial fit used to remove continuum when identifying strong spectral features
1121
+
1122
+ normalization_percentile
1123
+ Compute this percentile of the data along a specified axis
1124
+
1125
+ rel_height
1126
+ The relative height at which the peak width is measured as a percentage of its prominence. E.g., 1.0 measures
1127
+ the peak width at the lowest contour line.
1128
+
1129
+ Returns
1130
+ -------
1131
+ regions
1132
+ List of indices defining the found spectral lines
1133
+
1134
+ """
1135
+ logger.info(
1136
+ f"Finding zones using {prominence=}, {width=}, {bg_order=}, {normalization_percentile=}, and {rel_height=}"
1137
+ )
1138
+ # Compute average along slit to improve signal. Line smearing isn't important here
1139
+ avg_1d = np.mean(spec_2d, axis=1)
1140
+
1141
+ # Convert to an emission spectrum and remove baseline continuum so peakutils has an easier time
1142
+ em_spec = -1 * avg_1d + avg_1d.max()
1143
+ em_spec /= np.nanpercentile(em_spec, normalization_percentile)
1144
+ baseline = peakutils.baseline(em_spec, bg_order)
1145
+ em_spec -= baseline
1146
+
1147
+ # Find indices of peaks
1148
+ peak_idxs = sps.find_peaks(em_spec, prominence=prominence, width=width)[0]
1149
+
1150
+ # Find the rough width based only on the height of the peak
1151
+ # rips and lips are the right and left borders of the region around the peak
1152
+ _, _, rips, lips = sps.peak_widths(em_spec, peak_idxs, rel_height=rel_height)
1153
+
1154
+ # Convert to ints so they can be used as indices
1155
+ rips = np.floor(rips).astype(int)
1156
+ lips = np.ceil(lips).astype(int)
1157
+
1158
+ # Remove any regions that are contained within another region
1159
+ ranges_to_remove = self.identify_overlapping_zones(rips, lips)
1160
+ rips = np.delete(rips, ranges_to_remove)
1161
+ lips = np.delete(lips, ranges_to_remove)
1162
+
1163
+ return list(zip(rips, lips))
1164
+
1165
+ @staticmethod
1166
+ def identify_overlapping_zones(rips: np.ndarray, lips: np.ndarray) -> list[int]:
1167
+ """
1168
+ Identify line zones that overlap with other zones. Any overlap greater than 1 pixel is flagged.
1169
+
1170
+ Parameters
1171
+ ----------
1172
+ rips
1173
+ Right borders of the region around the peak
1174
+
1175
+ lips
1176
+ Left borders of the region around the peak
1177
+
1178
+ Returns
1179
+ -------
1180
+ overlapping regions
1181
+ List indices into the input arrays that represent an overlapped region that can be removed
1182
+ """
1183
+ all_ranges = [np.arange(zmin, zmax) for zmin, zmax in zip(rips, lips)]
1184
+ ranges_to_remove = []
1185
+ for i in range(len(all_ranges)):
1186
+ target_range = all_ranges[i]
1187
+ for j in range(i + 1, len(all_ranges)):
1188
+ if (
1189
+ np.intersect1d(target_range, all_ranges[j]).size > 1
1190
+ ): # Allow for a single overlap just to be nice
1191
+ if target_range.size > all_ranges[j].size:
1192
+ ranges_to_remove.append(j)
1193
+ logger.info(
1194
+ f"Zone ({all_ranges[j][0]}, {all_ranges[j][-1]}) inside zone ({target_range[0]}, {target_range[-1]})"
1195
+ )
1196
+ else:
1197
+ ranges_to_remove.append(i)
1198
+ logger.info(
1199
+ f"Zone ({target_range[0]}, {target_range[-1]}) inside zone ({all_ranges[j][0]}, {all_ranges[j][-1]})"
1200
+ )
1201
+
1202
+ return ranges_to_remove
1203
+
1092
1204
  @staticmethod
1093
1205
  def high_pass_filter_array(array: np.ndarray) -> np.ndarray:
1094
1206
  """Perform a simple high-pass filter to accentuate narrow features (hairlines and spectra)."""
@@ -1,4 +1,5 @@
1
1
  """ViSP instrument polarization task. See :doc:`this page </polarization_calibration>` for more information."""
2
+
2
3
  from collections import defaultdict
3
4
 
4
5
  import numpy as np
@@ -18,8 +19,8 @@ from dkist_processing_pac.input_data.drawer import Drawer
18
19
  from dkist_processing_pac.input_data.dresser import Dresser
19
20
  from dkist_service_configuration.logging import logger
20
21
  from sklearn.linear_model import RANSACRegressor
21
- from sklearn.pipeline import make_pipeline
22
22
  from sklearn.pipeline import Pipeline
23
+ from sklearn.pipeline import make_pipeline
23
24
  from sklearn.preprocessing import PolynomialFeatures
24
25
  from sklearn.preprocessing import RobustScaler
25
26
 
@@ -88,22 +89,22 @@ class InstrumentPolarizationCalibration(
88
89
  )
89
90
  remove_I_trend = self.parameters.pac_remove_linear_I_trend
90
91
  for beam in range(1, self.constants.num_beams + 1):
91
- with self.apm_processing_step("Generate polcal DARK frame"):
92
+ with self.telemetry_span("Generate polcal DARK frame"):
92
93
  logger.info("Generating polcal dark frame")
93
94
  self.generate_polcal_dark_calibration(
94
95
  readout_exp_times=polcal_readout_exposure_times, beam=beam
95
96
  )
96
97
 
97
- with self.apm_processing_step("Generate polcal GAIN frame"):
98
+ with self.telemetry_span("Generate polcal GAIN frame"):
98
99
  logger.info("Generating polcal gain frame")
99
100
  self.generate_polcal_gain_calibration(
100
101
  readout_exp_times=polcal_readout_exposure_times, beam=beam
101
102
  )
102
103
 
103
- with self.apm_processing_step(f"Reducing CS steps for {beam = }"):
104
+ with self.telemetry_span(f"Reducing CS steps for {beam = }"):
104
105
  local_reduced_arrays, global_reduced_arrays = self.reduce_cs_steps(beam)
105
106
 
106
- with self.apm_processing_step(f"Fit CU parameters for {beam = }"):
107
+ with self.telemetry_span(f"Fit CU parameters for {beam = }"):
107
108
  local_dresser = Dresser()
108
109
  local_dresser.add_drawer(
109
110
  Drawer(local_reduced_arrays, remove_I_trend=remove_I_trend)
@@ -122,7 +123,7 @@ class InstrumentPolarizationCalibration(
122
123
  fit_TM=False,
123
124
  )
124
125
 
125
- with self.apm_processing_step(f"Resampling demodulation matrices for {beam = }"):
126
+ with self.telemetry_span(f"Resampling demodulation matrices for {beam = }"):
126
127
  demod_matrices = pac_fitter.demodulation_matrices
127
128
 
128
129
  self.write(
@@ -148,7 +149,7 @@ class InstrumentPolarizationCalibration(
148
149
  demod_matrices = self.reshape_demod_matrices(smoothed_demod)
149
150
  logger.info(f"Shape of resampled demodulation matrices: {demod_matrices.shape}")
150
151
 
151
- with self.apm_writing_step(f"Writing demodulation matrices for {beam = }"):
152
+ with self.telemetry_span(f"Writing demodulation matrices for {beam = }"):
152
153
  # Save the demod matrices as intermediate products
153
154
  self.write(
154
155
  data=demod_matrices,
@@ -159,10 +160,10 @@ class InstrumentPolarizationCalibration(
159
160
  encoder=fits_array_encoder,
160
161
  )
161
162
 
162
- with self.apm_processing_step("Computing and recording polcal quality metrics"):
163
+ with self.telemetry_span("Computing and recording polcal quality metrics"):
163
164
  self.record_polcal_quality_metrics(beam, polcal_fitter=pac_fitter)
164
165
 
165
- with self.apm_processing_step("Computing and logging quality metrics"):
166
+ with self.telemetry_span("Computing and logging quality metrics"):
166
167
  no_of_raw_polcal_frames: int = self.scratch.count_all(
167
168
  tags=[
168
169
  VispTag.input(),
@@ -339,7 +340,7 @@ class InstrumentPolarizationCalibration(
339
340
  )
340
341
  avg_inst_pol_cal_array = average_numpy_arrays(readout_normalized_arrays)
341
342
 
342
- with self.apm_processing_step(f"Apply basic corrections for {apm_str}"):
343
+ with self.telemetry_span(f"Apply basic corrections for {apm_str}"):
343
344
  dark_corrected_array = subtract_array_from_arrays(avg_inst_pol_cal_array, dark_array)
344
345
 
345
346
  background_corrected_array = subtract_array_from_arrays(
@@ -368,7 +369,7 @@ class InstrumentPolarizationCalibration(
368
369
  self.corrections_remove_spec_geometry(geo_corrected_array, spec_shift)
369
370
  )
370
371
 
371
- with self.apm_processing_step(f"Extract macro pixels from {apm_str}"):
372
+ with self.telemetry_span(f"Extract macro pixels from {apm_str}"):
372
373
  self.set_original_beam_size(gain_corrected_array)
373
374
  filtered_array = self.corrections_mask_hairlines(spectral_corrected_array)
374
375
 
@@ -388,7 +389,7 @@ class InstrumentPolarizationCalibration(
388
389
  # Add two dummy dimensions just to keep it 2D.
389
390
  global_binned_array = np.nanmedian(filtered_array)[None, None]
390
391
 
391
- with self.apm_processing_step(f"Create reduced VispL0FitsAccess for {apm_str}"):
392
+ with self.telemetry_span(f"Create reduced VispL0FitsAccess for {apm_str}"):
392
393
  local_result = VispL0FitsAccess(
393
394
  fits.ImageHDU(local_array, avg_inst_pol_cal_header),
394
395
  auto_squeeze=False,
@@ -1,13 +1,216 @@
1
1
  """Subclass of AssembleQualityData that causes the correct polcal metrics to build."""
2
+
3
+ import numpy as np
4
+ from dkist_processing_common.codecs.asdf import asdf_decoder
5
+ from dkist_processing_common.models.quality import Plot2D
6
+ from dkist_processing_common.models.quality import ReportMetric
7
+ from dkist_processing_common.models.quality import VerticalMultiPanePlot2D
2
8
  from dkist_processing_common.tasks import AssembleQualityData
3
9
 
4
10
  __all__ = ["VispAssembleQualityData"]
5
11
 
12
+ from dkist_processing_visp.models.constants import VispConstants
13
+ from dkist_processing_visp.models.metric_code import VispMetricCode
14
+ from dkist_processing_visp.models.tags import VispTag
15
+
6
16
 
7
17
  class VispAssembleQualityData(AssembleQualityData):
8
18
  """Subclass just so that the polcal_label_list can be populated."""
9
19
 
20
+ constants: VispConstants
21
+
22
+ @property
23
+ def constants_model_class(self):
24
+ """Get ViSP pipeline constants."""
25
+ return VispConstants
26
+
10
27
  @property
11
28
  def polcal_label_list(self) -> list[str]:
12
29
  """Return labels for beams 1 and 2."""
13
30
  return ["Beam 1", "Beam 2"]
31
+
32
+ def quality_assemble_data(self, polcal_label_list: list[str] | None = None) -> list[dict]:
33
+ """
34
+ Assemble the full quality report and insert ViSP-specific metrics.
35
+
36
+ We try to place the new metrics right before default polcal ones, if possible.
37
+ """
38
+ vignette_metrics = []
39
+ for beam in range(1, self.constants.num_beams + 1):
40
+ vignette_metrics.append(self.build_first_vignette_metric(beam=beam))
41
+ vignette_metrics.append(self.build_final_vignette_metric(beam=beam))
42
+
43
+ report = super().quality_assemble_data(polcal_label_list=polcal_label_list)
44
+
45
+ # Look for the first "PolCal" metric
46
+ first_polcal_metric_index = 0
47
+ try:
48
+ while not report[first_polcal_metric_index]["name"].lower().startswith("polcal"):
49
+ first_polcal_metric_index += 1
50
+ except:
51
+ # Wasn't found for whatever reason. No big deal, just put the new metrics at the front of the list
52
+ first_polcal_metric_index = 0
53
+
54
+ final_report = (
55
+ report[:first_polcal_metric_index]
56
+ + vignette_metrics
57
+ + report[first_polcal_metric_index:]
58
+ )
59
+
60
+ return final_report
61
+
62
+ def build_first_vignette_metric(self, beam: int) -> dict:
63
+ """Build a ReportMetric showing the initial atlas-with-continuum fit and residuals."""
64
+ data = next(
65
+ self.read(
66
+ tags=[VispTag.quality(VispMetricCode.solar_first_vignette), VispTag.beam(beam)],
67
+ decoder=asdf_decoder,
68
+ )
69
+ )
70
+
71
+ wave_vec = data["output_wave_vec"].tolist()
72
+ input_spectrum = data["input_spectrum"].tolist()
73
+ best_fit_atlas = data["best_fit_atlas"].tolist()
74
+ continuum = data["best_fit_continuum"].tolist()
75
+ residuals = data["residuals"].tolist()
76
+
77
+ fit_series = {
78
+ "Raw input spectrum": [wave_vec, input_spectrum],
79
+ "Best fit atlas": [wave_vec, best_fit_atlas],
80
+ "Best fit continuum": [wave_vec, continuum],
81
+ }
82
+
83
+ fit_plot_kwargs = {
84
+ "Raw input spectrum": {
85
+ "ls": "-",
86
+ "ms": 0,
87
+ "color": "#FAA61C",
88
+ "zorder": 2.0,
89
+ "lw": 4,
90
+ "alpha": 0.6,
91
+ },
92
+ "Best fit atlas": {"color": "k", "ls": "-", "ms": 0, "zorder": 2.1},
93
+ "Best fit continuum": {"ls": "-", "ms": 0, "color": "g", "zorder": 2.2},
94
+ }
95
+
96
+ fit_plot = Plot2D(
97
+ xlabel="Wavelength [nm]",
98
+ ylabel="Signal",
99
+ series_data=fit_series,
100
+ plot_kwargs=fit_plot_kwargs,
101
+ sort_series=False,
102
+ )
103
+
104
+ residuals_series = {"Residuals": [wave_vec, residuals]}
105
+ residuals_plot_kwargs = {"Residuals": {"ls": "-", "color": "k", "ms": 0}}
106
+
107
+ y_min = np.nanpercentile(residuals, 2)
108
+ y_max = np.nanpercentile(residuals, 98)
109
+ y_range = y_max - y_min
110
+ y_min -= 0.1 * y_range
111
+ y_max += 0.1 * y_range
112
+ residuals_plot = Plot2D(
113
+ xlabel="Wavelength [nm]",
114
+ ylabel=r"$\frac{\mathrm{Obs - Atlas}}{\mathrm{Obs}}$",
115
+ series_data=residuals_series,
116
+ plot_kwargs=residuals_plot_kwargs,
117
+ ylim=(y_min, y_max),
118
+ )
119
+
120
+ plot_list = [fit_plot, residuals_plot]
121
+ height_ratios = [1.5, 1.0]
122
+
123
+ full_plot = VerticalMultiPanePlot2D(
124
+ top_to_bottom_plot_list=plot_list,
125
+ match_x_axes=True,
126
+ no_gap=True,
127
+ top_to_bottom_height_ratios=height_ratios,
128
+ )
129
+
130
+ metric = ReportMetric(
131
+ name=f"Initial Vignette Estimation - Beam {beam}",
132
+ description="These plots show the solar atlas fit used to estimate the initial, 1D spectral vignette "
133
+ "present in solar gain frames. The vignette signature is taken to be the fit continuum shown.",
134
+ metric_code=VispMetricCode.solar_first_vignette,
135
+ facet=self._format_facet(f"Beam {beam}"),
136
+ multi_plot_data=full_plot,
137
+ )
138
+
139
+ return metric.model_dump()
140
+
141
+ def build_final_vignette_metric(self, beam: int) -> dict:
142
+ """Build a ReportMetric showing the quality of the vignette correction on solar gain data."""
143
+ data = next(
144
+ self.read(
145
+ tags=[VispTag.quality(VispMetricCode.solar_final_vignette), VispTag.beam(beam)],
146
+ decoder=asdf_decoder,
147
+ )
148
+ )
149
+
150
+ wave_vec = data["output_wave_vec"].tolist()
151
+ median_spec = data["median_spec"].tolist()
152
+ low_deviation = data["low_deviation"]
153
+ high_deviation = data["high_deviation"]
154
+ diff = (high_deviation - low_deviation).tolist()
155
+ low_deviation = low_deviation.tolist()
156
+ high_deviation = high_deviation.tolist()
157
+
158
+ bounds_series = {
159
+ "Median solar signal": [wave_vec, median_spec],
160
+ "5th percentile bounds": [wave_vec, low_deviation],
161
+ "95th percentile bounds": [wave_vec, high_deviation],
162
+ }
163
+
164
+ bounds_plot_kwargs = {
165
+ "Median solar signal": {"ls": "-", "color": "k", "alpha": 0.8, "ms": 0, "zorder": 2.2},
166
+ "5th percentile bounds": {"color": "#1E317A", "ls": "-", "ms": 0, "zorder": 2.0},
167
+ "95th percentile bounds": {"ls": "-", "color": "#FAA61C", "ms": 0, "zorder": 2.1},
168
+ }
169
+
170
+ bounds_plot = Plot2D(
171
+ xlabel="Wavelength [nm]",
172
+ ylabel="Signal",
173
+ series_data=bounds_series,
174
+ plot_kwargs=bounds_plot_kwargs,
175
+ sort_series=False,
176
+ )
177
+
178
+ residuals_series = {"Residuals": [wave_vec, diff]}
179
+ residuals_plot_kwargs = {"Residuals": {"ls": "-", "color": "k", "ms": 0}}
180
+
181
+ y_min = np.nanpercentile(diff, 5)
182
+ y_max = np.nanpercentile(diff, 95)
183
+ y_range = y_max - y_min
184
+ y_min -= 0.1 * y_range
185
+ y_max += 0.1 * y_range
186
+ residuals_plot = Plot2D(
187
+ xlabel="Wavelength [nm]",
188
+ ylabel="95th - 5th percentile",
189
+ series_data=residuals_series,
190
+ plot_kwargs=residuals_plot_kwargs,
191
+ ylim=(y_min, y_max),
192
+ )
193
+
194
+ plot_list = [bounds_plot, residuals_plot]
195
+ height_ratios = [1.5, 1.0]
196
+
197
+ full_plot = VerticalMultiPanePlot2D(
198
+ top_to_bottom_plot_list=plot_list,
199
+ match_x_axes=True,
200
+ no_gap=True,
201
+ top_to_bottom_height_ratios=height_ratios,
202
+ )
203
+
204
+ metric = ReportMetric(
205
+ name=f"Final Vignette Estimation - Beam {beam}",
206
+ description="These plots show how well the full, 2D vignette signal was removed from solar gain frames. "
207
+ "The median solar signal shows a full spatial median of the vignette corrected solar gain; "
208
+ "this should be very close to the true solar spectrum incident on the DKIST optics. "
209
+ "The 5th and 9th percentile ranges show how stable this spectrum is along the spatial dimension "
210
+ "after removing the vignette signal.",
211
+ metric_code=VispMetricCode.solar_final_vignette,
212
+ facet=self._format_facet(f"Beam {beam}"),
213
+ multi_plot_data=full_plot,
214
+ )
215
+
216
+ return metric.model_dump()