dkist-processing-cryonirsp 1.3.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dkist-processing-cryonirsp might be problematic. Click here for more details.

Files changed (111) hide show
  1. changelog/.gitempty +0 -0
  2. dkist_processing_cryonirsp/__init__.py +11 -0
  3. dkist_processing_cryonirsp/config.py +12 -0
  4. dkist_processing_cryonirsp/models/__init__.py +1 -0
  5. dkist_processing_cryonirsp/models/constants.py +248 -0
  6. dkist_processing_cryonirsp/models/exposure_conditions.py +26 -0
  7. dkist_processing_cryonirsp/models/parameters.py +296 -0
  8. dkist_processing_cryonirsp/models/tags.py +168 -0
  9. dkist_processing_cryonirsp/models/task_name.py +14 -0
  10. dkist_processing_cryonirsp/parsers/__init__.py +1 -0
  11. dkist_processing_cryonirsp/parsers/cryonirsp_l0_fits_access.py +111 -0
  12. dkist_processing_cryonirsp/parsers/cryonirsp_l1_fits_access.py +30 -0
  13. dkist_processing_cryonirsp/parsers/exposure_conditions.py +163 -0
  14. dkist_processing_cryonirsp/parsers/map_repeats.py +40 -0
  15. dkist_processing_cryonirsp/parsers/measurements.py +55 -0
  16. dkist_processing_cryonirsp/parsers/modstates.py +31 -0
  17. dkist_processing_cryonirsp/parsers/optical_density_filters.py +40 -0
  18. dkist_processing_cryonirsp/parsers/polarimetric_check.py +120 -0
  19. dkist_processing_cryonirsp/parsers/scan_step.py +412 -0
  20. dkist_processing_cryonirsp/parsers/time.py +80 -0
  21. dkist_processing_cryonirsp/parsers/wavelength.py +26 -0
  22. dkist_processing_cryonirsp/tasks/__init__.py +19 -0
  23. dkist_processing_cryonirsp/tasks/assemble_movie.py +202 -0
  24. dkist_processing_cryonirsp/tasks/bad_pixel_map.py +96 -0
  25. dkist_processing_cryonirsp/tasks/beam_boundaries_base.py +279 -0
  26. dkist_processing_cryonirsp/tasks/ci_beam_boundaries.py +55 -0
  27. dkist_processing_cryonirsp/tasks/ci_science.py +169 -0
  28. dkist_processing_cryonirsp/tasks/cryonirsp_base.py +67 -0
  29. dkist_processing_cryonirsp/tasks/dark.py +98 -0
  30. dkist_processing_cryonirsp/tasks/gain.py +251 -0
  31. dkist_processing_cryonirsp/tasks/instrument_polarization.py +447 -0
  32. dkist_processing_cryonirsp/tasks/l1_output_data.py +44 -0
  33. dkist_processing_cryonirsp/tasks/linearity_correction.py +582 -0
  34. dkist_processing_cryonirsp/tasks/make_movie_frames.py +302 -0
  35. dkist_processing_cryonirsp/tasks/mixin/__init__.py +1 -0
  36. dkist_processing_cryonirsp/tasks/mixin/beam_access.py +52 -0
  37. dkist_processing_cryonirsp/tasks/mixin/corrections.py +177 -0
  38. dkist_processing_cryonirsp/tasks/mixin/intermediate_frame.py +193 -0
  39. dkist_processing_cryonirsp/tasks/mixin/linearized_frame.py +309 -0
  40. dkist_processing_cryonirsp/tasks/mixin/shift_measurements.py +297 -0
  41. dkist_processing_cryonirsp/tasks/parse.py +281 -0
  42. dkist_processing_cryonirsp/tasks/quality_metrics.py +271 -0
  43. dkist_processing_cryonirsp/tasks/science_base.py +511 -0
  44. dkist_processing_cryonirsp/tasks/sp_beam_boundaries.py +270 -0
  45. dkist_processing_cryonirsp/tasks/sp_dispersion_axis_correction.py +484 -0
  46. dkist_processing_cryonirsp/tasks/sp_geometric.py +585 -0
  47. dkist_processing_cryonirsp/tasks/sp_science.py +299 -0
  48. dkist_processing_cryonirsp/tasks/sp_solar_gain.py +475 -0
  49. dkist_processing_cryonirsp/tasks/trial_output_data.py +61 -0
  50. dkist_processing_cryonirsp/tasks/write_l1.py +1033 -0
  51. dkist_processing_cryonirsp/tests/__init__.py +1 -0
  52. dkist_processing_cryonirsp/tests/conftest.py +456 -0
  53. dkist_processing_cryonirsp/tests/header_models.py +592 -0
  54. dkist_processing_cryonirsp/tests/local_trial_workflows/__init__.py +0 -0
  55. dkist_processing_cryonirsp/tests/local_trial_workflows/l0_cals_only.py +541 -0
  56. dkist_processing_cryonirsp/tests/local_trial_workflows/l0_to_l1.py +615 -0
  57. dkist_processing_cryonirsp/tests/local_trial_workflows/linearize_only.py +96 -0
  58. dkist_processing_cryonirsp/tests/local_trial_workflows/local_trial_helpers.py +592 -0
  59. dkist_processing_cryonirsp/tests/test_assemble_movie.py +144 -0
  60. dkist_processing_cryonirsp/tests/test_assemble_qualilty.py +517 -0
  61. dkist_processing_cryonirsp/tests/test_bad_pixel_maps.py +115 -0
  62. dkist_processing_cryonirsp/tests/test_ci_beam_boundaries.py +106 -0
  63. dkist_processing_cryonirsp/tests/test_ci_science.py +355 -0
  64. dkist_processing_cryonirsp/tests/test_corrections.py +126 -0
  65. dkist_processing_cryonirsp/tests/test_cryo_base.py +202 -0
  66. dkist_processing_cryonirsp/tests/test_cryo_constants.py +76 -0
  67. dkist_processing_cryonirsp/tests/test_dark.py +287 -0
  68. dkist_processing_cryonirsp/tests/test_gain.py +278 -0
  69. dkist_processing_cryonirsp/tests/test_instrument_polarization.py +531 -0
  70. dkist_processing_cryonirsp/tests/test_linearity_correction.py +245 -0
  71. dkist_processing_cryonirsp/tests/test_make_movie_frames.py +111 -0
  72. dkist_processing_cryonirsp/tests/test_parameters.py +266 -0
  73. dkist_processing_cryonirsp/tests/test_parse.py +1439 -0
  74. dkist_processing_cryonirsp/tests/test_quality.py +203 -0
  75. dkist_processing_cryonirsp/tests/test_sp_beam_boundaries.py +112 -0
  76. dkist_processing_cryonirsp/tests/test_sp_dispersion_axis_correction.py +155 -0
  77. dkist_processing_cryonirsp/tests/test_sp_geometric.py +319 -0
  78. dkist_processing_cryonirsp/tests/test_sp_make_movie_frames.py +121 -0
  79. dkist_processing_cryonirsp/tests/test_sp_science.py +483 -0
  80. dkist_processing_cryonirsp/tests/test_sp_solar.py +198 -0
  81. dkist_processing_cryonirsp/tests/test_trial_create_quality_report.py +79 -0
  82. dkist_processing_cryonirsp/tests/test_trial_output_data.py +251 -0
  83. dkist_processing_cryonirsp/tests/test_workflows.py +9 -0
  84. dkist_processing_cryonirsp/tests/test_write_l1.py +436 -0
  85. dkist_processing_cryonirsp/workflows/__init__.py +2 -0
  86. dkist_processing_cryonirsp/workflows/ci_l0_processing.py +77 -0
  87. dkist_processing_cryonirsp/workflows/sp_l0_processing.py +84 -0
  88. dkist_processing_cryonirsp/workflows/trial_workflows.py +190 -0
  89. dkist_processing_cryonirsp-1.3.4.dist-info/METADATA +194 -0
  90. dkist_processing_cryonirsp-1.3.4.dist-info/RECORD +111 -0
  91. dkist_processing_cryonirsp-1.3.4.dist-info/WHEEL +5 -0
  92. dkist_processing_cryonirsp-1.3.4.dist-info/top_level.txt +4 -0
  93. docs/Makefile +134 -0
  94. docs/bad_pixel_calibration.rst +47 -0
  95. docs/beam_angle_calculation.rst +53 -0
  96. docs/beam_boundary_computation.rst +88 -0
  97. docs/changelog.rst +7 -0
  98. docs/ci_science_calibration.rst +33 -0
  99. docs/conf.py +52 -0
  100. docs/index.rst +21 -0
  101. docs/l0_to_l1_cryonirsp_ci-full-trial.rst +10 -0
  102. docs/l0_to_l1_cryonirsp_ci.rst +10 -0
  103. docs/l0_to_l1_cryonirsp_sp-full-trial.rst +10 -0
  104. docs/l0_to_l1_cryonirsp_sp.rst +10 -0
  105. docs/linearization.rst +43 -0
  106. docs/make.bat +170 -0
  107. docs/requirements.txt +1 -0
  108. docs/requirements_table.rst +8 -0
  109. docs/scientific_changelog.rst +10 -0
  110. docs/sp_science_calibration.rst +59 -0
  111. licenses/LICENSE.rst +11 -0
@@ -0,0 +1,1439 @@
1
+ import json
2
+ import re
3
+ from dataclasses import dataclass
4
+ from datetime import datetime
5
+ from datetime import timedelta
6
+ from typing import Callable
7
+ from typing import Type
8
+
9
+ import pytest
10
+ from astropy.io import fits
11
+ from dkist_processing_common._util.scratch import WorkflowFileSystem
12
+ from dkist_processing_common.models.constants import BudName
13
+ from dkist_processing_common.tasks import WorkflowTaskBase
14
+ from dkist_processing_common.tests.conftest import FakeGQLClient
15
+
16
+ from dkist_processing_cryonirsp.models.constants import CryonirspBudName
17
+ from dkist_processing_cryonirsp.models.exposure_conditions import AllowableOpticalDensityFilterNames
18
+ from dkist_processing_cryonirsp.models.exposure_conditions import ExposureConditions
19
+ from dkist_processing_cryonirsp.models.parameters import CryonirspParsingParameters
20
+ from dkist_processing_cryonirsp.models.tags import CryonirspTag
21
+ from dkist_processing_cryonirsp.parsers.polarimetric_check import PolarimetricCheckingUniqueBud
22
+ from dkist_processing_cryonirsp.tasks.parse import ParseL0CryonirspCILinearizedData
23
+ from dkist_processing_cryonirsp.tasks.parse import ParseL0CryonirspRampData
24
+ from dkist_processing_cryonirsp.tasks.parse import ParseL0CryonirspSPLinearizedData
25
+ from dkist_processing_cryonirsp.tests.conftest import _write_frames_to_task
26
+ from dkist_processing_cryonirsp.tests.conftest import cryonirsp_testing_parameters_factory
27
+ from dkist_processing_cryonirsp.tests.header_models import CryonirspHeaders
28
+ from dkist_processing_cryonirsp.tests.header_models import CryonirspHeadersValidNonLinearizedFrames
29
+ from dkist_processing_cryonirsp.tests.header_models import ModulatedDarkHeaders
30
+ from dkist_processing_cryonirsp.tests.header_models import ModulatedLampGainHeaders
31
+ from dkist_processing_cryonirsp.tests.header_models import ModulatedObserveHeaders
32
+ from dkist_processing_cryonirsp.tests.header_models import ModulatedPolcalHeaders
33
+ from dkist_processing_cryonirsp.tests.header_models import ModulatedSolarGainHeaders
34
+
35
+
36
+ def write_dark_frames_to_task(
37
+ task: Type[WorkflowTaskBase],
38
+ exposure_condition: ExposureConditions,
39
+ array_shape=(2, 2, 1),
40
+ tags: list[str] | None = None,
41
+ num_modstates: int = 1,
42
+ ):
43
+ num_frames = 0
44
+ for modstate in range(1, num_modstates + 1):
45
+ frame_generator = ModulatedDarkHeaders(
46
+ array_shape=array_shape,
47
+ exposure_condition=exposure_condition,
48
+ num_modstates=num_modstates,
49
+ modstate=modstate,
50
+ )
51
+ num_frames += _write_frames_to_task(
52
+ task=task, frame_generator=frame_generator, extra_tags=tags
53
+ )
54
+
55
+ return num_frames
56
+
57
+
58
+ def write_lamp_gain_frames_to_task(
59
+ task: Type[WorkflowTaskBase],
60
+ exposure_condition: ExposureConditions,
61
+ array_shape=(2, 2, 1),
62
+ tags: list[str] | None = None,
63
+ tag_func: Callable[[CryonirspHeaders], list[str]] = lambda x: [],
64
+ num_modstates: int = 1,
65
+ ):
66
+ num_frames = 0
67
+ for modstate in range(1, num_modstates + 1):
68
+ frame_generator = ModulatedLampGainHeaders(
69
+ array_shape=array_shape,
70
+ exposure_condition=exposure_condition,
71
+ num_modstates=num_modstates,
72
+ modstate=modstate,
73
+ )
74
+
75
+ num_frames += _write_frames_to_task(
76
+ task=task,
77
+ frame_generator=frame_generator,
78
+ extra_tags=tags,
79
+ tag_func=tag_func,
80
+ )
81
+
82
+ return num_frames
83
+
84
+
85
+ def write_solar_gain_frames_to_task(
86
+ task: Type[WorkflowTaskBase],
87
+ exposure_condition: ExposureConditions,
88
+ array_shape=(2, 2, 1),
89
+ tags: list[str] | None = None,
90
+ num_modstates: int = 1,
91
+ ):
92
+ num_frames = 0
93
+ for modstate in range(1, num_modstates + 1):
94
+ frame_generator = ModulatedSolarGainHeaders(
95
+ array_shape=array_shape,
96
+ exposure_condition=exposure_condition,
97
+ num_modstates=num_modstates,
98
+ modstate=modstate,
99
+ )
100
+
101
+ num_frames += _write_frames_to_task(
102
+ task=task, frame_generator=frame_generator, extra_tags=tags
103
+ )
104
+
105
+ return num_frames
106
+
107
+
108
+ def write_polcal_frames_to_task(
109
+ task: Type[WorkflowTaskBase],
110
+ num_modstates: int,
111
+ num_map_scans: int,
112
+ extra_headers: dict,
113
+ exposure_condition: ExposureConditions,
114
+ array_shape=(2, 2, 1),
115
+ tags: list[str] | None = None,
116
+ ):
117
+ num_frames = 0
118
+
119
+ modstates = [0] if num_modstates == 0 else range(1, num_modstates + 1)
120
+
121
+ for map_scan in range(1, num_map_scans + 1):
122
+ for mod_state in modstates:
123
+ frame_generator = ModulatedPolcalHeaders(
124
+ num_modstates=num_modstates,
125
+ modstate=mod_state,
126
+ array_shape=array_shape,
127
+ exposure_condition=exposure_condition,
128
+ extra_headers=extra_headers,
129
+ )
130
+
131
+ _write_frames_to_task(task=task, frame_generator=frame_generator, extra_tags=tags)
132
+ num_frames += 1
133
+
134
+ return num_frames
135
+
136
+
137
+ def write_observe_frames_to_task(
138
+ task: Type[WorkflowTaskBase],
139
+ num_modstates: int,
140
+ num_scan_steps: int,
141
+ num_map_scans: int,
142
+ num_sub_repeats: int,
143
+ num_measurements: int,
144
+ arm_id: str,
145
+ exposure_condition: ExposureConditions,
146
+ change_translated_headers: Callable[[fits.Header | None], fits.Header] = lambda x: x,
147
+ array_shape=(2, 2, 1),
148
+ tags: list[str] | None = None,
149
+ ):
150
+ num_frames = 0
151
+
152
+ modstates = [0] if num_modstates == 0 else range(1, num_modstates + 1)
153
+
154
+ start_time = datetime.now()
155
+ frame_delta_time = timedelta(seconds=10)
156
+ for map_scan in range(1, num_map_scans + 1):
157
+ for scan_step in range(1, num_scan_steps + 1):
158
+ for measurement in range(1, num_measurements + 1):
159
+ for mod_state in modstates:
160
+ for repeat in range(1, num_sub_repeats + 1):
161
+ frame_generator = ModulatedObserveHeaders(
162
+ start_date=start_time.isoformat(),
163
+ num_modstates=num_modstates,
164
+ modstate=mod_state,
165
+ num_map_scans=num_map_scans,
166
+ map_scan=map_scan,
167
+ num_sub_repeats=num_sub_repeats,
168
+ sub_repeat_num=repeat,
169
+ array_shape=array_shape,
170
+ exposure_condition=exposure_condition,
171
+ num_scan_steps=num_scan_steps,
172
+ scan_step=scan_step,
173
+ num_meas=num_measurements,
174
+ meas_num=measurement,
175
+ arm_id=arm_id,
176
+ )
177
+ start_time += frame_delta_time
178
+
179
+ _write_frames_to_task(
180
+ task=task,
181
+ frame_generator=frame_generator,
182
+ extra_tags=tags,
183
+ change_translated_headers=change_translated_headers,
184
+ )
185
+
186
+ num_frames += 1
187
+
188
+ return num_frames
189
+
190
+
191
+ def write_non_linearized_frames(
192
+ task: Type[WorkflowTaskBase],
193
+ arm_id: str,
194
+ start_time: str,
195
+ camera_readout_mode: str,
196
+ change_translated_headers: Callable[[fits.Header | None], fits.Header] = lambda x: x,
197
+ tags: list[str] | None = None,
198
+ ):
199
+ frame_generator = CryonirspHeadersValidNonLinearizedFrames(
200
+ arm_id=arm_id,
201
+ camera_readout_mode=camera_readout_mode,
202
+ dataset_shape=(2, 2, 2),
203
+ array_shape=(1, 2, 2),
204
+ time_delta=10,
205
+ roi_x_origin=0,
206
+ roi_y_origin=0,
207
+ roi_x_size=2,
208
+ roi_y_size=2,
209
+ date_obs=start_time,
210
+ exposure_time=0.01,
211
+ )
212
+
213
+ def tag_ramp_frames(translated_header):
214
+ ramp_tags = [
215
+ CryonirspTag.curr_frame_in_ramp(translated_header["CNCNDR"]),
216
+ ]
217
+
218
+ return ramp_tags
219
+
220
+ for frame in frame_generator:
221
+ _write_frames_to_task(
222
+ task=task,
223
+ frame_generator=frame,
224
+ change_translated_headers=change_translated_headers,
225
+ extra_tags=tags,
226
+ tag_ramp_frames=tag_ramp_frames,
227
+ )
228
+
229
+
230
+ def make_linearized_test_frames(
231
+ task,
232
+ arm_id: str,
233
+ dark_exposure_conditions: list[ExposureConditions],
234
+ num_modstates: int,
235
+ num_scan_steps: int,
236
+ change_translated_headers: Callable[[fits.Header | None], fits.Header] = lambda x: x,
237
+ lamp_exposure_condition: ExposureConditions = ExposureConditions(
238
+ 10.0, AllowableOpticalDensityFilterNames.OPEN.value
239
+ ),
240
+ solar_exposure_condition: ExposureConditions = ExposureConditions(
241
+ 5.0, AllowableOpticalDensityFilterNames.OPEN.value
242
+ ),
243
+ polcal_exposure_condition: ExposureConditions = ExposureConditions(
244
+ 7.0, AllowableOpticalDensityFilterNames.OPEN.value
245
+ ),
246
+ observe_exposure_condition: ExposureConditions = ExposureConditions(
247
+ 6.0, AllowableOpticalDensityFilterNames.OPEN.value
248
+ ),
249
+ num_map_scans: int = 1,
250
+ num_sub_repeats: int = 1,
251
+ num_measurements: int = 1,
252
+ extra_headers: dict | None = None,
253
+ ):
254
+ num_dark = 0
255
+ num_polcal = 0
256
+ num_obs = 0
257
+ lin_tag = [CryonirspTag.linearized()]
258
+
259
+ for condition in dark_exposure_conditions:
260
+ num_dark += write_dark_frames_to_task(
261
+ task,
262
+ exposure_condition=condition,
263
+ tags=lin_tag,
264
+ num_modstates=num_modstates or 1, # We *always* need dark frames
265
+ )
266
+
267
+ num_lamp = write_lamp_gain_frames_to_task(
268
+ task,
269
+ tags=lin_tag,
270
+ exposure_condition=lamp_exposure_condition,
271
+ num_modstates=num_modstates or 1, # We *always* need dark frames
272
+ )
273
+ num_solar = write_solar_gain_frames_to_task(
274
+ task,
275
+ tags=lin_tag,
276
+ exposure_condition=solar_exposure_condition,
277
+ num_modstates=num_modstates or 1, # We *always* need dark frames
278
+ )
279
+
280
+ num_polcal += write_polcal_frames_to_task(
281
+ task,
282
+ num_modstates=num_modstates,
283
+ num_map_scans=num_map_scans,
284
+ tags=lin_tag,
285
+ extra_headers=extra_headers,
286
+ exposure_condition=polcal_exposure_condition,
287
+ )
288
+ num_obs += write_observe_frames_to_task(
289
+ task,
290
+ arm_id=arm_id,
291
+ num_scan_steps=num_scan_steps,
292
+ num_map_scans=num_map_scans,
293
+ num_sub_repeats=num_sub_repeats,
294
+ num_modstates=num_modstates,
295
+ exposure_condition=observe_exposure_condition,
296
+ num_measurements=num_measurements,
297
+ tags=lin_tag,
298
+ change_translated_headers=change_translated_headers,
299
+ )
300
+
301
+ return num_dark, num_lamp, num_solar, num_polcal, num_obs
302
+
303
+
304
+ def make_non_linearized_test_frames(
305
+ task,
306
+ change_translated_headers: Callable[[fits.Header | None], fits.Header] = lambda x: x,
307
+ ):
308
+ arm_id = "SP"
309
+ camera_readout_mode = "FastUpTheRamp"
310
+
311
+ start_time = datetime(1946, 11, 20).isoformat("T")
312
+
313
+ extra_tags = [
314
+ CryonirspTag.input(),
315
+ # All frames in a ramp have the same date-obs
316
+ CryonirspTag.time_obs(str(start_time)),
317
+ ]
318
+
319
+ write_non_linearized_frames(
320
+ task,
321
+ start_time=start_time,
322
+ arm_id=arm_id,
323
+ camera_readout_mode=camera_readout_mode,
324
+ tags=extra_tags,
325
+ change_translated_headers=change_translated_headers,
326
+ )
327
+
328
+
329
+ @pytest.fixture
330
+ def parse_linearized_task(
331
+ tmp_path, recipe_run_id, assign_input_dataset_doc_to_task, mocker, arm_id
332
+ ):
333
+ mocker.patch(
334
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
335
+ )
336
+ if arm_id == "CI":
337
+ parsing_class = ParseL0CryonirspCILinearizedData
338
+ if arm_id == "SP":
339
+ parsing_class = ParseL0CryonirspSPLinearizedData
340
+ with parsing_class(
341
+ recipe_run_id=recipe_run_id,
342
+ workflow_name="parse_cryonirsp_input_data",
343
+ workflow_version="VX.Y",
344
+ ) as task:
345
+ try: # This try... block is here to make sure the dbs get cleaned up if there's a failure in the fixture
346
+ task.scratch = WorkflowFileSystem(
347
+ scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
348
+ )
349
+ param_dataclass = cryonirsp_testing_parameters_factory(param_path=tmp_path)
350
+ assign_input_dataset_doc_to_task(
351
+ task,
352
+ param_dataclass(),
353
+ parameter_class=CryonirspParsingParameters,
354
+ obs_ip_start_time=None,
355
+ )
356
+ yield task
357
+ finally:
358
+ task._purge()
359
+
360
+
361
+ @pytest.fixture
362
+ def parse_non_linearized_task(tmp_path, recipe_run_id, mocker):
363
+ mocker.patch(
364
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
365
+ )
366
+ with ParseL0CryonirspRampData(
367
+ recipe_run_id=recipe_run_id,
368
+ workflow_name="parse_cryonirsp_input_data",
369
+ workflow_version="VX.Y",
370
+ ) as task:
371
+ try: # This try... block is here to make sure the dbs get cleaned up if there's a failure in the fixture
372
+ task.scratch = WorkflowFileSystem(
373
+ scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
374
+ )
375
+ yield task
376
+ finally:
377
+ task._purge()
378
+
379
+
380
+ def test_parse_cryonirsp_non_linearized_data(parse_non_linearized_task):
381
+ """
382
+ Given: A ParseCryonirspRampData task
383
+ When: Calling the task instance
384
+ Then: All tagged files exist and individual task tags are applied
385
+ """
386
+
387
+ task = parse_non_linearized_task
388
+ make_non_linearized_test_frames(task)
389
+
390
+ task()
391
+
392
+ filepaths = list(task.read(tags=[CryonirspTag.input(), CryonirspTag.frame()]))
393
+ cncndr_list = []
394
+ for i, filepath in enumerate(filepaths):
395
+ assert filepath.exists()
396
+ hdul = fits.open(filepath)
397
+ cncndr_list.append(hdul[0].header["CNCNDR"])
398
+ assert len(filepaths) == 2
399
+ assert sorted(cncndr_list) == [1, 2]
400
+ assert task.constants._db_dict[CryonirspBudName.camera_readout_mode.value] == "FastUpTheRamp"
401
+ assert task.constants._db_dict[CryonirspBudName.arm_id.value] == "SP"
402
+ assert len(task.constants._db_dict[CryonirspBudName.time_obs_list]) == 1
403
+ assert task.constants._db_dict[CryonirspBudName.wavelength.value] == 1083.0
404
+ assert task.constants._db_dict[CryonirspBudName.time_obs_list][0] == datetime(
405
+ 1946, 11, 20
406
+ ).isoformat("T")
407
+ assert task.constants._db_dict[BudName.obs_ip_start_time.value] == "1999-12-31T23:59:59"
408
+
409
+
410
+ def test_parse_cryonirsp_non_linearized_data_bad_filter_name(parse_non_linearized_task):
411
+ """
412
+ Given: A ParseCryonirspRampData task with a bad filter name in the headers
413
+ When: Calling the task instance
414
+ Then: The task fails with a ValueError exception
415
+ """
416
+
417
+ task = parse_non_linearized_task
418
+
419
+ def insert_bad_filter_name_into_header(translated_header: fits.Header):
420
+ translated_header["CNFILTNP"] = "BAD_FILTER_NAME"
421
+ return translated_header
422
+
423
+ make_non_linearized_test_frames(
424
+ task, change_translated_headers=insert_bad_filter_name_into_header
425
+ )
426
+
427
+ with pytest.raises(
428
+ ValueError,
429
+ match=re.escape(
430
+ "Unknown Optical Density Filter Name(s): bad_filter_names = {'BAD_FILTER_NAME'}"
431
+ ),
432
+ ):
433
+ task()
434
+
435
+
436
+ @pytest.mark.parametrize("number_of_modstates", [0, 1, 8])
437
+ @pytest.mark.parametrize("arm_id", ["CI", "SP"])
438
+ def test_parse_cryonirsp_linearized_data(parse_linearized_task, arm_id, number_of_modstates):
439
+ """
440
+ Given: A ParseCryonirspInputData task
441
+ When: Calling the task instance
442
+ Then: All tagged files exist and individual task tags are applied
443
+ """
444
+
445
+ task = parse_linearized_task
446
+
447
+ lamp_exp_cond = ExposureConditions(10.0, AllowableOpticalDensityFilterNames.OPEN.value)
448
+ solar_exp_cond = ExposureConditions(5.0, AllowableOpticalDensityFilterNames.OPEN.value)
449
+ obs_exp_cond = ExposureConditions(6.0, AllowableOpticalDensityFilterNames.OPEN.value)
450
+ polcal_exp_cond = ExposureConditions(7.0, AllowableOpticalDensityFilterNames.OPEN.value)
451
+ dark_exp_conditions = [
452
+ lamp_exp_cond,
453
+ solar_exp_cond,
454
+ obs_exp_cond,
455
+ polcal_exp_cond,
456
+ ]
457
+
458
+ num_dark, num_lamp, num_solar, num_polcal, num_obs = make_linearized_test_frames(
459
+ task,
460
+ arm_id,
461
+ dark_exposure_conditions=dark_exp_conditions,
462
+ num_modstates=number_of_modstates,
463
+ num_scan_steps=3,
464
+ num_map_scans=1,
465
+ num_sub_repeats=1,
466
+ lamp_exposure_condition=lamp_exp_cond,
467
+ solar_exposure_condition=solar_exp_cond,
468
+ observe_exposure_condition=obs_exp_cond,
469
+ polcal_exposure_condition=polcal_exp_cond,
470
+ )
471
+
472
+ task()
473
+ num_actual_modstates = number_of_modstates or 1
474
+ for modstate in range(1, num_actual_modstates + 1):
475
+ assert (
476
+ len(
477
+ list(
478
+ task.read(
479
+ tags=[
480
+ CryonirspTag.linearized(),
481
+ CryonirspTag.task_dark(),
482
+ CryonirspTag.modstate(modstate),
483
+ ]
484
+ )
485
+ )
486
+ )
487
+ == num_dark / num_actual_modstates
488
+ )
489
+
490
+ assert (
491
+ len(
492
+ list(
493
+ task.read(
494
+ tags=[
495
+ CryonirspTag.linearized(),
496
+ CryonirspTag.task_lamp_gain(),
497
+ CryonirspTag.modstate(modstate),
498
+ ]
499
+ )
500
+ )
501
+ )
502
+ == num_lamp / num_actual_modstates
503
+ )
504
+
505
+ assert (
506
+ len(
507
+ list(
508
+ task.read(
509
+ tags=[
510
+ CryonirspTag.linearized(),
511
+ CryonirspTag.task_solar_gain(),
512
+ CryonirspTag.modstate(modstate),
513
+ ]
514
+ )
515
+ )
516
+ )
517
+ == num_solar / num_actual_modstates
518
+ )
519
+
520
+ assert (
521
+ len(
522
+ list(
523
+ task.read(
524
+ tags=[
525
+ CryonirspTag.linearized(),
526
+ CryonirspTag.task_polcal(),
527
+ CryonirspTag.modstate(modstate),
528
+ ]
529
+ )
530
+ )
531
+ )
532
+ == num_polcal / num_actual_modstates
533
+ )
534
+
535
+ assert (
536
+ len(
537
+ list(
538
+ task.read(
539
+ tags=[
540
+ CryonirspTag.linearized(),
541
+ CryonirspTag.task_observe(),
542
+ CryonirspTag.modstate(modstate),
543
+ ]
544
+ )
545
+ )
546
+ )
547
+ == num_obs / num_actual_modstates
548
+ )
549
+
550
+
551
+ @pytest.mark.parametrize("arm_id", ["CI", "SP"])
552
+ def test_parse_cryonirsp_linearized_data_mismatched_darks(parse_linearized_task, arm_id):
553
+ """
554
+ Given: A parse task with dark data that have mismatched exposure times
555
+ When: Calling the Parse task
556
+ Then: Raise the correct error
557
+ """
558
+
559
+ task = parse_linearized_task
560
+
561
+ lamp_exp_cond = ExposureConditions(10.0, AllowableOpticalDensityFilterNames.OPEN.value)
562
+ solar_exp_cond = ExposureConditions(5.0, AllowableOpticalDensityFilterNames.OPEN.value)
563
+ obs_exp_cond = ExposureConditions(6.0, AllowableOpticalDensityFilterNames.OPEN.value)
564
+ polcal_exp_cond = ExposureConditions(7.0, AllowableOpticalDensityFilterNames.OPEN.value)
565
+
566
+ # Make all of them differ by only one aspect of the exposure condition
567
+ dark_exp_conditions = [
568
+ ExposureConditions(11.0, AllowableOpticalDensityFilterNames.OPEN.value),
569
+ ExposureConditions(5.0, AllowableOpticalDensityFilterNames.NONE.value),
570
+ ExposureConditions(7.0, AllowableOpticalDensityFilterNames.G278.value),
571
+ ]
572
+
573
+ make_linearized_test_frames(
574
+ task,
575
+ arm_id,
576
+ dark_exposure_conditions=dark_exp_conditions,
577
+ num_modstates=8,
578
+ num_scan_steps=3,
579
+ num_map_scans=2,
580
+ num_sub_repeats=1,
581
+ lamp_exposure_condition=lamp_exp_cond,
582
+ solar_exposure_condition=solar_exp_cond,
583
+ polcal_exposure_condition=polcal_exp_cond,
584
+ observe_exposure_condition=obs_exp_cond,
585
+ )
586
+
587
+ with pytest.raises(
588
+ ValueError, match="Exposure conditions required in the set of dark frames not found.*"
589
+ ):
590
+ task()
591
+
592
+
593
+ @pytest.mark.parametrize("arm_id", ["CI", "SP"])
594
+ def test_parse_cryonirsp_linearized_data_multi_num_scan_steps(parse_linearized_task, arm_id):
595
+ """
596
+ Given: A parse task with data that has muliple num_scan_step values
597
+ When: Calling the Parse task
598
+ Then: Raise the correct error
599
+ """
600
+
601
+ task = parse_linearized_task
602
+
603
+ lamp_exp_cond = ExposureConditions(10.0, AllowableOpticalDensityFilterNames.OPEN.value)
604
+ solar_exp_cond = ExposureConditions(5.0, AllowableOpticalDensityFilterNames.OPEN.value)
605
+ obs_exp_cond = ExposureConditions(6.0, AllowableOpticalDensityFilterNames.OPEN.value)
606
+ polcal_exp_cond = ExposureConditions(7.0, AllowableOpticalDensityFilterNames.OPEN.value)
607
+ dark_exp_conditions = [
608
+ lamp_exp_cond,
609
+ solar_exp_cond,
610
+ obs_exp_cond,
611
+ polcal_exp_cond,
612
+ ]
613
+
614
+ def make_multi_num_scans(translated_header: fits.Header):
615
+ translated_header["CNNUMSCN"] = translated_header["CNCURSCN"] % 3
616
+ return translated_header
617
+
618
+ make_linearized_test_frames(
619
+ task,
620
+ arm_id,
621
+ dark_exposure_conditions=dark_exp_conditions,
622
+ num_modstates=8,
623
+ num_scan_steps=4,
624
+ num_map_scans=4,
625
+ num_sub_repeats=2,
626
+ change_translated_headers=make_multi_num_scans,
627
+ lamp_exposure_condition=lamp_exp_cond,
628
+ solar_exposure_condition=solar_exp_cond,
629
+ polcal_exposure_condition=polcal_exp_cond,
630
+ observe_exposure_condition=obs_exp_cond,
631
+ )
632
+
633
+ with pytest.raises(ValueError, match="Multiple NUM_SCAN_STEPS values found.*"):
634
+ task()
635
+
636
+
637
+ @pytest.mark.parametrize("arm_id", ["CI", "SP"])
638
+ @pytest.mark.parametrize(
639
+ "abort_loop_name",
640
+ [
641
+ pytest.param("scan_step", id="Missing_step"),
642
+ pytest.param("measurement", id="Missing_measurement"),
643
+ pytest.param("modstate", id="Missing_modstate"),
644
+ pytest.param("sub_repeat", id="Missing_sub_repeat"),
645
+ ],
646
+ )
647
+ def test_parse_cryonirsp_linearized_incomplete_final_map(
648
+ parse_linearized_task, arm_id, abort_loop_name
649
+ ):
650
+ """
651
+ Given: A parse task with data that has complete raster scans along with an incomplete raster scan
652
+ When: Calling the Parse task
653
+ Then: The correct number of scan steps and maps are found
654
+ """
655
+
656
+ task = parse_linearized_task
657
+
658
+ exp_cond = ExposureConditions(6.0, AllowableOpticalDensityFilterNames.OPEN.value)
659
+ lin_tag = [CryonirspTag.linearized()]
660
+
661
+ num_map_scans = 3
662
+ num_scan_steps = 3
663
+ num_measurements = 2
664
+ num_modstates = 2
665
+ num_sub_repeats = 2
666
+
667
+ # Needed so the dark picky bud is happy
668
+ write_dark_frames_to_task(task, exposure_condition=exp_cond, tags=lin_tag)
669
+
670
+ # Needed so the pol checking buds are happy
671
+ write_polcal_frames_to_task(
672
+ task,
673
+ num_modstates=num_modstates,
674
+ num_map_scans=num_map_scans,
675
+ tags=lin_tag,
676
+ extra_headers=dict(),
677
+ exposure_condition=exp_cond,
678
+ )
679
+
680
+ # Make all test frames except for last map scan
681
+ write_observe_frames_to_task(
682
+ task,
683
+ arm_id=arm_id,
684
+ num_scan_steps=num_scan_steps,
685
+ num_map_scans=num_map_scans - 1,
686
+ num_sub_repeats=num_sub_repeats,
687
+ num_modstates=num_modstates,
688
+ num_measurements=num_measurements,
689
+ tags=lin_tag,
690
+ exposure_condition=exp_cond,
691
+ )
692
+
693
+ # Make incomplete final map scan. The "abort_loop_name" sets the level of the instrument loop that has the abort.
694
+ final_map_scan_number = num_map_scans
695
+ aborted = False
696
+ for scan_step in range(1, num_scan_steps + 1):
697
+ if aborted or (abort_loop_name == "scan_step" and scan_step == num_scan_steps):
698
+ aborted = True
699
+ break
700
+
701
+ for measurement in range(1, num_measurements + 1):
702
+ if aborted or (abort_loop_name == "measurement" and measurement == num_measurements):
703
+ aborted = True
704
+ break
705
+
706
+ for mod_state in range(1, num_modstates + 1):
707
+ if aborted or (abort_loop_name == "modstate" and mod_state == num_modstates):
708
+ aborted = True
709
+ break
710
+
711
+ for repeat in range(1, num_sub_repeats + 1):
712
+ if aborted or (abort_loop_name == "sub_repeat" and repeat == num_sub_repeats):
713
+ aborted = True
714
+ break
715
+
716
+ frame_generator = ModulatedObserveHeaders(
717
+ num_modstates=num_modstates,
718
+ modstate=mod_state,
719
+ num_map_scans=num_map_scans,
720
+ map_scan=final_map_scan_number,
721
+ num_sub_repeats=num_sub_repeats,
722
+ sub_repeat_num=repeat,
723
+ array_shape=(1, 2, 2),
724
+ exposure_condition=exp_cond,
725
+ num_scan_steps=num_scan_steps,
726
+ scan_step=scan_step,
727
+ num_meas=num_measurements,
728
+ meas_num=measurement,
729
+ arm_id=arm_id,
730
+ )
731
+
732
+ _write_frames_to_task(
733
+ task=task,
734
+ frame_generator=frame_generator,
735
+ extra_tags=[CryonirspTag.linearized()],
736
+ )
737
+
738
+ task()
739
+ assert task.constants._db_dict[CryonirspBudName.num_scan_steps.value] == num_scan_steps
740
+ assert task.constants._db_dict[CryonirspBudName.num_map_scans.value] == num_map_scans - 1
741
+
742
+
743
+ @pytest.mark.parametrize("arm_id", ["CI", "SP"])
744
+ @pytest.mark.parametrize(
745
+ "abort_loop_name",
746
+ [
747
+ pytest.param("scan_step", id="Missing_step"),
748
+ pytest.param("measurement", id="Missing_measurement"),
749
+ pytest.param("modstate", id="Missing_modstate"),
750
+ pytest.param("sub_repeat", id="Missing_sub_repeat"),
751
+ ],
752
+ )
753
+ def test_parse_cryonirsp_linearized_incomplete_final_map_error(
754
+ parse_linearized_task, arm_id, abort_loop_name
755
+ ):
756
+ """
757
+ Given: A parse task with data that containing multiple aborted maps
758
+ When: Calling the Parse task
759
+ Then: The correct Error is raised
760
+ """
761
+
762
+ task = parse_linearized_task
763
+
764
+ exp_cond = ExposureConditions(4.0, AllowableOpticalDensityFilterNames.OPEN.value)
765
+ lin_tag = [CryonirspTag.linearized()]
766
+
767
+ num_map_scans = 3
768
+ num_scan_steps = 3
769
+ num_measurements = 2
770
+ num_modstates = 2
771
+ num_sub_repeats = 2
772
+
773
+ # Needed so the pol checking buds are happy
774
+ write_polcal_frames_to_task(
775
+ task,
776
+ num_modstates=num_modstates,
777
+ num_map_scans=num_map_scans,
778
+ tags=lin_tag,
779
+ extra_headers=dict(),
780
+ exposure_condition=exp_cond,
781
+ )
782
+
783
+ # Make one complete map
784
+ write_observe_frames_to_task(
785
+ task,
786
+ arm_id=arm_id,
787
+ num_scan_steps=1,
788
+ num_map_scans=num_map_scans,
789
+ num_sub_repeats=num_sub_repeats,
790
+ num_modstates=num_modstates,
791
+ num_measurements=num_measurements,
792
+ tags=lin_tag,
793
+ exposure_condition=exp_cond,
794
+ )
795
+
796
+ # Make 2 incomplete maps. The "abort_loop_name" sets the level of the instrument loop that has the abort.
797
+ final_map_scan_number = num_map_scans
798
+ for map_scan in range(2, num_map_scans + 1):
799
+ aborted = False
800
+ for scan_step in range(1, num_scan_steps + 1):
801
+ if aborted or (abort_loop_name == "scan_step" and scan_step == num_scan_steps):
802
+ aborted = True
803
+ break
804
+
805
+ for measurement in range(1, num_measurements + 1):
806
+ if aborted or (
807
+ abort_loop_name == "measurement" and measurement == num_measurements
808
+ ):
809
+ aborted = True
810
+ break
811
+
812
+ for mod_state in range(1, num_modstates + 1):
813
+ if aborted or (abort_loop_name == "modstate" and mod_state == num_modstates):
814
+ aborted = True
815
+ break
816
+
817
+ for repeat in range(1, num_sub_repeats + 1):
818
+ if aborted or (
819
+ abort_loop_name == "sub_repeat" and repeat == num_sub_repeats
820
+ ):
821
+ aborted = True
822
+ break
823
+
824
+ frame_generator = ModulatedObserveHeaders(
825
+ num_modstates=num_modstates,
826
+ modstate=mod_state,
827
+ num_map_scans=num_map_scans,
828
+ map_scan=final_map_scan_number,
829
+ num_sub_repeats=num_sub_repeats,
830
+ sub_repeat_num=repeat,
831
+ array_shape=(1, 2, 2),
832
+ exposure_condition=exp_cond,
833
+ num_scan_steps=num_scan_steps,
834
+ scan_step=scan_step,
835
+ num_meas=num_measurements,
836
+ meas_num=measurement,
837
+ arm_id=arm_id,
838
+ )
839
+
840
+ _write_frames_to_task(
841
+ task=task,
842
+ frame_generator=frame_generator,
843
+ extra_tags=[CryonirspTag.linearized()],
844
+ )
845
+
846
+ with pytest.raises(ValueError, match="More than one incomplete map exists in the data."):
847
+ task()
848
+
849
+
850
+ @pytest.mark.parametrize("arm_id", ["CI", "SP"])
851
+ @pytest.mark.parametrize(
852
+ "abort_loop_name",
853
+ [
854
+ pytest.param("scan_step", id="Missing_step"),
855
+ pytest.param("measurement", id="Missing_measurement"),
856
+ pytest.param("modstate", id="Missing_modstate"),
857
+ pytest.param("sub_repeat", id="Missing_sub_repeat"),
858
+ ],
859
+ )
860
+ def test_parse_cryonirsp_linearized_incomplete_raster_scan(
861
+ parse_linearized_task, arm_id, abort_loop_name
862
+ ):
863
+ """
864
+ Given: A parse task with data that has an incomplete raster scan
865
+ When: Calling the parse task
866
+ Then: The correct number of scan steps and maps are found
867
+ """
868
+
869
+ task = parse_linearized_task
870
+
871
+ exp_cond = ExposureConditions(6.0, AllowableOpticalDensityFilterNames.OPEN.value)
872
+ lin_tag = [CryonirspTag.linearized()]
873
+
874
+ num_scan_steps = 4
875
+ num_map_scans = 1
876
+ num_scan_steps = 3
877
+ num_measurements = 2
878
+ num_modstates = 2
879
+ num_sub_repeats = 2
880
+
881
+ # Needed so the frames from the complete and incomplete maps have the same value for
882
+ # CNNUMSCN (the number of scan steps)
883
+ def set_constant_num_scan_steps(translated_header):
884
+ translated_header["CNNUMSCN"] = num_scan_steps
885
+ return translated_header
886
+
887
+ # Needed so the dark picky bud is happy
888
+ write_dark_frames_to_task(task, exposure_condition=exp_cond, tags=lin_tag)
889
+
890
+ # Needed so the pol checking buds are happy
891
+ write_polcal_frames_to_task(
892
+ task,
893
+ num_modstates=num_modstates,
894
+ num_map_scans=num_map_scans,
895
+ tags=lin_tag,
896
+ extra_headers=dict(),
897
+ exposure_condition=exp_cond,
898
+ )
899
+
900
+ # Make all the complete scan steps
901
+ write_observe_frames_to_task(
902
+ task,
903
+ arm_id=arm_id,
904
+ num_scan_steps=num_scan_steps - 1,
905
+ num_map_scans=num_map_scans,
906
+ num_sub_repeats=num_sub_repeats,
907
+ num_modstates=num_modstates,
908
+ num_measurements=num_measurements,
909
+ tags=lin_tag,
910
+ exposure_condition=exp_cond,
911
+ change_translated_headers=set_constant_num_scan_steps,
912
+ )
913
+
914
+ # Now make the final scan step, which will be aborted at various points
915
+ final_scan_step_number = num_scan_steps
916
+
917
+ # If abort_loop_name == "scan_step" then don't make *any* files for the last step
918
+ if abort_loop_name != "scan_step":
919
+ aborted = False
920
+ for measurement in range(1, num_measurements + 1):
921
+ if aborted or (abort_loop_name == "measurement" and measurement == num_measurements):
922
+ aborted = True
923
+ break
924
+
925
+ for mod_state in range(1, num_modstates + 1):
926
+ if aborted or (abort_loop_name == "modstate" and mod_state == num_modstates):
927
+ aborted = True
928
+ break
929
+
930
+ for repeat in range(1, num_sub_repeats + 1):
931
+ if aborted or (abort_loop_name == "sub_repeat" and repeat == num_sub_repeats):
932
+ aborted = True
933
+ break
934
+
935
+ frame_generator = ModulatedObserveHeaders(
936
+ num_modstates=num_modstates,
937
+ modstate=mod_state,
938
+ num_map_scans=num_map_scans,
939
+ map_scan=1,
940
+ num_sub_repeats=num_sub_repeats,
941
+ sub_repeat_num=repeat,
942
+ array_shape=(1, 2, 2),
943
+ exposure_condition=exp_cond,
944
+ num_scan_steps=num_scan_steps,
945
+ scan_step=final_scan_step_number,
946
+ num_meas=num_measurements,
947
+ meas_num=measurement,
948
+ arm_id=arm_id,
949
+ )
950
+
951
+ _write_frames_to_task(
952
+ task=task,
953
+ frame_generator=frame_generator,
954
+ extra_tags=[CryonirspTag.linearized()],
955
+ )
956
+
957
+ task()
958
+
959
+ assert task.constants._db_dict[CryonirspBudName.num_scan_steps.value] == num_scan_steps - 1
960
+ assert task.constants._db_dict[CryonirspBudName.num_map_scans.value] == num_map_scans
961
+
962
+
963
+ @pytest.mark.parametrize("arm_id", ["CI", "SP"])
964
+ @pytest.mark.parametrize(
965
+ "abort_loop_name",
966
+ [
967
+ pytest.param("scan_step", id="Missing_step"),
968
+ pytest.param("measurement", id="Missing_measurement"),
969
+ pytest.param("modstate", id="Missing_modstate"),
970
+ pytest.param("sub_repeat", id="Missing_sub_repeat"),
971
+ ],
972
+ )
973
+ def test_parse_cryonirsp_linearized_incomplete_raster_scan_error(
974
+ parse_linearized_task, arm_id, abort_loop_name
975
+ ):
976
+ """
977
+ Given: A parse task with data representing a single map scan that was aborted and then continued
978
+ When: Calling the parse task
979
+ Then: The correct Error is raised
980
+ """
981
+ task = parse_linearized_task
982
+
983
+ num_map_scans = 1
984
+ num_scan_steps = 3
985
+ num_measurements = 2
986
+ num_modstates = 2
987
+ num_sub_repeats = 2
988
+
989
+ lin_tag = [CryonirspTag.linearized()]
990
+ exp_cond = ExposureConditions(6.0, AllowableOpticalDensityFilterNames.OPEN.value)
991
+
992
+ # Needed so the frames from the complete and incomplete maps have the same value for
993
+ # CNNUMSCN (the number of scan steps)
994
+ def set_constant_num_scan_steps(translated_header):
995
+ translated_header["CNNUMSCN"] = num_scan_steps
996
+ return translated_header
997
+
998
+ # Needed so the dark picky bud is happy
999
+ write_dark_frames_to_task(task, exposure_condition=exp_cond, tags=lin_tag)
1000
+
1001
+ # Needed so the pol checking buds are happy
1002
+ write_polcal_frames_to_task(
1003
+ task,
1004
+ num_modstates=num_modstates,
1005
+ num_map_scans=num_map_scans,
1006
+ tags=lin_tag,
1007
+ extra_headers=dict(),
1008
+ exposure_condition=exp_cond,
1009
+ )
1010
+
1011
+ # Make the first complete scan step
1012
+ write_observe_frames_to_task(
1013
+ task,
1014
+ arm_id=arm_id,
1015
+ num_scan_steps=1,
1016
+ num_map_scans=num_map_scans,
1017
+ num_sub_repeats=num_sub_repeats,
1018
+ num_modstates=num_modstates,
1019
+ num_measurements=num_measurements,
1020
+ tags=lin_tag,
1021
+ exposure_condition=exp_cond,
1022
+ change_translated_headers=set_constant_num_scan_steps,
1023
+ )
1024
+
1025
+ def set_last_scan_step(translated_header):
1026
+ translated_header["CNNUMSCN"] = num_scan_steps
1027
+ translated_header["CNCURSCN"] = 4
1028
+ return translated_header
1029
+
1030
+ # Make the final complete scan step
1031
+ write_observe_frames_to_task(
1032
+ task,
1033
+ arm_id=arm_id,
1034
+ num_scan_steps=1,
1035
+ num_map_scans=num_map_scans,
1036
+ num_sub_repeats=num_sub_repeats,
1037
+ num_modstates=num_modstates,
1038
+ num_measurements=num_measurements,
1039
+ tags=lin_tag,
1040
+ exposure_condition=exp_cond,
1041
+ change_translated_headers=set_last_scan_step,
1042
+ )
1043
+
1044
+ # Now make 2 scan steps that are aborted at various points
1045
+ for scan_step in [2, 3]: # Abort the middle 2 scan steps
1046
+ aborted = False
1047
+
1048
+ # If we're aborting at the "scan_step" level we don't need any frames at all
1049
+ if abort_loop_name != "scan_step":
1050
+ for measurement in range(1, num_measurements + 1):
1051
+ if aborted or (
1052
+ abort_loop_name == "measurement" and measurement == num_measurements
1053
+ ):
1054
+ aborted = True
1055
+ break
1056
+
1057
+ for mod_state in range(1, num_modstates + 1):
1058
+ if aborted or (abort_loop_name == "modstate" and mod_state == num_modstates):
1059
+ aborted = True
1060
+ break
1061
+
1062
+ for repeat in range(1, num_sub_repeats + 1):
1063
+ if aborted or (
1064
+ abort_loop_name == "sub_repeat" and repeat == num_sub_repeats
1065
+ ):
1066
+ aborted = True
1067
+ break
1068
+
1069
+ frame_generator = ModulatedObserveHeaders(
1070
+ num_modstates=num_modstates,
1071
+ modstate=mod_state,
1072
+ num_map_scans=num_map_scans,
1073
+ map_scan=1,
1074
+ num_sub_repeats=num_sub_repeats,
1075
+ sub_repeat_num=repeat,
1076
+ array_shape=(1, 2, 2),
1077
+ exposure_condition=exp_cond,
1078
+ num_scan_steps=num_scan_steps,
1079
+ scan_step=scan_step,
1080
+ num_meas=num_measurements,
1081
+ meas_num=measurement,
1082
+ arm_id=arm_id,
1083
+ )
1084
+
1085
+ _write_frames_to_task(
1086
+ task=task,
1087
+ frame_generator=frame_generator,
1088
+ extra_tags=[CryonirspTag.linearized()],
1089
+ )
1090
+
1091
+ # We aborted steps 2 and 3 so [1, 4] is the expected sequence of complete steps
1092
+ with pytest.raises(
1093
+ ValueError, match=re.escape("Not all sequential steps could be found. Found [1, 4]")
1094
+ ):
1095
+ task()
1096
+
1097
+
1098
+ @pytest.mark.parametrize("arm_id", ["CI", "SP"])
1099
+ def test_parse_cryonirsp_linearized_polcal_task_types(parse_linearized_task, arm_id):
1100
+ """
1101
+ Given: A Parse task with associated polcal files that include polcal gain and dark
1102
+ When: Tagging the task of each file
1103
+ Then: Polcal gain and darks are identified and tagged correctly
1104
+ """
1105
+
1106
+ task = parse_linearized_task
1107
+
1108
+ lamp_exp_cond = ExposureConditions(10.0, AllowableOpticalDensityFilterNames.OPEN.value)
1109
+ solar_exp_cond = ExposureConditions(5.0, AllowableOpticalDensityFilterNames.OPEN.value)
1110
+ obs_exp_cond = ExposureConditions(6.0, AllowableOpticalDensityFilterNames.OPEN.value)
1111
+ polcal_exp_cond = ExposureConditions(7.0, AllowableOpticalDensityFilterNames.OPEN.value)
1112
+ dark_exp_conditions = [
1113
+ lamp_exp_cond,
1114
+ solar_exp_cond,
1115
+ obs_exp_cond,
1116
+ polcal_exp_cond,
1117
+ ]
1118
+
1119
+ num_scan_steps = 0
1120
+ num_map_scans = 7
1121
+ num_modstates = 8
1122
+ num_sub_repeats = 1
1123
+
1124
+ polcal_dark_headers = {"PAC__008": "DarkShutter", "PAC__006": "clear", "PAC__004": "clear"}
1125
+ polcal_gain_headers = {"PAC__008": "FieldStopFoo", "PAC__006": "clear", "PAC__004": "clear"}
1126
+ polcal_data_headers = {
1127
+ "PAC__008": "FieldStopFoo",
1128
+ "PAC__006": "SiO2 SAR",
1129
+ "PAC__004": "Sapphire Polarizer",
1130
+ }
1131
+
1132
+ extra_headers = [polcal_dark_headers, polcal_gain_headers, polcal_data_headers]
1133
+
1134
+ for headers in extra_headers:
1135
+ make_linearized_test_frames(
1136
+ task,
1137
+ arm_id,
1138
+ dark_exposure_conditions=dark_exp_conditions,
1139
+ num_modstates=num_modstates,
1140
+ num_scan_steps=num_scan_steps,
1141
+ num_map_scans=num_map_scans,
1142
+ num_sub_repeats=num_sub_repeats,
1143
+ extra_headers=headers,
1144
+ lamp_exposure_condition=lamp_exp_cond,
1145
+ solar_exposure_condition=solar_exp_cond,
1146
+ polcal_exposure_condition=polcal_exp_cond,
1147
+ observe_exposure_condition=obs_exp_cond,
1148
+ )
1149
+
1150
+ task()
1151
+
1152
+ assert (
1153
+ task.scratch.count_all(tags=[CryonirspTag.task("POLCAL_DARK")])
1154
+ == num_map_scans * num_modstates
1155
+ )
1156
+ assert (
1157
+ task.scratch.count_all(tags=[CryonirspTag.task("POLCAL_GAIN")])
1158
+ == num_map_scans * num_modstates
1159
+ )
1160
+ assert (
1161
+ task.scratch.count_all(tags=[CryonirspTag.task("POLCAL")])
1162
+ == (num_map_scans * num_modstates) * 3
1163
+ )
1164
+
1165
+
1166
+ @pytest.mark.parametrize("number_of_modulator_states", [0, 1, 8])
1167
+ @pytest.mark.parametrize("arm_id", ["CI", "SP"])
1168
+ def test_parse_cryonirsp_linearized_data_constants(
1169
+ parse_linearized_task, arm_id, number_of_modulator_states
1170
+ ):
1171
+ """
1172
+ Given: A ParseCryonirspInputData task
1173
+ When: Calling the task instance
1174
+ Then: Constants are in the constants object as expected
1175
+ """
1176
+
1177
+ task = parse_linearized_task
1178
+
1179
+ lamp_exp_cond = ExposureConditions(10.0, AllowableOpticalDensityFilterNames.OPEN.value)
1180
+ solar_exp_cond = ExposureConditions(5.0, AllowableOpticalDensityFilterNames.OPEN.value)
1181
+ obs_exp_cond = ExposureConditions(6.0, AllowableOpticalDensityFilterNames.OPEN.value)
1182
+ polcal_exp_cond = ExposureConditions(7.0, AllowableOpticalDensityFilterNames.OPEN.value)
1183
+ dark_exp_conditions = [
1184
+ lamp_exp_cond,
1185
+ solar_exp_cond,
1186
+ obs_exp_cond,
1187
+ polcal_exp_cond,
1188
+ ]
1189
+
1190
+ num_modstates = number_of_modulator_states
1191
+ num_scan_steps = 3
1192
+ num_map_scans = 2
1193
+ num_sub_repeats = 2
1194
+
1195
+ make_linearized_test_frames(
1196
+ task,
1197
+ arm_id,
1198
+ dark_exposure_conditions=dark_exp_conditions,
1199
+ num_modstates=num_modstates,
1200
+ num_scan_steps=num_scan_steps,
1201
+ num_map_scans=num_map_scans,
1202
+ num_sub_repeats=num_sub_repeats,
1203
+ lamp_exposure_condition=lamp_exp_cond,
1204
+ solar_exposure_condition=solar_exp_cond,
1205
+ polcal_exposure_condition=polcal_exp_cond,
1206
+ observe_exposure_condition=obs_exp_cond,
1207
+ )
1208
+
1209
+ task()
1210
+
1211
+ if num_modstates == 0:
1212
+ assert task.constants._db_dict[CryonirspBudName.num_modstates.value] == 1
1213
+ else:
1214
+ assert task.constants._db_dict[CryonirspBudName.num_modstates.value] == num_modstates
1215
+ assert task.constants._db_dict[CryonirspBudName.num_map_scans.value] == num_map_scans
1216
+ assert task.constants._db_dict[CryonirspBudName.num_scan_steps.value] == num_scan_steps
1217
+ assert task.constants._db_dict[CryonirspBudName.modulator_spin_mode.value] == "Continuous"
1218
+
1219
+ assert sorted(task.constants._db_dict["DARK_FRAME_EXPOSURE_CONDITIONS_LIST"]) == sorted(
1220
+ [json.loads(json.dumps(condition)) for condition in dark_exp_conditions]
1221
+ )
1222
+
1223
+ assert task.constants._db_dict["LAMP_GAIN_EXPOSURE_CONDITIONS_LIST"] == [
1224
+ json.loads(json.dumps(lamp_exp_cond))
1225
+ ]
1226
+ assert task.constants._db_dict["SOLAR_GAIN_EXPOSURE_CONDITIONS_LIST"] == [
1227
+ json.loads(json.dumps(solar_exp_cond))
1228
+ ]
1229
+ assert task.constants._db_dict["POLCAL_EXPOSURE_CONDITIONS_LIST"] == [
1230
+ json.loads(json.dumps(polcal_exp_cond))
1231
+ ]
1232
+ assert task.constants._db_dict["OBSERVE_EXPOSURE_CONDITIONS_LIST"] == [
1233
+ json.loads(json.dumps(obs_exp_cond))
1234
+ ]
1235
+
1236
+ assert task.constants._db_dict["INSTRUMENT"] == "CRYO-NIRSP"
1237
+ assert task.constants._db_dict["AVERAGE_CADENCE"] == 10
1238
+ assert task.constants._db_dict["MAXIMUM_CADENCE"] == 10
1239
+ assert task.constants._db_dict["MINIMUM_CADENCE"] == 10
1240
+ assert task.constants._db_dict["VARIANCE_CADENCE"] == 0
1241
+
1242
+
1243
+ @pytest.mark.parametrize("arm_id", ["SP"])
1244
+ def test_parse_cryonirsp_linearized_data_internal_scan_loops_as_map_scan_and_scan_step(
1245
+ parse_linearized_task,
1246
+ ):
1247
+ """
1248
+ Given: A parse task for an SP dataset where the internal scan loops are being used as a proxy for
1249
+ map scans and scan steps.
1250
+ When: Calling the task instance
1251
+ Then: All tagged files exist and individual task tags are applied. Specifically test that the
1252
+ internal scan loop parameters map to num_map_scans and num_scan_steps.
1253
+ """
1254
+
1255
+ task = parse_linearized_task
1256
+
1257
+ lamp_exp_cond = ExposureConditions(10.0, AllowableOpticalDensityFilterNames.OPEN.value)
1258
+ solar_exp_cond = ExposureConditions(5.0, AllowableOpticalDensityFilterNames.OPEN.value)
1259
+ obs_exp_cond = ExposureConditions(6.0, AllowableOpticalDensityFilterNames.OPEN.value)
1260
+ polcal_exp_cond = ExposureConditions(7.0, AllowableOpticalDensityFilterNames.OPEN.value)
1261
+ dark_exp_conditions = [
1262
+ lamp_exp_cond,
1263
+ solar_exp_cond,
1264
+ obs_exp_cond,
1265
+ polcal_exp_cond,
1266
+ ]
1267
+
1268
+ num_map_scans = 1
1269
+ num_scan_steps = 6
1270
+ num_alt_maps = 2
1271
+ num_alt_scan_steps = 3
1272
+
1273
+ def make_dual_scan_loop_headers(translated_header):
1274
+ translated_header[
1275
+ "CNP2DSS"
1276
+ ] = 0.0 # This triggers the parsing of the dual internal scan loops
1277
+ translated_header["CNP1DNSP"] = num_alt_scan_steps # inner loop -- becomes num scan steps
1278
+ translated_header["CNP2DNSP"] = num_alt_maps # outer loop -- becomes num map scans
1279
+ translated_header["CNP1DCUR"] = (translated_header["CNCURSCN"] - 1) % num_alt_scan_steps + 1
1280
+ translated_header["CNP2DCUR"] = (
1281
+ translated_header["CNCURSCN"] - 1
1282
+ ) // num_alt_scan_steps + 1
1283
+ return translated_header
1284
+
1285
+ num_dark, num_lamp, num_solar, num_polcal, num_obs = make_linearized_test_frames(
1286
+ task,
1287
+ "SP",
1288
+ dark_exposure_conditions=dark_exp_conditions,
1289
+ num_modstates=1,
1290
+ num_scan_steps=num_scan_steps,
1291
+ num_map_scans=num_map_scans,
1292
+ change_translated_headers=make_dual_scan_loop_headers,
1293
+ lamp_exposure_condition=lamp_exp_cond,
1294
+ solar_exposure_condition=solar_exp_cond,
1295
+ polcal_exposure_condition=polcal_exp_cond,
1296
+ observe_exposure_condition=obs_exp_cond,
1297
+ )
1298
+
1299
+ task()
1300
+
1301
+ assert task.constants._db_dict[CryonirspBudName.num_scan_steps.value] == num_alt_scan_steps
1302
+ assert task.constants._db_dict[CryonirspBudName.num_map_scans.value] == num_alt_maps
1303
+
1304
+
1305
+ @pytest.mark.parametrize("arm_id", ["CI", "SP"])
1306
+ def test_parse_cryonirsp_not_polarimetric_obs(parse_linearized_task, arm_id):
1307
+ """
1308
+ Given: A ParseCryonirspInputData task
1309
+ When: Calling the task instance with non-polarimetric observe frames as input
1310
+ Then: PolarimetricCheckingUniqueBud has set the constants correctly
1311
+ """
1312
+
1313
+ task = parse_linearized_task
1314
+
1315
+ lin_tag = [CryonirspTag.linearized()]
1316
+ obs_exp_cond = ExposureConditions(6.0, AllowableOpticalDensityFilterNames.OPEN.value)
1317
+ polcal_exp_cond = ExposureConditions(7.0, AllowableOpticalDensityFilterNames.OPEN.value)
1318
+ dark_exp_conditions = [
1319
+ obs_exp_cond,
1320
+ polcal_exp_cond,
1321
+ ]
1322
+
1323
+ num_steps = 3
1324
+ num_map_scans = 2
1325
+ num_sub_repeats = 2
1326
+
1327
+ for condition in dark_exp_conditions:
1328
+ write_dark_frames_to_task(task, exposure_condition=condition, tags=lin_tag)
1329
+
1330
+ write_polcal_frames_to_task(
1331
+ task,
1332
+ num_modstates=8,
1333
+ num_map_scans=num_map_scans,
1334
+ tags=lin_tag,
1335
+ extra_headers=dict(),
1336
+ exposure_condition=polcal_exp_cond,
1337
+ )
1338
+ write_observe_frames_to_task(
1339
+ task,
1340
+ arm_id=arm_id,
1341
+ num_scan_steps=num_steps,
1342
+ num_map_scans=num_map_scans,
1343
+ num_sub_repeats=num_sub_repeats,
1344
+ num_modstates=1,
1345
+ exposure_condition=obs_exp_cond,
1346
+ num_measurements=1,
1347
+ tags=lin_tag,
1348
+ )
1349
+
1350
+ task()
1351
+
1352
+ assert task.constants._db_dict[CryonirspBudName.num_modstates.value] == 1
1353
+ assert task.constants._db_dict[CryonirspBudName.modulator_spin_mode.value] == "Continuous"
1354
+
1355
+
1356
+ @pytest.fixture
1357
+ def dummy_fits_obj():
1358
+ @dataclass
1359
+ class DummyFitsObj:
1360
+ ip_task_type: str
1361
+ number_of_modulator_states: int
1362
+ modulator_spin_mode: str
1363
+
1364
+ return DummyFitsObj
1365
+
1366
+
1367
+ def test_polarimetric_checking_unique_bud(dummy_fits_obj):
1368
+ """
1369
+ Given: A PolarimetricCheckingUniqueBud
1370
+ When: Ingesting various polcal and observe frames
1371
+ Then: The Bud functions as expected
1372
+ """
1373
+ pol_frame1 = dummy_fits_obj(
1374
+ ip_task_type="POLCAL", number_of_modulator_states=8, modulator_spin_mode="Continuous"
1375
+ )
1376
+ pol_frame2 = dummy_fits_obj(
1377
+ ip_task_type="POLCAL", number_of_modulator_states=3, modulator_spin_mode="Continuous"
1378
+ )
1379
+
1380
+ obs_frame1 = dummy_fits_obj(
1381
+ ip_task_type="OBSERVE", number_of_modulator_states=8, modulator_spin_mode="Continuous"
1382
+ )
1383
+ obs_frame2 = dummy_fits_obj(
1384
+ ip_task_type="OBSERVE", number_of_modulator_states=2, modulator_spin_mode="Continuous"
1385
+ )
1386
+
1387
+ nonpol_obs_frame1 = dummy_fits_obj(
1388
+ ip_task_type="OBSERVE", number_of_modulator_states=1, modulator_spin_mode="Continuous"
1389
+ )
1390
+ nonpol_obs_frame2 = dummy_fits_obj(
1391
+ ip_task_type="OBSERVE", number_of_modulator_states=1, modulator_spin_mode="Bad"
1392
+ )
1393
+
1394
+ # Test failures in `is_polarimetric
1395
+ Bud = PolarimetricCheckingUniqueBud("dummy_constant", "number_of_modulator_states")
1396
+ Bud.update("key1", obs_frame1)
1397
+ Bud.update("key2", obs_frame2)
1398
+ with pytest.raises(
1399
+ ValueError, match="Observe frames have more than one value of NUM_MODSTATES."
1400
+ ):
1401
+ Bud.is_polarimetric()
1402
+
1403
+ Bud = PolarimetricCheckingUniqueBud("dummy_constant", "number_of_modulator_states")
1404
+ Bud.update("key1", nonpol_obs_frame1)
1405
+ Bud.update("key2", nonpol_obs_frame2)
1406
+ with pytest.raises(
1407
+ ValueError, match="Observe frames have more than one value of MODULATOR_SPIN_MODE."
1408
+ ):
1409
+ Bud.is_polarimetric()
1410
+
1411
+ # Test correct operation of `is_polarimetric`
1412
+ Bud = PolarimetricCheckingUniqueBud("dummy_constant", "number_of_modulator_states")
1413
+ Bud.update("key1", nonpol_obs_frame1)
1414
+ assert not Bud.is_polarimetric()
1415
+
1416
+ Bud = PolarimetricCheckingUniqueBud("dummy_constant", "number_of_modulator_states")
1417
+ Bud.update("key1", obs_frame1)
1418
+ assert Bud.is_polarimetric()
1419
+
1420
+ # Test non-unique polcal values
1421
+ Bud = PolarimetricCheckingUniqueBud("dummy_constant", "number_of_modulator_states")
1422
+ Bud.update("key1", obs_frame1)
1423
+ Bud.update("key2", pol_frame1)
1424
+ Bud.update("key3", pol_frame2)
1425
+ with pytest.raises(ValueError, match="Polcal frames have more than one value of NUM_MODSTATES"):
1426
+ Bud.getter("key1")
1427
+
1428
+ # Test for correct error if polcal and observe frames have different values for polarimetric data
1429
+ Bud = PolarimetricCheckingUniqueBud("dummy_constant", "number_of_modulator_states")
1430
+ Bud.update("key1", obs_frame1)
1431
+ Bud.update("key2", pol_frame2)
1432
+ with pytest.raises(ValueError, match="Polcal and Observe frames have different values for"):
1433
+ Bud.getter("key1")
1434
+
1435
+ # Test that polcal and observe frames having different values doesn't matter for non-polarimetric datra
1436
+ Bud = PolarimetricCheckingUniqueBud("dummy_constant", "modulator_spin_mode")
1437
+ Bud.update("key1", nonpol_obs_frame2)
1438
+ Bud.update("key2", pol_frame2)
1439
+ assert Bud.getter("key1") == "Bad"