dkist-processing-cryonirsp 1.3.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dkist-processing-cryonirsp might be problematic. Click here for more details.

Files changed (111) hide show
  1. changelog/.gitempty +0 -0
  2. dkist_processing_cryonirsp/__init__.py +11 -0
  3. dkist_processing_cryonirsp/config.py +12 -0
  4. dkist_processing_cryonirsp/models/__init__.py +1 -0
  5. dkist_processing_cryonirsp/models/constants.py +248 -0
  6. dkist_processing_cryonirsp/models/exposure_conditions.py +26 -0
  7. dkist_processing_cryonirsp/models/parameters.py +296 -0
  8. dkist_processing_cryonirsp/models/tags.py +168 -0
  9. dkist_processing_cryonirsp/models/task_name.py +14 -0
  10. dkist_processing_cryonirsp/parsers/__init__.py +1 -0
  11. dkist_processing_cryonirsp/parsers/cryonirsp_l0_fits_access.py +111 -0
  12. dkist_processing_cryonirsp/parsers/cryonirsp_l1_fits_access.py +30 -0
  13. dkist_processing_cryonirsp/parsers/exposure_conditions.py +163 -0
  14. dkist_processing_cryonirsp/parsers/map_repeats.py +40 -0
  15. dkist_processing_cryonirsp/parsers/measurements.py +55 -0
  16. dkist_processing_cryonirsp/parsers/modstates.py +31 -0
  17. dkist_processing_cryonirsp/parsers/optical_density_filters.py +40 -0
  18. dkist_processing_cryonirsp/parsers/polarimetric_check.py +120 -0
  19. dkist_processing_cryonirsp/parsers/scan_step.py +412 -0
  20. dkist_processing_cryonirsp/parsers/time.py +80 -0
  21. dkist_processing_cryonirsp/parsers/wavelength.py +26 -0
  22. dkist_processing_cryonirsp/tasks/__init__.py +19 -0
  23. dkist_processing_cryonirsp/tasks/assemble_movie.py +202 -0
  24. dkist_processing_cryonirsp/tasks/bad_pixel_map.py +96 -0
  25. dkist_processing_cryonirsp/tasks/beam_boundaries_base.py +279 -0
  26. dkist_processing_cryonirsp/tasks/ci_beam_boundaries.py +55 -0
  27. dkist_processing_cryonirsp/tasks/ci_science.py +169 -0
  28. dkist_processing_cryonirsp/tasks/cryonirsp_base.py +67 -0
  29. dkist_processing_cryonirsp/tasks/dark.py +98 -0
  30. dkist_processing_cryonirsp/tasks/gain.py +251 -0
  31. dkist_processing_cryonirsp/tasks/instrument_polarization.py +447 -0
  32. dkist_processing_cryonirsp/tasks/l1_output_data.py +44 -0
  33. dkist_processing_cryonirsp/tasks/linearity_correction.py +582 -0
  34. dkist_processing_cryonirsp/tasks/make_movie_frames.py +302 -0
  35. dkist_processing_cryonirsp/tasks/mixin/__init__.py +1 -0
  36. dkist_processing_cryonirsp/tasks/mixin/beam_access.py +52 -0
  37. dkist_processing_cryonirsp/tasks/mixin/corrections.py +177 -0
  38. dkist_processing_cryonirsp/tasks/mixin/intermediate_frame.py +193 -0
  39. dkist_processing_cryonirsp/tasks/mixin/linearized_frame.py +309 -0
  40. dkist_processing_cryonirsp/tasks/mixin/shift_measurements.py +297 -0
  41. dkist_processing_cryonirsp/tasks/parse.py +281 -0
  42. dkist_processing_cryonirsp/tasks/quality_metrics.py +271 -0
  43. dkist_processing_cryonirsp/tasks/science_base.py +511 -0
  44. dkist_processing_cryonirsp/tasks/sp_beam_boundaries.py +270 -0
  45. dkist_processing_cryonirsp/tasks/sp_dispersion_axis_correction.py +484 -0
  46. dkist_processing_cryonirsp/tasks/sp_geometric.py +585 -0
  47. dkist_processing_cryonirsp/tasks/sp_science.py +299 -0
  48. dkist_processing_cryonirsp/tasks/sp_solar_gain.py +475 -0
  49. dkist_processing_cryonirsp/tasks/trial_output_data.py +61 -0
  50. dkist_processing_cryonirsp/tasks/write_l1.py +1033 -0
  51. dkist_processing_cryonirsp/tests/__init__.py +1 -0
  52. dkist_processing_cryonirsp/tests/conftest.py +456 -0
  53. dkist_processing_cryonirsp/tests/header_models.py +592 -0
  54. dkist_processing_cryonirsp/tests/local_trial_workflows/__init__.py +0 -0
  55. dkist_processing_cryonirsp/tests/local_trial_workflows/l0_cals_only.py +541 -0
  56. dkist_processing_cryonirsp/tests/local_trial_workflows/l0_to_l1.py +615 -0
  57. dkist_processing_cryonirsp/tests/local_trial_workflows/linearize_only.py +96 -0
  58. dkist_processing_cryonirsp/tests/local_trial_workflows/local_trial_helpers.py +592 -0
  59. dkist_processing_cryonirsp/tests/test_assemble_movie.py +144 -0
  60. dkist_processing_cryonirsp/tests/test_assemble_qualilty.py +517 -0
  61. dkist_processing_cryonirsp/tests/test_bad_pixel_maps.py +115 -0
  62. dkist_processing_cryonirsp/tests/test_ci_beam_boundaries.py +106 -0
  63. dkist_processing_cryonirsp/tests/test_ci_science.py +355 -0
  64. dkist_processing_cryonirsp/tests/test_corrections.py +126 -0
  65. dkist_processing_cryonirsp/tests/test_cryo_base.py +202 -0
  66. dkist_processing_cryonirsp/tests/test_cryo_constants.py +76 -0
  67. dkist_processing_cryonirsp/tests/test_dark.py +287 -0
  68. dkist_processing_cryonirsp/tests/test_gain.py +278 -0
  69. dkist_processing_cryonirsp/tests/test_instrument_polarization.py +531 -0
  70. dkist_processing_cryonirsp/tests/test_linearity_correction.py +245 -0
  71. dkist_processing_cryonirsp/tests/test_make_movie_frames.py +111 -0
  72. dkist_processing_cryonirsp/tests/test_parameters.py +266 -0
  73. dkist_processing_cryonirsp/tests/test_parse.py +1439 -0
  74. dkist_processing_cryonirsp/tests/test_quality.py +203 -0
  75. dkist_processing_cryonirsp/tests/test_sp_beam_boundaries.py +112 -0
  76. dkist_processing_cryonirsp/tests/test_sp_dispersion_axis_correction.py +155 -0
  77. dkist_processing_cryonirsp/tests/test_sp_geometric.py +319 -0
  78. dkist_processing_cryonirsp/tests/test_sp_make_movie_frames.py +121 -0
  79. dkist_processing_cryonirsp/tests/test_sp_science.py +483 -0
  80. dkist_processing_cryonirsp/tests/test_sp_solar.py +198 -0
  81. dkist_processing_cryonirsp/tests/test_trial_create_quality_report.py +79 -0
  82. dkist_processing_cryonirsp/tests/test_trial_output_data.py +251 -0
  83. dkist_processing_cryonirsp/tests/test_workflows.py +9 -0
  84. dkist_processing_cryonirsp/tests/test_write_l1.py +436 -0
  85. dkist_processing_cryonirsp/workflows/__init__.py +2 -0
  86. dkist_processing_cryonirsp/workflows/ci_l0_processing.py +77 -0
  87. dkist_processing_cryonirsp/workflows/sp_l0_processing.py +84 -0
  88. dkist_processing_cryonirsp/workflows/trial_workflows.py +190 -0
  89. dkist_processing_cryonirsp-1.3.4.dist-info/METADATA +194 -0
  90. dkist_processing_cryonirsp-1.3.4.dist-info/RECORD +111 -0
  91. dkist_processing_cryonirsp-1.3.4.dist-info/WHEEL +5 -0
  92. dkist_processing_cryonirsp-1.3.4.dist-info/top_level.txt +4 -0
  93. docs/Makefile +134 -0
  94. docs/bad_pixel_calibration.rst +47 -0
  95. docs/beam_angle_calculation.rst +53 -0
  96. docs/beam_boundary_computation.rst +88 -0
  97. docs/changelog.rst +7 -0
  98. docs/ci_science_calibration.rst +33 -0
  99. docs/conf.py +52 -0
  100. docs/index.rst +21 -0
  101. docs/l0_to_l1_cryonirsp_ci-full-trial.rst +10 -0
  102. docs/l0_to_l1_cryonirsp_ci.rst +10 -0
  103. docs/l0_to_l1_cryonirsp_sp-full-trial.rst +10 -0
  104. docs/l0_to_l1_cryonirsp_sp.rst +10 -0
  105. docs/linearization.rst +43 -0
  106. docs/make.bat +170 -0
  107. docs/requirements.txt +1 -0
  108. docs/requirements_table.rst +8 -0
  109. docs/scientific_changelog.rst +10 -0
  110. docs/sp_science_calibration.rst +59 -0
  111. licenses/LICENSE.rst +11 -0
@@ -0,0 +1,245 @@
1
+ """Test the linearity correction task."""
2
+ import re
3
+ from dataclasses import dataclass
4
+ from datetime import datetime
5
+
6
+ import numpy as np
7
+ import pytest
8
+ from astropy.io import fits
9
+ from dkist_header_validator import spec122_validator
10
+ from dkist_processing_common._util.scratch import WorkflowFileSystem
11
+ from dkist_processing_common.codecs.fits import fits_hdulist_encoder
12
+ from dkist_processing_common.models.tags import Tag
13
+ from dkist_processing_common.tests.conftest import FakeGQLClient
14
+ from dkist_service_configuration.logging import logger
15
+
16
+ from dkist_processing_cryonirsp.models.constants import CryonirspBudName
17
+ from dkist_processing_cryonirsp.models.exposure_conditions import AllowableOpticalDensityFilterNames
18
+ from dkist_processing_cryonirsp.models.tags import CryonirspTag
19
+ from dkist_processing_cryonirsp.tasks.linearity_correction import LinearityCorrection
20
+ from dkist_processing_cryonirsp.tests.conftest import cryonirsp_testing_parameters_factory
21
+ from dkist_processing_cryonirsp.tests.conftest import CryonirspConstantsDb
22
+ from dkist_processing_cryonirsp.tests.conftest import generate_fits_frame
23
+ from dkist_processing_cryonirsp.tests.header_models import CryonirspHeadersValidNonLinearizedFrames
24
+
25
+
26
+ @pytest.fixture(scope="function")
27
+ def linearity_correction(
28
+ tmp_path,
29
+ recipe_run_id,
30
+ assign_input_dataset_doc_to_task,
31
+ init_cryonirsp_constants_db,
32
+ arm_id,
33
+ frames_in_ramp,
34
+ filter_name,
35
+ ):
36
+ # time y x
37
+ dataset_shape = (frames_in_ramp, 10, 10)
38
+ # z y x
39
+ array_shape = (1, 10, 10)
40
+ time_delta = 0.1
41
+ start_time = datetime.now()
42
+ expected_num_frames_in_ramp = 10
43
+ constants_db = CryonirspConstantsDb(
44
+ TIME_OBS_LIST=(str(start_time),),
45
+ ARM_ID=arm_id,
46
+ ROI_1_SIZE_X=array_shape[2],
47
+ ROI_1_SIZE_Y=array_shape[1],
48
+ )
49
+ init_cryonirsp_constants_db(recipe_run_id, constants_db)
50
+ with LinearityCorrection(
51
+ recipe_run_id=recipe_run_id,
52
+ workflow_name="linearity_correction",
53
+ workflow_version="VX.Y",
54
+ ) as task:
55
+ try: # This try... block is here to make sure the dbs get cleaned up if there's a failure in the fixture
56
+ task.scratch = WorkflowFileSystem(
57
+ scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
58
+ )
59
+ param_class = cryonirsp_testing_parameters_factory(param_path=tmp_path)
60
+ assign_input_dataset_doc_to_task(task, param_class())
61
+ ds = CryonirspHeadersValidNonLinearizedFrames(
62
+ arm_id=arm_id,
63
+ camera_readout_mode="FastUpTheRamp",
64
+ dataset_shape=dataset_shape,
65
+ array_shape=array_shape,
66
+ time_delta=time_delta,
67
+ roi_x_origin=0,
68
+ roi_y_origin=0,
69
+ roi_x_size=array_shape[2],
70
+ roi_y_size=array_shape[1],
71
+ date_obs=start_time.isoformat("T"),
72
+ exposure_time=time_delta,
73
+ )
74
+ # Initial header creation...
75
+ header_generator = (
76
+ spec122_validator.validate_and_translate_to_214_l0(
77
+ d.header(), return_type=fits.HDUList
78
+ )[0].header
79
+ for d in ds
80
+ )
81
+ # Patch the headers for non-linearized Cryo data...
82
+ header_list = []
83
+ exp_time = 0.0
84
+ counter = 0
85
+ for header in header_generator:
86
+ # Set the integrated exposure time for this NDR
87
+ # This is a range from 0 to 90 in 10 steps
88
+ header["XPOSURE"] = 100 * counter * time_delta
89
+ # Set the current frame in ramp, 1-based
90
+ header["CNCNDR"] = counter + 1
91
+ header["CNNNDR"] = expected_num_frames_in_ramp
92
+ header["CNFILTNP"] = filter_name
93
+ header_list.append(header)
94
+ counter += 1
95
+ # Step on the old one with the new one
96
+ header_generator = (header for header in header_list)
97
+ # Iterate through the headers and create the frames...
98
+ for _ in header_list:
99
+ hdul = generate_fits_frame(header_generator=header_generator, shape=array_shape)
100
+ # Now tweak the data...
101
+ for hdu in hdul:
102
+ header = hdu.header
103
+ exp_time = header["XPOSURE"]
104
+ # Create a simple perfectly linear ramp
105
+ hdu.data.fill(exp_time)
106
+ task.write(
107
+ data=hdul,
108
+ tags=[
109
+ CryonirspTag.input(),
110
+ CryonirspTag.frame(),
111
+ CryonirspTag.curr_frame_in_ramp(header["CNCNDR"]),
112
+ # All frames in a ramp have the same date-obs
113
+ CryonirspTag.time_obs(str(start_time)),
114
+ Tag.readout_exp_time(exp_time),
115
+ ],
116
+ encoder=fits_hdulist_encoder,
117
+ )
118
+ task.constants._update({CryonirspBudName.camera_readout_mode.value: "FastUpTheRamp"})
119
+ yield task, filter_name, expected_num_frames_in_ramp
120
+ finally:
121
+ task._purge()
122
+
123
+
124
+ @pytest.mark.parametrize(
125
+ "filter_name",
126
+ [
127
+ pytest.param(AllowableOpticalDensityFilterNames.G358.value),
128
+ pytest.param(AllowableOpticalDensityFilterNames.OPEN.value),
129
+ ],
130
+ )
131
+ @pytest.mark.parametrize(
132
+ "frames_in_ramp",
133
+ [pytest.param(10, id="Full ramp"), pytest.param(5, id="Bad ramp")],
134
+ )
135
+ @pytest.mark.parametrize(
136
+ "arm_id",
137
+ [pytest.param("CI", id="CI"), pytest.param("SP", id="SP")],
138
+ )
139
+ @pytest.mark.parametrize(
140
+ "num_chunks",
141
+ [pytest.param(1, id="1 chunk"), pytest.param(2, id="2 chunks")],
142
+ )
143
+ def test_linearity_correction(
144
+ linearity_correction,
145
+ mocker,
146
+ arm_id,
147
+ frames_in_ramp,
148
+ num_chunks,
149
+ filter_name,
150
+ ):
151
+ """
152
+ Given: A LinearityCorrection task
153
+ When: Calling the task instance with known input data
154
+ Then: The non-linearized frames are linearized and produce the correct results.
155
+ """
156
+ mocker.patch(
157
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
158
+ )
159
+ if num_chunks == 2:
160
+ mocker.patch(
161
+ "dkist_processing_cryonirsp.tasks.linearity_correction.LinearityCorrection.compute_linear_chunk_size",
162
+ new=lambda self, frame_size, num_frames_in_ramp: frame_size // 2,
163
+ )
164
+ # Given
165
+ task, filter_name, expected_num_frames_in_ramp = linearity_correction
166
+ # When
167
+ task()
168
+ # Then
169
+ tags = [
170
+ CryonirspTag.linearized(),
171
+ CryonirspTag.frame(),
172
+ ]
173
+ # We used a perfect linear ramp from 0 to 90, where the ramp value is equal to the exposure time in ms
174
+ # The algorithm normalizes the linearized frame by the exposure time in seconds, so the expected value is:
175
+ # 90 / (90 / 1000) = 1000 / attenuation, where attenuation is the multiplicative attenuation due to the
176
+ # filter in use
177
+ attenuation = 10 ** (task.parameters.linearization_filter_attenuation_dict[filter_name])
178
+ expected_data = np.ones((10, 10)) * 1000.0 / attenuation
179
+ files_found = list(task.read(tags=tags))
180
+ if frames_in_ramp == expected_num_frames_in_ramp:
181
+ assert len(files_found) == 1
182
+ hdul = fits.open(files_found[0])
183
+ data = hdul[0].data
184
+ assert np.allclose(data, expected_data)
185
+ else:
186
+ assert len(files_found) == 0
187
+
188
+
189
+ @pytest.fixture
190
+ def simple_linearity_correction_task(recipe_run_id, arm_id, init_cryonirsp_constants_db, tmp_path):
191
+ constants_db = CryonirspConstantsDb(
192
+ ARM_ID=arm_id,
193
+ )
194
+ init_cryonirsp_constants_db(recipe_run_id, constants_db)
195
+ with LinearityCorrection(
196
+ recipe_run_id=recipe_run_id,
197
+ workflow_name="workflow_name",
198
+ workflow_version="workflow_version",
199
+ ) as task:
200
+ task.scratch = WorkflowFileSystem(recipe_run_id=recipe_run_id, scratch_base_path=tmp_path)
201
+
202
+ yield task
203
+
204
+ task._purge()
205
+
206
+
207
+ @dataclass
208
+ class DummyRampFitsAccess:
209
+ """Just a class that has the one property that is checked during ramp validation."""
210
+
211
+ num_frames_in_ramp: int
212
+ ip_task_type: str = "TASK"
213
+
214
+
215
+ @pytest.mark.parametrize(
216
+ "arm_id",
217
+ [pytest.param("CI", id="CI"), pytest.param("SP", id="SP")],
218
+ )
219
+ @pytest.mark.parametrize(
220
+ "ramp_list, valid, message",
221
+ [
222
+ pytest.param(
223
+ [
224
+ DummyRampFitsAccess(num_frames_in_ramp=2),
225
+ DummyRampFitsAccess(num_frames_in_ramp=3),
226
+ ],
227
+ False,
228
+ "Not all frames have the same FRAMES_IN_RAMP value. Set is {2, 3}. Ramp is task TASK. Skipping ramp.",
229
+ id="num_frames_mismatch_actual_frames",
230
+ ),
231
+ pytest.param(
232
+ [
233
+ DummyRampFitsAccess(num_frames_in_ramp=8),
234
+ ],
235
+ False,
236
+ "Missing some ramp frames. Expected 8 from header value, but only have 1. Ramp is task TASK. Skipping ramp.",
237
+ id="num_frames_in_set_mismatch",
238
+ ),
239
+ ],
240
+ )
241
+ def test_is_ramp_valid(simple_linearity_correction_task, ramp_list, valid, message, caplog):
242
+ logger.add(caplog.handler)
243
+ assert simple_linearity_correction_task.is_ramp_valid(ramp_list) is valid
244
+ if not valid:
245
+ assert re.search(message, caplog.text)
@@ -0,0 +1,111 @@
1
+ from datetime import datetime
2
+
3
+ import pytest
4
+ from astropy.io import fits
5
+ from dkist_header_validator import spec122_validator
6
+ from dkist_processing_common._util.scratch import WorkflowFileSystem
7
+ from dkist_processing_common.codecs.fits import fits_hdulist_encoder
8
+ from dkist_processing_common.tests.conftest import FakeGQLClient
9
+
10
+ from dkist_processing_cryonirsp.models.tags import CryonirspTag
11
+ from dkist_processing_cryonirsp.tasks.make_movie_frames import MakeCryonirspMovieFrames
12
+ from dkist_processing_cryonirsp.tests.conftest import CryonirspConstantsDb
13
+ from dkist_processing_cryonirsp.tests.conftest import generate_fits_frame
14
+ from dkist_processing_cryonirsp.tests.header_models import CryonirspHeadersValidObserveFrames
15
+
16
+
17
+ @pytest.fixture(
18
+ scope="function",
19
+ params=[pytest.param(True, id="polarimetric"), pytest.param(False, id="intensity-only")],
20
+ )
21
+ def movie_frames_task(tmp_path, recipe_run_id, init_cryonirsp_constants_db, request):
22
+ is_polarimetric = request.param
23
+ map_scans = 3
24
+ scan_steps = 2
25
+ array_shape = (3, 4)
26
+ if is_polarimetric:
27
+ num_mod = 8
28
+ spin_mode = "Continuous"
29
+ else:
30
+ num_mod = 1
31
+ spin_mode = "None"
32
+ constants_db = CryonirspConstantsDb(
33
+ NUM_MODSTATES=num_mod,
34
+ MODULATOR_SPIN_MODE=spin_mode,
35
+ NUM_SCAN_STEPS=scan_steps,
36
+ NUM_MAP_SCANS=map_scans,
37
+ TIME_OBS_LIST=(datetime.now().isoformat("T"),),
38
+ )
39
+ init_cryonirsp_constants_db(recipe_run_id, constants_db)
40
+ with MakeCryonirspMovieFrames(
41
+ recipe_run_id=recipe_run_id, workflow_name="make_movie_frames", workflow_version="VX.Y"
42
+ ) as task:
43
+ try: # This try... block is here to make sure the dbs get cleaned up if there's a failure in the fixture
44
+ meas_num = 1 # Use only the first measurement if there are multiple measurements.
45
+ task.scratch = WorkflowFileSystem(
46
+ scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
47
+ )
48
+ start_time = datetime.now()
49
+ for stokes_state in ["I", "Q", "U", "V"]:
50
+ for map_scan in range(1, map_scans + 1):
51
+ for scan_step in range(0, scan_steps + 1):
52
+ ds = CryonirspHeadersValidObserveFrames(
53
+ dataset_shape=(2, *array_shape),
54
+ array_shape=(1, *array_shape),
55
+ time_delta=10,
56
+ num_map_scans=map_scans,
57
+ map_scan=map_scan,
58
+ num_scan_steps=scan_steps,
59
+ scan_step=scan_step,
60
+ num_modstates=1,
61
+ modstate=1,
62
+ start_time=start_time,
63
+ num_meas=1,
64
+ meas_num=1,
65
+ arm_id="CI",
66
+ )
67
+ header_generator = (
68
+ spec122_validator.validate_and_translate_to_214_l0(
69
+ d.header(), return_type=fits.HDUList
70
+ )[0].header
71
+ for d in ds
72
+ )
73
+ hdul = generate_fits_frame(
74
+ header_generator=header_generator, shape=(1, *array_shape)
75
+ )
76
+ task.write(
77
+ data=hdul,
78
+ tags=[
79
+ CryonirspTag.calibrated(),
80
+ CryonirspTag.frame(),
81
+ CryonirspTag.map_scan(map_scan),
82
+ CryonirspTag.scan_step(scan_step),
83
+ CryonirspTag.stokes(stokes_state),
84
+ CryonirspTag.meas_num(meas_num),
85
+ ],
86
+ encoder=fits_hdulist_encoder,
87
+ )
88
+ yield task, map_scans, scan_steps, array_shape, is_polarimetric
89
+ finally:
90
+ task._purge()
91
+
92
+
93
+ def test_make_movie_frames(movie_frames_task, mocker):
94
+ """
95
+ Given: A MakeCryonirspMovieFrames task
96
+ When: Calling the task instance
97
+ Then: a fits file is made for each scan containing the movie frame for that scan
98
+ """
99
+ mocker.patch(
100
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
101
+ )
102
+ task, map_scans, scan_steps, array_shape, is_polarimetric = movie_frames_task
103
+ expected_shape = array_shape
104
+
105
+ task()
106
+ assert len(list(task.read(tags=[CryonirspTag.movie_frame()]))) == map_scans * scan_steps
107
+ for filepath in task.read(tags=[CryonirspTag.movie_frame()]):
108
+ assert filepath.exists()
109
+ hdul = fits.open(filepath)
110
+ assert hdul[0].header["INSTRUME"] == "CRYO-NIRSP"
111
+ assert hdul[0].data.shape == expected_shape
@@ -0,0 +1,266 @@
1
+ from dataclasses import asdict
2
+ from typing import Any
3
+
4
+ import numpy as np
5
+ import pytest
6
+ from astropy.units import Quantity
7
+ from dkist_processing_common._util.scratch import WorkflowFileSystem
8
+ from hypothesis import example
9
+ from hypothesis import given
10
+ from hypothesis import HealthCheck
11
+ from hypothesis import settings
12
+ from hypothesis import strategies as st
13
+
14
+ from dkist_processing_cryonirsp.models.parameters import CryonirspParameters
15
+ from dkist_processing_cryonirsp.models.parameters import CryonirspParsingParameters
16
+ from dkist_processing_cryonirsp.parsers.optical_density_filters import (
17
+ ALLOWABLE_OPTICAL_DENSITY_FILTERS,
18
+ )
19
+ from dkist_processing_cryonirsp.tasks.cryonirsp_base import CryonirspTaskBase
20
+ from dkist_processing_cryonirsp.tests.conftest import cryonirsp_testing_parameters_factory
21
+ from dkist_processing_cryonirsp.tests.conftest import CryonirspConstantsDb
22
+ from dkist_processing_cryonirsp.tests.conftest import FileParameter
23
+ from dkist_processing_cryonirsp.tests.conftest import TestingParameters
24
+
25
+ # The property names of all parameters on `CryonirspParsingParameters`
26
+ PARSE_PARAMETER_NAMES = [
27
+ k for k, v in vars(CryonirspParsingParameters).items() if isinstance(v, property)
28
+ ]
29
+
30
+
31
+ @pytest.fixture(scope="function")
32
+ def basic_science_task_with_parameter_mixin(
33
+ tmp_path,
34
+ recipe_run_id,
35
+ assign_input_dataset_doc_to_task,
36
+ init_cryonirsp_constants_db,
37
+ testing_obs_ip_start_time,
38
+ ):
39
+ def make_task(
40
+ parameter_class=CryonirspParameters,
41
+ arm_id: str = "SP",
42
+ obs_ip_start_time: str = testing_obs_ip_start_time,
43
+ ):
44
+ class Task(CryonirspTaskBase):
45
+ def run(self):
46
+ ...
47
+
48
+ init_cryonirsp_constants_db(recipe_run_id, CryonirspConstantsDb())
49
+ task = Task(
50
+ recipe_run_id=recipe_run_id,
51
+ workflow_name="parse_cryonirsp_input_data",
52
+ workflow_version="VX.Y",
53
+ )
54
+ try: # This try... block is here to make sure the dbs get cleaned up if there's a failure in the fixture
55
+ task.scratch = WorkflowFileSystem(
56
+ scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
57
+ )
58
+ test_params = cryonirsp_testing_parameters_factory(param_path=tmp_path)
59
+ param_dict = test_params()
60
+ assign_input_dataset_doc_to_task(
61
+ task,
62
+ param_dict,
63
+ parameter_class=parameter_class,
64
+ arm_id=arm_id,
65
+ obs_ip_start_time=obs_ip_start_time,
66
+ )
67
+ yield task, param_dict
68
+ finally:
69
+ task._purge()
70
+
71
+ return make_task
72
+
73
+
74
+ def _is_wavelength_param(param_value: Any) -> bool:
75
+ return isinstance(param_value, dict) and "wavelength" in param_value
76
+
77
+
78
+ @given(wave=st.floats(min_value=800.0, max_value=2000.0))
79
+ @settings(suppress_health_check=[HealthCheck.function_scoped_fixture], deadline=None)
80
+ @example(wave=1082.7)
81
+ def test_filter_parameters(basic_science_task_with_parameter_mixin, wave):
82
+ """
83
+ Given: A Science task with the parameter mixin
84
+ When: Accessing properties for the optical density filters
85
+ Then: The correct value is returned
86
+ """
87
+ task, expected = next(basic_science_task_with_parameter_mixin())
88
+ task_params = task.parameters
89
+ task_params._wavelength = wave
90
+ expected = {
91
+ "_linearization_optical_density_filter_attenuation_g278": -1.64,
92
+ "_linearization_optical_density_filter_attenuation_g358": -3.75,
93
+ "_linearization_optical_density_filter_attenuation_g408": -4.26,
94
+ }
95
+ for param in expected:
96
+ assert getattr(task_params, param) == expected[param]
97
+
98
+
99
+ def _is_file_param(param_value: Any) -> bool:
100
+ return isinstance(param_value, dict) and "is_file" in param_value and param_value["is_file"]
101
+
102
+
103
+ def test_file_parameters(basic_science_task_with_parameter_mixin):
104
+ """
105
+ Given: A Science task with the parameter mixin
106
+ When: Accessing parameters whose values are loaded from files
107
+ Then: The correct value is returned
108
+
109
+ This test exercises all aspects of file parameters from their names to the loading of values
110
+ """
111
+ task, test_params = next(basic_science_task_with_parameter_mixin())
112
+ task_params = task.parameters
113
+ # Iterate over the test parameters and check that each file param exists in the task parameters
114
+ # we load the actual parameter from the task param object using only the param name
115
+ for pn, pv in asdict(test_params).items():
116
+ # We want to test only file parameters
117
+ if _is_file_param(pv):
118
+ pn_no_prefix = pn.removeprefix("cryonirsp_")
119
+ # If the param name is an attribute in task_params, then load it directly
120
+ if hasattr(task_params, pn_no_prefix):
121
+ param_name = pn_no_prefix
122
+ actual = getattr(task_params, param_name)
123
+ # if the param name is not a task param attribute, then check that we can load the param
124
+ # using the value defined in the input_dataset_parameters list of the task param object
125
+ else:
126
+ param_dict = task_params._find_most_recent_past_value(pn)
127
+ actual = task_params._load_param_value_from_numpy_save(param_dict)
128
+ # Now get the expected value using the param value dict from the testing params
129
+ expected = np.load(pv["param_path"])
130
+ # Compare the actual and expected values
131
+ assert np.array_equal(actual, expected)
132
+
133
+
134
+ def _is_arm_param(
135
+ param_name: str,
136
+ task_params: CryonirspParameters,
137
+ testing_params: TestingParameters,
138
+ single_arm_only: str | None = None,
139
+ ):
140
+ """
141
+ Test if a parameter is an arm parameter.
142
+
143
+ An arm parameter is one which is present in the task_param class with no arm suffix and is also
144
+ present in the test_param class with suffixed forms only, one for each arm.
145
+ This allows a non-arm-specific name to be used as a property in the parameters class which
146
+ encapsulates the mechanism used to return the arm specific parameter value based on the arm in use.
147
+ """
148
+ # NB: param_name is assumed to have a prefix of "cryonirsp_"
149
+ arm_suffixes = ["_sp", "_ci"] if single_arm_only is None else [f"_{single_arm_only}".casefold()]
150
+ suffix = param_name[-3:]
151
+ if suffix not in arm_suffixes:
152
+ return False
153
+ param_name_no_suffix = param_name[:-3]
154
+ param_names_with_suffixes = [f"{param_name_no_suffix}{suffix}" for suffix in arm_suffixes]
155
+ suffixed_names_exist = all(
156
+ [hasattr(testing_params, pname) for pname in param_names_with_suffixes]
157
+ )
158
+ generic_param_name = param_name_no_suffix.removeprefix("cryonirsp_")
159
+ return hasattr(task_params, generic_param_name) and suffixed_names_exist
160
+
161
+
162
+ @pytest.mark.parametrize("arm_id", ["SP", "CI"])
163
+ def test_arm_parameters(basic_science_task_with_parameter_mixin, arm_id):
164
+ """
165
+ Given: A Science task with the parameter mixin
166
+ When: Accessing parameters that are "arm" parameters
167
+ Then: The correct value is returned
168
+
169
+ This test exercises all aspects of arm parameters from their names to the loading of values,
170
+ which includes exercising the method _find_parameter_for_arm
171
+ """
172
+ # An arm parameter is one which is present in the param class with no arm suffix
173
+ # and is also present in the testing param class with both suffix forms
174
+ task, test_params = next(basic_science_task_with_parameter_mixin(arm_id=arm_id))
175
+ task_params = task.parameters
176
+ # Iterate over the test parameters
177
+ for pn, pv in asdict(test_params).items():
178
+ suffix = f"_{arm_id}".casefold()
179
+ if _is_arm_param(pn, task_params, test_params, single_arm_only=arm_id):
180
+ generic_param_name = pn.removeprefix("cryonirsp_").removesuffix(suffix)
181
+
182
+ actual = getattr(task_params, generic_param_name)
183
+ expected = getattr(test_params, pn)
184
+ if isinstance(expected, FileParameter) and expected.is_file:
185
+ expected = task_params._load_param_value_from_numpy_save(asdict(expected))
186
+ assert np.array_equal(expected, actual)
187
+ elif isinstance(actual, np.ndarray):
188
+ assert expected == actual.tolist()
189
+ else:
190
+ assert expected == actual
191
+
192
+
193
+ def test_parameters(basic_science_task_with_parameter_mixin):
194
+ """
195
+ Given: A Science task with the parameter mixin
196
+ When: Accessing properties for parameters that are not wavelength, file or generic parameters
197
+ Then: The correct value is returned
198
+ """
199
+ task, test_params = next(basic_science_task_with_parameter_mixin())
200
+ task_params = task.parameters
201
+ for pn, pv in asdict(test_params).items():
202
+ parameter_name = pn.removeprefix("cryonirsp_")
203
+ if (
204
+ _is_file_param(pv)
205
+ or _is_wavelength_param(pv)
206
+ or _is_arm_param(pn, task_params, test_params)
207
+ or parameter_name in PARSE_PARAMETER_NAMES
208
+ ):
209
+ continue
210
+ accessed_parameter_value = getattr(task_params, parameter_name)
211
+ if isinstance(accessed_parameter_value, Quantity):
212
+ assert pv == accessed_parameter_value.value
213
+ else:
214
+ assert pv == getattr(task_params, parameter_name)
215
+
216
+
217
+ def test_parse_parameters(basic_science_task_with_parameter_mixin):
218
+ """
219
+ Given: A Science task with ParsingParameters
220
+ When: Accessing properties for the parsing parameters
221
+ Then: The correct value is returned
222
+ """
223
+ task, test_params = next(
224
+ basic_science_task_with_parameter_mixin(
225
+ parameter_class=CryonirspParsingParameters,
226
+ obs_ip_start_time=None,
227
+ )
228
+ )
229
+ task_param_attr = task.parameters
230
+ for pn, pv in asdict(test_params).items():
231
+ property_name = pn.removeprefix("cryonirsp_")
232
+ if property_name in PARSE_PARAMETER_NAMES and type(pv) is not dict:
233
+ assert getattr(task_param_attr, property_name) == pv
234
+
235
+
236
+ @pytest.mark.parametrize("arm", [pytest.param("CI"), pytest.param("SP")])
237
+ def test_linearization_threshold_parameters(
238
+ basic_science_task_with_parameter_mixin, arm, init_cryonirsp_constants_db
239
+ ):
240
+ """
241
+ Given: A Science task with the parameter mixin
242
+ When: Accessing properties for the linearization thresholds
243
+ Then: The correct type is returned
244
+ """
245
+ task, _ = next(basic_science_task_with_parameter_mixin())
246
+ recipe_run_id = task.recipe_run_id
247
+ init_cryonirsp_constants_db(recipe_run_id, CryonirspConstantsDb(ARM_ID=arm))
248
+ linearization_threshold_array = task.parameters.linearization_thresholds
249
+
250
+ assert linearization_threshold_array.dtype == np.float32
251
+
252
+
253
+ def test_optical_density_filter_names(basic_science_task_with_parameter_mixin):
254
+ task, _ = next(basic_science_task_with_parameter_mixin())
255
+ # List of filter attenuation parameters defined in CryonirspParameters:
256
+ defined_filter_params = {
257
+ item[-4:].upper()
258
+ for item in dir(task.parameters)
259
+ if item.startswith("_linearization_optical_density_filter_attenuation_")
260
+ }
261
+ # List of filters in the filter map:
262
+ filter_map_params = {k for k in task.parameters.linearization_filter_attenuation_dict.keys()}
263
+ # Make sure all filter parameters match the allowable list
264
+ assert not defined_filter_params.symmetric_difference(ALLOWABLE_OPTICAL_DENSITY_FILTERS)
265
+ # Make sure all filter map keys match the allowable list
266
+ assert not filter_map_params.symmetric_difference(ALLOWABLE_OPTICAL_DENSITY_FILTERS)