dkist-processing-cryonirsp 1.3.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dkist-processing-cryonirsp might be problematic. Click here for more details.
- changelog/.gitempty +0 -0
- dkist_processing_cryonirsp/__init__.py +11 -0
- dkist_processing_cryonirsp/config.py +12 -0
- dkist_processing_cryonirsp/models/__init__.py +1 -0
- dkist_processing_cryonirsp/models/constants.py +248 -0
- dkist_processing_cryonirsp/models/exposure_conditions.py +26 -0
- dkist_processing_cryonirsp/models/parameters.py +296 -0
- dkist_processing_cryonirsp/models/tags.py +168 -0
- dkist_processing_cryonirsp/models/task_name.py +14 -0
- dkist_processing_cryonirsp/parsers/__init__.py +1 -0
- dkist_processing_cryonirsp/parsers/cryonirsp_l0_fits_access.py +111 -0
- dkist_processing_cryonirsp/parsers/cryonirsp_l1_fits_access.py +30 -0
- dkist_processing_cryonirsp/parsers/exposure_conditions.py +163 -0
- dkist_processing_cryonirsp/parsers/map_repeats.py +40 -0
- dkist_processing_cryonirsp/parsers/measurements.py +55 -0
- dkist_processing_cryonirsp/parsers/modstates.py +31 -0
- dkist_processing_cryonirsp/parsers/optical_density_filters.py +40 -0
- dkist_processing_cryonirsp/parsers/polarimetric_check.py +120 -0
- dkist_processing_cryonirsp/parsers/scan_step.py +412 -0
- dkist_processing_cryonirsp/parsers/time.py +80 -0
- dkist_processing_cryonirsp/parsers/wavelength.py +26 -0
- dkist_processing_cryonirsp/tasks/__init__.py +19 -0
- dkist_processing_cryonirsp/tasks/assemble_movie.py +202 -0
- dkist_processing_cryonirsp/tasks/bad_pixel_map.py +96 -0
- dkist_processing_cryonirsp/tasks/beam_boundaries_base.py +279 -0
- dkist_processing_cryonirsp/tasks/ci_beam_boundaries.py +55 -0
- dkist_processing_cryonirsp/tasks/ci_science.py +169 -0
- dkist_processing_cryonirsp/tasks/cryonirsp_base.py +67 -0
- dkist_processing_cryonirsp/tasks/dark.py +98 -0
- dkist_processing_cryonirsp/tasks/gain.py +251 -0
- dkist_processing_cryonirsp/tasks/instrument_polarization.py +447 -0
- dkist_processing_cryonirsp/tasks/l1_output_data.py +44 -0
- dkist_processing_cryonirsp/tasks/linearity_correction.py +582 -0
- dkist_processing_cryonirsp/tasks/make_movie_frames.py +302 -0
- dkist_processing_cryonirsp/tasks/mixin/__init__.py +1 -0
- dkist_processing_cryonirsp/tasks/mixin/beam_access.py +52 -0
- dkist_processing_cryonirsp/tasks/mixin/corrections.py +177 -0
- dkist_processing_cryonirsp/tasks/mixin/intermediate_frame.py +193 -0
- dkist_processing_cryonirsp/tasks/mixin/linearized_frame.py +309 -0
- dkist_processing_cryonirsp/tasks/mixin/shift_measurements.py +297 -0
- dkist_processing_cryonirsp/tasks/parse.py +281 -0
- dkist_processing_cryonirsp/tasks/quality_metrics.py +271 -0
- dkist_processing_cryonirsp/tasks/science_base.py +511 -0
- dkist_processing_cryonirsp/tasks/sp_beam_boundaries.py +270 -0
- dkist_processing_cryonirsp/tasks/sp_dispersion_axis_correction.py +484 -0
- dkist_processing_cryonirsp/tasks/sp_geometric.py +585 -0
- dkist_processing_cryonirsp/tasks/sp_science.py +299 -0
- dkist_processing_cryonirsp/tasks/sp_solar_gain.py +475 -0
- dkist_processing_cryonirsp/tasks/trial_output_data.py +61 -0
- dkist_processing_cryonirsp/tasks/write_l1.py +1033 -0
- dkist_processing_cryonirsp/tests/__init__.py +1 -0
- dkist_processing_cryonirsp/tests/conftest.py +456 -0
- dkist_processing_cryonirsp/tests/header_models.py +592 -0
- dkist_processing_cryonirsp/tests/local_trial_workflows/__init__.py +0 -0
- dkist_processing_cryonirsp/tests/local_trial_workflows/l0_cals_only.py +541 -0
- dkist_processing_cryonirsp/tests/local_trial_workflows/l0_to_l1.py +615 -0
- dkist_processing_cryonirsp/tests/local_trial_workflows/linearize_only.py +96 -0
- dkist_processing_cryonirsp/tests/local_trial_workflows/local_trial_helpers.py +592 -0
- dkist_processing_cryonirsp/tests/test_assemble_movie.py +144 -0
- dkist_processing_cryonirsp/tests/test_assemble_qualilty.py +517 -0
- dkist_processing_cryonirsp/tests/test_bad_pixel_maps.py +115 -0
- dkist_processing_cryonirsp/tests/test_ci_beam_boundaries.py +106 -0
- dkist_processing_cryonirsp/tests/test_ci_science.py +355 -0
- dkist_processing_cryonirsp/tests/test_corrections.py +126 -0
- dkist_processing_cryonirsp/tests/test_cryo_base.py +202 -0
- dkist_processing_cryonirsp/tests/test_cryo_constants.py +76 -0
- dkist_processing_cryonirsp/tests/test_dark.py +287 -0
- dkist_processing_cryonirsp/tests/test_gain.py +278 -0
- dkist_processing_cryonirsp/tests/test_instrument_polarization.py +531 -0
- dkist_processing_cryonirsp/tests/test_linearity_correction.py +245 -0
- dkist_processing_cryonirsp/tests/test_make_movie_frames.py +111 -0
- dkist_processing_cryonirsp/tests/test_parameters.py +266 -0
- dkist_processing_cryonirsp/tests/test_parse.py +1439 -0
- dkist_processing_cryonirsp/tests/test_quality.py +203 -0
- dkist_processing_cryonirsp/tests/test_sp_beam_boundaries.py +112 -0
- dkist_processing_cryonirsp/tests/test_sp_dispersion_axis_correction.py +155 -0
- dkist_processing_cryonirsp/tests/test_sp_geometric.py +319 -0
- dkist_processing_cryonirsp/tests/test_sp_make_movie_frames.py +121 -0
- dkist_processing_cryonirsp/tests/test_sp_science.py +483 -0
- dkist_processing_cryonirsp/tests/test_sp_solar.py +198 -0
- dkist_processing_cryonirsp/tests/test_trial_create_quality_report.py +79 -0
- dkist_processing_cryonirsp/tests/test_trial_output_data.py +251 -0
- dkist_processing_cryonirsp/tests/test_workflows.py +9 -0
- dkist_processing_cryonirsp/tests/test_write_l1.py +436 -0
- dkist_processing_cryonirsp/workflows/__init__.py +2 -0
- dkist_processing_cryonirsp/workflows/ci_l0_processing.py +77 -0
- dkist_processing_cryonirsp/workflows/sp_l0_processing.py +84 -0
- dkist_processing_cryonirsp/workflows/trial_workflows.py +190 -0
- dkist_processing_cryonirsp-1.3.4.dist-info/METADATA +194 -0
- dkist_processing_cryonirsp-1.3.4.dist-info/RECORD +111 -0
- dkist_processing_cryonirsp-1.3.4.dist-info/WHEEL +5 -0
- dkist_processing_cryonirsp-1.3.4.dist-info/top_level.txt +4 -0
- docs/Makefile +134 -0
- docs/bad_pixel_calibration.rst +47 -0
- docs/beam_angle_calculation.rst +53 -0
- docs/beam_boundary_computation.rst +88 -0
- docs/changelog.rst +7 -0
- docs/ci_science_calibration.rst +33 -0
- docs/conf.py +52 -0
- docs/index.rst +21 -0
- docs/l0_to_l1_cryonirsp_ci-full-trial.rst +10 -0
- docs/l0_to_l1_cryonirsp_ci.rst +10 -0
- docs/l0_to_l1_cryonirsp_sp-full-trial.rst +10 -0
- docs/l0_to_l1_cryonirsp_sp.rst +10 -0
- docs/linearization.rst +43 -0
- docs/make.bat +170 -0
- docs/requirements.txt +1 -0
- docs/requirements_table.rst +8 -0
- docs/scientific_changelog.rst +10 -0
- docs/sp_science_calibration.rst +59 -0
- licenses/LICENSE.rst +11 -0
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
"""Test the linearity correction task."""
|
|
2
|
+
import re
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from datetime import datetime
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
import pytest
|
|
8
|
+
from astropy.io import fits
|
|
9
|
+
from dkist_header_validator import spec122_validator
|
|
10
|
+
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
11
|
+
from dkist_processing_common.codecs.fits import fits_hdulist_encoder
|
|
12
|
+
from dkist_processing_common.models.tags import Tag
|
|
13
|
+
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
14
|
+
from dkist_service_configuration.logging import logger
|
|
15
|
+
|
|
16
|
+
from dkist_processing_cryonirsp.models.constants import CryonirspBudName
|
|
17
|
+
from dkist_processing_cryonirsp.models.exposure_conditions import AllowableOpticalDensityFilterNames
|
|
18
|
+
from dkist_processing_cryonirsp.models.tags import CryonirspTag
|
|
19
|
+
from dkist_processing_cryonirsp.tasks.linearity_correction import LinearityCorrection
|
|
20
|
+
from dkist_processing_cryonirsp.tests.conftest import cryonirsp_testing_parameters_factory
|
|
21
|
+
from dkist_processing_cryonirsp.tests.conftest import CryonirspConstantsDb
|
|
22
|
+
from dkist_processing_cryonirsp.tests.conftest import generate_fits_frame
|
|
23
|
+
from dkist_processing_cryonirsp.tests.header_models import CryonirspHeadersValidNonLinearizedFrames
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@pytest.fixture(scope="function")
|
|
27
|
+
def linearity_correction(
|
|
28
|
+
tmp_path,
|
|
29
|
+
recipe_run_id,
|
|
30
|
+
assign_input_dataset_doc_to_task,
|
|
31
|
+
init_cryonirsp_constants_db,
|
|
32
|
+
arm_id,
|
|
33
|
+
frames_in_ramp,
|
|
34
|
+
filter_name,
|
|
35
|
+
):
|
|
36
|
+
# time y x
|
|
37
|
+
dataset_shape = (frames_in_ramp, 10, 10)
|
|
38
|
+
# z y x
|
|
39
|
+
array_shape = (1, 10, 10)
|
|
40
|
+
time_delta = 0.1
|
|
41
|
+
start_time = datetime.now()
|
|
42
|
+
expected_num_frames_in_ramp = 10
|
|
43
|
+
constants_db = CryonirspConstantsDb(
|
|
44
|
+
TIME_OBS_LIST=(str(start_time),),
|
|
45
|
+
ARM_ID=arm_id,
|
|
46
|
+
ROI_1_SIZE_X=array_shape[2],
|
|
47
|
+
ROI_1_SIZE_Y=array_shape[1],
|
|
48
|
+
)
|
|
49
|
+
init_cryonirsp_constants_db(recipe_run_id, constants_db)
|
|
50
|
+
with LinearityCorrection(
|
|
51
|
+
recipe_run_id=recipe_run_id,
|
|
52
|
+
workflow_name="linearity_correction",
|
|
53
|
+
workflow_version="VX.Y",
|
|
54
|
+
) as task:
|
|
55
|
+
try: # This try... block is here to make sure the dbs get cleaned up if there's a failure in the fixture
|
|
56
|
+
task.scratch = WorkflowFileSystem(
|
|
57
|
+
scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
|
|
58
|
+
)
|
|
59
|
+
param_class = cryonirsp_testing_parameters_factory(param_path=tmp_path)
|
|
60
|
+
assign_input_dataset_doc_to_task(task, param_class())
|
|
61
|
+
ds = CryonirspHeadersValidNonLinearizedFrames(
|
|
62
|
+
arm_id=arm_id,
|
|
63
|
+
camera_readout_mode="FastUpTheRamp",
|
|
64
|
+
dataset_shape=dataset_shape,
|
|
65
|
+
array_shape=array_shape,
|
|
66
|
+
time_delta=time_delta,
|
|
67
|
+
roi_x_origin=0,
|
|
68
|
+
roi_y_origin=0,
|
|
69
|
+
roi_x_size=array_shape[2],
|
|
70
|
+
roi_y_size=array_shape[1],
|
|
71
|
+
date_obs=start_time.isoformat("T"),
|
|
72
|
+
exposure_time=time_delta,
|
|
73
|
+
)
|
|
74
|
+
# Initial header creation...
|
|
75
|
+
header_generator = (
|
|
76
|
+
spec122_validator.validate_and_translate_to_214_l0(
|
|
77
|
+
d.header(), return_type=fits.HDUList
|
|
78
|
+
)[0].header
|
|
79
|
+
for d in ds
|
|
80
|
+
)
|
|
81
|
+
# Patch the headers for non-linearized Cryo data...
|
|
82
|
+
header_list = []
|
|
83
|
+
exp_time = 0.0
|
|
84
|
+
counter = 0
|
|
85
|
+
for header in header_generator:
|
|
86
|
+
# Set the integrated exposure time for this NDR
|
|
87
|
+
# This is a range from 0 to 90 in 10 steps
|
|
88
|
+
header["XPOSURE"] = 100 * counter * time_delta
|
|
89
|
+
# Set the current frame in ramp, 1-based
|
|
90
|
+
header["CNCNDR"] = counter + 1
|
|
91
|
+
header["CNNNDR"] = expected_num_frames_in_ramp
|
|
92
|
+
header["CNFILTNP"] = filter_name
|
|
93
|
+
header_list.append(header)
|
|
94
|
+
counter += 1
|
|
95
|
+
# Step on the old one with the new one
|
|
96
|
+
header_generator = (header for header in header_list)
|
|
97
|
+
# Iterate through the headers and create the frames...
|
|
98
|
+
for _ in header_list:
|
|
99
|
+
hdul = generate_fits_frame(header_generator=header_generator, shape=array_shape)
|
|
100
|
+
# Now tweak the data...
|
|
101
|
+
for hdu in hdul:
|
|
102
|
+
header = hdu.header
|
|
103
|
+
exp_time = header["XPOSURE"]
|
|
104
|
+
# Create a simple perfectly linear ramp
|
|
105
|
+
hdu.data.fill(exp_time)
|
|
106
|
+
task.write(
|
|
107
|
+
data=hdul,
|
|
108
|
+
tags=[
|
|
109
|
+
CryonirspTag.input(),
|
|
110
|
+
CryonirspTag.frame(),
|
|
111
|
+
CryonirspTag.curr_frame_in_ramp(header["CNCNDR"]),
|
|
112
|
+
# All frames in a ramp have the same date-obs
|
|
113
|
+
CryonirspTag.time_obs(str(start_time)),
|
|
114
|
+
Tag.readout_exp_time(exp_time),
|
|
115
|
+
],
|
|
116
|
+
encoder=fits_hdulist_encoder,
|
|
117
|
+
)
|
|
118
|
+
task.constants._update({CryonirspBudName.camera_readout_mode.value: "FastUpTheRamp"})
|
|
119
|
+
yield task, filter_name, expected_num_frames_in_ramp
|
|
120
|
+
finally:
|
|
121
|
+
task._purge()
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@pytest.mark.parametrize(
|
|
125
|
+
"filter_name",
|
|
126
|
+
[
|
|
127
|
+
pytest.param(AllowableOpticalDensityFilterNames.G358.value),
|
|
128
|
+
pytest.param(AllowableOpticalDensityFilterNames.OPEN.value),
|
|
129
|
+
],
|
|
130
|
+
)
|
|
131
|
+
@pytest.mark.parametrize(
|
|
132
|
+
"frames_in_ramp",
|
|
133
|
+
[pytest.param(10, id="Full ramp"), pytest.param(5, id="Bad ramp")],
|
|
134
|
+
)
|
|
135
|
+
@pytest.mark.parametrize(
|
|
136
|
+
"arm_id",
|
|
137
|
+
[pytest.param("CI", id="CI"), pytest.param("SP", id="SP")],
|
|
138
|
+
)
|
|
139
|
+
@pytest.mark.parametrize(
|
|
140
|
+
"num_chunks",
|
|
141
|
+
[pytest.param(1, id="1 chunk"), pytest.param(2, id="2 chunks")],
|
|
142
|
+
)
|
|
143
|
+
def test_linearity_correction(
|
|
144
|
+
linearity_correction,
|
|
145
|
+
mocker,
|
|
146
|
+
arm_id,
|
|
147
|
+
frames_in_ramp,
|
|
148
|
+
num_chunks,
|
|
149
|
+
filter_name,
|
|
150
|
+
):
|
|
151
|
+
"""
|
|
152
|
+
Given: A LinearityCorrection task
|
|
153
|
+
When: Calling the task instance with known input data
|
|
154
|
+
Then: The non-linearized frames are linearized and produce the correct results.
|
|
155
|
+
"""
|
|
156
|
+
mocker.patch(
|
|
157
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
|
|
158
|
+
)
|
|
159
|
+
if num_chunks == 2:
|
|
160
|
+
mocker.patch(
|
|
161
|
+
"dkist_processing_cryonirsp.tasks.linearity_correction.LinearityCorrection.compute_linear_chunk_size",
|
|
162
|
+
new=lambda self, frame_size, num_frames_in_ramp: frame_size // 2,
|
|
163
|
+
)
|
|
164
|
+
# Given
|
|
165
|
+
task, filter_name, expected_num_frames_in_ramp = linearity_correction
|
|
166
|
+
# When
|
|
167
|
+
task()
|
|
168
|
+
# Then
|
|
169
|
+
tags = [
|
|
170
|
+
CryonirspTag.linearized(),
|
|
171
|
+
CryonirspTag.frame(),
|
|
172
|
+
]
|
|
173
|
+
# We used a perfect linear ramp from 0 to 90, where the ramp value is equal to the exposure time in ms
|
|
174
|
+
# The algorithm normalizes the linearized frame by the exposure time in seconds, so the expected value is:
|
|
175
|
+
# 90 / (90 / 1000) = 1000 / attenuation, where attenuation is the multiplicative attenuation due to the
|
|
176
|
+
# filter in use
|
|
177
|
+
attenuation = 10 ** (task.parameters.linearization_filter_attenuation_dict[filter_name])
|
|
178
|
+
expected_data = np.ones((10, 10)) * 1000.0 / attenuation
|
|
179
|
+
files_found = list(task.read(tags=tags))
|
|
180
|
+
if frames_in_ramp == expected_num_frames_in_ramp:
|
|
181
|
+
assert len(files_found) == 1
|
|
182
|
+
hdul = fits.open(files_found[0])
|
|
183
|
+
data = hdul[0].data
|
|
184
|
+
assert np.allclose(data, expected_data)
|
|
185
|
+
else:
|
|
186
|
+
assert len(files_found) == 0
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
@pytest.fixture
|
|
190
|
+
def simple_linearity_correction_task(recipe_run_id, arm_id, init_cryonirsp_constants_db, tmp_path):
|
|
191
|
+
constants_db = CryonirspConstantsDb(
|
|
192
|
+
ARM_ID=arm_id,
|
|
193
|
+
)
|
|
194
|
+
init_cryonirsp_constants_db(recipe_run_id, constants_db)
|
|
195
|
+
with LinearityCorrection(
|
|
196
|
+
recipe_run_id=recipe_run_id,
|
|
197
|
+
workflow_name="workflow_name",
|
|
198
|
+
workflow_version="workflow_version",
|
|
199
|
+
) as task:
|
|
200
|
+
task.scratch = WorkflowFileSystem(recipe_run_id=recipe_run_id, scratch_base_path=tmp_path)
|
|
201
|
+
|
|
202
|
+
yield task
|
|
203
|
+
|
|
204
|
+
task._purge()
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
@dataclass
|
|
208
|
+
class DummyRampFitsAccess:
|
|
209
|
+
"""Just a class that has the one property that is checked during ramp validation."""
|
|
210
|
+
|
|
211
|
+
num_frames_in_ramp: int
|
|
212
|
+
ip_task_type: str = "TASK"
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
@pytest.mark.parametrize(
|
|
216
|
+
"arm_id",
|
|
217
|
+
[pytest.param("CI", id="CI"), pytest.param("SP", id="SP")],
|
|
218
|
+
)
|
|
219
|
+
@pytest.mark.parametrize(
|
|
220
|
+
"ramp_list, valid, message",
|
|
221
|
+
[
|
|
222
|
+
pytest.param(
|
|
223
|
+
[
|
|
224
|
+
DummyRampFitsAccess(num_frames_in_ramp=2),
|
|
225
|
+
DummyRampFitsAccess(num_frames_in_ramp=3),
|
|
226
|
+
],
|
|
227
|
+
False,
|
|
228
|
+
"Not all frames have the same FRAMES_IN_RAMP value. Set is {2, 3}. Ramp is task TASK. Skipping ramp.",
|
|
229
|
+
id="num_frames_mismatch_actual_frames",
|
|
230
|
+
),
|
|
231
|
+
pytest.param(
|
|
232
|
+
[
|
|
233
|
+
DummyRampFitsAccess(num_frames_in_ramp=8),
|
|
234
|
+
],
|
|
235
|
+
False,
|
|
236
|
+
"Missing some ramp frames. Expected 8 from header value, but only have 1. Ramp is task TASK. Skipping ramp.",
|
|
237
|
+
id="num_frames_in_set_mismatch",
|
|
238
|
+
),
|
|
239
|
+
],
|
|
240
|
+
)
|
|
241
|
+
def test_is_ramp_valid(simple_linearity_correction_task, ramp_list, valid, message, caplog):
|
|
242
|
+
logger.add(caplog.handler)
|
|
243
|
+
assert simple_linearity_correction_task.is_ramp_valid(ramp_list) is valid
|
|
244
|
+
if not valid:
|
|
245
|
+
assert re.search(message, caplog.text)
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
from datetime import datetime
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
from astropy.io import fits
|
|
5
|
+
from dkist_header_validator import spec122_validator
|
|
6
|
+
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
7
|
+
from dkist_processing_common.codecs.fits import fits_hdulist_encoder
|
|
8
|
+
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
9
|
+
|
|
10
|
+
from dkist_processing_cryonirsp.models.tags import CryonirspTag
|
|
11
|
+
from dkist_processing_cryonirsp.tasks.make_movie_frames import MakeCryonirspMovieFrames
|
|
12
|
+
from dkist_processing_cryonirsp.tests.conftest import CryonirspConstantsDb
|
|
13
|
+
from dkist_processing_cryonirsp.tests.conftest import generate_fits_frame
|
|
14
|
+
from dkist_processing_cryonirsp.tests.header_models import CryonirspHeadersValidObserveFrames
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@pytest.fixture(
|
|
18
|
+
scope="function",
|
|
19
|
+
params=[pytest.param(True, id="polarimetric"), pytest.param(False, id="intensity-only")],
|
|
20
|
+
)
|
|
21
|
+
def movie_frames_task(tmp_path, recipe_run_id, init_cryonirsp_constants_db, request):
|
|
22
|
+
is_polarimetric = request.param
|
|
23
|
+
map_scans = 3
|
|
24
|
+
scan_steps = 2
|
|
25
|
+
array_shape = (3, 4)
|
|
26
|
+
if is_polarimetric:
|
|
27
|
+
num_mod = 8
|
|
28
|
+
spin_mode = "Continuous"
|
|
29
|
+
else:
|
|
30
|
+
num_mod = 1
|
|
31
|
+
spin_mode = "None"
|
|
32
|
+
constants_db = CryonirspConstantsDb(
|
|
33
|
+
NUM_MODSTATES=num_mod,
|
|
34
|
+
MODULATOR_SPIN_MODE=spin_mode,
|
|
35
|
+
NUM_SCAN_STEPS=scan_steps,
|
|
36
|
+
NUM_MAP_SCANS=map_scans,
|
|
37
|
+
TIME_OBS_LIST=(datetime.now().isoformat("T"),),
|
|
38
|
+
)
|
|
39
|
+
init_cryonirsp_constants_db(recipe_run_id, constants_db)
|
|
40
|
+
with MakeCryonirspMovieFrames(
|
|
41
|
+
recipe_run_id=recipe_run_id, workflow_name="make_movie_frames", workflow_version="VX.Y"
|
|
42
|
+
) as task:
|
|
43
|
+
try: # This try... block is here to make sure the dbs get cleaned up if there's a failure in the fixture
|
|
44
|
+
meas_num = 1 # Use only the first measurement if there are multiple measurements.
|
|
45
|
+
task.scratch = WorkflowFileSystem(
|
|
46
|
+
scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
|
|
47
|
+
)
|
|
48
|
+
start_time = datetime.now()
|
|
49
|
+
for stokes_state in ["I", "Q", "U", "V"]:
|
|
50
|
+
for map_scan in range(1, map_scans + 1):
|
|
51
|
+
for scan_step in range(0, scan_steps + 1):
|
|
52
|
+
ds = CryonirspHeadersValidObserveFrames(
|
|
53
|
+
dataset_shape=(2, *array_shape),
|
|
54
|
+
array_shape=(1, *array_shape),
|
|
55
|
+
time_delta=10,
|
|
56
|
+
num_map_scans=map_scans,
|
|
57
|
+
map_scan=map_scan,
|
|
58
|
+
num_scan_steps=scan_steps,
|
|
59
|
+
scan_step=scan_step,
|
|
60
|
+
num_modstates=1,
|
|
61
|
+
modstate=1,
|
|
62
|
+
start_time=start_time,
|
|
63
|
+
num_meas=1,
|
|
64
|
+
meas_num=1,
|
|
65
|
+
arm_id="CI",
|
|
66
|
+
)
|
|
67
|
+
header_generator = (
|
|
68
|
+
spec122_validator.validate_and_translate_to_214_l0(
|
|
69
|
+
d.header(), return_type=fits.HDUList
|
|
70
|
+
)[0].header
|
|
71
|
+
for d in ds
|
|
72
|
+
)
|
|
73
|
+
hdul = generate_fits_frame(
|
|
74
|
+
header_generator=header_generator, shape=(1, *array_shape)
|
|
75
|
+
)
|
|
76
|
+
task.write(
|
|
77
|
+
data=hdul,
|
|
78
|
+
tags=[
|
|
79
|
+
CryonirspTag.calibrated(),
|
|
80
|
+
CryonirspTag.frame(),
|
|
81
|
+
CryonirspTag.map_scan(map_scan),
|
|
82
|
+
CryonirspTag.scan_step(scan_step),
|
|
83
|
+
CryonirspTag.stokes(stokes_state),
|
|
84
|
+
CryonirspTag.meas_num(meas_num),
|
|
85
|
+
],
|
|
86
|
+
encoder=fits_hdulist_encoder,
|
|
87
|
+
)
|
|
88
|
+
yield task, map_scans, scan_steps, array_shape, is_polarimetric
|
|
89
|
+
finally:
|
|
90
|
+
task._purge()
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_make_movie_frames(movie_frames_task, mocker):
|
|
94
|
+
"""
|
|
95
|
+
Given: A MakeCryonirspMovieFrames task
|
|
96
|
+
When: Calling the task instance
|
|
97
|
+
Then: a fits file is made for each scan containing the movie frame for that scan
|
|
98
|
+
"""
|
|
99
|
+
mocker.patch(
|
|
100
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
|
|
101
|
+
)
|
|
102
|
+
task, map_scans, scan_steps, array_shape, is_polarimetric = movie_frames_task
|
|
103
|
+
expected_shape = array_shape
|
|
104
|
+
|
|
105
|
+
task()
|
|
106
|
+
assert len(list(task.read(tags=[CryonirspTag.movie_frame()]))) == map_scans * scan_steps
|
|
107
|
+
for filepath in task.read(tags=[CryonirspTag.movie_frame()]):
|
|
108
|
+
assert filepath.exists()
|
|
109
|
+
hdul = fits.open(filepath)
|
|
110
|
+
assert hdul[0].header["INSTRUME"] == "CRYO-NIRSP"
|
|
111
|
+
assert hdul[0].data.shape == expected_shape
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
from dataclasses import asdict
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import pytest
|
|
6
|
+
from astropy.units import Quantity
|
|
7
|
+
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
8
|
+
from hypothesis import example
|
|
9
|
+
from hypothesis import given
|
|
10
|
+
from hypothesis import HealthCheck
|
|
11
|
+
from hypothesis import settings
|
|
12
|
+
from hypothesis import strategies as st
|
|
13
|
+
|
|
14
|
+
from dkist_processing_cryonirsp.models.parameters import CryonirspParameters
|
|
15
|
+
from dkist_processing_cryonirsp.models.parameters import CryonirspParsingParameters
|
|
16
|
+
from dkist_processing_cryonirsp.parsers.optical_density_filters import (
|
|
17
|
+
ALLOWABLE_OPTICAL_DENSITY_FILTERS,
|
|
18
|
+
)
|
|
19
|
+
from dkist_processing_cryonirsp.tasks.cryonirsp_base import CryonirspTaskBase
|
|
20
|
+
from dkist_processing_cryonirsp.tests.conftest import cryonirsp_testing_parameters_factory
|
|
21
|
+
from dkist_processing_cryonirsp.tests.conftest import CryonirspConstantsDb
|
|
22
|
+
from dkist_processing_cryonirsp.tests.conftest import FileParameter
|
|
23
|
+
from dkist_processing_cryonirsp.tests.conftest import TestingParameters
|
|
24
|
+
|
|
25
|
+
# The property names of all parameters on `CryonirspParsingParameters`
|
|
26
|
+
PARSE_PARAMETER_NAMES = [
|
|
27
|
+
k for k, v in vars(CryonirspParsingParameters).items() if isinstance(v, property)
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@pytest.fixture(scope="function")
|
|
32
|
+
def basic_science_task_with_parameter_mixin(
|
|
33
|
+
tmp_path,
|
|
34
|
+
recipe_run_id,
|
|
35
|
+
assign_input_dataset_doc_to_task,
|
|
36
|
+
init_cryonirsp_constants_db,
|
|
37
|
+
testing_obs_ip_start_time,
|
|
38
|
+
):
|
|
39
|
+
def make_task(
|
|
40
|
+
parameter_class=CryonirspParameters,
|
|
41
|
+
arm_id: str = "SP",
|
|
42
|
+
obs_ip_start_time: str = testing_obs_ip_start_time,
|
|
43
|
+
):
|
|
44
|
+
class Task(CryonirspTaskBase):
|
|
45
|
+
def run(self):
|
|
46
|
+
...
|
|
47
|
+
|
|
48
|
+
init_cryonirsp_constants_db(recipe_run_id, CryonirspConstantsDb())
|
|
49
|
+
task = Task(
|
|
50
|
+
recipe_run_id=recipe_run_id,
|
|
51
|
+
workflow_name="parse_cryonirsp_input_data",
|
|
52
|
+
workflow_version="VX.Y",
|
|
53
|
+
)
|
|
54
|
+
try: # This try... block is here to make sure the dbs get cleaned up if there's a failure in the fixture
|
|
55
|
+
task.scratch = WorkflowFileSystem(
|
|
56
|
+
scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
|
|
57
|
+
)
|
|
58
|
+
test_params = cryonirsp_testing_parameters_factory(param_path=tmp_path)
|
|
59
|
+
param_dict = test_params()
|
|
60
|
+
assign_input_dataset_doc_to_task(
|
|
61
|
+
task,
|
|
62
|
+
param_dict,
|
|
63
|
+
parameter_class=parameter_class,
|
|
64
|
+
arm_id=arm_id,
|
|
65
|
+
obs_ip_start_time=obs_ip_start_time,
|
|
66
|
+
)
|
|
67
|
+
yield task, param_dict
|
|
68
|
+
finally:
|
|
69
|
+
task._purge()
|
|
70
|
+
|
|
71
|
+
return make_task
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _is_wavelength_param(param_value: Any) -> bool:
|
|
75
|
+
return isinstance(param_value, dict) and "wavelength" in param_value
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@given(wave=st.floats(min_value=800.0, max_value=2000.0))
|
|
79
|
+
@settings(suppress_health_check=[HealthCheck.function_scoped_fixture], deadline=None)
|
|
80
|
+
@example(wave=1082.7)
|
|
81
|
+
def test_filter_parameters(basic_science_task_with_parameter_mixin, wave):
|
|
82
|
+
"""
|
|
83
|
+
Given: A Science task with the parameter mixin
|
|
84
|
+
When: Accessing properties for the optical density filters
|
|
85
|
+
Then: The correct value is returned
|
|
86
|
+
"""
|
|
87
|
+
task, expected = next(basic_science_task_with_parameter_mixin())
|
|
88
|
+
task_params = task.parameters
|
|
89
|
+
task_params._wavelength = wave
|
|
90
|
+
expected = {
|
|
91
|
+
"_linearization_optical_density_filter_attenuation_g278": -1.64,
|
|
92
|
+
"_linearization_optical_density_filter_attenuation_g358": -3.75,
|
|
93
|
+
"_linearization_optical_density_filter_attenuation_g408": -4.26,
|
|
94
|
+
}
|
|
95
|
+
for param in expected:
|
|
96
|
+
assert getattr(task_params, param) == expected[param]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _is_file_param(param_value: Any) -> bool:
|
|
100
|
+
return isinstance(param_value, dict) and "is_file" in param_value and param_value["is_file"]
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def test_file_parameters(basic_science_task_with_parameter_mixin):
|
|
104
|
+
"""
|
|
105
|
+
Given: A Science task with the parameter mixin
|
|
106
|
+
When: Accessing parameters whose values are loaded from files
|
|
107
|
+
Then: The correct value is returned
|
|
108
|
+
|
|
109
|
+
This test exercises all aspects of file parameters from their names to the loading of values
|
|
110
|
+
"""
|
|
111
|
+
task, test_params = next(basic_science_task_with_parameter_mixin())
|
|
112
|
+
task_params = task.parameters
|
|
113
|
+
# Iterate over the test parameters and check that each file param exists in the task parameters
|
|
114
|
+
# we load the actual parameter from the task param object using only the param name
|
|
115
|
+
for pn, pv in asdict(test_params).items():
|
|
116
|
+
# We want to test only file parameters
|
|
117
|
+
if _is_file_param(pv):
|
|
118
|
+
pn_no_prefix = pn.removeprefix("cryonirsp_")
|
|
119
|
+
# If the param name is an attribute in task_params, then load it directly
|
|
120
|
+
if hasattr(task_params, pn_no_prefix):
|
|
121
|
+
param_name = pn_no_prefix
|
|
122
|
+
actual = getattr(task_params, param_name)
|
|
123
|
+
# if the param name is not a task param attribute, then check that we can load the param
|
|
124
|
+
# using the value defined in the input_dataset_parameters list of the task param object
|
|
125
|
+
else:
|
|
126
|
+
param_dict = task_params._find_most_recent_past_value(pn)
|
|
127
|
+
actual = task_params._load_param_value_from_numpy_save(param_dict)
|
|
128
|
+
# Now get the expected value using the param value dict from the testing params
|
|
129
|
+
expected = np.load(pv["param_path"])
|
|
130
|
+
# Compare the actual and expected values
|
|
131
|
+
assert np.array_equal(actual, expected)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _is_arm_param(
|
|
135
|
+
param_name: str,
|
|
136
|
+
task_params: CryonirspParameters,
|
|
137
|
+
testing_params: TestingParameters,
|
|
138
|
+
single_arm_only: str | None = None,
|
|
139
|
+
):
|
|
140
|
+
"""
|
|
141
|
+
Test if a parameter is an arm parameter.
|
|
142
|
+
|
|
143
|
+
An arm parameter is one which is present in the task_param class with no arm suffix and is also
|
|
144
|
+
present in the test_param class with suffixed forms only, one for each arm.
|
|
145
|
+
This allows a non-arm-specific name to be used as a property in the parameters class which
|
|
146
|
+
encapsulates the mechanism used to return the arm specific parameter value based on the arm in use.
|
|
147
|
+
"""
|
|
148
|
+
# NB: param_name is assumed to have a prefix of "cryonirsp_"
|
|
149
|
+
arm_suffixes = ["_sp", "_ci"] if single_arm_only is None else [f"_{single_arm_only}".casefold()]
|
|
150
|
+
suffix = param_name[-3:]
|
|
151
|
+
if suffix not in arm_suffixes:
|
|
152
|
+
return False
|
|
153
|
+
param_name_no_suffix = param_name[:-3]
|
|
154
|
+
param_names_with_suffixes = [f"{param_name_no_suffix}{suffix}" for suffix in arm_suffixes]
|
|
155
|
+
suffixed_names_exist = all(
|
|
156
|
+
[hasattr(testing_params, pname) for pname in param_names_with_suffixes]
|
|
157
|
+
)
|
|
158
|
+
generic_param_name = param_name_no_suffix.removeprefix("cryonirsp_")
|
|
159
|
+
return hasattr(task_params, generic_param_name) and suffixed_names_exist
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
@pytest.mark.parametrize("arm_id", ["SP", "CI"])
|
|
163
|
+
def test_arm_parameters(basic_science_task_with_parameter_mixin, arm_id):
|
|
164
|
+
"""
|
|
165
|
+
Given: A Science task with the parameter mixin
|
|
166
|
+
When: Accessing parameters that are "arm" parameters
|
|
167
|
+
Then: The correct value is returned
|
|
168
|
+
|
|
169
|
+
This test exercises all aspects of arm parameters from their names to the loading of values,
|
|
170
|
+
which includes exercising the method _find_parameter_for_arm
|
|
171
|
+
"""
|
|
172
|
+
# An arm parameter is one which is present in the param class with no arm suffix
|
|
173
|
+
# and is also present in the testing param class with both suffix forms
|
|
174
|
+
task, test_params = next(basic_science_task_with_parameter_mixin(arm_id=arm_id))
|
|
175
|
+
task_params = task.parameters
|
|
176
|
+
# Iterate over the test parameters
|
|
177
|
+
for pn, pv in asdict(test_params).items():
|
|
178
|
+
suffix = f"_{arm_id}".casefold()
|
|
179
|
+
if _is_arm_param(pn, task_params, test_params, single_arm_only=arm_id):
|
|
180
|
+
generic_param_name = pn.removeprefix("cryonirsp_").removesuffix(suffix)
|
|
181
|
+
|
|
182
|
+
actual = getattr(task_params, generic_param_name)
|
|
183
|
+
expected = getattr(test_params, pn)
|
|
184
|
+
if isinstance(expected, FileParameter) and expected.is_file:
|
|
185
|
+
expected = task_params._load_param_value_from_numpy_save(asdict(expected))
|
|
186
|
+
assert np.array_equal(expected, actual)
|
|
187
|
+
elif isinstance(actual, np.ndarray):
|
|
188
|
+
assert expected == actual.tolist()
|
|
189
|
+
else:
|
|
190
|
+
assert expected == actual
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def test_parameters(basic_science_task_with_parameter_mixin):
|
|
194
|
+
"""
|
|
195
|
+
Given: A Science task with the parameter mixin
|
|
196
|
+
When: Accessing properties for parameters that are not wavelength, file or generic parameters
|
|
197
|
+
Then: The correct value is returned
|
|
198
|
+
"""
|
|
199
|
+
task, test_params = next(basic_science_task_with_parameter_mixin())
|
|
200
|
+
task_params = task.parameters
|
|
201
|
+
for pn, pv in asdict(test_params).items():
|
|
202
|
+
parameter_name = pn.removeprefix("cryonirsp_")
|
|
203
|
+
if (
|
|
204
|
+
_is_file_param(pv)
|
|
205
|
+
or _is_wavelength_param(pv)
|
|
206
|
+
or _is_arm_param(pn, task_params, test_params)
|
|
207
|
+
or parameter_name in PARSE_PARAMETER_NAMES
|
|
208
|
+
):
|
|
209
|
+
continue
|
|
210
|
+
accessed_parameter_value = getattr(task_params, parameter_name)
|
|
211
|
+
if isinstance(accessed_parameter_value, Quantity):
|
|
212
|
+
assert pv == accessed_parameter_value.value
|
|
213
|
+
else:
|
|
214
|
+
assert pv == getattr(task_params, parameter_name)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def test_parse_parameters(basic_science_task_with_parameter_mixin):
|
|
218
|
+
"""
|
|
219
|
+
Given: A Science task with ParsingParameters
|
|
220
|
+
When: Accessing properties for the parsing parameters
|
|
221
|
+
Then: The correct value is returned
|
|
222
|
+
"""
|
|
223
|
+
task, test_params = next(
|
|
224
|
+
basic_science_task_with_parameter_mixin(
|
|
225
|
+
parameter_class=CryonirspParsingParameters,
|
|
226
|
+
obs_ip_start_time=None,
|
|
227
|
+
)
|
|
228
|
+
)
|
|
229
|
+
task_param_attr = task.parameters
|
|
230
|
+
for pn, pv in asdict(test_params).items():
|
|
231
|
+
property_name = pn.removeprefix("cryonirsp_")
|
|
232
|
+
if property_name in PARSE_PARAMETER_NAMES and type(pv) is not dict:
|
|
233
|
+
assert getattr(task_param_attr, property_name) == pv
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
@pytest.mark.parametrize("arm", [pytest.param("CI"), pytest.param("SP")])
|
|
237
|
+
def test_linearization_threshold_parameters(
|
|
238
|
+
basic_science_task_with_parameter_mixin, arm, init_cryonirsp_constants_db
|
|
239
|
+
):
|
|
240
|
+
"""
|
|
241
|
+
Given: A Science task with the parameter mixin
|
|
242
|
+
When: Accessing properties for the linearization thresholds
|
|
243
|
+
Then: The correct type is returned
|
|
244
|
+
"""
|
|
245
|
+
task, _ = next(basic_science_task_with_parameter_mixin())
|
|
246
|
+
recipe_run_id = task.recipe_run_id
|
|
247
|
+
init_cryonirsp_constants_db(recipe_run_id, CryonirspConstantsDb(ARM_ID=arm))
|
|
248
|
+
linearization_threshold_array = task.parameters.linearization_thresholds
|
|
249
|
+
|
|
250
|
+
assert linearization_threshold_array.dtype == np.float32
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def test_optical_density_filter_names(basic_science_task_with_parameter_mixin):
|
|
254
|
+
task, _ = next(basic_science_task_with_parameter_mixin())
|
|
255
|
+
# List of filter attenuation parameters defined in CryonirspParameters:
|
|
256
|
+
defined_filter_params = {
|
|
257
|
+
item[-4:].upper()
|
|
258
|
+
for item in dir(task.parameters)
|
|
259
|
+
if item.startswith("_linearization_optical_density_filter_attenuation_")
|
|
260
|
+
}
|
|
261
|
+
# List of filters in the filter map:
|
|
262
|
+
filter_map_params = {k for k in task.parameters.linearization_filter_attenuation_dict.keys()}
|
|
263
|
+
# Make sure all filter parameters match the allowable list
|
|
264
|
+
assert not defined_filter_params.symmetric_difference(ALLOWABLE_OPTICAL_DENSITY_FILTERS)
|
|
265
|
+
# Make sure all filter map keys match the allowable list
|
|
266
|
+
assert not filter_map_params.symmetric_difference(ALLOWABLE_OPTICAL_DENSITY_FILTERS)
|