dkist-processing-trend 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- changelog/.gitempty +0 -0
- dkist_processing_trend/__init__.py +10 -0
- dkist_processing_trend/config.py +11 -0
- dkist_processing_trend/models/__init__.py +1 -0
- dkist_processing_trend/models/constants.py +143 -0
- dkist_processing_trend/models/fit_options.py +15 -0
- dkist_processing_trend/models/fits_access.py +65 -0
- dkist_processing_trend/models/instrument.py +35 -0
- dkist_processing_trend/models/instrument_options.py +35 -0
- dkist_processing_trend/models/parameters.py +134 -0
- dkist_processing_trend/models/tags.py +117 -0
- dkist_processing_trend/models/task_name.py +20 -0
- dkist_processing_trend/parsers/__init__.py +1 -0
- dkist_processing_trend/parsers/arm_id.py +103 -0
- dkist_processing_trend/parsers/instrument_unique_bud.py +40 -0
- dkist_processing_trend/parsers/time.py +27 -0
- dkist_processing_trend/parsers/trend_l0_fits_access.py +123 -0
- dkist_processing_trend/tasks/__init__.py +27 -0
- dkist_processing_trend/tasks/arm_task_factory.py +57 -0
- dkist_processing_trend/tasks/dark.py +56 -0
- dkist_processing_trend/tasks/gain.py +74 -0
- dkist_processing_trend/tasks/initialize_arm_tasks.py +50 -0
- dkist_processing_trend/tasks/parse.py +169 -0
- dkist_processing_trend/tasks/prepare_fit_data_base.py +257 -0
- dkist_processing_trend/tasks/run_pac_fitter.py +392 -0
- dkist_processing_trend/tasks/trend_base.py +97 -0
- dkist_processing_trend/tasks/trend_output_data.py +167 -0
- dkist_processing_trend/tasks/visp/__init__.py +6 -0
- dkist_processing_trend/tasks/visp/visp_dmpd.py +410 -0
- dkist_processing_trend/tasks/visp/visp_extract_beam.py +14 -0
- dkist_processing_trend/tasks/visp/visp_geometric.py +260 -0
- dkist_processing_trend/tasks/visp/visp_prep_fit_data.py +162 -0
- dkist_processing_trend/tasks/visp/visp_process_demod.py +236 -0
- dkist_processing_trend/tasks/write_trend.py +663 -0
- dkist_processing_trend/tests/__init__.py +1 -0
- dkist_processing_trend/tests/conftest.py +718 -0
- dkist_processing_trend/tests/local_trial_workflows/__init__.py +0 -0
- dkist_processing_trend/tests/local_trial_workflows/l0_to_trend_visp_polcal.py +294 -0
- dkist_processing_trend/tests/local_trial_workflows/local_trial_helpers.py +488 -0
- dkist_processing_trend/tests/test_arm_task_factory.py +82 -0
- dkist_processing_trend/tests/test_base_tasks.py +86 -0
- dkist_processing_trend/tests/test_constants.py +120 -0
- dkist_processing_trend/tests/test_dark.py +97 -0
- dkist_processing_trend/tests/test_gain.py +135 -0
- dkist_processing_trend/tests/test_parameters.py +149 -0
- dkist_processing_trend/tests/test_parse.py +276 -0
- dkist_processing_trend/tests/test_prep_fit_data_base.py +233 -0
- dkist_processing_trend/tests/test_publish_catalog_messages.py +45 -0
- dkist_processing_trend/tests/test_run_pac_fitter.py +371 -0
- dkist_processing_trend/tests/test_stems.py +75 -0
- dkist_processing_trend/tests/test_transfer_output_data.py +76 -0
- dkist_processing_trend/tests/test_trend_fits_access.py +173 -0
- dkist_processing_trend/tests/test_visp.py +874 -0
- dkist_processing_trend/tests/test_workflows.py +10 -0
- dkist_processing_trend/tests/test_write_trend.py +460 -0
- dkist_processing_trend/workflows/__init__.py +3 -0
- dkist_processing_trend/workflows/visp.py +58 -0
- dkist_processing_trend-0.1.0.dist-info/METADATA +549 -0
- dkist_processing_trend-0.1.0.dist-info/RECORD +66 -0
- dkist_processing_trend-0.1.0.dist-info/WHEEL +5 -0
- dkist_processing_trend-0.1.0.dist-info/top_level.txt +3 -0
- docs/conf.py +57 -0
- docs/index.rst +10 -0
- docs/l0_to_trend_visp_polcal.rst +4 -0
- docs/landing_page.rst +11 -0
- docs/requirements_table.rst +8 -0
|
@@ -0,0 +1,371 @@
|
|
|
1
|
+
from collections import defaultdict
|
|
2
|
+
|
|
3
|
+
import astropy.units as u
|
|
4
|
+
import numpy as np
|
|
5
|
+
import pytest
|
|
6
|
+
from astropy.time import Time
|
|
7
|
+
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
8
|
+
from dkist_processing_common.codecs.asdf import asdf_encoder
|
|
9
|
+
from dkist_processing_common.codecs.fits import fits_array_decoder
|
|
10
|
+
from dkist_processing_common.tests.conftest import CalibrationUnit
|
|
11
|
+
from dkist_processing_common.tests.conftest import Telescope
|
|
12
|
+
from dkist_processing_pac.fitter.fitter_parameters import PolcalDresserParameters
|
|
13
|
+
from dkist_processing_pac.input_data.drawer import Drawer
|
|
14
|
+
from dkist_processing_pac.input_data.dresser import Dresser
|
|
15
|
+
|
|
16
|
+
from dkist_processing_trend.models.instrument import InstrumentName
|
|
17
|
+
from dkist_processing_trend.models.parameters import TrendParameters
|
|
18
|
+
from dkist_processing_trend.models.tags import TrendTag
|
|
19
|
+
from dkist_processing_trend.parsers.trend_l0_fits_access import TrendL0FitsAccess
|
|
20
|
+
from dkist_processing_trend.tasks.run_pac_fitter import RunPacFitter
|
|
21
|
+
from dkist_processing_trend.tests.conftest import ModulatedCSStepHeaders
|
|
22
|
+
from dkist_processing_trend.tests.conftest import TrendInputDatasetParameterValues
|
|
23
|
+
from dkist_processing_trend.tests.conftest import TrendTestingConstantsDatabase
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@pytest.fixture(scope="session")
|
|
27
|
+
def cs_with_correct_geometry():
|
|
28
|
+
pol_status = [
|
|
29
|
+
"clear",
|
|
30
|
+
"clear",
|
|
31
|
+
"Sapphire Polarizer",
|
|
32
|
+
"Sapphire Polarizer",
|
|
33
|
+
"Sapphire Polarizer",
|
|
34
|
+
"Sapphire Polarizer",
|
|
35
|
+
"Sapphire Polarizer",
|
|
36
|
+
"Sapphire Polarizer",
|
|
37
|
+
"Sapphire Polarizer",
|
|
38
|
+
"Sapphire Polarizer",
|
|
39
|
+
"Sapphire Polarizer",
|
|
40
|
+
"Sapphire Polarizer",
|
|
41
|
+
"clear",
|
|
42
|
+
"clear",
|
|
43
|
+
]
|
|
44
|
+
pol_theta = [0, 0, 0, 60, 120, 0, 0, 0, 45, 45, 45, 45, 0, 0]
|
|
45
|
+
ret_status = [
|
|
46
|
+
"clear",
|
|
47
|
+
"clear",
|
|
48
|
+
"clear",
|
|
49
|
+
"clear",
|
|
50
|
+
"clear",
|
|
51
|
+
"SiO2 SAR",
|
|
52
|
+
"SiO2 SAR",
|
|
53
|
+
"SiO2 SAR",
|
|
54
|
+
"SiO2 SAR",
|
|
55
|
+
"SiO2 SAR",
|
|
56
|
+
"SiO2 SAR",
|
|
57
|
+
"SiO2 SAR",
|
|
58
|
+
"clear",
|
|
59
|
+
"clear",
|
|
60
|
+
]
|
|
61
|
+
ret_theta = [0, 0, 0, 0, 0, 0, 60, 120, 30, 90, 150, 0, 0, 0]
|
|
62
|
+
dark_status = [
|
|
63
|
+
"DarkShutter",
|
|
64
|
+
"FieldStop (5arcmin)",
|
|
65
|
+
"FieldStop (5arcmin)",
|
|
66
|
+
"FieldStop (5arcmin)",
|
|
67
|
+
"FieldStop (5arcmin)",
|
|
68
|
+
"FieldStop (5arcmin)",
|
|
69
|
+
"FieldStop (5arcmin)",
|
|
70
|
+
"FieldStop (5arcmin)",
|
|
71
|
+
"FieldStop (5arcmin)",
|
|
72
|
+
"FieldStop (5arcmin)",
|
|
73
|
+
"FieldStop (5arcmin)",
|
|
74
|
+
"FieldStop (5arcmin)",
|
|
75
|
+
"FieldStop (5arcmin)",
|
|
76
|
+
"DarkShutter",
|
|
77
|
+
]
|
|
78
|
+
return pol_status, pol_theta, ret_status, ret_theta, dark_status
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@pytest.fixture(scope="session")
|
|
82
|
+
def visp_modulation_matrix() -> np.ndarray:
|
|
83
|
+
# Modulation matrix for AdW's synthetic ViSP data from mod_matrix_630.out
|
|
84
|
+
return np.array(
|
|
85
|
+
[
|
|
86
|
+
[1.0, 0.19155013, 0.80446989, -0.47479524],
|
|
87
|
+
[1.0, -0.65839661, 0.68433984, 0.00466389],
|
|
88
|
+
[1.0, -0.80679413, -0.16112977, 0.48234158],
|
|
89
|
+
[1.0, -0.04856211, -0.56352868, 0.77578117],
|
|
90
|
+
[1.0, 0.56844858, 0.03324473, 0.77289873],
|
|
91
|
+
[1.0, 0.19155013, 0.80446989, 0.47479524],
|
|
92
|
+
[1.0, -0.65839661, 0.68433984, -0.00466389],
|
|
93
|
+
[1.0, -0.80679413, -0.16112977, -0.48234158],
|
|
94
|
+
[1.0, -0.04856211, -0.56352868, -0.77578117],
|
|
95
|
+
[1.0, 0.56844858, 0.03324473, -0.77289873],
|
|
96
|
+
],
|
|
97
|
+
dtype=np.float64,
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@pytest.fixture(scope="session")
|
|
102
|
+
def num_cs_steps(cs_with_correct_geometry) -> int:
|
|
103
|
+
return len(cs_with_correct_geometry[0])
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@pytest.fixture(scope="session")
|
|
107
|
+
def num_modstates(visp_modulation_matrix) -> int:
|
|
108
|
+
return visp_modulation_matrix.shape[0]
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@pytest.fixture(scope="session")
|
|
112
|
+
def arm_id() -> int:
|
|
113
|
+
return 1
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@pytest.fixture(scope="session")
|
|
117
|
+
def full_cs_fits_obj_dict_no_data(
|
|
118
|
+
cs_with_correct_geometry, num_modstates, arm_id
|
|
119
|
+
) -> dict[int, list[TrendL0FitsAccess]]:
|
|
120
|
+
|
|
121
|
+
pol_status, pol_theta, ret_status, ret_theta, dark_status = cs_with_correct_geometry
|
|
122
|
+
|
|
123
|
+
step_dict: dict[int, list[TrendL0FitsAccess]] = defaultdict(list)
|
|
124
|
+
|
|
125
|
+
base_time = Time("2026-01-01")
|
|
126
|
+
|
|
127
|
+
for step, (ps, pt, rs, rt, ds) in enumerate(
|
|
128
|
+
zip(pol_status, pol_theta, ret_status, ret_theta, dark_status)
|
|
129
|
+
):
|
|
130
|
+
|
|
131
|
+
step_start = (base_time + (step * 60) * u.s).fits
|
|
132
|
+
|
|
133
|
+
ds = ModulatedCSStepHeaders(
|
|
134
|
+
instrument=InstrumentName.visp,
|
|
135
|
+
arm_id=arm_id,
|
|
136
|
+
wavelength=366.0,
|
|
137
|
+
num_modstates=num_modstates,
|
|
138
|
+
pol_status=ps,
|
|
139
|
+
pol_theta=pt,
|
|
140
|
+
ret_status=rs,
|
|
141
|
+
ret_theta=rt,
|
|
142
|
+
dark_status=ds,
|
|
143
|
+
cs_step_num=step,
|
|
144
|
+
array_shape=(2, 2),
|
|
145
|
+
exp_time_ms=6.28,
|
|
146
|
+
start_date=step_start,
|
|
147
|
+
)
|
|
148
|
+
for modstate_frame in ds:
|
|
149
|
+
step_dict[step].append(
|
|
150
|
+
TrendL0FitsAccess(header=modstate_frame.header(), data=np.ones((3, 3)))
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
return step_dict
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
@pytest.fixture(scope="session")
|
|
157
|
+
def fit_options_list() -> list[dict[str, str | bool]]:
|
|
158
|
+
return [
|
|
159
|
+
{
|
|
160
|
+
"name": "option1",
|
|
161
|
+
"remove_I_trend": True,
|
|
162
|
+
"fit_mode_name": "use_M12",
|
|
163
|
+
"init_set_name": "OCCal_VIS",
|
|
164
|
+
},
|
|
165
|
+
{
|
|
166
|
+
"name": "option2",
|
|
167
|
+
"remove_I_trend": False,
|
|
168
|
+
"fit_mode_name": "use_M12_I_sys_per_step",
|
|
169
|
+
"init_set_name": "OCCal_VIS",
|
|
170
|
+
},
|
|
171
|
+
]
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
@pytest.fixture(scope="session")
|
|
175
|
+
def full_observed_values(full_cs_fits_obj_dict_no_data, visp_modulation_matrix) -> np.ndarray:
|
|
176
|
+
|
|
177
|
+
fit_mode = "use_M12"
|
|
178
|
+
init_set = "OCCal_VIS"
|
|
179
|
+
cs_dict = full_cs_fits_obj_dict_no_data
|
|
180
|
+
dresser = Dresser()
|
|
181
|
+
dresser.add_drawer(Drawer(cs_dict, skip_darks=False, remove_I_trend=False))
|
|
182
|
+
CM = CalibrationUnit(dresser)
|
|
183
|
+
TM = Telescope(dresser)
|
|
184
|
+
full_params = PolcalDresserParameters(dresser, fit_mode, init_set)
|
|
185
|
+
|
|
186
|
+
global_params = full_params.init_params._all_parameters[0]
|
|
187
|
+
pardict = global_params.valuesdict()
|
|
188
|
+
CM.load_pars_from_dict(pardict)
|
|
189
|
+
TM.load_pars_from_dict(pardict)
|
|
190
|
+
|
|
191
|
+
CM.I_sys[0] = 1e4
|
|
192
|
+
|
|
193
|
+
# Has shape (4, N)
|
|
194
|
+
S = np.sum((TM.TM @ CM.CM @ TM.M12) * CM.S_in[:, None, :], axis=2).T
|
|
195
|
+
|
|
196
|
+
# Has shape (M, N)
|
|
197
|
+
observed = visp_modulation_matrix @ S
|
|
198
|
+
|
|
199
|
+
return observed
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def write_pac_inputs_to_task(
|
|
203
|
+
task,
|
|
204
|
+
step_to_fits_access_dict: dict[int, list[TrendL0FitsAccess]],
|
|
205
|
+
observed_data: np.ndarray,
|
|
206
|
+
beam: int,
|
|
207
|
+
arm_id: str | int,
|
|
208
|
+
local_fov_shape: tuple[int, ...],
|
|
209
|
+
) -> None:
|
|
210
|
+
global_dict = defaultdict(list)
|
|
211
|
+
local_dict = defaultdict(list)
|
|
212
|
+
|
|
213
|
+
for step, fits_access_list in step_to_fits_access_dict.items():
|
|
214
|
+
for modstate, fits_obj in enumerate(fits_access_list):
|
|
215
|
+
header = fits_obj.header
|
|
216
|
+
obs_value = observed_data[modstate, step]
|
|
217
|
+
global_data = np.full((1, 1), obs_value)
|
|
218
|
+
local_data = np.full(local_fov_shape, obs_value)
|
|
219
|
+
|
|
220
|
+
global_dict[step].append({"header": header, "data": global_data})
|
|
221
|
+
local_dict[step].append({"header": header, "data": local_data})
|
|
222
|
+
|
|
223
|
+
# Cheat a little bit and just write the same data for each instrument processing option
|
|
224
|
+
inst_option_names = [i.name for i in task.parameters.instrument_processing_options]
|
|
225
|
+
for inst_option_name in inst_option_names:
|
|
226
|
+
task.write(
|
|
227
|
+
data=global_dict,
|
|
228
|
+
tags=[
|
|
229
|
+
TrendTag.intermediate(),
|
|
230
|
+
TrendTag.beam(beam),
|
|
231
|
+
TrendTag.arm_id(arm_id),
|
|
232
|
+
TrendTag.instrument_processing_options(inst_option_name),
|
|
233
|
+
TrendTag.task_global_pac_input(),
|
|
234
|
+
],
|
|
235
|
+
encoder=asdf_encoder,
|
|
236
|
+
)
|
|
237
|
+
task.write(
|
|
238
|
+
data=local_dict,
|
|
239
|
+
tags=[
|
|
240
|
+
TrendTag.intermediate(),
|
|
241
|
+
TrendTag.beam(beam),
|
|
242
|
+
TrendTag.arm_id(arm_id),
|
|
243
|
+
TrendTag.instrument_processing_options(inst_option_name),
|
|
244
|
+
TrendTag.task_local_pac_input(),
|
|
245
|
+
],
|
|
246
|
+
encoder=asdf_encoder,
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
@pytest.fixture
|
|
251
|
+
def run_pac_fitter_task(
|
|
252
|
+
tmp_path,
|
|
253
|
+
recipe_run_id,
|
|
254
|
+
link_constants_db,
|
|
255
|
+
assign_input_dataset_doc_to_task,
|
|
256
|
+
num_cs_steps,
|
|
257
|
+
num_modstates,
|
|
258
|
+
arm_id,
|
|
259
|
+
fit_options_list,
|
|
260
|
+
):
|
|
261
|
+
link_constants_db(
|
|
262
|
+
recipe_run_id,
|
|
263
|
+
TrendTestingConstantsDatabase(NUM_CS_STEPS=num_cs_steps, NUM_MODSTATES=num_modstates),
|
|
264
|
+
)
|
|
265
|
+
with RunPacFitter(
|
|
266
|
+
arm_id=arm_id, recipe_run_id=recipe_run_id, workflow_name="name", workflow_version="version"
|
|
267
|
+
) as task:
|
|
268
|
+
task.scratch = WorkflowFileSystem(recipe_run_id=recipe_run_id, scratch_base_path=tmp_path)
|
|
269
|
+
assign_input_dataset_doc_to_task(
|
|
270
|
+
task, TrendInputDatasetParameterValues(trend_fit_options_list=fit_options_list)
|
|
271
|
+
)
|
|
272
|
+
task.parameters = TrendParameters(
|
|
273
|
+
scratch=task.scratch,
|
|
274
|
+
obs_ip_start_time=task.constants.earliest_ip_start_time,
|
|
275
|
+
instrument=task.constants.instrument,
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
yield task
|
|
279
|
+
task._purge()
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def test_run_pac_fitter(
|
|
283
|
+
run_pac_fitter_task,
|
|
284
|
+
mocker,
|
|
285
|
+
fake_gql_client,
|
|
286
|
+
num_cs_steps,
|
|
287
|
+
num_modstates,
|
|
288
|
+
full_cs_fits_obj_dict_no_data,
|
|
289
|
+
full_observed_values,
|
|
290
|
+
arm_id,
|
|
291
|
+
fit_options_list,
|
|
292
|
+
):
|
|
293
|
+
"""
|
|
294
|
+
Given: A set of PAC input data and a `RunPacFitter` task
|
|
295
|
+
When: Running the task
|
|
296
|
+
Then: The task completes, the expected outputs exist, and they have the correct shapes
|
|
297
|
+
"""
|
|
298
|
+
mocker.patch(
|
|
299
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
task = run_pac_fitter_task
|
|
303
|
+
fov_shape = (3, 2)
|
|
304
|
+
num_beams = 2
|
|
305
|
+
inst_options_name_list = [i.name for i in task.parameters.instrument_processing_options]
|
|
306
|
+
|
|
307
|
+
for beam in range(1, num_beams + 1):
|
|
308
|
+
write_pac_inputs_to_task(
|
|
309
|
+
task=task,
|
|
310
|
+
step_to_fits_access_dict=full_cs_fits_obj_dict_no_data,
|
|
311
|
+
observed_data=full_observed_values,
|
|
312
|
+
beam=beam,
|
|
313
|
+
arm_id=arm_id,
|
|
314
|
+
local_fov_shape=fov_shape,
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
task()
|
|
318
|
+
|
|
319
|
+
cal_seq_tags = [
|
|
320
|
+
TrendTag.intermediate(),
|
|
321
|
+
TrendTag.arm_id(arm_id),
|
|
322
|
+
TrendTag.task_calibration_sequence(),
|
|
323
|
+
]
|
|
324
|
+
assert task.count(tags=cal_seq_tags) == 1
|
|
325
|
+
cal_seq_data = next(
|
|
326
|
+
task.read(tags=cal_seq_tags, decoder=fits_array_decoder, auto_squeeze=False)
|
|
327
|
+
)
|
|
328
|
+
assert cal_seq_data.shape == (1, 8, num_cs_steps)
|
|
329
|
+
|
|
330
|
+
for inst_option_name in inst_options_name_list:
|
|
331
|
+
for beam in range(1, num_beams + 1):
|
|
332
|
+
for fit_options in fit_options_list:
|
|
333
|
+
fit_opt_name = fit_options["name"]
|
|
334
|
+
common_tags = [
|
|
335
|
+
TrendTag.intermediate(),
|
|
336
|
+
TrendTag.beam(beam),
|
|
337
|
+
TrendTag.arm_id(arm_id),
|
|
338
|
+
TrendTag.instrument_processing_options(inst_option_name),
|
|
339
|
+
TrendTag.pac_fit_options(fit_opt_name),
|
|
340
|
+
]
|
|
341
|
+
|
|
342
|
+
best_fit_param_tags = common_tags + [TrendTag.task_best_fit_parameters()]
|
|
343
|
+
assert task.count(best_fit_param_tags) == 1
|
|
344
|
+
best_fit_param_data = next(
|
|
345
|
+
task.read(
|
|
346
|
+
tags=best_fit_param_tags, decoder=fits_array_decoder, auto_squeeze=False
|
|
347
|
+
)
|
|
348
|
+
)
|
|
349
|
+
num_params = 15 + num_cs_steps - 2 # -2 because we don't fit dark steps
|
|
350
|
+
assert best_fit_param_data.shape == fov_shape + (1, num_params, 3)
|
|
351
|
+
|
|
352
|
+
best_fit_demod_tags = common_tags + [TrendTag.task_best_fit_demodulation_matrices()]
|
|
353
|
+
assert task.count(best_fit_demod_tags) == 1
|
|
354
|
+
best_fit_demod = next(
|
|
355
|
+
task.read(best_fit_demod_tags, decoder=fits_array_decoder, auto_squeeze=False)
|
|
356
|
+
)
|
|
357
|
+
assert best_fit_demod.shape == fov_shape + (4, num_modstates)
|
|
358
|
+
|
|
359
|
+
best_fit_flux_tags = common_tags + [TrendTag.task_best_fit_flux()]
|
|
360
|
+
assert task.count(best_fit_flux_tags) == 1
|
|
361
|
+
best_fit_flux_data = next(
|
|
362
|
+
task.read(best_fit_flux_tags, decoder=fits_array_decoder, auto_squeeze=False)
|
|
363
|
+
)
|
|
364
|
+
assert best_fit_flux_data.shape == fov_shape + (num_modstates, num_cs_steps - 2)
|
|
365
|
+
|
|
366
|
+
fit_residuals_tags = common_tags + [TrendTag.task_fit_residuals()]
|
|
367
|
+
assert task.count(fit_residuals_tags) == 1
|
|
368
|
+
fit_residuals_data = next(
|
|
369
|
+
task.read(fit_residuals_tags, decoder=fits_array_decoder, auto_squeeze=True)
|
|
370
|
+
)
|
|
371
|
+
assert fit_residuals_data.shape == best_fit_flux_data.shape
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
from dkist_processing_common.models.fits_access import MetadataKey
|
|
5
|
+
|
|
6
|
+
from dkist_processing_trend.parsers.arm_id import ArmIdLookupBud
|
|
7
|
+
from dkist_processing_trend.parsers.arm_id import InstrumentArmIdLookupBud
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class DummyFitsAccess:
|
|
12
|
+
|
|
13
|
+
arm_id: str | int
|
|
14
|
+
wavelength: float
|
|
15
|
+
instrument: str = "foo"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def test_arm_id_lookup_bud():
|
|
19
|
+
"""
|
|
20
|
+
Given: A set of `FitsAccess` objects corresponding to multiple arms and wavelengths
|
|
21
|
+
When: Processing these objects through an `ArmIdLookupBud`
|
|
22
|
+
Then: The correct value is returned
|
|
23
|
+
"""
|
|
24
|
+
fo_list = (
|
|
25
|
+
[DummyFitsAccess(arm_id=1, wavelength=123.4) for _ in range(3)]
|
|
26
|
+
+ [DummyFitsAccess(arm_id="SP", wavelength=1079) for _ in range(5)]
|
|
27
|
+
+ [DummyFitsAccess(arm_id=3, wavelength=567)]
|
|
28
|
+
)
|
|
29
|
+
bud = ArmIdLookupBud(constant_name="foo", metadata_key=MetadataKey.wavelength)
|
|
30
|
+
for fo in fo_list:
|
|
31
|
+
bud.update(key="doesn't matter", value=fo)
|
|
32
|
+
|
|
33
|
+
assert bud.bud.value == {"1": 123.4, '"SP"': 1079, "3": 567}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_arm_id_to_wavelength_non_singular_wave():
|
|
37
|
+
"""
|
|
38
|
+
Given: A set of `FitsAccess` objects corresponding to multiple arms where some arms have multiple wavelength values
|
|
39
|
+
When: Processing these objects through an `ArmIdLookupBud`
|
|
40
|
+
Then: The error is raised
|
|
41
|
+
"""
|
|
42
|
+
fo_list = (
|
|
43
|
+
[DummyFitsAccess(arm_id=1, wavelength=123.4) for _ in range(3)]
|
|
44
|
+
+ [DummyFitsAccess(arm_id=1, wavelength=1079) for _ in range(5)]
|
|
45
|
+
+ [DummyFitsAccess(arm_id=3, wavelength=567)]
|
|
46
|
+
)
|
|
47
|
+
bud = ArmIdLookupBud(constant_name="foo", metadata_key=MetadataKey.wavelength)
|
|
48
|
+
for fo in fo_list:
|
|
49
|
+
bud.update(key="doesn't matter", value=fo)
|
|
50
|
+
|
|
51
|
+
with pytest.raises(
|
|
52
|
+
ValueError,
|
|
53
|
+
match=f"Some arms have multiple values for wavelength: arm_id = 1: {{123.4, 1079}}",
|
|
54
|
+
):
|
|
55
|
+
bud.bud
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def test_instrument_arm_id_lookup_bud():
|
|
59
|
+
"""
|
|
60
|
+
Given: A set of `FitsAccess` objects from different instruments corresponding to multiple arms and wavelengths
|
|
61
|
+
When: Processing these objects through an `InstrumentArmIdLookupBud` for only one of the input instruments
|
|
62
|
+
Then: The correct value for the single instrument is returned
|
|
63
|
+
"""
|
|
64
|
+
fo_list = (
|
|
65
|
+
[DummyFitsAccess(arm_id=1, wavelength=123.4, instrument="foo") for _ in range(3)]
|
|
66
|
+
+ [DummyFitsAccess(arm_id="SP", wavelength=1079, instrument="foo") for _ in range(5)]
|
|
67
|
+
+ [DummyFitsAccess(arm_id=1, wavelength=-9999, instrument="bar") for _ in range(3)]
|
|
68
|
+
)
|
|
69
|
+
bud = InstrumentArmIdLookupBud(
|
|
70
|
+
constant_name="asdf", metadata_key=MetadataKey.wavelength, instruments="foo"
|
|
71
|
+
)
|
|
72
|
+
for fo in fo_list:
|
|
73
|
+
bud.update(key="doesn't matter", value=fo)
|
|
74
|
+
|
|
75
|
+
assert bud.bud.value == {"1": 123.4, '"SP"': 1079}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
5
|
+
|
|
6
|
+
from dkist_processing_trend.models.tags import TrendTag
|
|
7
|
+
from dkist_processing_trend.tasks.trend_output_data import TransferTrendData
|
|
8
|
+
from dkist_processing_trend.tasks.trend_output_data import TrendOutputDataBase
|
|
9
|
+
from dkist_processing_trend.tests.conftest import TrendTestingConstantsDatabase
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def fake_list_objects(self, bucket, prefix=None):
|
|
13
|
+
return list()
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@pytest.fixture
|
|
17
|
+
def transfer_l1_data_task(recipe_run_id, tmp_path, link_constants_db):
|
|
18
|
+
def make_task(
|
|
19
|
+
instrument,
|
|
20
|
+
earliest_ip_start_time,
|
|
21
|
+
):
|
|
22
|
+
link_constants_db(
|
|
23
|
+
recipe_run_id,
|
|
24
|
+
TrendTestingConstantsDatabase(
|
|
25
|
+
INSTRUMENT=instrument, EARLIEST_IP_START_TIME=earliest_ip_start_time
|
|
26
|
+
),
|
|
27
|
+
)
|
|
28
|
+
with TransferTrendData(
|
|
29
|
+
recipe_run_id=recipe_run_id,
|
|
30
|
+
workflow_name="workflow_name",
|
|
31
|
+
workflow_version="workflow_version",
|
|
32
|
+
) as task:
|
|
33
|
+
task.scratch = WorkflowFileSystem(
|
|
34
|
+
recipe_run_id=recipe_run_id,
|
|
35
|
+
scratch_base_path=tmp_path,
|
|
36
|
+
)
|
|
37
|
+
frame_path = task.scratch.workflow_base_path / Path("frame.fits")
|
|
38
|
+
with open(frame_path, "w") as f:
|
|
39
|
+
f.write("Frame")
|
|
40
|
+
task.tag(path=frame_path, tags=[TrendTag.frame(), TrendTag.output()])
|
|
41
|
+
|
|
42
|
+
yield task
|
|
43
|
+
task._purge()
|
|
44
|
+
|
|
45
|
+
return make_task
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_transfer_l1_data(transfer_l1_data_task, mocker, fake_gql_client):
|
|
49
|
+
"""
|
|
50
|
+
Given: A task with frames and movies tagged as output
|
|
51
|
+
When: Transfering the L1 data
|
|
52
|
+
Then: The task completes without errors and the destination object keys are correct
|
|
53
|
+
"""
|
|
54
|
+
# Yeah, we mock a whole bunch of stuff here, but this test at least confirms that the setup to these calls is correct
|
|
55
|
+
mocker.patch(
|
|
56
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
|
|
57
|
+
)
|
|
58
|
+
mocker.patch(
|
|
59
|
+
"dkist_processing_common.tasks.mixin.globus.GlobusMixin.globus_transfer_scratch_to_object_store"
|
|
60
|
+
)
|
|
61
|
+
mocker.patch("dkist_processing_common.tasks.mixin.object_store.ObjectClerk.upload_object")
|
|
62
|
+
mocker.patch(
|
|
63
|
+
"dkist_processing_common.tasks.mixin.object_store.ObjectClerk.list_objects",
|
|
64
|
+
fake_list_objects,
|
|
65
|
+
)
|
|
66
|
+
instrument = "HELLO"
|
|
67
|
+
earliest_ip_start_time = "1900-01-01T12:34:45.0002"
|
|
68
|
+
task = next(
|
|
69
|
+
transfer_l1_data_task(instrument=instrument, earliest_ip_start_time=earliest_ip_start_time)
|
|
70
|
+
)
|
|
71
|
+
task()
|
|
72
|
+
|
|
73
|
+
assert (
|
|
74
|
+
task.format_object_key(Path("scratch/foo/bar/thing.fits"))
|
|
75
|
+
== "trend/polcal/19000101T123445/HELLO/thing.fits"
|
|
76
|
+
)
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
from inspect import get_annotations
|
|
2
|
+
from random import randint
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
from astropy.io import fits
|
|
6
|
+
|
|
7
|
+
from dkist_processing_trend.models.fits_access import TrendMetadataKey
|
|
8
|
+
from dkist_processing_trend.models.instrument import InstrumentName
|
|
9
|
+
from dkist_processing_trend.parsers.trend_l0_fits_access import IncorrectInstrumentError
|
|
10
|
+
from dkist_processing_trend.parsers.trend_l0_fits_access import TrendL0FitsAccess
|
|
11
|
+
from dkist_processing_trend.tests.conftest import WAVELENGTH_FOR_ARM
|
|
12
|
+
from dkist_processing_trend.tests.conftest import ModulatedCSStepHeaders
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@pytest.fixture(scope="function") # So we get new random for each test
|
|
16
|
+
def random_cs_step_config(
|
|
17
|
+
small_calibration_sequence, small_num_cs_steps
|
|
18
|
+
) -> tuple[str, float, str, float, str]:
|
|
19
|
+
idx = randint(0, small_num_cs_steps - 1)
|
|
20
|
+
return tuple(i[idx] for i in small_calibration_sequence)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@pytest.fixture
|
|
24
|
+
def num_modstates() -> int:
|
|
25
|
+
return 2
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@pytest.fixture
|
|
29
|
+
def metadata_keys_from_wrong_instruments(instrument):
|
|
30
|
+
visp_metadata_keys = {
|
|
31
|
+
TrendMetadataKey.visp_grating_angle,
|
|
32
|
+
TrendMetadataKey.visp_arm_position,
|
|
33
|
+
TrendMetadataKey.visp_grating_constant,
|
|
34
|
+
}
|
|
35
|
+
cryonirsp_metadata_keys = {
|
|
36
|
+
TrendMetadataKey.cryonirsp_od_filter_name,
|
|
37
|
+
TrendMetadataKey.cryonirsp_grating_angle,
|
|
38
|
+
TrendMetadataKey.cryonirsp_grating_littrow_angle,
|
|
39
|
+
TrendMetadataKey.cryonirsp_grating_constant,
|
|
40
|
+
TrendMetadataKey.cryonirsp_slit_width,
|
|
41
|
+
}
|
|
42
|
+
dlnirsp_metadata_keys = {
|
|
43
|
+
TrendMetadataKey.dlnirsp_grating_angle,
|
|
44
|
+
TrendMetadataKey.dlnirsp_arm_position,
|
|
45
|
+
TrendMetadataKey.dlnirsp_grating_constant,
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
wrong_inst_metadata_keys = visp_metadata_keys | cryonirsp_metadata_keys | dlnirsp_metadata_keys
|
|
49
|
+
match instrument:
|
|
50
|
+
case InstrumentName.visp:
|
|
51
|
+
wrong_inst_metadata_keys -= visp_metadata_keys
|
|
52
|
+
case InstrumentName.cryonirsp:
|
|
53
|
+
wrong_inst_metadata_keys -= cryonirsp_metadata_keys
|
|
54
|
+
case InstrumentName.dlnirsp:
|
|
55
|
+
wrong_inst_metadata_keys -= dlnirsp_metadata_keys
|
|
56
|
+
|
|
57
|
+
return wrong_inst_metadata_keys
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@pytest.fixture
|
|
61
|
+
def input_header(
|
|
62
|
+
instrument: InstrumentName,
|
|
63
|
+
arm_id: str | int,
|
|
64
|
+
num_modstates: int,
|
|
65
|
+
random_cs_step_config: tuple[str, float, str, float, str],
|
|
66
|
+
) -> fits.Header:
|
|
67
|
+
pol_status, pol_theta, ret_status, ret_theta, dark_status = random_cs_step_config
|
|
68
|
+
ds = ModulatedCSStepHeaders(
|
|
69
|
+
instrument=instrument,
|
|
70
|
+
arm_id=arm_id,
|
|
71
|
+
wavelength=WAVELENGTH_FOR_ARM[arm_id],
|
|
72
|
+
num_modstates=num_modstates,
|
|
73
|
+
pol_status=pol_status,
|
|
74
|
+
pol_theta=pol_theta,
|
|
75
|
+
ret_status=ret_status,
|
|
76
|
+
ret_theta=ret_theta,
|
|
77
|
+
dark_status=dark_status,
|
|
78
|
+
array_shape=(2, 2),
|
|
79
|
+
cs_step_num=1,
|
|
80
|
+
exp_time_ms=34.0,
|
|
81
|
+
)
|
|
82
|
+
return ds.header()
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@pytest.mark.parametrize(
|
|
86
|
+
"instrument, arm_id",
|
|
87
|
+
[
|
|
88
|
+
pytest.param(InstrumentName.visp, 1, id="visp"),
|
|
89
|
+
pytest.param(InstrumentName.cryonirsp, "SP", id="cryonirsp"),
|
|
90
|
+
pytest.param(InstrumentName.dlnirsp, "JBand", id="JBand"),
|
|
91
|
+
],
|
|
92
|
+
)
|
|
93
|
+
def test_trend_l0_fits_access(input_header, num_modstates, arm_id):
|
|
94
|
+
"""
|
|
95
|
+
Given: A Header with instrument-specific trend values
|
|
96
|
+
When: Parsing the header with `TrendL0FitsAccess`
|
|
97
|
+
Then: The keys for each instrument are correctly used such that the FitsAccess properties return the correct value
|
|
98
|
+
"""
|
|
99
|
+
fits_obj = TrendL0FitsAccess(header=input_header)
|
|
100
|
+
|
|
101
|
+
assert fits_obj.arm_id == arm_id
|
|
102
|
+
assert fits_obj.number_of_modulator_states == num_modstates
|
|
103
|
+
assert fits_obj.modulator_state == 1 # Because we took the first header from the dataset
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@pytest.mark.parametrize("instrument, arm_id", [pytest.param(InstrumentName.visp, 1, id="")])
|
|
107
|
+
def test_error_on_unknown_instrument(input_header):
|
|
108
|
+
"""
|
|
109
|
+
Given: A Header where the INSTRUME key is not one of the recognized values
|
|
110
|
+
When: Parsing the header with `TrendL0FitsAccess`
|
|
111
|
+
Then: The correct Error is raised
|
|
112
|
+
"""
|
|
113
|
+
input_header["INSTRUME"] = "WRONG"
|
|
114
|
+
with pytest.raises(ValueError, match="Did not expect instrument = WRONG"):
|
|
115
|
+
TrendL0FitsAccess(header=input_header)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
@pytest.mark.parametrize(
|
|
119
|
+
"instrument, arm_id",
|
|
120
|
+
[
|
|
121
|
+
pytest.param(InstrumentName.visp, 1, id="visp"),
|
|
122
|
+
pytest.param(InstrumentName.cryonirsp, "SP", id="cryonirsp"),
|
|
123
|
+
pytest.param(InstrumentName.dlnirsp, "VIS", id="dlnirsp"),
|
|
124
|
+
],
|
|
125
|
+
)
|
|
126
|
+
def test_metadata_keys_in_access_bases(input_header):
|
|
127
|
+
"""
|
|
128
|
+
Given: The set of metadata key names in TrendMetadataKey
|
|
129
|
+
When: The `TrendL0FitsAccess` class defines a set of new attributes
|
|
130
|
+
Then: The sets are the same
|
|
131
|
+
"""
|
|
132
|
+
trend_metadata_key_names = {tmk.name for tmk in TrendMetadataKey}
|
|
133
|
+
all_trend_fits_access_attrs = set()
|
|
134
|
+
for access_class in [TrendL0FitsAccess]:
|
|
135
|
+
fits_obj = access_class(header=input_header)
|
|
136
|
+
trend_instance_attrs = set(vars(fits_obj).keys())
|
|
137
|
+
|
|
138
|
+
# This next line grabs the `InstrumentSpecificProperty` for the instruments not represented in the header
|
|
139
|
+
trend_instance_attrs |= set(get_annotations(access_class).keys())
|
|
140
|
+
parent_class = access_class.mro()[1]
|
|
141
|
+
parent_fits_obj = parent_class(header=input_header)
|
|
142
|
+
parent_instance_attrs = set(vars(parent_fits_obj).keys())
|
|
143
|
+
trend_fits_access_attrs = trend_instance_attrs - parent_instance_attrs
|
|
144
|
+
all_trend_fits_access_attrs |= trend_fits_access_attrs
|
|
145
|
+
assert trend_metadata_key_names == all_trend_fits_access_attrs
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
@pytest.mark.parametrize(
|
|
149
|
+
"instrument, arm_id",
|
|
150
|
+
[
|
|
151
|
+
pytest.param(InstrumentName.visp, 1, id="visp"),
|
|
152
|
+
pytest.param(InstrumentName.cryonirsp, "SP", id="cryonirsp"),
|
|
153
|
+
pytest.param(InstrumentName.dlnirsp, "VIS", id="dlnirsp"),
|
|
154
|
+
],
|
|
155
|
+
)
|
|
156
|
+
def test_instrument_specific_properties(
|
|
157
|
+
input_header, instrument, metadata_keys_from_wrong_instruments
|
|
158
|
+
):
|
|
159
|
+
"""
|
|
160
|
+
Given: A TrendL0FitsAccess object generated from an input header corresponding to a certain instrument
|
|
161
|
+
When: Trying to access properties that are not defined for that certain instrument
|
|
162
|
+
Then: The correct error is raised
|
|
163
|
+
"""
|
|
164
|
+
fits_obj = TrendL0FitsAccess(header=input_header)
|
|
165
|
+
|
|
166
|
+
for metadata_key in metadata_keys_from_wrong_instruments:
|
|
167
|
+
with pytest.raises(
|
|
168
|
+
IncorrectInstrumentError,
|
|
169
|
+
match="This property is only defined for data from .*, which this frame is not.",
|
|
170
|
+
) as exc_info:
|
|
171
|
+
_ = getattr(fits_obj, metadata_key.name)
|
|
172
|
+
pytest.fail(f"{metadata_key = } did not raise expected error")
|
|
173
|
+
assert instrument not in exc_info.value.args[0]
|