dkist-processing-trend 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. changelog/.gitempty +0 -0
  2. dkist_processing_trend/__init__.py +10 -0
  3. dkist_processing_trend/config.py +11 -0
  4. dkist_processing_trend/models/__init__.py +1 -0
  5. dkist_processing_trend/models/constants.py +143 -0
  6. dkist_processing_trend/models/fit_options.py +15 -0
  7. dkist_processing_trend/models/fits_access.py +65 -0
  8. dkist_processing_trend/models/instrument.py +35 -0
  9. dkist_processing_trend/models/instrument_options.py +35 -0
  10. dkist_processing_trend/models/parameters.py +134 -0
  11. dkist_processing_trend/models/tags.py +117 -0
  12. dkist_processing_trend/models/task_name.py +20 -0
  13. dkist_processing_trend/parsers/__init__.py +1 -0
  14. dkist_processing_trend/parsers/arm_id.py +103 -0
  15. dkist_processing_trend/parsers/instrument_unique_bud.py +40 -0
  16. dkist_processing_trend/parsers/time.py +27 -0
  17. dkist_processing_trend/parsers/trend_l0_fits_access.py +123 -0
  18. dkist_processing_trend/tasks/__init__.py +27 -0
  19. dkist_processing_trend/tasks/arm_task_factory.py +57 -0
  20. dkist_processing_trend/tasks/dark.py +56 -0
  21. dkist_processing_trend/tasks/gain.py +74 -0
  22. dkist_processing_trend/tasks/initialize_arm_tasks.py +50 -0
  23. dkist_processing_trend/tasks/parse.py +169 -0
  24. dkist_processing_trend/tasks/prepare_fit_data_base.py +257 -0
  25. dkist_processing_trend/tasks/run_pac_fitter.py +392 -0
  26. dkist_processing_trend/tasks/trend_base.py +97 -0
  27. dkist_processing_trend/tasks/trend_output_data.py +167 -0
  28. dkist_processing_trend/tasks/visp/__init__.py +6 -0
  29. dkist_processing_trend/tasks/visp/visp_dmpd.py +410 -0
  30. dkist_processing_trend/tasks/visp/visp_extract_beam.py +14 -0
  31. dkist_processing_trend/tasks/visp/visp_geometric.py +260 -0
  32. dkist_processing_trend/tasks/visp/visp_prep_fit_data.py +162 -0
  33. dkist_processing_trend/tasks/visp/visp_process_demod.py +236 -0
  34. dkist_processing_trend/tasks/write_trend.py +663 -0
  35. dkist_processing_trend/tests/__init__.py +1 -0
  36. dkist_processing_trend/tests/conftest.py +718 -0
  37. dkist_processing_trend/tests/local_trial_workflows/__init__.py +0 -0
  38. dkist_processing_trend/tests/local_trial_workflows/l0_to_trend_visp_polcal.py +294 -0
  39. dkist_processing_trend/tests/local_trial_workflows/local_trial_helpers.py +488 -0
  40. dkist_processing_trend/tests/test_arm_task_factory.py +82 -0
  41. dkist_processing_trend/tests/test_base_tasks.py +86 -0
  42. dkist_processing_trend/tests/test_constants.py +120 -0
  43. dkist_processing_trend/tests/test_dark.py +97 -0
  44. dkist_processing_trend/tests/test_gain.py +135 -0
  45. dkist_processing_trend/tests/test_parameters.py +149 -0
  46. dkist_processing_trend/tests/test_parse.py +276 -0
  47. dkist_processing_trend/tests/test_prep_fit_data_base.py +233 -0
  48. dkist_processing_trend/tests/test_publish_catalog_messages.py +45 -0
  49. dkist_processing_trend/tests/test_run_pac_fitter.py +371 -0
  50. dkist_processing_trend/tests/test_stems.py +75 -0
  51. dkist_processing_trend/tests/test_transfer_output_data.py +76 -0
  52. dkist_processing_trend/tests/test_trend_fits_access.py +173 -0
  53. dkist_processing_trend/tests/test_visp.py +874 -0
  54. dkist_processing_trend/tests/test_workflows.py +10 -0
  55. dkist_processing_trend/tests/test_write_trend.py +460 -0
  56. dkist_processing_trend/workflows/__init__.py +3 -0
  57. dkist_processing_trend/workflows/visp.py +58 -0
  58. dkist_processing_trend-0.1.0.dist-info/METADATA +549 -0
  59. dkist_processing_trend-0.1.0.dist-info/RECORD +66 -0
  60. dkist_processing_trend-0.1.0.dist-info/WHEEL +5 -0
  61. dkist_processing_trend-0.1.0.dist-info/top_level.txt +3 -0
  62. docs/conf.py +57 -0
  63. docs/index.rst +10 -0
  64. docs/l0_to_trend_visp_polcal.rst +4 -0
  65. docs/landing_page.rst +11 -0
  66. docs/requirements_table.rst +8 -0
@@ -0,0 +1,371 @@
1
+ from collections import defaultdict
2
+
3
+ import astropy.units as u
4
+ import numpy as np
5
+ import pytest
6
+ from astropy.time import Time
7
+ from dkist_processing_common._util.scratch import WorkflowFileSystem
8
+ from dkist_processing_common.codecs.asdf import asdf_encoder
9
+ from dkist_processing_common.codecs.fits import fits_array_decoder
10
+ from dkist_processing_common.tests.conftest import CalibrationUnit
11
+ from dkist_processing_common.tests.conftest import Telescope
12
+ from dkist_processing_pac.fitter.fitter_parameters import PolcalDresserParameters
13
+ from dkist_processing_pac.input_data.drawer import Drawer
14
+ from dkist_processing_pac.input_data.dresser import Dresser
15
+
16
+ from dkist_processing_trend.models.instrument import InstrumentName
17
+ from dkist_processing_trend.models.parameters import TrendParameters
18
+ from dkist_processing_trend.models.tags import TrendTag
19
+ from dkist_processing_trend.parsers.trend_l0_fits_access import TrendL0FitsAccess
20
+ from dkist_processing_trend.tasks.run_pac_fitter import RunPacFitter
21
+ from dkist_processing_trend.tests.conftest import ModulatedCSStepHeaders
22
+ from dkist_processing_trend.tests.conftest import TrendInputDatasetParameterValues
23
+ from dkist_processing_trend.tests.conftest import TrendTestingConstantsDatabase
24
+
25
+
26
+ @pytest.fixture(scope="session")
27
+ def cs_with_correct_geometry():
28
+ pol_status = [
29
+ "clear",
30
+ "clear",
31
+ "Sapphire Polarizer",
32
+ "Sapphire Polarizer",
33
+ "Sapphire Polarizer",
34
+ "Sapphire Polarizer",
35
+ "Sapphire Polarizer",
36
+ "Sapphire Polarizer",
37
+ "Sapphire Polarizer",
38
+ "Sapphire Polarizer",
39
+ "Sapphire Polarizer",
40
+ "Sapphire Polarizer",
41
+ "clear",
42
+ "clear",
43
+ ]
44
+ pol_theta = [0, 0, 0, 60, 120, 0, 0, 0, 45, 45, 45, 45, 0, 0]
45
+ ret_status = [
46
+ "clear",
47
+ "clear",
48
+ "clear",
49
+ "clear",
50
+ "clear",
51
+ "SiO2 SAR",
52
+ "SiO2 SAR",
53
+ "SiO2 SAR",
54
+ "SiO2 SAR",
55
+ "SiO2 SAR",
56
+ "SiO2 SAR",
57
+ "SiO2 SAR",
58
+ "clear",
59
+ "clear",
60
+ ]
61
+ ret_theta = [0, 0, 0, 0, 0, 0, 60, 120, 30, 90, 150, 0, 0, 0]
62
+ dark_status = [
63
+ "DarkShutter",
64
+ "FieldStop (5arcmin)",
65
+ "FieldStop (5arcmin)",
66
+ "FieldStop (5arcmin)",
67
+ "FieldStop (5arcmin)",
68
+ "FieldStop (5arcmin)",
69
+ "FieldStop (5arcmin)",
70
+ "FieldStop (5arcmin)",
71
+ "FieldStop (5arcmin)",
72
+ "FieldStop (5arcmin)",
73
+ "FieldStop (5arcmin)",
74
+ "FieldStop (5arcmin)",
75
+ "FieldStop (5arcmin)",
76
+ "DarkShutter",
77
+ ]
78
+ return pol_status, pol_theta, ret_status, ret_theta, dark_status
79
+
80
+
81
+ @pytest.fixture(scope="session")
82
+ def visp_modulation_matrix() -> np.ndarray:
83
+ # Modulation matrix for AdW's synthetic ViSP data from mod_matrix_630.out
84
+ return np.array(
85
+ [
86
+ [1.0, 0.19155013, 0.80446989, -0.47479524],
87
+ [1.0, -0.65839661, 0.68433984, 0.00466389],
88
+ [1.0, -0.80679413, -0.16112977, 0.48234158],
89
+ [1.0, -0.04856211, -0.56352868, 0.77578117],
90
+ [1.0, 0.56844858, 0.03324473, 0.77289873],
91
+ [1.0, 0.19155013, 0.80446989, 0.47479524],
92
+ [1.0, -0.65839661, 0.68433984, -0.00466389],
93
+ [1.0, -0.80679413, -0.16112977, -0.48234158],
94
+ [1.0, -0.04856211, -0.56352868, -0.77578117],
95
+ [1.0, 0.56844858, 0.03324473, -0.77289873],
96
+ ],
97
+ dtype=np.float64,
98
+ )
99
+
100
+
101
+ @pytest.fixture(scope="session")
102
+ def num_cs_steps(cs_with_correct_geometry) -> int:
103
+ return len(cs_with_correct_geometry[0])
104
+
105
+
106
+ @pytest.fixture(scope="session")
107
+ def num_modstates(visp_modulation_matrix) -> int:
108
+ return visp_modulation_matrix.shape[0]
109
+
110
+
111
+ @pytest.fixture(scope="session")
112
+ def arm_id() -> int:
113
+ return 1
114
+
115
+
116
+ @pytest.fixture(scope="session")
117
+ def full_cs_fits_obj_dict_no_data(
118
+ cs_with_correct_geometry, num_modstates, arm_id
119
+ ) -> dict[int, list[TrendL0FitsAccess]]:
120
+
121
+ pol_status, pol_theta, ret_status, ret_theta, dark_status = cs_with_correct_geometry
122
+
123
+ step_dict: dict[int, list[TrendL0FitsAccess]] = defaultdict(list)
124
+
125
+ base_time = Time("2026-01-01")
126
+
127
+ for step, (ps, pt, rs, rt, ds) in enumerate(
128
+ zip(pol_status, pol_theta, ret_status, ret_theta, dark_status)
129
+ ):
130
+
131
+ step_start = (base_time + (step * 60) * u.s).fits
132
+
133
+ ds = ModulatedCSStepHeaders(
134
+ instrument=InstrumentName.visp,
135
+ arm_id=arm_id,
136
+ wavelength=366.0,
137
+ num_modstates=num_modstates,
138
+ pol_status=ps,
139
+ pol_theta=pt,
140
+ ret_status=rs,
141
+ ret_theta=rt,
142
+ dark_status=ds,
143
+ cs_step_num=step,
144
+ array_shape=(2, 2),
145
+ exp_time_ms=6.28,
146
+ start_date=step_start,
147
+ )
148
+ for modstate_frame in ds:
149
+ step_dict[step].append(
150
+ TrendL0FitsAccess(header=modstate_frame.header(), data=np.ones((3, 3)))
151
+ )
152
+
153
+ return step_dict
154
+
155
+
156
+ @pytest.fixture(scope="session")
157
+ def fit_options_list() -> list[dict[str, str | bool]]:
158
+ return [
159
+ {
160
+ "name": "option1",
161
+ "remove_I_trend": True,
162
+ "fit_mode_name": "use_M12",
163
+ "init_set_name": "OCCal_VIS",
164
+ },
165
+ {
166
+ "name": "option2",
167
+ "remove_I_trend": False,
168
+ "fit_mode_name": "use_M12_I_sys_per_step",
169
+ "init_set_name": "OCCal_VIS",
170
+ },
171
+ ]
172
+
173
+
174
+ @pytest.fixture(scope="session")
175
+ def full_observed_values(full_cs_fits_obj_dict_no_data, visp_modulation_matrix) -> np.ndarray:
176
+
177
+ fit_mode = "use_M12"
178
+ init_set = "OCCal_VIS"
179
+ cs_dict = full_cs_fits_obj_dict_no_data
180
+ dresser = Dresser()
181
+ dresser.add_drawer(Drawer(cs_dict, skip_darks=False, remove_I_trend=False))
182
+ CM = CalibrationUnit(dresser)
183
+ TM = Telescope(dresser)
184
+ full_params = PolcalDresserParameters(dresser, fit_mode, init_set)
185
+
186
+ global_params = full_params.init_params._all_parameters[0]
187
+ pardict = global_params.valuesdict()
188
+ CM.load_pars_from_dict(pardict)
189
+ TM.load_pars_from_dict(pardict)
190
+
191
+ CM.I_sys[0] = 1e4
192
+
193
+ # Has shape (4, N)
194
+ S = np.sum((TM.TM @ CM.CM @ TM.M12) * CM.S_in[:, None, :], axis=2).T
195
+
196
+ # Has shape (M, N)
197
+ observed = visp_modulation_matrix @ S
198
+
199
+ return observed
200
+
201
+
202
+ def write_pac_inputs_to_task(
203
+ task,
204
+ step_to_fits_access_dict: dict[int, list[TrendL0FitsAccess]],
205
+ observed_data: np.ndarray,
206
+ beam: int,
207
+ arm_id: str | int,
208
+ local_fov_shape: tuple[int, ...],
209
+ ) -> None:
210
+ global_dict = defaultdict(list)
211
+ local_dict = defaultdict(list)
212
+
213
+ for step, fits_access_list in step_to_fits_access_dict.items():
214
+ for modstate, fits_obj in enumerate(fits_access_list):
215
+ header = fits_obj.header
216
+ obs_value = observed_data[modstate, step]
217
+ global_data = np.full((1, 1), obs_value)
218
+ local_data = np.full(local_fov_shape, obs_value)
219
+
220
+ global_dict[step].append({"header": header, "data": global_data})
221
+ local_dict[step].append({"header": header, "data": local_data})
222
+
223
+ # Cheat a little bit and just write the same data for each instrument processing option
224
+ inst_option_names = [i.name for i in task.parameters.instrument_processing_options]
225
+ for inst_option_name in inst_option_names:
226
+ task.write(
227
+ data=global_dict,
228
+ tags=[
229
+ TrendTag.intermediate(),
230
+ TrendTag.beam(beam),
231
+ TrendTag.arm_id(arm_id),
232
+ TrendTag.instrument_processing_options(inst_option_name),
233
+ TrendTag.task_global_pac_input(),
234
+ ],
235
+ encoder=asdf_encoder,
236
+ )
237
+ task.write(
238
+ data=local_dict,
239
+ tags=[
240
+ TrendTag.intermediate(),
241
+ TrendTag.beam(beam),
242
+ TrendTag.arm_id(arm_id),
243
+ TrendTag.instrument_processing_options(inst_option_name),
244
+ TrendTag.task_local_pac_input(),
245
+ ],
246
+ encoder=asdf_encoder,
247
+ )
248
+
249
+
250
+ @pytest.fixture
251
+ def run_pac_fitter_task(
252
+ tmp_path,
253
+ recipe_run_id,
254
+ link_constants_db,
255
+ assign_input_dataset_doc_to_task,
256
+ num_cs_steps,
257
+ num_modstates,
258
+ arm_id,
259
+ fit_options_list,
260
+ ):
261
+ link_constants_db(
262
+ recipe_run_id,
263
+ TrendTestingConstantsDatabase(NUM_CS_STEPS=num_cs_steps, NUM_MODSTATES=num_modstates),
264
+ )
265
+ with RunPacFitter(
266
+ arm_id=arm_id, recipe_run_id=recipe_run_id, workflow_name="name", workflow_version="version"
267
+ ) as task:
268
+ task.scratch = WorkflowFileSystem(recipe_run_id=recipe_run_id, scratch_base_path=tmp_path)
269
+ assign_input_dataset_doc_to_task(
270
+ task, TrendInputDatasetParameterValues(trend_fit_options_list=fit_options_list)
271
+ )
272
+ task.parameters = TrendParameters(
273
+ scratch=task.scratch,
274
+ obs_ip_start_time=task.constants.earliest_ip_start_time,
275
+ instrument=task.constants.instrument,
276
+ )
277
+
278
+ yield task
279
+ task._purge()
280
+
281
+
282
+ def test_run_pac_fitter(
283
+ run_pac_fitter_task,
284
+ mocker,
285
+ fake_gql_client,
286
+ num_cs_steps,
287
+ num_modstates,
288
+ full_cs_fits_obj_dict_no_data,
289
+ full_observed_values,
290
+ arm_id,
291
+ fit_options_list,
292
+ ):
293
+ """
294
+ Given: A set of PAC input data and a `RunPacFitter` task
295
+ When: Running the task
296
+ Then: The task completes, the expected outputs exist, and they have the correct shapes
297
+ """
298
+ mocker.patch(
299
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
300
+ )
301
+
302
+ task = run_pac_fitter_task
303
+ fov_shape = (3, 2)
304
+ num_beams = 2
305
+ inst_options_name_list = [i.name for i in task.parameters.instrument_processing_options]
306
+
307
+ for beam in range(1, num_beams + 1):
308
+ write_pac_inputs_to_task(
309
+ task=task,
310
+ step_to_fits_access_dict=full_cs_fits_obj_dict_no_data,
311
+ observed_data=full_observed_values,
312
+ beam=beam,
313
+ arm_id=arm_id,
314
+ local_fov_shape=fov_shape,
315
+ )
316
+
317
+ task()
318
+
319
+ cal_seq_tags = [
320
+ TrendTag.intermediate(),
321
+ TrendTag.arm_id(arm_id),
322
+ TrendTag.task_calibration_sequence(),
323
+ ]
324
+ assert task.count(tags=cal_seq_tags) == 1
325
+ cal_seq_data = next(
326
+ task.read(tags=cal_seq_tags, decoder=fits_array_decoder, auto_squeeze=False)
327
+ )
328
+ assert cal_seq_data.shape == (1, 8, num_cs_steps)
329
+
330
+ for inst_option_name in inst_options_name_list:
331
+ for beam in range(1, num_beams + 1):
332
+ for fit_options in fit_options_list:
333
+ fit_opt_name = fit_options["name"]
334
+ common_tags = [
335
+ TrendTag.intermediate(),
336
+ TrendTag.beam(beam),
337
+ TrendTag.arm_id(arm_id),
338
+ TrendTag.instrument_processing_options(inst_option_name),
339
+ TrendTag.pac_fit_options(fit_opt_name),
340
+ ]
341
+
342
+ best_fit_param_tags = common_tags + [TrendTag.task_best_fit_parameters()]
343
+ assert task.count(best_fit_param_tags) == 1
344
+ best_fit_param_data = next(
345
+ task.read(
346
+ tags=best_fit_param_tags, decoder=fits_array_decoder, auto_squeeze=False
347
+ )
348
+ )
349
+ num_params = 15 + num_cs_steps - 2 # -2 because we don't fit dark steps
350
+ assert best_fit_param_data.shape == fov_shape + (1, num_params, 3)
351
+
352
+ best_fit_demod_tags = common_tags + [TrendTag.task_best_fit_demodulation_matrices()]
353
+ assert task.count(best_fit_demod_tags) == 1
354
+ best_fit_demod = next(
355
+ task.read(best_fit_demod_tags, decoder=fits_array_decoder, auto_squeeze=False)
356
+ )
357
+ assert best_fit_demod.shape == fov_shape + (4, num_modstates)
358
+
359
+ best_fit_flux_tags = common_tags + [TrendTag.task_best_fit_flux()]
360
+ assert task.count(best_fit_flux_tags) == 1
361
+ best_fit_flux_data = next(
362
+ task.read(best_fit_flux_tags, decoder=fits_array_decoder, auto_squeeze=False)
363
+ )
364
+ assert best_fit_flux_data.shape == fov_shape + (num_modstates, num_cs_steps - 2)
365
+
366
+ fit_residuals_tags = common_tags + [TrendTag.task_fit_residuals()]
367
+ assert task.count(fit_residuals_tags) == 1
368
+ fit_residuals_data = next(
369
+ task.read(fit_residuals_tags, decoder=fits_array_decoder, auto_squeeze=True)
370
+ )
371
+ assert fit_residuals_data.shape == best_fit_flux_data.shape
@@ -0,0 +1,75 @@
1
+ from dataclasses import dataclass
2
+
3
+ import pytest
4
+ from dkist_processing_common.models.fits_access import MetadataKey
5
+
6
+ from dkist_processing_trend.parsers.arm_id import ArmIdLookupBud
7
+ from dkist_processing_trend.parsers.arm_id import InstrumentArmIdLookupBud
8
+
9
+
10
+ @dataclass
11
+ class DummyFitsAccess:
12
+
13
+ arm_id: str | int
14
+ wavelength: float
15
+ instrument: str = "foo"
16
+
17
+
18
+ def test_arm_id_lookup_bud():
19
+ """
20
+ Given: A set of `FitsAccess` objects corresponding to multiple arms and wavelengths
21
+ When: Processing these objects through an `ArmIdLookupBud`
22
+ Then: The correct value is returned
23
+ """
24
+ fo_list = (
25
+ [DummyFitsAccess(arm_id=1, wavelength=123.4) for _ in range(3)]
26
+ + [DummyFitsAccess(arm_id="SP", wavelength=1079) for _ in range(5)]
27
+ + [DummyFitsAccess(arm_id=3, wavelength=567)]
28
+ )
29
+ bud = ArmIdLookupBud(constant_name="foo", metadata_key=MetadataKey.wavelength)
30
+ for fo in fo_list:
31
+ bud.update(key="doesn't matter", value=fo)
32
+
33
+ assert bud.bud.value == {"1": 123.4, '"SP"': 1079, "3": 567}
34
+
35
+
36
+ def test_arm_id_to_wavelength_non_singular_wave():
37
+ """
38
+ Given: A set of `FitsAccess` objects corresponding to multiple arms where some arms have multiple wavelength values
39
+ When: Processing these objects through an `ArmIdLookupBud`
40
+ Then: The error is raised
41
+ """
42
+ fo_list = (
43
+ [DummyFitsAccess(arm_id=1, wavelength=123.4) for _ in range(3)]
44
+ + [DummyFitsAccess(arm_id=1, wavelength=1079) for _ in range(5)]
45
+ + [DummyFitsAccess(arm_id=3, wavelength=567)]
46
+ )
47
+ bud = ArmIdLookupBud(constant_name="foo", metadata_key=MetadataKey.wavelength)
48
+ for fo in fo_list:
49
+ bud.update(key="doesn't matter", value=fo)
50
+
51
+ with pytest.raises(
52
+ ValueError,
53
+ match=f"Some arms have multiple values for wavelength: arm_id = 1: {{123.4, 1079}}",
54
+ ):
55
+ bud.bud
56
+
57
+
58
+ def test_instrument_arm_id_lookup_bud():
59
+ """
60
+ Given: A set of `FitsAccess` objects from different instruments corresponding to multiple arms and wavelengths
61
+ When: Processing these objects through an `InstrumentArmIdLookupBud` for only one of the input instruments
62
+ Then: The correct value for the single instrument is returned
63
+ """
64
+ fo_list = (
65
+ [DummyFitsAccess(arm_id=1, wavelength=123.4, instrument="foo") for _ in range(3)]
66
+ + [DummyFitsAccess(arm_id="SP", wavelength=1079, instrument="foo") for _ in range(5)]
67
+ + [DummyFitsAccess(arm_id=1, wavelength=-9999, instrument="bar") for _ in range(3)]
68
+ )
69
+ bud = InstrumentArmIdLookupBud(
70
+ constant_name="asdf", metadata_key=MetadataKey.wavelength, instruments="foo"
71
+ )
72
+ for fo in fo_list:
73
+ bud.update(key="doesn't matter", value=fo)
74
+
75
+ assert bud.bud.value == {"1": 123.4, '"SP"': 1079}
@@ -0,0 +1,76 @@
1
+ from pathlib import Path
2
+
3
+ import pytest
4
+ from dkist_processing_common._util.scratch import WorkflowFileSystem
5
+
6
+ from dkist_processing_trend.models.tags import TrendTag
7
+ from dkist_processing_trend.tasks.trend_output_data import TransferTrendData
8
+ from dkist_processing_trend.tasks.trend_output_data import TrendOutputDataBase
9
+ from dkist_processing_trend.tests.conftest import TrendTestingConstantsDatabase
10
+
11
+
12
+ def fake_list_objects(self, bucket, prefix=None):
13
+ return list()
14
+
15
+
16
+ @pytest.fixture
17
+ def transfer_l1_data_task(recipe_run_id, tmp_path, link_constants_db):
18
+ def make_task(
19
+ instrument,
20
+ earliest_ip_start_time,
21
+ ):
22
+ link_constants_db(
23
+ recipe_run_id,
24
+ TrendTestingConstantsDatabase(
25
+ INSTRUMENT=instrument, EARLIEST_IP_START_TIME=earliest_ip_start_time
26
+ ),
27
+ )
28
+ with TransferTrendData(
29
+ recipe_run_id=recipe_run_id,
30
+ workflow_name="workflow_name",
31
+ workflow_version="workflow_version",
32
+ ) as task:
33
+ task.scratch = WorkflowFileSystem(
34
+ recipe_run_id=recipe_run_id,
35
+ scratch_base_path=tmp_path,
36
+ )
37
+ frame_path = task.scratch.workflow_base_path / Path("frame.fits")
38
+ with open(frame_path, "w") as f:
39
+ f.write("Frame")
40
+ task.tag(path=frame_path, tags=[TrendTag.frame(), TrendTag.output()])
41
+
42
+ yield task
43
+ task._purge()
44
+
45
+ return make_task
46
+
47
+
48
+ def test_transfer_l1_data(transfer_l1_data_task, mocker, fake_gql_client):
49
+ """
50
+ Given: A task with frames and movies tagged as output
51
+ When: Transfering the L1 data
52
+ Then: The task completes without errors and the destination object keys are correct
53
+ """
54
+ # Yeah, we mock a whole bunch of stuff here, but this test at least confirms that the setup to these calls is correct
55
+ mocker.patch(
56
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
57
+ )
58
+ mocker.patch(
59
+ "dkist_processing_common.tasks.mixin.globus.GlobusMixin.globus_transfer_scratch_to_object_store"
60
+ )
61
+ mocker.patch("dkist_processing_common.tasks.mixin.object_store.ObjectClerk.upload_object")
62
+ mocker.patch(
63
+ "dkist_processing_common.tasks.mixin.object_store.ObjectClerk.list_objects",
64
+ fake_list_objects,
65
+ )
66
+ instrument = "HELLO"
67
+ earliest_ip_start_time = "1900-01-01T12:34:45.0002"
68
+ task = next(
69
+ transfer_l1_data_task(instrument=instrument, earliest_ip_start_time=earliest_ip_start_time)
70
+ )
71
+ task()
72
+
73
+ assert (
74
+ task.format_object_key(Path("scratch/foo/bar/thing.fits"))
75
+ == "trend/polcal/19000101T123445/HELLO/thing.fits"
76
+ )
@@ -0,0 +1,173 @@
1
+ from inspect import get_annotations
2
+ from random import randint
3
+
4
+ import pytest
5
+ from astropy.io import fits
6
+
7
+ from dkist_processing_trend.models.fits_access import TrendMetadataKey
8
+ from dkist_processing_trend.models.instrument import InstrumentName
9
+ from dkist_processing_trend.parsers.trend_l0_fits_access import IncorrectInstrumentError
10
+ from dkist_processing_trend.parsers.trend_l0_fits_access import TrendL0FitsAccess
11
+ from dkist_processing_trend.tests.conftest import WAVELENGTH_FOR_ARM
12
+ from dkist_processing_trend.tests.conftest import ModulatedCSStepHeaders
13
+
14
+
15
+ @pytest.fixture(scope="function") # So we get new random for each test
16
+ def random_cs_step_config(
17
+ small_calibration_sequence, small_num_cs_steps
18
+ ) -> tuple[str, float, str, float, str]:
19
+ idx = randint(0, small_num_cs_steps - 1)
20
+ return tuple(i[idx] for i in small_calibration_sequence)
21
+
22
+
23
+ @pytest.fixture
24
+ def num_modstates() -> int:
25
+ return 2
26
+
27
+
28
+ @pytest.fixture
29
+ def metadata_keys_from_wrong_instruments(instrument):
30
+ visp_metadata_keys = {
31
+ TrendMetadataKey.visp_grating_angle,
32
+ TrendMetadataKey.visp_arm_position,
33
+ TrendMetadataKey.visp_grating_constant,
34
+ }
35
+ cryonirsp_metadata_keys = {
36
+ TrendMetadataKey.cryonirsp_od_filter_name,
37
+ TrendMetadataKey.cryonirsp_grating_angle,
38
+ TrendMetadataKey.cryonirsp_grating_littrow_angle,
39
+ TrendMetadataKey.cryonirsp_grating_constant,
40
+ TrendMetadataKey.cryonirsp_slit_width,
41
+ }
42
+ dlnirsp_metadata_keys = {
43
+ TrendMetadataKey.dlnirsp_grating_angle,
44
+ TrendMetadataKey.dlnirsp_arm_position,
45
+ TrendMetadataKey.dlnirsp_grating_constant,
46
+ }
47
+
48
+ wrong_inst_metadata_keys = visp_metadata_keys | cryonirsp_metadata_keys | dlnirsp_metadata_keys
49
+ match instrument:
50
+ case InstrumentName.visp:
51
+ wrong_inst_metadata_keys -= visp_metadata_keys
52
+ case InstrumentName.cryonirsp:
53
+ wrong_inst_metadata_keys -= cryonirsp_metadata_keys
54
+ case InstrumentName.dlnirsp:
55
+ wrong_inst_metadata_keys -= dlnirsp_metadata_keys
56
+
57
+ return wrong_inst_metadata_keys
58
+
59
+
60
+ @pytest.fixture
61
+ def input_header(
62
+ instrument: InstrumentName,
63
+ arm_id: str | int,
64
+ num_modstates: int,
65
+ random_cs_step_config: tuple[str, float, str, float, str],
66
+ ) -> fits.Header:
67
+ pol_status, pol_theta, ret_status, ret_theta, dark_status = random_cs_step_config
68
+ ds = ModulatedCSStepHeaders(
69
+ instrument=instrument,
70
+ arm_id=arm_id,
71
+ wavelength=WAVELENGTH_FOR_ARM[arm_id],
72
+ num_modstates=num_modstates,
73
+ pol_status=pol_status,
74
+ pol_theta=pol_theta,
75
+ ret_status=ret_status,
76
+ ret_theta=ret_theta,
77
+ dark_status=dark_status,
78
+ array_shape=(2, 2),
79
+ cs_step_num=1,
80
+ exp_time_ms=34.0,
81
+ )
82
+ return ds.header()
83
+
84
+
85
+ @pytest.mark.parametrize(
86
+ "instrument, arm_id",
87
+ [
88
+ pytest.param(InstrumentName.visp, 1, id="visp"),
89
+ pytest.param(InstrumentName.cryonirsp, "SP", id="cryonirsp"),
90
+ pytest.param(InstrumentName.dlnirsp, "JBand", id="JBand"),
91
+ ],
92
+ )
93
+ def test_trend_l0_fits_access(input_header, num_modstates, arm_id):
94
+ """
95
+ Given: A Header with instrument-specific trend values
96
+ When: Parsing the header with `TrendL0FitsAccess`
97
+ Then: The keys for each instrument are correctly used such that the FitsAccess properties return the correct value
98
+ """
99
+ fits_obj = TrendL0FitsAccess(header=input_header)
100
+
101
+ assert fits_obj.arm_id == arm_id
102
+ assert fits_obj.number_of_modulator_states == num_modstates
103
+ assert fits_obj.modulator_state == 1 # Because we took the first header from the dataset
104
+
105
+
106
+ @pytest.mark.parametrize("instrument, arm_id", [pytest.param(InstrumentName.visp, 1, id="")])
107
+ def test_error_on_unknown_instrument(input_header):
108
+ """
109
+ Given: A Header where the INSTRUME key is not one of the recognized values
110
+ When: Parsing the header with `TrendL0FitsAccess`
111
+ Then: The correct Error is raised
112
+ """
113
+ input_header["INSTRUME"] = "WRONG"
114
+ with pytest.raises(ValueError, match="Did not expect instrument = WRONG"):
115
+ TrendL0FitsAccess(header=input_header)
116
+
117
+
118
+ @pytest.mark.parametrize(
119
+ "instrument, arm_id",
120
+ [
121
+ pytest.param(InstrumentName.visp, 1, id="visp"),
122
+ pytest.param(InstrumentName.cryonirsp, "SP", id="cryonirsp"),
123
+ pytest.param(InstrumentName.dlnirsp, "VIS", id="dlnirsp"),
124
+ ],
125
+ )
126
+ def test_metadata_keys_in_access_bases(input_header):
127
+ """
128
+ Given: The set of metadata key names in TrendMetadataKey
129
+ When: The `TrendL0FitsAccess` class defines a set of new attributes
130
+ Then: The sets are the same
131
+ """
132
+ trend_metadata_key_names = {tmk.name for tmk in TrendMetadataKey}
133
+ all_trend_fits_access_attrs = set()
134
+ for access_class in [TrendL0FitsAccess]:
135
+ fits_obj = access_class(header=input_header)
136
+ trend_instance_attrs = set(vars(fits_obj).keys())
137
+
138
+ # This next line grabs the `InstrumentSpecificProperty` for the instruments not represented in the header
139
+ trend_instance_attrs |= set(get_annotations(access_class).keys())
140
+ parent_class = access_class.mro()[1]
141
+ parent_fits_obj = parent_class(header=input_header)
142
+ parent_instance_attrs = set(vars(parent_fits_obj).keys())
143
+ trend_fits_access_attrs = trend_instance_attrs - parent_instance_attrs
144
+ all_trend_fits_access_attrs |= trend_fits_access_attrs
145
+ assert trend_metadata_key_names == all_trend_fits_access_attrs
146
+
147
+
148
+ @pytest.mark.parametrize(
149
+ "instrument, arm_id",
150
+ [
151
+ pytest.param(InstrumentName.visp, 1, id="visp"),
152
+ pytest.param(InstrumentName.cryonirsp, "SP", id="cryonirsp"),
153
+ pytest.param(InstrumentName.dlnirsp, "VIS", id="dlnirsp"),
154
+ ],
155
+ )
156
+ def test_instrument_specific_properties(
157
+ input_header, instrument, metadata_keys_from_wrong_instruments
158
+ ):
159
+ """
160
+ Given: A TrendL0FitsAccess object generated from an input header corresponding to a certain instrument
161
+ When: Trying to access properties that are not defined for that certain instrument
162
+ Then: The correct error is raised
163
+ """
164
+ fits_obj = TrendL0FitsAccess(header=input_header)
165
+
166
+ for metadata_key in metadata_keys_from_wrong_instruments:
167
+ with pytest.raises(
168
+ IncorrectInstrumentError,
169
+ match="This property is only defined for data from .*, which this frame is not.",
170
+ ) as exc_info:
171
+ _ = getattr(fits_obj, metadata_key.name)
172
+ pytest.fail(f"{metadata_key = } did not raise expected error")
173
+ assert instrument not in exc_info.value.args[0]