dkist-processing-cryonirsp 1.4.20__py3-none-any.whl → 1.14.9rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. changelog/232.misc.rst +1 -0
  2. dkist_processing_cryonirsp/__init__.py +1 -0
  3. dkist_processing_cryonirsp/codecs/fits.py +1 -0
  4. dkist_processing_cryonirsp/config.py +5 -1
  5. dkist_processing_cryonirsp/models/beam_boundaries.py +1 -0
  6. dkist_processing_cryonirsp/models/constants.py +31 -30
  7. dkist_processing_cryonirsp/models/exposure_conditions.py +6 -5
  8. dkist_processing_cryonirsp/models/fits_access.py +40 -0
  9. dkist_processing_cryonirsp/models/parameters.py +14 -26
  10. dkist_processing_cryonirsp/models/tags.py +1 -0
  11. dkist_processing_cryonirsp/models/task_name.py +1 -0
  12. dkist_processing_cryonirsp/parsers/check_for_gains.py +1 -0
  13. dkist_processing_cryonirsp/parsers/cryonirsp_l0_fits_access.py +40 -47
  14. dkist_processing_cryonirsp/parsers/cryonirsp_l1_fits_access.py +1 -0
  15. dkist_processing_cryonirsp/parsers/exposure_conditions.py +14 -13
  16. dkist_processing_cryonirsp/parsers/map_repeats.py +1 -0
  17. dkist_processing_cryonirsp/parsers/measurements.py +29 -16
  18. dkist_processing_cryonirsp/parsers/modstates.py +5 -1
  19. dkist_processing_cryonirsp/parsers/optical_density_filters.py +1 -0
  20. dkist_processing_cryonirsp/parsers/polarimetric_check.py +18 -7
  21. dkist_processing_cryonirsp/parsers/scan_step.py +12 -4
  22. dkist_processing_cryonirsp/parsers/time.py +7 -7
  23. dkist_processing_cryonirsp/parsers/wavelength.py +6 -1
  24. dkist_processing_cryonirsp/tasks/__init__.py +2 -1
  25. dkist_processing_cryonirsp/tasks/assemble_movie.py +1 -0
  26. dkist_processing_cryonirsp/tasks/bad_pixel_map.py +6 -5
  27. dkist_processing_cryonirsp/tasks/beam_boundaries_base.py +12 -11
  28. dkist_processing_cryonirsp/tasks/ci_beam_boundaries.py +1 -0
  29. dkist_processing_cryonirsp/tasks/ci_science.py +1 -0
  30. dkist_processing_cryonirsp/tasks/cryonirsp_base.py +2 -3
  31. dkist_processing_cryonirsp/tasks/dark.py +5 -4
  32. dkist_processing_cryonirsp/tasks/gain.py +7 -6
  33. dkist_processing_cryonirsp/tasks/instrument_polarization.py +17 -16
  34. dkist_processing_cryonirsp/tasks/l1_output_data.py +1 -0
  35. dkist_processing_cryonirsp/tasks/linearity_correction.py +1 -0
  36. dkist_processing_cryonirsp/tasks/make_movie_frames.py +3 -2
  37. dkist_processing_cryonirsp/tasks/mixin/corrections.py +1 -0
  38. dkist_processing_cryonirsp/tasks/mixin/shift_measurements.py +9 -2
  39. dkist_processing_cryonirsp/tasks/parse.py +70 -52
  40. dkist_processing_cryonirsp/tasks/quality_metrics.py +15 -14
  41. dkist_processing_cryonirsp/tasks/science_base.py +8 -6
  42. dkist_processing_cryonirsp/tasks/sp_beam_boundaries.py +2 -1
  43. dkist_processing_cryonirsp/tasks/sp_geometric.py +11 -10
  44. dkist_processing_cryonirsp/tasks/sp_science.py +1 -0
  45. dkist_processing_cryonirsp/tasks/sp_solar_gain.py +15 -12
  46. dkist_processing_cryonirsp/tasks/sp_wavelength_calibration.py +300 -0
  47. dkist_processing_cryonirsp/tasks/write_l1.py +59 -38
  48. dkist_processing_cryonirsp/tests/conftest.py +75 -53
  49. dkist_processing_cryonirsp/tests/header_models.py +62 -11
  50. dkist_processing_cryonirsp/tests/local_trial_workflows/l0_cals_only.py +26 -46
  51. dkist_processing_cryonirsp/tests/local_trial_workflows/l0_to_l1.py +26 -47
  52. dkist_processing_cryonirsp/tests/local_trial_workflows/linearize_only.py +3 -3
  53. dkist_processing_cryonirsp/tests/local_trial_workflows/local_trial_helpers.py +57 -26
  54. dkist_processing_cryonirsp/tests/test_assemble_movie.py +4 -5
  55. dkist_processing_cryonirsp/tests/test_assemble_qualilty.py +5 -1
  56. dkist_processing_cryonirsp/tests/test_bad_pixel_maps.py +4 -5
  57. dkist_processing_cryonirsp/tests/test_ci_beam_boundaries.py +4 -5
  58. dkist_processing_cryonirsp/tests/test_ci_science.py +4 -5
  59. dkist_processing_cryonirsp/tests/test_corrections.py +5 -6
  60. dkist_processing_cryonirsp/tests/test_cryo_base.py +4 -6
  61. dkist_processing_cryonirsp/tests/test_cryo_constants.py +7 -3
  62. dkist_processing_cryonirsp/tests/test_dark.py +7 -8
  63. dkist_processing_cryonirsp/tests/test_fits_access.py +44 -0
  64. dkist_processing_cryonirsp/tests/test_gain.py +7 -8
  65. dkist_processing_cryonirsp/tests/test_instrument_polarization.py +19 -10
  66. dkist_processing_cryonirsp/tests/test_linearity_correction.py +5 -4
  67. dkist_processing_cryonirsp/tests/test_make_movie_frames.py +2 -3
  68. dkist_processing_cryonirsp/tests/test_parameters.py +23 -28
  69. dkist_processing_cryonirsp/tests/test_parse.py +48 -12
  70. dkist_processing_cryonirsp/tests/test_quality.py +2 -3
  71. dkist_processing_cryonirsp/tests/test_sp_beam_boundaries.py +5 -5
  72. dkist_processing_cryonirsp/tests/test_sp_geometric.py +5 -6
  73. dkist_processing_cryonirsp/tests/test_sp_make_movie_frames.py +2 -3
  74. dkist_processing_cryonirsp/tests/test_sp_science.py +4 -5
  75. dkist_processing_cryonirsp/tests/test_sp_solar.py +6 -5
  76. dkist_processing_cryonirsp/tests/{test_sp_dispersion_axis_correction.py → test_sp_wavelength_calibration.py} +11 -29
  77. dkist_processing_cryonirsp/tests/test_trial_create_quality_report.py +1 -1
  78. dkist_processing_cryonirsp/tests/test_workflows.py +1 -0
  79. dkist_processing_cryonirsp/tests/test_write_l1.py +29 -31
  80. dkist_processing_cryonirsp/workflows/__init__.py +1 -0
  81. dkist_processing_cryonirsp/workflows/ci_l0_processing.py +9 -5
  82. dkist_processing_cryonirsp/workflows/sp_l0_processing.py +12 -8
  83. dkist_processing_cryonirsp/workflows/trial_workflows.py +12 -11
  84. dkist_processing_cryonirsp-1.14.9rc1.dist-info/METADATA +552 -0
  85. dkist_processing_cryonirsp-1.14.9rc1.dist-info/RECORD +115 -0
  86. {dkist_processing_cryonirsp-1.4.20.dist-info → dkist_processing_cryonirsp-1.14.9rc1.dist-info}/WHEEL +1 -1
  87. docs/ci_science_calibration.rst +10 -0
  88. docs/conf.py +1 -0
  89. docs/index.rst +1 -0
  90. docs/sp_science_calibration.rst +7 -0
  91. docs/wavelength_calibration.rst +62 -0
  92. dkist_processing_cryonirsp/tasks/sp_dispersion_axis_correction.py +0 -492
  93. dkist_processing_cryonirsp-1.4.20.dist-info/METADATA +0 -452
  94. dkist_processing_cryonirsp-1.4.20.dist-info/RECORD +0 -111
  95. {dkist_processing_cryonirsp-1.4.20.dist-info → dkist_processing_cryonirsp-1.14.9rc1.dist-info}/top_level.txt +0 -0
@@ -7,14 +7,13 @@ from dkist_header_validator import spec122_validator
7
7
  from dkist_processing_common._util.scratch import WorkflowFileSystem
8
8
  from dkist_processing_common.codecs.fits import fits_array_encoder
9
9
  from dkist_processing_common.codecs.fits import fits_hdulist_encoder
10
- from dkist_processing_common.tests.conftest import FakeGQLClient
11
10
 
12
11
  from dkist_processing_cryonirsp.models.exposure_conditions import AllowableOpticalDensityFilterNames
13
12
  from dkist_processing_cryonirsp.models.exposure_conditions import ExposureConditions
14
13
  from dkist_processing_cryonirsp.models.tags import CryonirspTag
15
14
  from dkist_processing_cryonirsp.tasks.dark import DarkCalibration
16
- from dkist_processing_cryonirsp.tests.conftest import cryonirsp_testing_parameters_factory
17
15
  from dkist_processing_cryonirsp.tests.conftest import CryonirspConstantsDb
16
+ from dkist_processing_cryonirsp.tests.conftest import cryonirsp_testing_parameters_factory
18
17
  from dkist_processing_cryonirsp.tests.conftest import generate_fits_frame
19
18
  from dkist_processing_cryonirsp.tests.header_models import CryonirspHeadersValidDarkFrames
20
19
 
@@ -52,7 +51,7 @@ def sp_dark_calibration_task(
52
51
  task.scratch = WorkflowFileSystem(
53
52
  scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
54
53
  )
55
- param_class = cryonirsp_testing_parameters_factory(param_path=tmp_path)
54
+ param_class = cryonirsp_testing_parameters_factory(task)
56
55
  assign_input_dataset_doc_to_task(task, param_class())
57
56
 
58
57
  # Create fake beam border intermediate arrays
@@ -137,7 +136,7 @@ def ci_dark_calibration_task(
137
136
  task.scratch = WorkflowFileSystem(
138
137
  scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
139
138
  )
140
- param_class = cryonirsp_testing_parameters_factory(param_path=tmp_path)
139
+ param_class = cryonirsp_testing_parameters_factory(task)
141
140
  assign_input_dataset_doc_to_task(task, param_class())
142
141
 
143
142
  # Need a beam boundary file
@@ -176,14 +175,14 @@ def ci_dark_calibration_task(
176
175
  task._purge()
177
176
 
178
177
 
179
- def test_sp_dark_calibration_task(sp_dark_calibration_task, mocker):
178
+ def test_sp_dark_calibration_task(sp_dark_calibration_task, mocker, fake_gql_client):
180
179
  """
181
180
  Given: A DarkCalibration task with multiple task exposure times
182
181
  When: Calling the task instance
183
182
  Then: Only one average intermediate dark frame exists for each exposure time and unused times are not made
184
183
  """
185
184
  mocker.patch(
186
- "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
185
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
187
186
  )
188
187
  # When
189
188
  (
@@ -232,14 +231,14 @@ def test_sp_dark_calibration_task(sp_dark_calibration_task, mocker):
232
231
  assert data["frames_not_used"] == 3
233
232
 
234
233
 
235
- def test_ci_dark_calibration_task(ci_dark_calibration_task, mocker):
234
+ def test_ci_dark_calibration_task(ci_dark_calibration_task, mocker, fake_gql_client):
236
235
  """
237
236
  Given: A DarkCalibration task with multiple task exposure times
238
237
  When: Calling the task instance
239
238
  Then: Only one average intermediate dark frame exists for each exposure time and unused times are not made
240
239
  """
241
240
  mocker.patch(
242
- "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
241
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
243
242
  )
244
243
  # When
245
244
  task, exp_conditions, unused_condition = ci_dark_calibration_task
@@ -0,0 +1,44 @@
1
+ import pytest
2
+ from dkist_header_validator.translator import translate_spec122_to_spec214_l0
3
+
4
+ from dkist_processing_cryonirsp.models.fits_access import CryonirspMetadataKey
5
+ from dkist_processing_cryonirsp.parsers.cryonirsp_l0_fits_access import CryonirspL0FitsAccess
6
+ from dkist_processing_cryonirsp.parsers.cryonirsp_l0_fits_access import CryonirspRampFitsAccess
7
+ from dkist_processing_cryonirsp.parsers.cryonirsp_l1_fits_access import CryonirspL1FitsAccess
8
+ from dkist_processing_cryonirsp.tests.header_models import Cryonirsp122ObserveFrames
9
+
10
+
11
+ @pytest.fixture(scope="session")
12
+ def complete_header():
13
+ dataset = Cryonirsp122ObserveFrames(array_shape=(1, 2, 2))
14
+ header = [translate_spec122_to_spec214_l0(d.header()) for d in dataset]
15
+ return header
16
+
17
+
18
+ def test_metadata_keys_in_access_bases(complete_header):
19
+ """
20
+ Given: the set of metadata key names in CryonirspMetadataKey
21
+ When: the Cryo FITS access classes define a set of new attributes
22
+ Then: the sets are the same and the attributes have the correct values
23
+ """
24
+ cryo_metadata_key_names = {cmk.name for cmk in CryonirspMetadataKey}
25
+ all_cryo_fits_access_attrs = set()
26
+ for access_class in [CryonirspRampFitsAccess, CryonirspL0FitsAccess, CryonirspL1FitsAccess]:
27
+ fits_obj = access_class.from_header(complete_header[0])
28
+ cryo_instance_attrs = set(vars(fits_obj).keys())
29
+ parent_class = access_class.mro()[1]
30
+ parent_fits_obj = parent_class.from_header(complete_header[0])
31
+ parent_instance_attrs = set(vars(parent_fits_obj).keys())
32
+ cryo_fits_access_attrs = cryo_instance_attrs - parent_instance_attrs
33
+ # exposure_conditions is created from multiple header keys, so it is not in CryonirspMetadataKey:
34
+ cryo_fits_access_attrs -= {"exposure_conditions"}
35
+ for attr in cryo_fits_access_attrs:
36
+ match attr:
37
+ case "cn1_scan_step":
38
+ assert getattr(fits_obj, attr) == int(
39
+ fits_obj.header[CryonirspMetadataKey[attr]]
40
+ )
41
+ case _:
42
+ assert getattr(fits_obj, attr) == fits_obj.header[CryonirspMetadataKey[attr]]
43
+ all_cryo_fits_access_attrs |= cryo_fits_access_attrs
44
+ assert cryo_metadata_key_names == all_cryo_fits_access_attrs
@@ -8,15 +8,14 @@ from dkist_header_validator import spec122_validator
8
8
  from dkist_processing_common._util.scratch import WorkflowFileSystem
9
9
  from dkist_processing_common.codecs.fits import fits_array_encoder
10
10
  from dkist_processing_common.codecs.fits import fits_hdulist_encoder
11
- from dkist_processing_common.tests.conftest import FakeGQLClient
12
11
 
13
12
  from dkist_processing_cryonirsp.models.exposure_conditions import AllowableOpticalDensityFilterNames
14
13
  from dkist_processing_cryonirsp.models.exposure_conditions import ExposureConditions
15
14
  from dkist_processing_cryonirsp.models.tags import CryonirspTag
16
15
  from dkist_processing_cryonirsp.tasks.gain import CISolarGainCalibration
17
16
  from dkist_processing_cryonirsp.tasks.gain import LampGainCalibration
18
- from dkist_processing_cryonirsp.tests.conftest import cryonirsp_testing_parameters_factory
19
17
  from dkist_processing_cryonirsp.tests.conftest import CryonirspConstantsDb
18
+ from dkist_processing_cryonirsp.tests.conftest import cryonirsp_testing_parameters_factory
20
19
  from dkist_processing_cryonirsp.tests.conftest import generate_fits_frame
21
20
  from dkist_processing_cryonirsp.tests.header_models import CryonirspHeadersValidLampGainFrames
22
21
 
@@ -43,7 +42,7 @@ def ci_solar_gain_calibration_task(
43
42
  task.scratch = WorkflowFileSystem(
44
43
  scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
45
44
  )
46
- param_class = cryonirsp_testing_parameters_factory(param_path=tmp_path)
45
+ param_class = cryonirsp_testing_parameters_factory(task)
47
46
  assign_input_dataset_doc_to_task(task, param_class())
48
47
  # Need a beam boundary file
49
48
  task.write(
@@ -126,7 +125,7 @@ def lamp_calibration_task(
126
125
  task.scratch = WorkflowFileSystem(
127
126
  scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
128
127
  )
129
- param_class = cryonirsp_testing_parameters_factory(param_path=tmp_path)
128
+ param_class = cryonirsp_testing_parameters_factory(task)
130
129
  assign_input_dataset_doc_to_task(task, param_class())
131
130
  # Need a beam boundary file
132
131
  task.write(
@@ -201,14 +200,14 @@ def lamp_calibration_task(
201
200
  task._purge()
202
201
 
203
202
 
204
- def test_ci_solar_gain_calibration_task(ci_solar_gain_calibration_task, mocker):
203
+ def test_ci_solar_gain_calibration_task(ci_solar_gain_calibration_task, mocker, fake_gql_client):
205
204
  """
206
205
  Given: A CISolarGainCalibration task
207
206
  When: Calling the task instance
208
207
  Then: The correct number of output solar gain frames exists, are tagged correctly, and are not normalized
209
208
  """
210
209
  mocker.patch(
211
- "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
210
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
212
211
  )
213
212
  # When
214
213
  task, solar_signal = ci_solar_gain_calibration_task
@@ -242,14 +241,14 @@ def test_ci_solar_gain_calibration_task(ci_solar_gain_calibration_task, mocker):
242
241
 
243
242
 
244
243
  @pytest.mark.parametrize("number_of_beams", [pytest.param(1, id="CI"), pytest.param(2, id="SP")])
245
- def test_lamp_calibration_task(lamp_calibration_task, number_of_beams, mocker):
244
+ def test_lamp_calibration_task(lamp_calibration_task, number_of_beams, mocker, fake_gql_client):
246
245
  """
247
246
  Given: A LampGainCalibration task
248
247
  When: Calling the task instance
249
248
  Then: The correct number of output lamp gain frames exists, are tagged correctly, and are normalized
250
249
  """
251
250
  mocker.patch(
252
- "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
251
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
253
252
  )
254
253
  # When
255
254
  task = lamp_calibration_task
@@ -10,7 +10,6 @@ from dkist_processing_common._util.scratch import WorkflowFileSystem
10
10
  from dkist_processing_common.codecs.fits import fits_array_encoder
11
11
  from dkist_processing_common.codecs.fits import fits_hdulist_encoder
12
12
  from dkist_processing_common.models.task_name import TaskName
13
- from dkist_processing_common.tests.conftest import FakeGQLClient
14
13
  from dkist_processing_pac.fitter.polcal_fitter import PolcalFitter
15
14
  from dkist_processing_pac.input_data.dresser import Dresser
16
15
 
@@ -23,8 +22,8 @@ from dkist_processing_cryonirsp.tasks.instrument_polarization import (
23
22
  from dkist_processing_cryonirsp.tasks.instrument_polarization import (
24
23
  SPInstrumentPolarizationCalibration,
25
24
  )
26
- from dkist_processing_cryonirsp.tests.conftest import cryonirsp_testing_parameters_factory
27
25
  from dkist_processing_cryonirsp.tests.conftest import CryonirspConstantsDb
26
+ from dkist_processing_cryonirsp.tests.conftest import cryonirsp_testing_parameters_factory
28
27
  from dkist_processing_cryonirsp.tests.conftest import generate_fits_frame
29
28
  from dkist_processing_cryonirsp.tests.header_models import CryonirspHeadersValidPolcalFrames
30
29
 
@@ -172,7 +171,7 @@ def ci_instrument_polarization_calibration_task(
172
171
  task.scratch = WorkflowFileSystem(
173
172
  scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
174
173
  )
175
- param_class = cryonirsp_testing_parameters_factory(param_path=tmp_path)
174
+ param_class = cryonirsp_testing_parameters_factory(task)
176
175
  assign_input_dataset_doc_to_task(task, param_class())
177
176
  mocker.patch(
178
177
  "dkist_processing_cryonirsp.tasks.instrument_polarization.PolcalFitter",
@@ -247,7 +246,10 @@ def ci_instrument_polarization_calibration_task_with_no_data(
247
246
  workflow_version="VX.Y",
248
247
  ) as task:
249
248
  try: # This try... block is here to make sure the dbs get cleaned up if there's a failure in the fixture
250
- param_class = cryonirsp_testing_parameters_factory(param_path=tmp_path)
249
+ task.scratch = WorkflowFileSystem(
250
+ scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
251
+ )
252
+ param_class = cryonirsp_testing_parameters_factory(task)
251
253
  assign_input_dataset_doc_to_task(task, param_class())
252
254
  yield task
253
255
  finally:
@@ -291,7 +293,7 @@ def sp_instrument_polarization_calibration_task(
291
293
  task.scratch = WorkflowFileSystem(
292
294
  scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
293
295
  )
294
- param_class = cryonirsp_testing_parameters_factory(param_path=tmp_path)
296
+ param_class = cryonirsp_testing_parameters_factory(task)
295
297
  assign_input_dataset_doc_to_task(task, param_class())
296
298
  mocker.patch(
297
299
  "dkist_processing_cryonirsp.tasks.instrument_polarization.PolcalFitter",
@@ -370,7 +372,10 @@ def sp_instrument_polarization_calibration_task_with_no_data(
370
372
  workflow_version="VX.Y",
371
373
  ) as task:
372
374
  try: # This try... block is here to make sure the dbs get cleaned up if there's a failure in the fixture
373
- param_class = cryonirsp_testing_parameters_factory(param_path=tmp_path)
375
+ task.scratch = WorkflowFileSystem(
376
+ scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
377
+ )
378
+ param_class = cryonirsp_testing_parameters_factory(task)
374
379
  assign_input_dataset_doc_to_task(task, param_class())
375
380
  yield task
376
381
  finally:
@@ -393,7 +398,9 @@ def multiple_demodulation_matrices() -> np.ndarray:
393
398
 
394
399
 
395
400
  def test_ci_instrument_polarization_calibration_task(
396
- ci_instrument_polarization_calibration_task, mocker
401
+ ci_instrument_polarization_calibration_task,
402
+ mocker,
403
+ fake_gql_client,
397
404
  ):
398
405
  """
399
406
  Given: An InstrumentPolarizationCalibration task
@@ -402,7 +409,7 @@ def test_ci_instrument_polarization_calibration_task(
402
409
  """
403
410
 
404
411
  mocker.patch(
405
- "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
412
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
406
413
  )
407
414
 
408
415
  # When
@@ -432,7 +439,9 @@ def test_ci_instrument_polarization_calibration_task(
432
439
 
433
440
 
434
441
  def test_sp_instrument_polarization_calibration_task(
435
- sp_instrument_polarization_calibration_task, mocker
442
+ sp_instrument_polarization_calibration_task,
443
+ mocker,
444
+ fake_gql_client,
436
445
  ):
437
446
  """
438
447
  Given: An InstrumentPolarizationCalibration task
@@ -441,7 +450,7 @@ def test_sp_instrument_polarization_calibration_task(
441
450
  """
442
451
 
443
452
  mocker.patch(
444
- "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
453
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
445
454
  )
446
455
 
447
456
  # When
@@ -1,4 +1,5 @@
1
1
  """Test the linearity correction task."""
2
+
2
3
  import re
3
4
  from dataclasses import dataclass
4
5
  from datetime import datetime
@@ -10,15 +11,14 @@ from dkist_header_validator import spec122_validator
10
11
  from dkist_processing_common._util.scratch import WorkflowFileSystem
11
12
  from dkist_processing_common.codecs.fits import fits_hdulist_encoder
12
13
  from dkist_processing_common.models.tags import Tag
13
- from dkist_processing_common.tests.conftest import FakeGQLClient
14
14
  from dkist_service_configuration.logging import logger
15
15
 
16
16
  from dkist_processing_cryonirsp.models.constants import CryonirspBudName
17
17
  from dkist_processing_cryonirsp.models.exposure_conditions import AllowableOpticalDensityFilterNames
18
18
  from dkist_processing_cryonirsp.models.tags import CryonirspTag
19
19
  from dkist_processing_cryonirsp.tasks.linearity_correction import LinearityCorrection
20
- from dkist_processing_cryonirsp.tests.conftest import cryonirsp_testing_parameters_factory
21
20
  from dkist_processing_cryonirsp.tests.conftest import CryonirspConstantsDb
21
+ from dkist_processing_cryonirsp.tests.conftest import cryonirsp_testing_parameters_factory
22
22
  from dkist_processing_cryonirsp.tests.conftest import generate_fits_frame
23
23
  from dkist_processing_cryonirsp.tests.header_models import CryonirspHeadersValidNonLinearizedFrames
24
24
 
@@ -56,7 +56,7 @@ def linearity_correction(
56
56
  task.scratch = WorkflowFileSystem(
57
57
  scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
58
58
  )
59
- param_class = cryonirsp_testing_parameters_factory(param_path=tmp_path)
59
+ param_class = cryonirsp_testing_parameters_factory(task)
60
60
  assign_input_dataset_doc_to_task(task, param_class())
61
61
  ds = CryonirspHeadersValidNonLinearizedFrames(
62
62
  arm_id=arm_id,
@@ -143,6 +143,7 @@ def linearity_correction(
143
143
  def test_linearity_correction(
144
144
  linearity_correction,
145
145
  mocker,
146
+ fake_gql_client,
146
147
  arm_id,
147
148
  frames_in_ramp,
148
149
  num_chunks,
@@ -154,7 +155,7 @@ def test_linearity_correction(
154
155
  Then: The non-linearized frames are linearized and produce the correct results.
155
156
  """
156
157
  mocker.patch(
157
- "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
158
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
158
159
  )
159
160
  if num_chunks == 2:
160
161
  mocker.patch(
@@ -5,7 +5,6 @@ from astropy.io import fits
5
5
  from dkist_header_validator import spec122_validator
6
6
  from dkist_processing_common._util.scratch import WorkflowFileSystem
7
7
  from dkist_processing_common.codecs.fits import fits_hdulist_encoder
8
- from dkist_processing_common.tests.conftest import FakeGQLClient
9
8
 
10
9
  from dkist_processing_cryonirsp.models.tags import CryonirspTag
11
10
  from dkist_processing_cryonirsp.tasks.make_movie_frames import MakeCryonirspMovieFrames
@@ -90,14 +89,14 @@ def movie_frames_task(tmp_path, recipe_run_id, init_cryonirsp_constants_db, requ
90
89
  task._purge()
91
90
 
92
91
 
93
- def test_make_movie_frames(movie_frames_task, mocker):
92
+ def test_make_movie_frames(movie_frames_task, mocker, fake_gql_client):
94
93
  """
95
94
  Given: A MakeCryonirspMovieFrames task
96
95
  When: Calling the task instance
97
96
  Then: a fits file is made for each scan containing the movie frame for that scan
98
97
  """
99
98
  mocker.patch(
100
- "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
99
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
101
100
  )
102
101
  task, map_scans, scan_steps, array_shape, is_polarimetric = movie_frames_task
103
102
  expected_shape = array_shape
@@ -5,11 +5,14 @@ import numpy as np
5
5
  import pytest
6
6
  from astropy.units import Quantity
7
7
  from dkist_processing_common._util.scratch import WorkflowFileSystem
8
+ from dkist_processing_common.codecs.array import array_decoder
9
+ from dkist_processing_common.models.input_dataset import InputDatasetFilePointer
10
+ from hypothesis import HealthCheck
8
11
  from hypothesis import example
9
12
  from hypothesis import given
10
- from hypothesis import HealthCheck
11
13
  from hypothesis import settings
12
14
  from hypothesis import strategies as st
15
+ from pydantic import BaseModel
13
16
 
14
17
  from dkist_processing_cryonirsp.models.parameters import CryonirspParameters
15
18
  from dkist_processing_cryonirsp.models.parameters import CryonirspParsingParameters
@@ -17,10 +20,9 @@ from dkist_processing_cryonirsp.parsers.optical_density_filters import (
17
20
  ALLOWABLE_OPTICAL_DENSITY_FILTERS,
18
21
  )
19
22
  from dkist_processing_cryonirsp.tasks.cryonirsp_base import CryonirspTaskBase
20
- from dkist_processing_cryonirsp.tests.conftest import cryonirsp_testing_parameters_factory
21
23
  from dkist_processing_cryonirsp.tests.conftest import CryonirspConstantsDb
22
- from dkist_processing_cryonirsp.tests.conftest import FileParameter
23
24
  from dkist_processing_cryonirsp.tests.conftest import TestingParameters
25
+ from dkist_processing_cryonirsp.tests.conftest import cryonirsp_testing_parameters_factory
24
26
 
25
27
  # The property names of all parameters on `CryonirspParsingParameters`
26
28
  PARSE_PARAMETER_NAMES = [
@@ -42,20 +44,18 @@ def basic_science_task_with_parameter_mixin(
42
44
  obs_ip_start_time: str = testing_obs_ip_start_time,
43
45
  ):
44
46
  class Task(CryonirspTaskBase):
45
- def run(self):
46
- ...
47
+ def run(self): ...
47
48
 
48
49
  init_cryonirsp_constants_db(recipe_run_id, CryonirspConstantsDb())
49
- task = Task(
50
+ with Task(
50
51
  recipe_run_id=recipe_run_id,
51
52
  workflow_name="parse_cryonirsp_input_data",
52
53
  workflow_version="VX.Y",
53
- )
54
- try: # This try... block is here to make sure the dbs get cleaned up if there's a failure in the fixture
54
+ ) as task:
55
55
  task.scratch = WorkflowFileSystem(
56
- scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
56
+ scratch_base_path=tmp_path, recipe_run_id=task.recipe_run_id
57
57
  )
58
- test_params = cryonirsp_testing_parameters_factory(param_path=tmp_path)
58
+ test_params = cryonirsp_testing_parameters_factory(task)
59
59
  param_dict = test_params()
60
60
  assign_input_dataset_doc_to_task(
61
61
  task,
@@ -65,7 +65,6 @@ def basic_science_task_with_parameter_mixin(
65
65
  obs_ip_start_time=obs_ip_start_time,
66
66
  )
67
67
  yield task, param_dict
68
- finally:
69
68
  task._purge()
70
69
 
71
70
  return make_task
@@ -94,10 +93,11 @@ def test_filter_parameters(basic_science_task_with_parameter_mixin, wave):
94
93
  }
95
94
  for param in expected:
96
95
  assert getattr(task_params, param) == expected[param]
96
+ task._purge()
97
97
 
98
98
 
99
99
  def _is_file_param(param_value: Any) -> bool:
100
- return isinstance(param_value, dict) and "is_file" in param_value and param_value["is_file"]
100
+ return isinstance(param_value, InputDatasetFilePointer)
101
101
 
102
102
 
103
103
  def test_file_parameters(basic_science_task_with_parameter_mixin):
@@ -115,18 +115,11 @@ def test_file_parameters(basic_science_task_with_parameter_mixin):
115
115
  for pn, pv in asdict(test_params).items():
116
116
  # We want to test only file parameters
117
117
  if _is_file_param(pv):
118
- pn_no_prefix = pn.removeprefix("cryonirsp_")
119
- # If the param name is an attribute in task_params, then load it directly
120
- if hasattr(task_params, pn_no_prefix):
121
- param_name = pn_no_prefix
122
- actual = getattr(task_params, param_name)
123
- # if the param name is not a task param attribute, then check that we can load the param
124
- # using the value defined in the input_dataset_parameters list of the task param object
125
- else:
126
- param_dict = task_params._find_most_recent_past_value(pn)
127
- actual = task_params._load_param_value_from_numpy_save(param_dict)
128
- # Now get the expected value using the param value dict from the testing params
129
- expected = np.load(pv["param_path"])
118
+ # Get the parameter value the way the tasks do
119
+ param_obj = task_params._find_most_recent_past_value(pn)
120
+ actual = task_params._load_param_value_from_numpy_save(param_obj=param_obj)
121
+ # Now get the expected value using the tag in the value from the testing params
122
+ expected = next(task.read(tags=pv.file_pointer.tag, decoder=array_decoder))
130
123
  # Compare the actual and expected values
131
124
  assert np.array_equal(actual, expected)
132
125
 
@@ -178,12 +171,11 @@ def test_arm_parameters(basic_science_task_with_parameter_mixin, arm_id):
178
171
  suffix = f"_{arm_id}".casefold()
179
172
  if _is_arm_param(pn, task_params, test_params, single_arm_only=arm_id):
180
173
  generic_param_name = pn.removeprefix("cryonirsp_").removesuffix(suffix)
181
-
182
174
  actual = getattr(task_params, generic_param_name)
183
175
  expected = getattr(test_params, pn)
184
- if isinstance(expected, FileParameter) and expected.is_file:
185
- expected = task_params._load_param_value_from_numpy_save(asdict(expected))
186
- assert np.array_equal(expected, actual)
176
+ if isinstance(expected, InputDatasetFilePointer):
177
+ expected_array = task_params._load_param_value_from_numpy_save(param_obj=expected)
178
+ assert np.array_equal(expected_array, actual)
187
179
  elif isinstance(actual, np.ndarray):
188
180
  assert expected == actual.tolist()
189
181
  else:
@@ -207,9 +199,12 @@ def test_parameters(basic_science_task_with_parameter_mixin):
207
199
  or parameter_name in PARSE_PARAMETER_NAMES
208
200
  ):
209
201
  continue
202
+
210
203
  accessed_parameter_value = getattr(task_params, parameter_name)
211
204
  if isinstance(accessed_parameter_value, Quantity):
212
205
  assert pv == accessed_parameter_value.value
206
+ elif isinstance(accessed_parameter_value, BaseModel):
207
+ assert accessed_parameter_value.model_dump() == pv
213
208
  else:
214
209
  assert pv == getattr(task_params, parameter_name)
215
210
 
@@ -11,7 +11,6 @@ from astropy.io import fits
11
11
  from dkist_processing_common._util.scratch import WorkflowFileSystem
12
12
  from dkist_processing_common.models.constants import BudName
13
13
  from dkist_processing_common.tasks import WorkflowTaskBase
14
- from dkist_processing_common.tests.conftest import FakeGQLClient
15
14
 
16
15
  from dkist_processing_cryonirsp.models.constants import CryonirspBudName
17
16
  from dkist_processing_cryonirsp.models.exposure_conditions import AllowableOpticalDensityFilterNames
@@ -88,6 +87,8 @@ def write_solar_gain_frames_to_task(
88
87
  array_shape=(2, 2, 1),
89
88
  tags: list[str] | None = None,
90
89
  num_modstates: int = 1,
90
+ center_wavelength: float = 1080.0,
91
+ slit_width: float = 52.0,
91
92
  ):
92
93
  num_frames = 0
93
94
  for modstate in range(1, num_modstates + 1):
@@ -96,6 +97,8 @@ def write_solar_gain_frames_to_task(
96
97
  exposure_condition=exposure_condition,
97
98
  num_modstates=num_modstates,
98
99
  modstate=modstate,
100
+ center_wavelength=center_wavelength,
101
+ slit_width=slit_width,
99
102
  )
100
103
 
101
104
  num_frames += _write_frames_to_task(
@@ -125,7 +128,8 @@ def write_polcal_frames_to_task(
125
128
  modstate=mod_state,
126
129
  array_shape=array_shape,
127
130
  exposure_condition=exposure_condition,
128
- extra_headers=extra_headers,
131
+ extra_headers={"PAC__006": "clear" if num_frames % 2 else "Cool retarder"}
132
+ | extra_headers,
129
133
  )
130
134
 
131
135
  _write_frames_to_task(task=task, frame_generator=frame_generator, extra_tags=tags)
@@ -145,6 +149,8 @@ def write_observe_frames_to_task(
145
149
  exposure_condition: ExposureConditions,
146
150
  change_translated_headers: Callable[[fits.Header | None], fits.Header] = lambda x: x,
147
151
  array_shape=(2, 2, 1),
152
+ center_wavelength: float = 1080.0,
153
+ slit_width: float = 52.0,
148
154
  tags: list[str] | None = None,
149
155
  ):
150
156
  num_frames = 0
@@ -173,6 +179,8 @@ def write_observe_frames_to_task(
173
179
  num_meas=num_measurements,
174
180
  meas_num=measurement,
175
181
  arm_id=arm_id,
182
+ center_wavelength=center_wavelength,
183
+ slit_width=slit_width,
176
184
  )
177
185
  start_time += frame_delta_time
178
186
 
@@ -249,8 +257,12 @@ def make_linearized_test_frames(
249
257
  num_map_scans: int = 1,
250
258
  num_sub_repeats: int = 1,
251
259
  num_measurements: int = 1,
260
+ center_wavelength: float = 1080.0,
261
+ slit_width: float = 52.0,
252
262
  extra_headers: dict | None = None,
253
263
  ):
264
+ if extra_headers is None:
265
+ extra_headers = dict()
254
266
  num_dark = 0
255
267
  num_polcal = 0
256
268
  num_obs = 0
@@ -268,13 +280,15 @@ def make_linearized_test_frames(
268
280
  task,
269
281
  tags=lin_tag,
270
282
  exposure_condition=lamp_exposure_condition,
271
- num_modstates=num_modstates or 1, # We *always* need dark frames
283
+ num_modstates=num_modstates or 1, # We *always* need lamp frames
272
284
  )
273
285
  num_solar = write_solar_gain_frames_to_task(
274
286
  task,
275
287
  tags=lin_tag,
276
288
  exposure_condition=solar_exposure_condition,
277
- num_modstates=num_modstates or 1, # We *always* need dark frames
289
+ num_modstates=num_modstates or 1, # We *always* need solar frames
290
+ center_wavelength=center_wavelength,
291
+ slit_width=slit_width,
278
292
  )
279
293
 
280
294
  num_polcal += write_polcal_frames_to_task(
@@ -296,6 +310,8 @@ def make_linearized_test_frames(
296
310
  num_measurements=num_measurements,
297
311
  tags=lin_tag,
298
312
  change_translated_headers=change_translated_headers,
313
+ center_wavelength=center_wavelength,
314
+ slit_width=slit_width,
299
315
  )
300
316
 
301
317
  return num_dark, num_lamp, num_solar, num_polcal, num_obs
@@ -328,10 +344,10 @@ def make_non_linearized_test_frames(
328
344
 
329
345
  @pytest.fixture
330
346
  def parse_linearized_task(
331
- tmp_path, recipe_run_id, assign_input_dataset_doc_to_task, mocker, arm_id
347
+ tmp_path, recipe_run_id, assign_input_dataset_doc_to_task, mocker, fake_gql_client, arm_id
332
348
  ):
333
349
  mocker.patch(
334
- "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
350
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
335
351
  )
336
352
  if arm_id == "CI":
337
353
  parsing_class = ParseL0CryonirspCILinearizedData
@@ -346,7 +362,7 @@ def parse_linearized_task(
346
362
  task.scratch = WorkflowFileSystem(
347
363
  scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
348
364
  )
349
- param_dataclass = cryonirsp_testing_parameters_factory(param_path=tmp_path)
365
+ param_dataclass = cryonirsp_testing_parameters_factory(task)
350
366
  assign_input_dataset_doc_to_task(
351
367
  task,
352
368
  param_dataclass(),
@@ -359,9 +375,9 @@ def parse_linearized_task(
359
375
 
360
376
 
361
377
  @pytest.fixture
362
- def parse_non_linearized_task(tmp_path, recipe_run_id, mocker):
378
+ def parse_non_linearized_task(tmp_path, recipe_run_id, mocker, fake_gql_client):
363
379
  mocker.patch(
364
- "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
380
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
365
381
  )
366
382
  with ParseL0CryonirspRampData(
367
383
  recipe_run_id=recipe_run_id,
@@ -964,6 +980,7 @@ def test_parse_cryonirsp_linearized_incomplete_raster_scan(
964
980
 
965
981
  assert task.constants._db_dict[CryonirspBudName.num_scan_steps.value] == num_scan_steps - 1
966
982
  assert task.constants._db_dict[CryonirspBudName.num_map_scans.value] == num_map_scans
983
+ assert task.constants._db_dict[CryonirspBudName.num_meas.value] == 2
967
984
 
968
985
 
969
986
  @pytest.mark.parametrize("arm_id", ["CI", "SP"])
@@ -1197,6 +1214,8 @@ def test_parse_cryonirsp_linearized_data_constants(
1197
1214
  num_scan_steps = 3
1198
1215
  num_map_scans = 2
1199
1216
  num_sub_repeats = 2
1217
+ center_wavelength = 1234.0
1218
+ slit_width = 6.28
1200
1219
 
1201
1220
  make_linearized_test_frames(
1202
1221
  task,
@@ -1210,6 +1229,8 @@ def test_parse_cryonirsp_linearized_data_constants(
1210
1229
  solar_exposure_condition=solar_exp_cond,
1211
1230
  polcal_exposure_condition=polcal_exp_cond,
1212
1231
  observe_exposure_condition=obs_exp_cond,
1232
+ center_wavelength=center_wavelength,
1233
+ slit_width=slit_width,
1213
1234
  )
1214
1235
 
1215
1236
  task()
@@ -1244,6 +1265,21 @@ def test_parse_cryonirsp_linearized_data_constants(
1244
1265
  assert task.constants._db_dict["MAXIMUM_CADENCE"] == 10
1245
1266
  assert task.constants._db_dict["MINIMUM_CADENCE"] == 10
1246
1267
  assert task.constants._db_dict["VARIANCE_CADENCE"] == 0
1268
+ assert task.constants._db_dict[BudName.retarder_name] == "Cool retarder"
1269
+ if arm_id == "SP":
1270
+ assert (
1271
+ task.constants._db_dict[CryonirspBudName.center_wavelength.value] == center_wavelength
1272
+ )
1273
+ assert task.constants._db_dict[CryonirspBudName.slit_width.value] == slit_width
1274
+ assert (
1275
+ task.constants._db_dict[CryonirspBudName.grating_constant.value] == 770970.3576216539
1276
+ ) # From `SimpleModulatedHeaders`
1277
+
1278
+ assert task.constants._db_dict["CAMERA_NAME"] == "camera_name"
1279
+ assert task.constants._db_dict["DARK_GOS_LEVEL3_STATUS"] == "lamp"
1280
+ assert task.constants._db_dict["SOLAR_GAIN_GOS_LEVEL3_STATUS"] == "clear"
1281
+ assert task.constants._db_dict["SOLAR_GAIN_NUM_RAW_FRAMES_PER_FPA"] == 10
1282
+ assert task.constants._db_dict["POLCAL_NUM_RAW_FRAMES_PER_FPA"] == 10
1247
1283
 
1248
1284
 
1249
1285
  @pytest.mark.parametrize("arm_id", ["SP"])
@@ -1277,9 +1313,9 @@ def test_parse_cryonirsp_linearized_data_internal_scan_loops_as_map_scan_and_sca
1277
1313
  num_alt_scan_steps = 3
1278
1314
 
1279
1315
  def make_dual_scan_loop_headers(translated_header):
1280
- translated_header[
1281
- "CNP2DSS"
1282
- ] = 0.0 # This triggers the parsing of the dual internal scan loops
1316
+ translated_header["CNP2DSS"] = (
1317
+ 0.0 # This triggers the parsing of the dual internal scan loops
1318
+ )
1283
1319
  translated_header["CNP1DNSP"] = num_alt_scan_steps # inner loop -- becomes num scan steps
1284
1320
  translated_header["CNP2DNSP"] = num_alt_maps # outer loop -- becomes num map scans
1285
1321
  translated_header["CNP1DCUR"] = (translated_header["CNCURSCN"] - 1) % num_alt_scan_steps + 1