dkist-processing-visp 3.3.0__py3-none-any.whl → 5.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. dkist_processing_visp/__init__.py +1 -0
  2. dkist_processing_visp/config.py +1 -0
  3. dkist_processing_visp/models/constants.py +52 -21
  4. dkist_processing_visp/models/fits_access.py +20 -0
  5. dkist_processing_visp/models/metric_code.py +10 -0
  6. dkist_processing_visp/models/parameters.py +129 -19
  7. dkist_processing_visp/models/tags.py +1 -0
  8. dkist_processing_visp/models/task_name.py +1 -0
  9. dkist_processing_visp/parsers/map_repeats.py +1 -0
  10. dkist_processing_visp/parsers/modulator_states.py +1 -0
  11. dkist_processing_visp/parsers/polarimeter_mode.py +3 -1
  12. dkist_processing_visp/parsers/raster_step.py +4 -1
  13. dkist_processing_visp/parsers/spectrograph_configuration.py +75 -0
  14. dkist_processing_visp/parsers/time.py +15 -7
  15. dkist_processing_visp/parsers/visp_l0_fits_access.py +19 -8
  16. dkist_processing_visp/parsers/visp_l1_fits_access.py +1 -0
  17. dkist_processing_visp/tasks/__init__.py +1 -0
  18. dkist_processing_visp/tasks/assemble_movie.py +1 -0
  19. dkist_processing_visp/tasks/background_light.py +2 -1
  20. dkist_processing_visp/tasks/dark.py +5 -4
  21. dkist_processing_visp/tasks/geometric.py +132 -20
  22. dkist_processing_visp/tasks/instrument_polarization.py +13 -12
  23. dkist_processing_visp/tasks/l1_output_data.py +203 -0
  24. dkist_processing_visp/tasks/lamp.py +53 -93
  25. dkist_processing_visp/tasks/make_movie_frames.py +8 -6
  26. dkist_processing_visp/tasks/mixin/beam_access.py +1 -0
  27. dkist_processing_visp/tasks/mixin/corrections.py +54 -4
  28. dkist_processing_visp/tasks/mixin/downsample.py +1 -0
  29. dkist_processing_visp/tasks/parse.py +34 -4
  30. dkist_processing_visp/tasks/quality_metrics.py +5 -4
  31. dkist_processing_visp/tasks/science.py +126 -46
  32. dkist_processing_visp/tasks/solar.py +896 -456
  33. dkist_processing_visp/tasks/visp_base.py +2 -0
  34. dkist_processing_visp/tasks/write_l1.py +25 -5
  35. dkist_processing_visp/tests/conftest.py +99 -35
  36. dkist_processing_visp/tests/header_models.py +92 -20
  37. dkist_processing_visp/tests/local_trial_workflows/l0_cals_only.py +4 -23
  38. dkist_processing_visp/tests/local_trial_workflows/l0_polcals_as_science.py +421 -0
  39. dkist_processing_visp/tests/local_trial_workflows/l0_solar_gain_as_science.py +10 -29
  40. dkist_processing_visp/tests/local_trial_workflows/l0_to_l1.py +1 -21
  41. dkist_processing_visp/tests/local_trial_workflows/local_trial_helpers.py +98 -14
  42. dkist_processing_visp/tests/test_assemble_movie.py +2 -3
  43. dkist_processing_visp/tests/test_assemble_quality.py +89 -4
  44. dkist_processing_visp/tests/test_background_light.py +8 -5
  45. dkist_processing_visp/tests/test_dark.py +4 -3
  46. dkist_processing_visp/tests/test_fits_access.py +43 -0
  47. dkist_processing_visp/tests/test_geometric.py +45 -4
  48. dkist_processing_visp/tests/test_instrument_polarization.py +4 -3
  49. dkist_processing_visp/tests/test_lamp.py +22 -26
  50. dkist_processing_visp/tests/test_make_movie_frames.py +4 -4
  51. dkist_processing_visp/tests/test_map_repeats.py +3 -1
  52. dkist_processing_visp/tests/test_parameters.py +122 -21
  53. dkist_processing_visp/tests/test_parse.py +98 -14
  54. dkist_processing_visp/tests/test_quality.py +2 -3
  55. dkist_processing_visp/tests/test_science.py +113 -15
  56. dkist_processing_visp/tests/test_solar.py +318 -99
  57. dkist_processing_visp/tests/test_visp_constants.py +36 -8
  58. dkist_processing_visp/tests/test_workflows.py +1 -0
  59. dkist_processing_visp/tests/test_write_l1.py +17 -3
  60. dkist_processing_visp/workflows/__init__.py +1 -0
  61. dkist_processing_visp/workflows/l0_processing.py +8 -2
  62. dkist_processing_visp/workflows/trial_workflows.py +8 -2
  63. dkist_processing_visp-5.1.1.dist-info/METADATA +552 -0
  64. dkist_processing_visp-5.1.1.dist-info/RECORD +94 -0
  65. docs/conf.py +5 -1
  66. docs/gain_correction.rst +50 -42
  67. dkist_processing_visp/tasks/mixin/line_zones.py +0 -115
  68. dkist_processing_visp-3.3.0.dist-info/METADATA +0 -459
  69. dkist_processing_visp-3.3.0.dist-info/RECORD +0 -90
  70. {dkist_processing_visp-3.3.0.dist-info → dkist_processing_visp-5.1.1.dist-info}/WHEEL +0 -0
  71. {dkist_processing_visp-3.3.0.dist-info → dkist_processing_visp-5.1.1.dist-info}/top_level.txt +0 -0
@@ -10,17 +10,20 @@ from dkist_header_validator import spec214_validator
10
10
  from dkist_processing_common.codecs.fits import fits_array_decoder
11
11
  from dkist_processing_common.codecs.fits import fits_hdulist_encoder
12
12
  from dkist_processing_common.models.constants import BudName
13
+ from dkist_processing_common.models.fits_access import MetadataKey
13
14
  from dkist_processing_common.models.task_name import TaskName
14
15
  from dkist_processing_common.parsers.cs_step import CSStepFlower
15
16
  from dkist_processing_common.parsers.cs_step import NumCSStepBud
16
17
  from dkist_processing_common.parsers.retarder import RetarderNameBud
17
- from dkist_processing_common.parsers.task import parse_header_ip_task_with_gains
18
18
  from dkist_processing_common.parsers.task import PolcalTaskFlower
19
19
  from dkist_processing_common.parsers.task import TaskTypeFlower
20
+ from dkist_processing_common.parsers.task import parse_header_ip_task_with_gains
20
21
  from dkist_processing_common.parsers.time import ExposureTimeFlower
21
22
  from dkist_processing_common.parsers.time import ReadoutExpTimeFlower
22
23
  from dkist_processing_common.parsers.time import TaskExposureTimesBud
23
24
  from dkist_processing_common.parsers.time import TaskReadoutExpTimesBud
25
+ from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
26
+ from dkist_processing_common.parsers.unique_bud import UniqueBud
24
27
  from dkist_processing_common.tasks import ParseL0InputDataBase
25
28
  from dkist_processing_common.tasks import WorkflowTaskBase
26
29
  from dkist_processing_common.tasks.mixin.globus import GlobusTransferItem
@@ -30,10 +33,14 @@ from loguru import logger
30
33
 
31
34
  from dkist_processing_visp.models.constants import VispBudName
32
35
  from dkist_processing_visp.models.constants import VispConstants
36
+ from dkist_processing_visp.models.fits_access import VispMetadataKey
37
+ from dkist_processing_visp.models.metric_code import VispMetricCode
33
38
  from dkist_processing_visp.models.parameters import VispParsingParameters
34
39
  from dkist_processing_visp.models.tags import VispTag
35
40
  from dkist_processing_visp.models.task_name import VispTaskName
36
41
  from dkist_processing_visp.parsers.modulator_states import ModulatorStateFlower
42
+ from dkist_processing_visp.parsers.spectrograph_configuration import IncidentLightAngleBud
43
+ from dkist_processing_visp.parsers.spectrograph_configuration import ReflectedLightAngleBud
37
44
  from dkist_processing_visp.parsers.time import DarkReadoutExpTimePickyBud
38
45
  from dkist_processing_visp.parsers.time import NonDarkNonPolcalTaskReadoutExpTimesBud
39
46
  from dkist_processing_visp.parsers.visp_l0_fits_access import VispL0FitsAccess
@@ -263,7 +270,9 @@ class SaveSolarCal(SaveTaskTags):
263
270
  @property
264
271
  def tag_lists_to_save(self) -> list[list[str]]:
265
272
  return super().tag_lists_to_save + [
266
- [VispTag.quality("TASK_TYPES"), VispTag.workflow_task("SolarCalibration")]
273
+ [VispTag.quality("TASK_TYPES"), VispTag.workflow_task("SolarCalibration")],
274
+ [VispTag.quality(VispMetricCode.solar_first_vignette)],
275
+ [VispTag.quality(VispMetricCode.solar_final_vignette)],
267
276
  ]
268
277
 
269
278
  @property
@@ -415,39 +424,55 @@ class ParseCalOnlyL0InputData(ParseL0InputDataBase):
415
424
  @property
416
425
  def constant_buds(self):
417
426
  """Add ViSP specific constants to common constants."""
427
+ # TODO: Subclass ViSP parse task and *remove* unneeded things from this list
418
428
  return super().constant_buds + [
429
+ UniqueBud(constant_name=VispBudName.arm_id.value, metadata_key=VispMetadataKey.arm_id),
419
430
  NumCSStepBud(self.parameters.max_cs_step_time_sec),
420
431
  NonDarkNonPolcalTaskReadoutExpTimesBud(),
421
432
  DarkReadoutExpTimePickyBud(),
422
433
  RetarderNameBud(),
434
+ IncidentLightAngleBud(),
435
+ ReflectedLightAngleBud(),
436
+ TaskUniqueBud(
437
+ constant_name=VispBudName.grating_constant_inverse_mm.value,
438
+ metadata_key=VispMetadataKey.grating_constant_inverse_mm,
439
+ ip_task_types=[TaskName.observe.value, TaskName.solar_gain.value],
440
+ task_type_parsing_function=parse_header_ip_task_with_gains,
441
+ ),
442
+ TaskUniqueBud(
443
+ constant_name=VispBudName.solar_gain_ip_start_time.value,
444
+ metadata_key=MetadataKey.ip_start_time,
445
+ ip_task_types=TaskName.solar_gain,
446
+ task_type_parsing_function=parse_header_ip_task_with_gains,
447
+ ),
423
448
  TaskExposureTimesBud(
424
449
  stem_name=VispBudName.lamp_exposure_times.value,
425
- ip_task_type=TaskName.lamp_gain.value,
450
+ ip_task_types=TaskName.lamp_gain.value,
426
451
  header_task_parsing_func=parse_header_ip_task_with_gains,
427
452
  ),
428
453
  TaskExposureTimesBud(
429
454
  stem_name=VispBudName.solar_exposure_times.value,
430
- ip_task_type=TaskName.solar_gain.value,
455
+ ip_task_types=TaskName.solar_gain.value,
431
456
  header_task_parsing_func=parse_header_ip_task_with_gains,
432
457
  ),
433
458
  TaskExposureTimesBud(
434
459
  stem_name=VispBudName.polcal_exposure_times.value,
435
- ip_task_type=TaskName.polcal.value,
460
+ ip_task_types=TaskName.polcal.value,
436
461
  header_task_parsing_func=parse_header_ip_task_with_gains,
437
462
  ),
438
463
  TaskReadoutExpTimesBud(
439
464
  stem_name=VispBudName.lamp_readout_exp_times.value,
440
- ip_task_type=TaskName.lamp_gain.value,
465
+ ip_task_types=TaskName.lamp_gain.value,
441
466
  header_task_parsing_func=parse_header_ip_task_with_gains,
442
467
  ),
443
468
  TaskReadoutExpTimesBud(
444
469
  stem_name=VispBudName.solar_readout_exp_times.value,
445
- ip_task_type=TaskName.solar_gain.value,
470
+ ip_task_types=TaskName.solar_gain.value,
446
471
  header_task_parsing_func=parse_header_ip_task_with_gains,
447
472
  ),
448
473
  TaskReadoutExpTimesBud(
449
474
  stem_name=VispBudName.polcal_readout_exp_times.value,
450
- ip_task_type=TaskName.polcal.value,
475
+ ip_task_types=TaskName.polcal.value,
451
476
  header_task_parsing_func=parse_header_ip_task_with_gains,
452
477
  ),
453
478
  ]
@@ -575,9 +600,9 @@ class TagSingleSolarGainAsScience(VispTaskBase):
575
600
  avg_array = average_numpy_arrays(arrays=arrays)
576
601
 
577
602
  hdul = fits.HDUList([fits.PrimaryHDU(data=avg_array, header=first_header)])
578
- hdul[0].header["VSPSTP"] = 0
579
- hdul[0].header["VSPNSTP"] = 1
580
- hdul[0].header["VSPSTNUM"] = 1
603
+ hdul[0].header[VispMetadataKey.raster_scan_step] = 0
604
+ hdul[0].header[VispMetadataKey.total_raster_steps] = 1
605
+ hdul[0].header[VispMetadataKey.modulator_state] = 1
581
606
  hdul[0].header["VSPPOLMD"] = "observe_intensity"
582
607
  # hdul[0].header["POL_NOIS"] = 0.666
583
608
  # hdul[0].header["POL_SENS"] = 0.666
@@ -626,9 +651,9 @@ class TagModulatedSolarGainsAsScience(VispTaskBase):
626
651
  avg_array = average_numpy_arrays(arrays=arrays)
627
652
 
628
653
  hdul = fits.HDUList([fits.PrimaryHDU(data=avg_array, header=first_header)])
629
- hdul[0].header["VSPSTP"] = 0
630
- hdul[0].header["VSPNSTP"] = 1
631
- hdul[0].header["VSPSTNUM"] = modstate
654
+ hdul[0].header[VispMetadataKey.raster_scan_step] = 0
655
+ hdul[0].header[VispMetadataKey.total_raster_steps] = 1
656
+ hdul[0].header[VispMetadataKey.modulator_state] = modstate
632
657
  hdul[0].header["VSPPOLMD"] = "observe_polarimetric"
633
658
 
634
659
  new_tags = [
@@ -691,3 +716,62 @@ def load_solar_gain_as_science_task(force_intensity_only: bool):
691
716
  logger.info(f"{self.constants.correct_for_polarization = }")
692
717
 
693
718
  return LoadSolarGainAsScience
719
+
720
+
721
+ class SavePolcalAsScience(WorkflowTaskBase):
722
+ constants: VispConstants
723
+
724
+ @property
725
+ def constants_model_class(self):
726
+ """Get ViSP pipeline constants."""
727
+ return VispConstants
728
+
729
+ @property
730
+ def tag_lists_to_save(self) -> list[str]:
731
+ return [VispTag.task_observe(), VispTag.input(), VispTag.frame()]
732
+
733
+ @property
734
+ def relative_save_file(self) -> str:
735
+ return "polcal_as_science.asdf"
736
+
737
+ def run(self):
738
+ file_tag_dict = dict()
739
+ tag_list_list = self.tag_lists_to_save
740
+ if isinstance(tag_list_list[0], str):
741
+ tag_list_list = [tag_list_list]
742
+
743
+ pcas_constants = {
744
+ VispBudName.num_map_scans.value: self.constants.num_map_scans,
745
+ VispBudName.num_raster_steps.value: self.constants.num_raster_steps,
746
+ }
747
+
748
+ for tags_to_save in tag_list_list:
749
+ path_list = self.read(tags=tags_to_save)
750
+ save_dir = self.scratch.workflow_base_path / Path(self.relative_save_file).stem
751
+ save_dir.mkdir(exist_ok=True)
752
+ for p in path_list:
753
+ copied_path = shutil.copy(str(p), save_dir)
754
+ tags = self.tags(p)
755
+ file_tag_dict[copied_path] = tags
756
+
757
+ full_save_file = self.scratch.workflow_base_path / self.relative_save_file
758
+ tree = {"file_tag_dict": file_tag_dict, "pcas_constants": pcas_constants}
759
+ af = asdf.AsdfFile(tree)
760
+ af.write_to(full_save_file)
761
+ logger.info(f"Saved polcal science frames to {full_save_file}")
762
+
763
+
764
+ class LoadPolcalAsScience(WorkflowTaskBase):
765
+ @property
766
+ def relative_save_file(self) -> str:
767
+ return "polcal_as_science.asdf"
768
+
769
+ def run(self):
770
+ full_save_file = self.scratch.workflow_base_path / self.relative_save_file
771
+ with asdf.open(full_save_file) as af:
772
+ for f, t in af.tree["file_tag_dict"].items():
773
+ self.tag(path=f, tags=t)
774
+
775
+ self.constants._db_dict.update(**af.tree["pcas_constants"])
776
+
777
+ logger.info(f"Loaded database entries from {full_save_file}")
@@ -1,6 +1,5 @@
1
1
  import pytest
2
2
  from dkist_processing_common._util.scratch import WorkflowFileSystem
3
- from dkist_processing_common.tests.conftest import FakeGQLClient
4
3
 
5
4
  from dkist_processing_visp.models.tags import VispTag
6
5
  from dkist_processing_visp.tasks.assemble_movie import AssembleVispMovie
@@ -49,9 +48,9 @@ def assemble_task_with_tagged_movie_frames(tmp_path, recipe_run_id, init_visp_co
49
48
  task._purge()
50
49
 
51
50
 
52
- def test_assemble_movie(assemble_task_with_tagged_movie_frames, mocker):
51
+ def test_assemble_movie(assemble_task_with_tagged_movie_frames, mocker, fake_gql_client):
53
52
  mocker.patch(
54
- "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
53
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
55
54
  )
56
55
  task, num_maps = assemble_task_with_tagged_movie_frames
57
56
  write_movie_frames_to_task(task, num_maps)
@@ -1,27 +1,65 @@
1
+ from typing import Generator
1
2
  from unittest.mock import MagicMock
2
3
 
4
+ import numpy as np
3
5
  import pytest
6
+ from dkist_processing_common._util.scratch import WorkflowFileSystem
7
+ from dkist_processing_common.codecs.asdf import asdf_encoder
8
+ from dkist_processing_common.codecs.quality import quality_data_decoder
4
9
 
10
+ from dkist_processing_visp.models.metric_code import VispMetricCode
11
+ from dkist_processing_visp.models.tags import VispTag
5
12
  from dkist_processing_visp.tasks.l1_output_data import VispAssembleQualityData
6
13
 
7
14
 
8
15
  @pytest.fixture
9
- def visp_assemble_quality_data_task(tmp_path, recipe_run_id) -> VispAssembleQualityData:
16
+ def visp_assemble_quality_data_task(
17
+ tmp_path, recipe_run_id
18
+ ) -> Generator[VispAssembleQualityData, None, None]:
10
19
 
11
20
  with VispAssembleQualityData(
12
21
  recipe_run_id=recipe_run_id, workflow_name="visp_assemble_quality", workflow_version="VX.Y"
13
22
  ) as task:
23
+ task.scratch = WorkflowFileSystem(scratch_base_path=tmp_path, recipe_run_id=recipe_run_id)
14
24
  yield task
15
25
  task._purge()
16
26
 
17
27
 
28
+ def write_raw_vignette_metrics_to_task(task):
29
+ for beam in [1, 2]:
30
+ dummy_vec = np.arange(10)
31
+ first_vignette_quality_outputs = {
32
+ "output_wave_vec": dummy_vec,
33
+ "input_spectrum": dummy_vec,
34
+ "best_fit_atlas": dummy_vec,
35
+ "best_fit_continuum": dummy_vec,
36
+ "residuals": dummy_vec,
37
+ }
38
+ task.write(
39
+ data=first_vignette_quality_outputs,
40
+ tags=[VispTag.quality(VispMetricCode.solar_first_vignette), VispTag.beam(beam)],
41
+ encoder=asdf_encoder,
42
+ )
43
+ final_correction_quality_outputs = {
44
+ "output_wave_vec": dummy_vec,
45
+ "median_spec": dummy_vec,
46
+ "low_deviation": dummy_vec,
47
+ "high_deviation": dummy_vec,
48
+ }
49
+ task.write(
50
+ data=final_correction_quality_outputs,
51
+ tags=[VispTag.quality(VispMetricCode.solar_final_vignette), VispTag.beam(beam)],
52
+ encoder=asdf_encoder,
53
+ )
54
+
55
+
18
56
  @pytest.fixture
19
57
  def dummy_quality_data() -> list[dict]:
20
58
  return [{"dummy_key": "dummy_value"}]
21
59
 
22
60
 
23
61
  @pytest.fixture
24
- def quality_assemble_data_mock(mocker, dummy_quality_data) -> MagicMock:
62
+ def common_quality_assemble_data_mock(mocker, dummy_quality_data) -> MagicMock:
25
63
  yield mocker.patch(
26
64
  "dkist_processing_common.tasks.mixin.quality.QualityMixin.quality_assemble_data",
27
65
  return_value=dummy_quality_data,
@@ -29,13 +67,60 @@ def quality_assemble_data_mock(mocker, dummy_quality_data) -> MagicMock:
29
67
  )
30
68
 
31
69
 
32
- def test_correct_polcal_label_list(visp_assemble_quality_data_task, quality_assemble_data_mock):
70
+ def test_vignette_metrics_built(visp_assemble_quality_data_task):
71
+ """
72
+ Given: A `VispAssembleQualityData` task with raw vignette metrics in scratch
73
+ When: Building the quality report data
74
+ Then: The vignette metrics are included in the data
75
+ """
76
+ task = visp_assemble_quality_data_task
77
+ write_raw_vignette_metrics_to_task(task)
78
+
79
+ task()
80
+
81
+ final_report_list = list(task.read(tags=VispTag.quality_data(), decoder=quality_data_decoder))
82
+ assert len(final_report_list) == 1
83
+ final_report = final_report_list[0]
84
+
85
+ initial_vignette_metrics = list(
86
+ filter(lambda i: i["name"].startswith("Initial Vignette Estimation"), final_report)
87
+ )
88
+ assert len(initial_vignette_metrics) == 2
89
+ facet_set = set()
90
+ for m in initial_vignette_metrics:
91
+ assert m["metric_code"] == VispMetricCode.solar_first_vignette.value
92
+ assert m["description"]
93
+ assert m["multi_plot_data"]
94
+ facet_set.add(m["facet"])
95
+
96
+ assert facet_set == {"BEAM_1", "BEAM_2"}
97
+
98
+ final_vignette_metrics = list(
99
+ filter(lambda i: i["name"].startswith("Final Vignette Estimation"), final_report)
100
+ )
101
+ assert len(final_vignette_metrics) == 2
102
+ facet_set = set()
103
+ for m in final_vignette_metrics:
104
+ assert m["metric_code"] == VispMetricCode.solar_final_vignette.value
105
+ assert m["description"]
106
+ assert m["multi_plot_data"]
107
+ facet_set.add(m["facet"])
108
+
109
+ assert facet_set == {"BEAM_1", "BEAM_2"}
110
+
111
+
112
+ def test_correct_polcal_label_list(
113
+ visp_assemble_quality_data_task, common_quality_assemble_data_mock
114
+ ):
33
115
  """
34
116
  Given: A VispAssembleQualityData task
35
117
  When: Calling the task
36
118
  Then: The correct polcal_label_list property is passed to .quality_assemble_data
37
119
  """
38
120
  task = visp_assemble_quality_data_task
121
+ write_raw_vignette_metrics_to_task(task)
39
122
 
40
123
  task()
41
- quality_assemble_data_mock.assert_called_once_with(task, polcal_label_list=["Beam 1", "Beam 2"])
124
+ common_quality_assemble_data_mock.assert_called_once_with(
125
+ task, polcal_label_list=["Beam 1", "Beam 2"]
126
+ )
@@ -12,7 +12,6 @@ from dkist_header_validator.translator import translate_spec122_to_spec214_l0
12
12
  from dkist_processing_common._util.scratch import WorkflowFileSystem
13
13
  from dkist_processing_common.codecs.fits import fits_array_encoder
14
14
  from dkist_processing_common.models.task_name import TaskName
15
- from dkist_processing_common.tests.conftest import FakeGQLClient
16
15
  from dkist_processing_pac.fitter.fitter_parameters import PolcalDresserParameters
17
16
  from dkist_processing_pac.input_data.drawer import Drawer
18
17
  from dkist_processing_pac.input_data.dresser import Dresser
@@ -401,6 +400,7 @@ def test_background_light_calibration_task(
401
400
  num_polcal_dark_files,
402
401
  background_light,
403
402
  mocker,
403
+ fake_gql_client,
404
404
  ):
405
405
  """
406
406
  Give: a BackgroundLightCalibrationTask with a valid set of polcal data
@@ -408,7 +408,7 @@ def test_background_light_calibration_task(
408
408
  Then: BACKGROUND intermediate frames are generated for each beam
409
409
  """
410
410
  mocker.patch(
411
- "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
411
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
412
412
  )
413
413
  task_generator = background_light_calibration_task_factory()
414
414
  task = next(task_generator)
@@ -442,6 +442,7 @@ def test_background_light_bad_exposure_times(
442
442
  background_light_calibration_task_factory,
443
443
  constants_bad_exp_times,
444
444
  mocker,
445
+ fake_gql_client,
445
446
  ):
446
447
  """
447
448
  Given: A set of data where the polcal exposure times don't match those of solar gain and observe
@@ -449,7 +450,7 @@ def test_background_light_bad_exposure_times(
449
450
  Then: An error is raised
450
451
  """
451
452
  mocker.patch(
452
- "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
453
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
453
454
  )
454
455
  task_generator = background_light_calibration_task_factory(constants=constants_bad_exp_times)
455
456
  task = next(task_generator)
@@ -461,6 +462,7 @@ def test_background_light_non_polarimetric_dataset(
461
462
  background_light_calibration_task_factory,
462
463
  constants_nor_polarimetric,
463
464
  mocker,
465
+ fake_gql_client,
464
466
  ):
465
467
  """
466
468
  Given: A dataset that is non-polarimetric (i.e., Stokes-I only)
@@ -468,7 +470,7 @@ def test_background_light_non_polarimetric_dataset(
468
470
  Then: Nothing is done and no files are written
469
471
  """
470
472
  mocker.patch(
471
- "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
473
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
472
474
  )
473
475
 
474
476
  task_generator = background_light_calibration_task_factory(constants=constants_nor_polarimetric)
@@ -485,6 +487,7 @@ def test_background_light_non_polarimetric_dataset(
485
487
  def test_background_light_switch_off(
486
488
  background_light_calibration_task_factory,
487
489
  mocker,
490
+ fake_gql_client,
488
491
  ):
489
492
  """
490
493
  Given: A task with the background light switch turned off
@@ -492,7 +495,7 @@ def test_background_light_switch_off(
492
495
  Then: Nothing is done and no files are written
493
496
  """
494
497
  mocker.patch(
495
- "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
498
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
496
499
  )
497
500
  task_generator = background_light_calibration_task_factory(background_on=False)
498
501
  task = next(task_generator)
@@ -6,7 +6,6 @@ import pytest
6
6
  from astropy.io import fits
7
7
  from dkist_processing_common._util.scratch import WorkflowFileSystem
8
8
  from dkist_processing_common.models.tags import Tag
9
- from dkist_processing_common.tests.conftest import FakeGQLClient
10
9
 
11
10
  from dkist_processing_visp.models.tags import VispTag
12
11
  from dkist_processing_visp.tasks.dark import DarkCalibration
@@ -90,14 +89,16 @@ def dark_calibration_task(tmp_path, init_visp_constants_db, recipe_run_id):
90
89
  task._purge()
91
90
 
92
91
 
93
- def test_dark_calibration_task(dark_calibration_task, assign_input_dataset_doc_to_task, mocker):
92
+ def test_dark_calibration_task(
93
+ dark_calibration_task, assign_input_dataset_doc_to_task, mocker, fake_gql_client
94
+ ):
94
95
  """
95
96
  Given: A DarkCalibration task with multiple task exposure times
96
97
  When: Calling the task instance
97
98
  Then: Only one average intermediate dark frame exists for each exposure time and unused times are not made
98
99
  """
99
100
  mocker.patch(
100
- "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
101
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
101
102
  )
102
103
  # Given
103
104
  task, readout_exp_times = dark_calibration_task
@@ -0,0 +1,43 @@
1
+ import pytest
2
+ from dkist_header_validator.translator import translate_spec122_to_spec214_l0
3
+
4
+ from dkist_processing_visp.models.fits_access import VispMetadataKey
5
+ from dkist_processing_visp.parsers.visp_l0_fits_access import VispL0FitsAccess
6
+ from dkist_processing_visp.parsers.visp_l1_fits_access import VispL1FitsAccess
7
+ from dkist_processing_visp.tests.header_models import VispHeadersValidObserveFrames
8
+
9
+
10
+ @pytest.fixture(scope="session")
11
+ def complete_header():
12
+ dataset = VispHeadersValidObserveFrames(
13
+ array_shape=(1, 1, 1),
14
+ time_delta=10,
15
+ num_maps=1,
16
+ num_raster_steps=1,
17
+ num_modstates=1,
18
+ )
19
+ header = translate_spec122_to_spec214_l0(dataset.header())
20
+ return header
21
+
22
+
23
+ def test_metadata_keys_in_access_bases(complete_header):
24
+ """
25
+ Given: the set of metadata key names in VispMetadataKey
26
+ When: the ViSP FITS access classes define a set of new attributes
27
+ Then: the sets are the same and the attributes have the correct values
28
+ """
29
+ # Given
30
+ visp_metadata_key_names = {vmk.name for vmk in VispMetadataKey}
31
+ # When
32
+ all_visp_fits_access_attrs = set()
33
+ for access_class in [VispL0FitsAccess, VispL1FitsAccess]:
34
+ fits_obj = access_class.from_header(complete_header)
35
+ visp_instance_attrs = set(vars(fits_obj).keys())
36
+ parent_class = access_class.mro()[1]
37
+ parent_fits_obj = parent_class.from_header(complete_header)
38
+ parent_instance_attrs = set(vars(parent_fits_obj).keys())
39
+ visp_fits_access_attrs = visp_instance_attrs - parent_instance_attrs
40
+ for attr in visp_fits_access_attrs:
41
+ assert getattr(fits_obj, attr) == fits_obj.header[VispMetadataKey[attr]]
42
+ all_visp_fits_access_attrs |= visp_fits_access_attrs
43
+ assert visp_metadata_key_names == all_visp_fits_access_attrs
@@ -7,14 +7,13 @@ import pytest
7
7
  from dkist_processing_common._util.scratch import WorkflowFileSystem
8
8
  from dkist_processing_common.codecs.fits import fits_array_decoder
9
9
  from dkist_processing_common.models.tags import Tag
10
- from dkist_processing_common.tests.conftest import FakeGQLClient
11
10
  from dkist_processing_math import transform
12
11
 
13
12
  from dkist_processing_visp.models.tags import VispTag
14
13
  from dkist_processing_visp.tasks.geometric import GeometricCalibration
15
- from dkist_processing_visp.tests.conftest import tag_on_modstate
16
14
  from dkist_processing_visp.tests.conftest import VispConstantsDb
17
15
  from dkist_processing_visp.tests.conftest import VispInputDatasetParameterValues
16
+ from dkist_processing_visp.tests.conftest import tag_on_modstate
18
17
  from dkist_processing_visp.tests.conftest import write_frames_to_task
19
18
  from dkist_processing_visp.tests.conftest import write_intermediate_darks_to_task
20
19
  from dkist_processing_visp.tests.header_models import VispHeadersInputLampGainFrames
@@ -189,7 +188,9 @@ def geometric_calibration_task(tmp_path, recipe_run_id, init_visp_constants_db):
189
188
  task._purge()
190
189
 
191
190
 
192
- def test_geometric_task(geometric_calibration_task, assign_input_dataset_doc_to_task, mocker):
191
+ def test_geometric_task(
192
+ geometric_calibration_task, assign_input_dataset_doc_to_task, mocker, fake_gql_client
193
+ ):
193
194
  """
194
195
  Given: A set of raw solar gain images and necessary intermediate calibrations
195
196
  When: Running the geometric task
@@ -201,7 +202,7 @@ def test_geometric_task(geometric_calibration_task, assign_input_dataset_doc_to_
201
202
  # tests for that. In other words, this fixture just tests if the machinery of the task completes and some object
202
203
  # (ANY object) is written correctly.
203
204
  mocker.patch(
204
- "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
205
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
205
206
  )
206
207
  task, readout_exp_times, num_modstates = geometric_calibration_task
207
208
  dark_signal = 3.0
@@ -357,3 +358,43 @@ def test_basic_corrections(geometric_calibration_task, assign_input_dataset_doc_
357
358
  lamp_array = task.basic_corrected_lamp_data(beam=beam, modstate=modstate)
358
359
  np.testing.assert_equal(expected, solar_array)
359
360
  np.testing.assert_equal(expected, lamp_array)
361
+
362
+
363
+ def test_line_zones(geometric_calibration_task):
364
+ """
365
+ Given: A spectrum with some absorption lines
366
+ When: Computing zones around the lines
367
+ Then: Correct results are returned
368
+ """
369
+
370
+ # NOTE that it does not test for removal of overlapping regions
371
+ def gaussian(x, amp, mu, sig):
372
+ return amp * np.exp(-np.power(x - mu, 2.0) / (2 * np.power(sig, 2.0)))
373
+
374
+ spec = np.ones(1000) * 100
375
+ x = np.arange(1000.0)
376
+ expected = []
377
+ for m, s in zip([100.0, 300.0, 700], [10.0, 20.0, 5.0]):
378
+ spec -= gaussian(x, 40, m, s)
379
+ hwhm = s * 2.355 / 2
380
+ expected.append((np.floor(m - hwhm).astype(int), np.ceil(m + hwhm).astype(int)))
381
+
382
+ task = geometric_calibration_task[0]
383
+
384
+ zones = task.compute_line_zones(spec[:, None], bg_order=0, rel_height=0.5)
385
+ assert zones == expected
386
+
387
+
388
+ def test_identify_overlapping_zones(geometric_calibration_task):
389
+ """
390
+ Given: A list of zone borders that contain overlapping zones
391
+ When: Identifying zones that overlap
392
+ Then: The smaller of the overlapping zones are identified for removal
393
+ """
394
+ rips = np.array([100, 110, 220, 200])
395
+ lips = np.array([150, 120, 230, 250])
396
+
397
+ task = geometric_calibration_task[0]
398
+
399
+ idx_to_remove = task.identify_overlapping_zones(rips, lips)
400
+ assert idx_to_remove == [1, 2]
@@ -6,7 +6,6 @@ import pytest
6
6
  from astropy.io import fits
7
7
  from dkist_data_simulator.spec122 import Spec122Dataset
8
8
  from dkist_processing_common._util.scratch import WorkflowFileSystem
9
- from dkist_processing_common.tests.conftest import FakeGQLClient
10
9
  from dkist_processing_pac.fitter.polcal_fitter import PolcalFitter
11
10
  from dkist_processing_pac.input_data.dresser import Dresser
12
11
 
@@ -213,6 +212,7 @@ def test_instrument_polarization_calibration_task(
213
212
  background_on,
214
213
  assign_input_dataset_doc_to_task,
215
214
  mocker,
215
+ fake_gql_client,
216
216
  ):
217
217
  """
218
218
  Given: An InstrumentPolarizationCalibration task
@@ -221,7 +221,7 @@ def test_instrument_polarization_calibration_task(
221
221
  """
222
222
 
223
223
  mocker.patch(
224
- "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
224
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
225
225
  )
226
226
  mocker.patch(
227
227
  "dkist_processing_visp.tasks.instrument_polarization.PolcalFitter",
@@ -255,7 +255,8 @@ def test_instrument_polarization_calibration_task(
255
255
  task=task, num_modstates=num_modstates, data_shape=intermediate_shape
256
256
  )
257
257
  write_dummy_intermediate_solar_cals_to_task(
258
- task=task, data_shape=intermediate_shape, num_modstates=num_modstates
258
+ task=task,
259
+ data_shape=intermediate_shape,
259
260
  )
260
261
  write_input_polcals_to_task(
261
262
  task=task,