dkist-processing-visp 2.20.14__py3-none-any.whl → 5.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. dkist_processing_visp/__init__.py +1 -0
  2. dkist_processing_visp/config.py +1 -0
  3. dkist_processing_visp/models/constants.py +61 -20
  4. dkist_processing_visp/models/fits_access.py +20 -0
  5. dkist_processing_visp/models/metric_code.py +10 -0
  6. dkist_processing_visp/models/parameters.py +129 -24
  7. dkist_processing_visp/models/tags.py +22 -1
  8. dkist_processing_visp/models/task_name.py +1 -0
  9. dkist_processing_visp/parsers/map_repeats.py +1 -0
  10. dkist_processing_visp/parsers/modulator_states.py +1 -0
  11. dkist_processing_visp/parsers/polarimeter_mode.py +4 -2
  12. dkist_processing_visp/parsers/raster_step.py +4 -1
  13. dkist_processing_visp/parsers/spectrograph_configuration.py +75 -0
  14. dkist_processing_visp/parsers/time.py +24 -14
  15. dkist_processing_visp/parsers/visp_l0_fits_access.py +19 -8
  16. dkist_processing_visp/parsers/visp_l1_fits_access.py +1 -0
  17. dkist_processing_visp/tasks/__init__.py +1 -0
  18. dkist_processing_visp/tasks/assemble_movie.py +1 -0
  19. dkist_processing_visp/tasks/background_light.py +2 -1
  20. dkist_processing_visp/tasks/dark.py +5 -4
  21. dkist_processing_visp/tasks/geometric.py +132 -20
  22. dkist_processing_visp/tasks/instrument_polarization.py +128 -18
  23. dkist_processing_visp/tasks/l1_output_data.py +203 -0
  24. dkist_processing_visp/tasks/lamp.py +53 -93
  25. dkist_processing_visp/tasks/make_movie_frames.py +8 -6
  26. dkist_processing_visp/tasks/mixin/beam_access.py +1 -0
  27. dkist_processing_visp/tasks/mixin/corrections.py +54 -4
  28. dkist_processing_visp/tasks/mixin/downsample.py +1 -0
  29. dkist_processing_visp/tasks/parse.py +50 -17
  30. dkist_processing_visp/tasks/quality_metrics.py +5 -4
  31. dkist_processing_visp/tasks/science.py +126 -46
  32. dkist_processing_visp/tasks/solar.py +896 -456
  33. dkist_processing_visp/tasks/visp_base.py +4 -3
  34. dkist_processing_visp/tasks/write_l1.py +38 -10
  35. dkist_processing_visp/tests/conftest.py +145 -47
  36. dkist_processing_visp/tests/header_models.py +157 -20
  37. dkist_processing_visp/tests/local_trial_workflows/l0_cals_only.py +21 -78
  38. dkist_processing_visp/tests/local_trial_workflows/l0_polcals_as_science.py +421 -0
  39. dkist_processing_visp/tests/local_trial_workflows/l0_solar_gain_as_science.py +387 -0
  40. dkist_processing_visp/tests/local_trial_workflows/l0_to_l1.py +18 -75
  41. dkist_processing_visp/tests/local_trial_workflows/local_trial_helpers.py +346 -14
  42. dkist_processing_visp/tests/test_assemble_movie.py +2 -3
  43. dkist_processing_visp/tests/test_assemble_quality.py +89 -4
  44. dkist_processing_visp/tests/test_background_light.py +51 -44
  45. dkist_processing_visp/tests/test_dark.py +4 -3
  46. dkist_processing_visp/tests/test_downsample.py +1 -0
  47. dkist_processing_visp/tests/test_fits_access.py +43 -0
  48. dkist_processing_visp/tests/test_geometric.py +45 -4
  49. dkist_processing_visp/tests/test_instrument_polarization.py +72 -9
  50. dkist_processing_visp/tests/test_lamp.py +22 -26
  51. dkist_processing_visp/tests/test_make_movie_frames.py +4 -4
  52. dkist_processing_visp/tests/test_map_repeats.py +3 -1
  53. dkist_processing_visp/tests/test_parameters.py +122 -21
  54. dkist_processing_visp/tests/test_parse.py +164 -18
  55. dkist_processing_visp/tests/test_quality.py +3 -4
  56. dkist_processing_visp/tests/test_science.py +113 -15
  57. dkist_processing_visp/tests/test_solar.py +318 -99
  58. dkist_processing_visp/tests/test_visp_constants.py +38 -8
  59. dkist_processing_visp/tests/test_workflows.py +1 -0
  60. dkist_processing_visp/tests/test_write_l1.py +22 -3
  61. dkist_processing_visp/workflows/__init__.py +1 -0
  62. dkist_processing_visp/workflows/l0_processing.py +10 -3
  63. dkist_processing_visp/workflows/trial_workflows.py +8 -2
  64. dkist_processing_visp-5.1.1.dist-info/METADATA +552 -0
  65. dkist_processing_visp-5.1.1.dist-info/RECORD +94 -0
  66. {dkist_processing_visp-2.20.14.dist-info → dkist_processing_visp-5.1.1.dist-info}/WHEEL +1 -1
  67. docs/conf.py +5 -1
  68. docs/gain_correction.rst +52 -44
  69. docs/science_calibration.rst +7 -0
  70. dkist_processing_visp/tasks/mixin/line_zones.py +0 -115
  71. dkist_processing_visp-2.20.14.dist-info/METADATA +0 -196
  72. dkist_processing_visp-2.20.14.dist-info/RECORD +0 -89
  73. {dkist_processing_visp-2.20.14.dist-info → dkist_processing_visp-5.1.1.dist-info}/top_level.txt +0 -0
@@ -4,30 +4,45 @@ from datetime import datetime
4
4
  from pathlib import Path
5
5
 
6
6
  import asdf
7
+ from astropy.io import fits
8
+ from dkist_header_validator import spec122_validator
9
+ from dkist_header_validator import spec214_validator
10
+ from dkist_processing_common.codecs.fits import fits_array_decoder
11
+ from dkist_processing_common.codecs.fits import fits_hdulist_encoder
7
12
  from dkist_processing_common.models.constants import BudName
13
+ from dkist_processing_common.models.fits_access import MetadataKey
8
14
  from dkist_processing_common.models.task_name import TaskName
9
15
  from dkist_processing_common.parsers.cs_step import CSStepFlower
10
16
  from dkist_processing_common.parsers.cs_step import NumCSStepBud
11
- from dkist_processing_common.parsers.task import parse_header_ip_task_with_gains
17
+ from dkist_processing_common.parsers.retarder import RetarderNameBud
18
+ from dkist_processing_common.parsers.task import PolcalTaskFlower
12
19
  from dkist_processing_common.parsers.task import TaskTypeFlower
20
+ from dkist_processing_common.parsers.task import parse_header_ip_task_with_gains
13
21
  from dkist_processing_common.parsers.time import ExposureTimeFlower
14
22
  from dkist_processing_common.parsers.time import ReadoutExpTimeFlower
15
23
  from dkist_processing_common.parsers.time import TaskExposureTimesBud
16
24
  from dkist_processing_common.parsers.time import TaskReadoutExpTimesBud
25
+ from dkist_processing_common.parsers.unique_bud import TaskUniqueBud
26
+ from dkist_processing_common.parsers.unique_bud import UniqueBud
17
27
  from dkist_processing_common.tasks import ParseL0InputDataBase
18
28
  from dkist_processing_common.tasks import WorkflowTaskBase
19
29
  from dkist_processing_common.tasks.mixin.globus import GlobusTransferItem
20
- from dkist_processing_common.tasks.mixin.input_dataset import InputDatasetMixin
21
30
  from dkist_processing_common.tasks.trial_output_data import TransferTrialData
22
- from dkist_service_configuration.logging import logger
31
+ from dkist_processing_math.statistics import average_numpy_arrays
32
+ from loguru import logger
23
33
 
24
34
  from dkist_processing_visp.models.constants import VispBudName
35
+ from dkist_processing_visp.models.constants import VispConstants
36
+ from dkist_processing_visp.models.fits_access import VispMetadataKey
37
+ from dkist_processing_visp.models.metric_code import VispMetricCode
25
38
  from dkist_processing_visp.models.parameters import VispParsingParameters
26
39
  from dkist_processing_visp.models.tags import VispTag
27
40
  from dkist_processing_visp.models.task_name import VispTaskName
28
41
  from dkist_processing_visp.parsers.modulator_states import ModulatorStateFlower
42
+ from dkist_processing_visp.parsers.spectrograph_configuration import IncidentLightAngleBud
43
+ from dkist_processing_visp.parsers.spectrograph_configuration import ReflectedLightAngleBud
29
44
  from dkist_processing_visp.parsers.time import DarkReadoutExpTimePickyBud
30
- from dkist_processing_visp.parsers.time import NonDarkTaskReadoutExpTimesBud
45
+ from dkist_processing_visp.parsers.time import NonDarkNonPolcalTaskReadoutExpTimesBud
31
46
  from dkist_processing_visp.parsers.visp_l0_fits_access import VispL0FitsAccess
32
47
  from dkist_processing_visp.tasks.visp_base import VispTaskBase
33
48
 
@@ -255,7 +270,9 @@ class SaveSolarCal(SaveTaskTags):
255
270
  @property
256
271
  def tag_lists_to_save(self) -> list[list[str]]:
257
272
  return super().tag_lists_to_save + [
258
- [VispTag.quality("TASK_TYPES"), VispTag.workflow_task("SolarCalibration")]
273
+ [VispTag.quality("TASK_TYPES"), VispTag.workflow_task("SolarCalibration")],
274
+ [VispTag.quality(VispMetricCode.solar_first_vignette)],
275
+ [VispTag.quality(VispMetricCode.solar_final_vignette)],
259
276
  ]
260
277
 
261
278
  @property
@@ -299,6 +316,22 @@ class LoadInstPolCal(LoadTaskTags):
299
316
  return "inst_pol_cal.asdf"
300
317
 
301
318
 
319
+ class SaveCalibratedData(SaveTaskTags):
320
+ @property
321
+ def tag_lists_to_save(self) -> list[str]:
322
+ return [VispTag.frame(), VispTag.calibrated()]
323
+
324
+ @property
325
+ def relative_save_file(self) -> str:
326
+ return "calibrated_science.asdf"
327
+
328
+
329
+ class LoadCalibratedData(LoadTaskTags):
330
+ @property
331
+ def relative_save_file(self) -> str:
332
+ return "calibrated_science.asdf"
333
+
334
+
302
335
  def set_observe_wavelength_task(wavelength: float = 630.0):
303
336
  class SetObserveWavelength(WorkflowTaskBase):
304
337
  def run(self):
@@ -322,6 +355,29 @@ class SetObserveExpTime(VispTaskBase):
322
355
  )
323
356
 
324
357
 
358
+ class SetCadenceConstants(WorkflowTaskBase):
359
+ def run(self):
360
+ self.constants._update(
361
+ {
362
+ BudName.average_cadence.value: 1.0,
363
+ BudName.minimum_cadence.value: 0.0,
364
+ BudName.maximum_cadence.value: 3.0,
365
+ BudName.variance_cadence.value: 1,
366
+ }
367
+ )
368
+
369
+
370
+ class SetAxesTypes(WorkflowTaskBase):
371
+ def run(self):
372
+ self.constants._update(
373
+ {
374
+ VispBudName.axis_1_type.value: "HPLT-TAN",
375
+ VispBudName.axis_2_type.value: "AWAV",
376
+ VispBudName.axis_3_type.value: "HPLN-TAN",
377
+ }
378
+ )
379
+
380
+
325
381
  class SetPolarimeterMode(VispTaskBase):
326
382
  def run(self):
327
383
  self.constants._update({VispBudName.polarimeter_mode.value: "observe_polarimetric"})
@@ -332,7 +388,7 @@ class SetNumModstates(VispTaskBase):
332
388
  self.constants._update({BudName.num_modstates.value: 10})
333
389
 
334
390
 
335
- class ParseCalOnlyL0InputData(ParseL0InputDataBase, InputDatasetMixin):
391
+ class ParseCalOnlyL0InputData(ParseL0InputDataBase):
336
392
  """
337
393
  Parse input ViSP data. Subclassed from the ParseL0InputDataBase task in dkist_processing_common to add ViSP specific parameters.
338
394
 
@@ -358,7 +414,7 @@ class ParseCalOnlyL0InputData(ParseL0InputDataBase, InputDatasetMixin):
358
414
  workflow_name=workflow_name,
359
415
  workflow_version=workflow_version,
360
416
  )
361
- self.parameters = VispParsingParameters(self.input_dataset_parameters)
417
+ self.parameters = VispParsingParameters(scratch=self.scratch)
362
418
 
363
419
  @property
364
420
  def fits_parsing_class(self):
@@ -368,38 +424,55 @@ class ParseCalOnlyL0InputData(ParseL0InputDataBase, InputDatasetMixin):
368
424
  @property
369
425
  def constant_buds(self):
370
426
  """Add ViSP specific constants to common constants."""
427
+ # TODO: Subclass ViSP parse task and *remove* unneeded things from this list
371
428
  return super().constant_buds + [
429
+ UniqueBud(constant_name=VispBudName.arm_id.value, metadata_key=VispMetadataKey.arm_id),
372
430
  NumCSStepBud(self.parameters.max_cs_step_time_sec),
373
- NonDarkTaskReadoutExpTimesBud(),
431
+ NonDarkNonPolcalTaskReadoutExpTimesBud(),
374
432
  DarkReadoutExpTimePickyBud(),
433
+ RetarderNameBud(),
434
+ IncidentLightAngleBud(),
435
+ ReflectedLightAngleBud(),
436
+ TaskUniqueBud(
437
+ constant_name=VispBudName.grating_constant_inverse_mm.value,
438
+ metadata_key=VispMetadataKey.grating_constant_inverse_mm,
439
+ ip_task_types=[TaskName.observe.value, TaskName.solar_gain.value],
440
+ task_type_parsing_function=parse_header_ip_task_with_gains,
441
+ ),
442
+ TaskUniqueBud(
443
+ constant_name=VispBudName.solar_gain_ip_start_time.value,
444
+ metadata_key=MetadataKey.ip_start_time,
445
+ ip_task_types=TaskName.solar_gain,
446
+ task_type_parsing_function=parse_header_ip_task_with_gains,
447
+ ),
375
448
  TaskExposureTimesBud(
376
449
  stem_name=VispBudName.lamp_exposure_times.value,
377
- ip_task_type=TaskName.lamp_gain.value,
450
+ ip_task_types=TaskName.lamp_gain.value,
378
451
  header_task_parsing_func=parse_header_ip_task_with_gains,
379
452
  ),
380
453
  TaskExposureTimesBud(
381
454
  stem_name=VispBudName.solar_exposure_times.value,
382
- ip_task_type=TaskName.solar_gain.value,
455
+ ip_task_types=TaskName.solar_gain.value,
383
456
  header_task_parsing_func=parse_header_ip_task_with_gains,
384
457
  ),
385
458
  TaskExposureTimesBud(
386
459
  stem_name=VispBudName.polcal_exposure_times.value,
387
- ip_task_type=TaskName.polcal.value,
460
+ ip_task_types=TaskName.polcal.value,
388
461
  header_task_parsing_func=parse_header_ip_task_with_gains,
389
462
  ),
390
463
  TaskReadoutExpTimesBud(
391
464
  stem_name=VispBudName.lamp_readout_exp_times.value,
392
- ip_task_type=TaskName.lamp_gain.value,
465
+ ip_task_types=TaskName.lamp_gain.value,
393
466
  header_task_parsing_func=parse_header_ip_task_with_gains,
394
467
  ),
395
468
  TaskReadoutExpTimesBud(
396
469
  stem_name=VispBudName.solar_readout_exp_times.value,
397
- ip_task_type=TaskName.solar_gain.value,
470
+ ip_task_types=TaskName.solar_gain.value,
398
471
  header_task_parsing_func=parse_header_ip_task_with_gains,
399
472
  ),
400
473
  TaskReadoutExpTimesBud(
401
474
  stem_name=VispBudName.polcal_readout_exp_times.value,
402
- ip_task_type=TaskName.polcal.value,
475
+ ip_task_types=TaskName.polcal.value,
403
476
  header_task_parsing_func=parse_header_ip_task_with_gains,
404
477
  ),
405
478
  ]
@@ -410,12 +483,21 @@ class ParseCalOnlyL0InputData(ParseL0InputDataBase, InputDatasetMixin):
410
483
  return super().tag_flowers + [
411
484
  CSStepFlower(max_cs_step_time_sec=self.parameters.max_cs_step_time_sec),
412
485
  TaskTypeFlower(header_task_parsing_func=parse_header_ip_task_with_gains),
486
+ PolcalTaskFlower(),
413
487
  ModulatorStateFlower(),
414
488
  ExposureTimeFlower(),
415
489
  ReadoutExpTimeFlower(),
416
490
  ]
417
491
 
418
492
 
493
+ class ValidateL1Output(VispTaskBase):
494
+ def run(self) -> None:
495
+ files = self.read(tags=[VispTag.output(), VispTag.frame()])
496
+ for f in files:
497
+ logger.info(f"Validating {f}")
498
+ spec214_validator.validate(f, extra=False)
499
+
500
+
419
501
  def transfer_trial_data_locally_task(
420
502
  trial_dir: str | Path,
421
503
  ):
@@ -443,3 +525,253 @@ def transfer_trial_data_locally_task(
443
525
  os.system(f"cp {frame.source_path} {frame.destination_path}")
444
526
 
445
527
  return LocalTrialData
528
+
529
+
530
+ def translate_122_to_214l0_task(suffix: str):
531
+ class Translate122To214L0(WorkflowTaskBase):
532
+ def run(self) -> None:
533
+ raw_dir = Path(self.scratch.scratch_base_path) / f"VISP{self.recipe_run_id:03n}"
534
+ if not os.path.exists(self.scratch.workflow_base_path):
535
+ os.makedirs(self.scratch.workflow_base_path)
536
+
537
+ if not raw_dir.exists():
538
+ raise FileNotFoundError(
539
+ f"Expected to find a raw VISP{{run_id:03n}} folder in {self.scratch.scratch_base_path}"
540
+ )
541
+
542
+ for file in raw_dir.glob(f"*{suffix}"):
543
+ translated_file_name = Path(self.scratch.workflow_base_path) / os.path.basename(
544
+ file
545
+ )
546
+ logger.info(f"Translating {file} -> {translated_file_name}")
547
+ hdl = fits.open(file)
548
+ i = 0
549
+ if hdl[i].data is None:
550
+ i = 1
551
+
552
+ header = spec122_validator.validate_and_translate_to_214_l0(
553
+ hdl[i].header, return_type=fits.HDUList
554
+ )[0].header
555
+
556
+ comp_hdu = fits.CompImageHDU(header=header, data=hdl[i].data)
557
+ comp_hdl = fits.HDUList([fits.PrimaryHDU(), comp_hdu])
558
+ comp_hdl.writeto(translated_file_name, overwrite=True)
559
+
560
+ hdl.close()
561
+ del hdl
562
+ comp_hdl.close()
563
+ del comp_hdl
564
+
565
+ return Translate122To214L0
566
+
567
+
568
+ def tag_inputs_task(suffix: str):
569
+ class TagInputs(WorkflowTaskBase):
570
+ def run(self) -> None:
571
+ logger.info(f"Looking in {os.path.abspath(self.scratch.workflow_base_path)}")
572
+ input_file_list = list(self.scratch.workflow_base_path.glob(f"*.{suffix}"))
573
+ if len(input_file_list) == 0:
574
+ raise FileNotFoundError(
575
+ f"Did not find any files matching '*.{suffix}' in {self.scratch.workflow_base_path}"
576
+ )
577
+ for file in input_file_list:
578
+ logger.info(f"Found {file}")
579
+ self.tag(path=file, tags=[VispTag.input(), VispTag.frame()])
580
+
581
+ return TagInputs
582
+
583
+
584
+ class TagSingleSolarGainAsScience(VispTaskBase):
585
+ """Do."""
586
+
587
+ def run(self) -> None:
588
+ """Do."""
589
+ tags = [
590
+ VispTag.input(),
591
+ VispTag.frame(),
592
+ VispTag.task_solar_gain(),
593
+ ]
594
+ file_list = list(self.read(tags=tags))
595
+ first_hdul = fits.open(file_list[0])
596
+ idx = 1 if first_hdul[0].data is None else 0
597
+ first_header = first_hdul[idx].header
598
+ logger.info(f"Averaging {len(file_list)} files")
599
+ arrays = self.read(tags=tags, decoder=fits_array_decoder)
600
+ avg_array = average_numpy_arrays(arrays=arrays)
601
+
602
+ hdul = fits.HDUList([fits.PrimaryHDU(data=avg_array, header=first_header)])
603
+ hdul[0].header[VispMetadataKey.raster_scan_step] = 0
604
+ hdul[0].header[VispMetadataKey.total_raster_steps] = 1
605
+ hdul[0].header[VispMetadataKey.modulator_state] = 1
606
+ hdul[0].header["VSPPOLMD"] = "observe_intensity"
607
+ # hdul[0].header["POL_NOIS"] = 0.666
608
+ # hdul[0].header["POL_SENS"] = 0.666
609
+
610
+ new_tags = [
611
+ VispTag.task_observe(),
612
+ VispTag.input(),
613
+ VispTag.frame(),
614
+ VispTag.map_scan(1),
615
+ VispTag.raster_step(0),
616
+ VispTag.modstate(1),
617
+ VispTag.readout_exp_time(self.constants.solar_readout_exp_times[0]),
618
+ ]
619
+ file_name = self.write(data=hdul, tags=new_tags, encoder=fits_hdulist_encoder)
620
+ final_tags = self.tags(self.scratch.workflow_base_path / file_name)
621
+ logger.info(f"after re-tagging tags for {str(file_name) = } are {final_tags}")
622
+
623
+ del self.constants._db_dict[VispBudName.polarimeter_mode.value]
624
+ self.constants._update(
625
+ {
626
+ VispBudName.num_map_scans.value: 1,
627
+ VispBudName.num_raster_steps.value: 1,
628
+ VispBudName.polarimeter_mode.value: "observe_intensity",
629
+ }
630
+ )
631
+
632
+
633
+ class TagModulatedSolarGainsAsScience(VispTaskBase):
634
+ """Do."""
635
+
636
+ def run(self) -> None:
637
+ """Do."""
638
+ for modstate in range(1, self.constants.num_modstates + 1):
639
+ tags = [
640
+ VispTag.task_solar_gain(),
641
+ VispTag.input(),
642
+ VispTag.frame(),
643
+ VispTag.modstate(modstate),
644
+ ]
645
+ file_list = list(self.read(tags=tags))
646
+ first_hdul = fits.open(file_list[0])
647
+ idx = 1 if first_hdul[0].data is None else 0
648
+ first_header = first_hdul[idx].header
649
+ logger.info(f"Averaging {len(file_list)} files")
650
+ arrays = self.read(tags=tags, decoder=fits_array_decoder)
651
+ avg_array = average_numpy_arrays(arrays=arrays)
652
+
653
+ hdul = fits.HDUList([fits.PrimaryHDU(data=avg_array, header=first_header)])
654
+ hdul[0].header[VispMetadataKey.raster_scan_step] = 0
655
+ hdul[0].header[VispMetadataKey.total_raster_steps] = 1
656
+ hdul[0].header[VispMetadataKey.modulator_state] = modstate
657
+ hdul[0].header["VSPPOLMD"] = "observe_polarimetric"
658
+
659
+ new_tags = [
660
+ VispTag.task_observe(),
661
+ VispTag.input(),
662
+ VispTag.frame(),
663
+ VispTag.map_scan(1),
664
+ VispTag.raster_step(0),
665
+ VispTag.modstate(modstate),
666
+ VispTag.readout_exp_time(self.constants.solar_readout_exp_times[0]),
667
+ ]
668
+ file_name = self.write(data=hdul, tags=new_tags, encoder=fits_hdulist_encoder)
669
+ final_tags = self.tags(self.scratch.workflow_base_path / file_name)
670
+ logger.info(f"after re-tagging tags for {str(file_name) = } are {final_tags}")
671
+
672
+ del self.constants._db_dict[VispBudName.polarimeter_mode.value]
673
+ self.constants._update(
674
+ {
675
+ VispBudName.num_map_scans.value: 1,
676
+ VispBudName.num_raster_steps.value: 1,
677
+ VispBudName.polarimeter_mode.value: "observe_polarimetric",
678
+ }
679
+ )
680
+ logger.info(f"{self.constants.correct_for_polarization = }")
681
+
682
+
683
+ class SaveSolarGainAsScience(SaveTaskTags):
684
+ @property
685
+ def tag_lists_to_save(self) -> list[str]:
686
+ return [VispTag.task_observe(), VispTag.input(), VispTag.frame()]
687
+
688
+ @property
689
+ def relative_save_file(self) -> str:
690
+ return "solar_gain_as_science.asdf"
691
+
692
+
693
+ def load_solar_gain_as_science_task(force_intensity_only: bool):
694
+ class LoadSolarGainAsScience(LoadTaskTags):
695
+ constants: VispConstants
696
+
697
+ @property
698
+ def constants_model_class(self):
699
+ """Get ViSP pipeline constants."""
700
+ return VispConstants
701
+
702
+ @property
703
+ def relative_save_file(self) -> str:
704
+ return "solar_gain_as_science.asdf"
705
+
706
+ def run(self):
707
+ super().run()
708
+ del self.constants._db_dict[VispBudName.polarimeter_mode.value]
709
+ self.constants._update(
710
+ {
711
+ VispBudName.num_map_scans.value: 1,
712
+ VispBudName.num_raster_steps.value: 1,
713
+ VispBudName.polarimeter_mode.value: "observe_intensity" if force_intensity_only else "observe_polarimetric", # fmt: skip
714
+ }
715
+ )
716
+ logger.info(f"{self.constants.correct_for_polarization = }")
717
+
718
+ return LoadSolarGainAsScience
719
+
720
+
721
+ class SavePolcalAsScience(WorkflowTaskBase):
722
+ constants: VispConstants
723
+
724
+ @property
725
+ def constants_model_class(self):
726
+ """Get ViSP pipeline constants."""
727
+ return VispConstants
728
+
729
+ @property
730
+ def tag_lists_to_save(self) -> list[str]:
731
+ return [VispTag.task_observe(), VispTag.input(), VispTag.frame()]
732
+
733
+ @property
734
+ def relative_save_file(self) -> str:
735
+ return "polcal_as_science.asdf"
736
+
737
+ def run(self):
738
+ file_tag_dict = dict()
739
+ tag_list_list = self.tag_lists_to_save
740
+ if isinstance(tag_list_list[0], str):
741
+ tag_list_list = [tag_list_list]
742
+
743
+ pcas_constants = {
744
+ VispBudName.num_map_scans.value: self.constants.num_map_scans,
745
+ VispBudName.num_raster_steps.value: self.constants.num_raster_steps,
746
+ }
747
+
748
+ for tags_to_save in tag_list_list:
749
+ path_list = self.read(tags=tags_to_save)
750
+ save_dir = self.scratch.workflow_base_path / Path(self.relative_save_file).stem
751
+ save_dir.mkdir(exist_ok=True)
752
+ for p in path_list:
753
+ copied_path = shutil.copy(str(p), save_dir)
754
+ tags = self.tags(p)
755
+ file_tag_dict[copied_path] = tags
756
+
757
+ full_save_file = self.scratch.workflow_base_path / self.relative_save_file
758
+ tree = {"file_tag_dict": file_tag_dict, "pcas_constants": pcas_constants}
759
+ af = asdf.AsdfFile(tree)
760
+ af.write_to(full_save_file)
761
+ logger.info(f"Saved polcal science frames to {full_save_file}")
762
+
763
+
764
+ class LoadPolcalAsScience(WorkflowTaskBase):
765
+ @property
766
+ def relative_save_file(self) -> str:
767
+ return "polcal_as_science.asdf"
768
+
769
+ def run(self):
770
+ full_save_file = self.scratch.workflow_base_path / self.relative_save_file
771
+ with asdf.open(full_save_file) as af:
772
+ for f, t in af.tree["file_tag_dict"].items():
773
+ self.tag(path=f, tags=t)
774
+
775
+ self.constants._db_dict.update(**af.tree["pcas_constants"])
776
+
777
+ logger.info(f"Loaded database entries from {full_save_file}")
@@ -1,6 +1,5 @@
1
1
  import pytest
2
2
  from dkist_processing_common._util.scratch import WorkflowFileSystem
3
- from dkist_processing_common.tests.conftest import FakeGQLClient
4
3
 
5
4
  from dkist_processing_visp.models.tags import VispTag
6
5
  from dkist_processing_visp.tasks.assemble_movie import AssembleVispMovie
@@ -49,9 +48,9 @@ def assemble_task_with_tagged_movie_frames(tmp_path, recipe_run_id, init_visp_co
49
48
  task._purge()
50
49
 
51
50
 
52
- def test_assemble_movie(assemble_task_with_tagged_movie_frames, mocker):
51
+ def test_assemble_movie(assemble_task_with_tagged_movie_frames, mocker, fake_gql_client):
53
52
  mocker.patch(
54
- "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
53
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
55
54
  )
56
55
  task, num_maps = assemble_task_with_tagged_movie_frames
57
56
  write_movie_frames_to_task(task, num_maps)
@@ -1,27 +1,65 @@
1
+ from typing import Generator
1
2
  from unittest.mock import MagicMock
2
3
 
4
+ import numpy as np
3
5
  import pytest
6
+ from dkist_processing_common._util.scratch import WorkflowFileSystem
7
+ from dkist_processing_common.codecs.asdf import asdf_encoder
8
+ from dkist_processing_common.codecs.quality import quality_data_decoder
4
9
 
10
+ from dkist_processing_visp.models.metric_code import VispMetricCode
11
+ from dkist_processing_visp.models.tags import VispTag
5
12
  from dkist_processing_visp.tasks.l1_output_data import VispAssembleQualityData
6
13
 
7
14
 
8
15
  @pytest.fixture
9
- def visp_assemble_quality_data_task(tmp_path, recipe_run_id) -> VispAssembleQualityData:
16
+ def visp_assemble_quality_data_task(
17
+ tmp_path, recipe_run_id
18
+ ) -> Generator[VispAssembleQualityData, None, None]:
10
19
 
11
20
  with VispAssembleQualityData(
12
21
  recipe_run_id=recipe_run_id, workflow_name="visp_assemble_quality", workflow_version="VX.Y"
13
22
  ) as task:
23
+ task.scratch = WorkflowFileSystem(scratch_base_path=tmp_path, recipe_run_id=recipe_run_id)
14
24
  yield task
15
25
  task._purge()
16
26
 
17
27
 
28
+ def write_raw_vignette_metrics_to_task(task):
29
+ for beam in [1, 2]:
30
+ dummy_vec = np.arange(10)
31
+ first_vignette_quality_outputs = {
32
+ "output_wave_vec": dummy_vec,
33
+ "input_spectrum": dummy_vec,
34
+ "best_fit_atlas": dummy_vec,
35
+ "best_fit_continuum": dummy_vec,
36
+ "residuals": dummy_vec,
37
+ }
38
+ task.write(
39
+ data=first_vignette_quality_outputs,
40
+ tags=[VispTag.quality(VispMetricCode.solar_first_vignette), VispTag.beam(beam)],
41
+ encoder=asdf_encoder,
42
+ )
43
+ final_correction_quality_outputs = {
44
+ "output_wave_vec": dummy_vec,
45
+ "median_spec": dummy_vec,
46
+ "low_deviation": dummy_vec,
47
+ "high_deviation": dummy_vec,
48
+ }
49
+ task.write(
50
+ data=final_correction_quality_outputs,
51
+ tags=[VispTag.quality(VispMetricCode.solar_final_vignette), VispTag.beam(beam)],
52
+ encoder=asdf_encoder,
53
+ )
54
+
55
+
18
56
  @pytest.fixture
19
57
  def dummy_quality_data() -> list[dict]:
20
58
  return [{"dummy_key": "dummy_value"}]
21
59
 
22
60
 
23
61
  @pytest.fixture
24
- def quality_assemble_data_mock(mocker, dummy_quality_data) -> MagicMock:
62
+ def common_quality_assemble_data_mock(mocker, dummy_quality_data) -> MagicMock:
25
63
  yield mocker.patch(
26
64
  "dkist_processing_common.tasks.mixin.quality.QualityMixin.quality_assemble_data",
27
65
  return_value=dummy_quality_data,
@@ -29,13 +67,60 @@ def quality_assemble_data_mock(mocker, dummy_quality_data) -> MagicMock:
29
67
  )
30
68
 
31
69
 
32
- def test_correct_polcal_label_list(visp_assemble_quality_data_task, quality_assemble_data_mock):
70
+ def test_vignette_metrics_built(visp_assemble_quality_data_task):
71
+ """
72
+ Given: A `VispAssembleQualityData` task with raw vignette metrics in scratch
73
+ When: Building the quality report data
74
+ Then: The vignette metrics are included in the data
75
+ """
76
+ task = visp_assemble_quality_data_task
77
+ write_raw_vignette_metrics_to_task(task)
78
+
79
+ task()
80
+
81
+ final_report_list = list(task.read(tags=VispTag.quality_data(), decoder=quality_data_decoder))
82
+ assert len(final_report_list) == 1
83
+ final_report = final_report_list[0]
84
+
85
+ initial_vignette_metrics = list(
86
+ filter(lambda i: i["name"].startswith("Initial Vignette Estimation"), final_report)
87
+ )
88
+ assert len(initial_vignette_metrics) == 2
89
+ facet_set = set()
90
+ for m in initial_vignette_metrics:
91
+ assert m["metric_code"] == VispMetricCode.solar_first_vignette.value
92
+ assert m["description"]
93
+ assert m["multi_plot_data"]
94
+ facet_set.add(m["facet"])
95
+
96
+ assert facet_set == {"BEAM_1", "BEAM_2"}
97
+
98
+ final_vignette_metrics = list(
99
+ filter(lambda i: i["name"].startswith("Final Vignette Estimation"), final_report)
100
+ )
101
+ assert len(final_vignette_metrics) == 2
102
+ facet_set = set()
103
+ for m in final_vignette_metrics:
104
+ assert m["metric_code"] == VispMetricCode.solar_final_vignette.value
105
+ assert m["description"]
106
+ assert m["multi_plot_data"]
107
+ facet_set.add(m["facet"])
108
+
109
+ assert facet_set == {"BEAM_1", "BEAM_2"}
110
+
111
+
112
+ def test_correct_polcal_label_list(
113
+ visp_assemble_quality_data_task, common_quality_assemble_data_mock
114
+ ):
33
115
  """
34
116
  Given: A VispAssembleQualityData task
35
117
  When: Calling the task
36
118
  Then: The correct polcal_label_list property is passed to .quality_assemble_data
37
119
  """
38
120
  task = visp_assemble_quality_data_task
121
+ write_raw_vignette_metrics_to_task(task)
39
122
 
40
123
  task()
41
- quality_assemble_data_mock.assert_called_once_with(task, polcal_label_list=["Beam 1", "Beam 2"])
124
+ common_quality_assemble_data_mock.assert_called_once_with(
125
+ task, polcal_label_list=["Beam 1", "Beam 2"]
126
+ )